gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-05 12:57:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics

Browse Source 
# Conflicts:
#	.github/workflows/build.yml
#	.github/workflows/ci.yml
#	Cargo.lock
#	Cargo.toml
#	appauth/src/token.rs
#	crates/config/src/config.rs
#	crates/event-notifier/examples/simple.rs
#	crates/event-notifier/src/global.rs
#	crates/event-notifier/src/lib.rs
#	crates/event-notifier/src/notifier.rs
#	crates/event-notifier/src/store.rs
#	crates/filemeta/src/filemeta.rs
#	crates/notify/examples/webhook.rs
#	crates/utils/Cargo.toml
#	ecstore/Cargo.toml
#	ecstore/src/cmd/bucket_replication.rs
#	ecstore/src/config/com.rs
#	ecstore/src/disk/error.rs
#	ecstore/src/disk/mod.rs
#	ecstore/src/set_disk.rs
#	ecstore/src/store_api.rs
#	ecstore/src/store_list_objects.rs
#	iam/Cargo.toml
#	iam/src/manager.rs
#	policy/Cargo.toml
#	rustfs/src/admin/rpc.rs
#	rustfs/src/main.rs
#	rustfs/src/storage/mod.rs
This commit is contained in:
houseme
2025-06-19 13:16:48 +08:00
parent 09e8bc8f02 450db14305
commit e6b019c29d
249 changed files with 25137 additions and 11731 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/utils/src/certs.rs
+12 -8
View File
@@ -396,10 +396,12 @@ mod tests {
// Should fail because no certificates found
let result = load_all_certs_from_directory(temp_dir.path().to_str().unwrap());
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("No valid certificate/private key pair found"));
assert!(
result
.unwrap_err()
.to_string()
.contains("No valid certificate/private key pair found")
);
}
#[test]
@@ -412,10 +414,12 @@ mod tests {
let result = load_all_certs_from_directory(unicode_dir.to_str().unwrap());
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("No valid certificate/private key pair found"));
assert!(
result
.unwrap_err()
.to_string()
.contains("No valid certificate/private key pair found")
);
}
#[test]
crates/utils/src/compress.rs
+318
View File
@@ -0,0 +1,318 @@
use std::io::Write;
use tokio::io;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub enum CompressionAlgorithm {
None,
Gzip,
Deflate,
Zstd,
#[default]
Lz4,
Brotli,
Snappy,
}
impl CompressionAlgorithm {
pub fn as_str(&self) -> &str {
match self {
CompressionAlgorithm::None => "none",
CompressionAlgorithm::Gzip => "gzip",
CompressionAlgorithm::Deflate => "deflate",
CompressionAlgorithm::Zstd => "zstd",
CompressionAlgorithm::Lz4 => "lz4",
CompressionAlgorithm::Brotli => "brotli",
CompressionAlgorithm::Snappy => "snappy",
}
}
}
impl std::fmt::Display for CompressionAlgorithm {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.as_str())
}
}
impl std::str::FromStr for CompressionAlgorithm {
type Err = std::io::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_lowercase().as_str() {
"gzip" => Ok(CompressionAlgorithm::Gzip),
"deflate" => Ok(CompressionAlgorithm::Deflate),
"zstd" => Ok(CompressionAlgorithm::Zstd),
"lz4" => Ok(CompressionAlgorithm::Lz4),
"brotli" => Ok(CompressionAlgorithm::Brotli),
"snappy" => Ok(CompressionAlgorithm::Snappy),
"none" => Ok(CompressionAlgorithm::None),
_ => Err(std::io::Error::other(format!("Unsupported compression algorithm: {}", s))),
}
}
}
pub fn compress_block(input: &[u8], algorithm: CompressionAlgorithm) -> Vec<u8> {
match algorithm {
CompressionAlgorithm::Gzip => {
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
let _ = encoder.write_all(input);
let _ = encoder.flush();
encoder.finish().unwrap_or_default()
}
CompressionAlgorithm::Deflate => {
let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
let _ = encoder.write_all(input);
let _ = encoder.flush();
encoder.finish().unwrap_or_default()
}
CompressionAlgorithm::Zstd => {
let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder");
let _ = encoder.write_all(input);
encoder.finish().unwrap_or_default()
}
CompressionAlgorithm::Lz4 => {
let mut encoder = lz4::EncoderBuilder::new().build(Vec::new()).expect("lz4 encoder");
let _ = encoder.write_all(input);
let (out, result) = encoder.finish();
result.expect("lz4 finish");
out
}
CompressionAlgorithm::Brotli => {
let mut out = Vec::new();
brotli::CompressorWriter::new(&mut out, 4096, 5, 22)
.write_all(input)
.expect("brotli compress");
out
}
CompressionAlgorithm::Snappy => {
let mut encoder = snap::write::FrameEncoder::new(Vec::new());
let _ = encoder.write_all(input);
encoder.into_inner().unwrap_or_default()
}
CompressionAlgorithm::None => input.to_vec(),
}
}
pub fn decompress_block(compressed: &[u8], algorithm: CompressionAlgorithm) -> io::Result<Vec<u8>> {
match algorithm {
CompressionAlgorithm::Gzip => {
let mut decoder = flate2::read::GzDecoder::new(std::io::Cursor::new(compressed));
let mut out = Vec::new();
std::io::Read::read_to_end(&mut decoder, &mut out)?;
Ok(out)
}
CompressionAlgorithm::Deflate => {
let mut decoder = flate2::read::DeflateDecoder::new(std::io::Cursor::new(compressed));
let mut out = Vec::new();
std::io::Read::read_to_end(&mut decoder, &mut out)?;
Ok(out)
}
CompressionAlgorithm::Zstd => {
let mut decoder = zstd::Decoder::new(std::io::Cursor::new(compressed))?;
let mut out = Vec::new();
std::io::Read::read_to_end(&mut decoder, &mut out)?;
Ok(out)
}
CompressionAlgorithm::Lz4 => {
let mut decoder = lz4::Decoder::new(std::io::Cursor::new(compressed)).expect("lz4 decoder");
let mut out = Vec::new();
std::io::Read::read_to_end(&mut decoder, &mut out)?;
Ok(out)
}
CompressionAlgorithm::Brotli => {
let mut out = Vec::new();
let mut decoder = brotli::Decompressor::new(std::io::Cursor::new(compressed), 4096);
std::io::Read::read_to_end(&mut decoder, &mut out)?;
Ok(out)
}
CompressionAlgorithm::Snappy => {
let mut decoder = snap::read::FrameDecoder::new(std::io::Cursor::new(compressed));
let mut out = Vec::new();
std::io::Read::read_to_end(&mut decoder, &mut out)?;
Ok(out)
}
CompressionAlgorithm::None => Ok(Vec::new()),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::str::FromStr;
use std::time::Instant;
#[test]
fn test_compress_decompress_gzip() {
let data = b"hello gzip compress";
let compressed = compress_block(data, CompressionAlgorithm::Gzip);
let decompressed = decompress_block(&compressed, CompressionAlgorithm::Gzip).unwrap();
assert_eq!(decompressed, data);
}
#[test]
fn test_compress_decompress_deflate() {
let data = b"hello deflate compress";
let compressed = compress_block(data, CompressionAlgorithm::Deflate);
let decompressed = decompress_block(&compressed, CompressionAlgorithm::Deflate).unwrap();
assert_eq!(decompressed, data);
}
#[test]
fn test_compress_decompress_zstd() {
let data = b"hello zstd compress";
let compressed = compress_block(data, CompressionAlgorithm::Zstd);
let decompressed = decompress_block(&compressed, CompressionAlgorithm::Zstd).unwrap();
assert_eq!(decompressed, data);
}
#[test]
fn test_compress_decompress_lz4() {
let data = b"hello lz4 compress";
let compressed = compress_block(data, CompressionAlgorithm::Lz4);
let decompressed = decompress_block(&compressed, CompressionAlgorithm::Lz4).unwrap();
assert_eq!(decompressed, data);
}
#[test]
fn test_compress_decompress_brotli() {
let data = b"hello brotli compress";
let compressed = compress_block(data, CompressionAlgorithm::Brotli);
let decompressed = decompress_block(&compressed, CompressionAlgorithm::Brotli).unwrap();
assert_eq!(decompressed, data);
}
#[test]
fn test_compress_decompress_snappy() {
let data = b"hello snappy compress";
let compressed = compress_block(data, CompressionAlgorithm::Snappy);
let decompressed = decompress_block(&compressed, CompressionAlgorithm::Snappy).unwrap();
assert_eq!(decompressed, data);
}
#[test]
fn test_from_str() {
assert_eq!(CompressionAlgorithm::from_str("gzip").unwrap(), CompressionAlgorithm::Gzip);
assert_eq!(CompressionAlgorithm::from_str("deflate").unwrap(), CompressionAlgorithm::Deflate);
assert_eq!(CompressionAlgorithm::from_str("zstd").unwrap(), CompressionAlgorithm::Zstd);
assert_eq!(CompressionAlgorithm::from_str("lz4").unwrap(), CompressionAlgorithm::Lz4);
assert_eq!(CompressionAlgorithm::from_str("brotli").unwrap(), CompressionAlgorithm::Brotli);
assert_eq!(CompressionAlgorithm::from_str("snappy").unwrap(), CompressionAlgorithm::Snappy);
assert!(CompressionAlgorithm::from_str("unknown").is_err());
}
#[test]
fn test_compare_compression_algorithms() {
use std::time::Instant;
let data = vec![42u8; 1024 * 100]; // 100KB of repetitive data
// let mut data = vec![0u8; 1024 * 1024];
// rand::thread_rng().fill(&mut data[..]);
let start = Instant::now();
let mut times = Vec::new();
times.push(("original", start.elapsed(), data.len()));
let start = Instant::now();
let gzip = compress_block(&data, CompressionAlgorithm::Gzip);
let gzip_time = start.elapsed();
times.push(("gzip", gzip_time, gzip.len()));
let start = Instant::now();
let deflate = compress_block(&data, CompressionAlgorithm::Deflate);
let deflate_time = start.elapsed();
times.push(("deflate", deflate_time, deflate.len()));
let start = Instant::now();
let zstd = compress_block(&data, CompressionAlgorithm::Zstd);
let zstd_time = start.elapsed();
times.push(("zstd", zstd_time, zstd.len()));
let start = Instant::now();
let lz4 = compress_block(&data, CompressionAlgorithm::Lz4);
let lz4_time = start.elapsed();
times.push(("lz4", lz4_time, lz4.len()));
let start = Instant::now();
let brotli = compress_block(&data, CompressionAlgorithm::Brotli);
let brotli_time = start.elapsed();
times.push(("brotli", brotli_time, brotli.len()));
let start = Instant::now();
let snappy = compress_block(&data, CompressionAlgorithm::Snappy);
let snappy_time = start.elapsed();
times.push(("snappy", snappy_time, snappy.len()));
println!("Compression results:");
for (name, dur, size) in &times {
println!("{}: {} bytes, {:?}", name, size, dur);
}
// All should decompress to the original
assert_eq!(decompress_block(&gzip, CompressionAlgorithm::Gzip).unwrap(), data);
assert_eq!(decompress_block(&deflate, CompressionAlgorithm::Deflate).unwrap(), data);
assert_eq!(decompress_block(&zstd, CompressionAlgorithm::Zstd).unwrap(), data);
assert_eq!(decompress_block(&lz4, CompressionAlgorithm::Lz4).unwrap(), data);
assert_eq!(decompress_block(&brotli, CompressionAlgorithm::Brotli).unwrap(), data);
assert_eq!(decompress_block(&snappy, CompressionAlgorithm::Snappy).unwrap(), data);
// All compressed results should not be empty
assert!(
!gzip.is_empty()
&& !deflate.is_empty()
&& !zstd.is_empty()
&& !lz4.is_empty()
&& !brotli.is_empty()
&& !snappy.is_empty()
);
}
#[test]
fn test_compression_benchmark() {
let sizes = [128 * 1024, 512 * 1024, 1024 * 1024];
let algorithms = [
CompressionAlgorithm::Gzip,
CompressionAlgorithm::Deflate,
CompressionAlgorithm::Zstd,
CompressionAlgorithm::Lz4,
CompressionAlgorithm::Brotli,
CompressionAlgorithm::Snappy,
];
println!("\n压缩算法基准测试结果:");
println!(
"{:<10} {:<10} {:<15} {:<15} {:<15}",
"数据大小", "算法", "压缩时间(ms)", "压缩后大小", "压缩率"
);
for size in sizes {
// 生成可压缩的数据（重复的文本模式）
let pattern = b"Hello, this is a test pattern that will be repeated multiple times to create compressible data. ";
let data: Vec<u8> = pattern.iter().cycle().take(size).copied().collect();
for algo in algorithms {
// 压缩测试
let start = Instant::now();
let compressed = compress_block(&data, algo);
let compress_time = start.elapsed();
// 解压测试
let start = Instant::now();
let _decompressed = decompress_block(&compressed, algo).unwrap();
let _decompress_time = start.elapsed();
// 计算压缩率
let compression_ratio = (size as f64 / compressed.len() as f64) as f32;
println!(
"{:<10} {:<10} {:<15.2} {:<15} {:<15.2}x",
format!("{}KB", size / 1024),
algo.as_str(),
compress_time.as_secs_f64() * 1000.0,
compressed.len(),
compression_ratio
);
// 验证解压结果
assert_eq!(_decompressed, data);
}
println!(); // 添加空行分隔不同大小的结果
}
}
}
crates/utils/src/crypto.rs
+39
View File
@@ -0,0 +1,39 @@
pub fn base64_encode(input: &[u8]) -> String {
base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
}
pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
}
pub fn hex(data: impl AsRef<[u8]>) -> String {
hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
}
// #[cfg(windows)]
// pub fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> {
// use sha2::{Digest, Sha256};
// <Sha256 as Digest>::digest(data)
// }
// #[cfg(not(windows))]
// pub fn sha256(data: &[u8]) -> impl AsRef<[u8]> {
// use openssl::hash::{Hasher, MessageDigest};
// let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
// h.update(data).unwrap();
// h.finish().unwrap()
// }
#[test]
fn test_base64_encoding_decoding() {
let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000";
let encoded_string = base64_encode(original_uuid_timestamp.as_bytes());
println!("Encoded: {}", &encoded_string);
let decoded_bytes = base64_decode(encoded_string.clone().as_bytes()).unwrap();
let decoded_string = String::from_utf8(decoded_bytes).unwrap();
assert_eq!(decoded_string, original_uuid_timestamp)
}
crates/utils/src/hash.rs
+206
View File
@@ -0,0 +1,206 @@
use highway::{HighwayHash, HighwayHasher, Key};
use md5::{Digest, Md5};
use serde::{Deserialize, Serialize};
use sha2::Sha256;
/// The fixed key for HighwayHash256. DO NOT change for compatibility.
const HIGHWAY_HASH256_KEY: [u64; 4] = [3, 4, 2, 1];
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)]
/// Supported hash algorithms for bitrot protection.
pub enum HashAlgorithm {
// SHA256 represents the SHA-256 hash function
SHA256,
// HighwayHash256 represents the HighwayHash-256 hash function
HighwayHash256,
// HighwayHash256S represents the Streaming HighwayHash-256 hash function
#[default]
HighwayHash256S,
// BLAKE2b512 represents the BLAKE2b-512 hash function
BLAKE2b512,
/// MD5 (128-bit)
Md5,
/// No hash (for testing or unprotected data)
None,
}
enum HashEncoded {
Md5([u8; 16]),
Sha256([u8; 32]),
HighwayHash256([u8; 32]),
HighwayHash256S([u8; 32]),
Blake2b512(blake3::Hash),
None,
}
impl AsRef<[u8]> for HashEncoded {
#[inline]
fn as_ref(&self) -> &[u8] {
match self {
HashEncoded::Md5(hash) => hash.as_ref(),
HashEncoded::Sha256(hash) => hash.as_ref(),
HashEncoded::HighwayHash256(hash) => hash.as_ref(),
HashEncoded::HighwayHash256S(hash) => hash.as_ref(),
HashEncoded::Blake2b512(hash) => hash.as_bytes(),
HashEncoded::None => &[],
}
}
}
#[inline]
fn u8x32_from_u64x4(input: [u64; 4]) -> [u8; 32] {
let mut output = [0u8; 32];
for (i, &n) in input.iter().enumerate() {
output[i * 8..(i + 1) * 8].copy_from_slice(&n.to_le_bytes());
}
output
}
impl HashAlgorithm {
/// Hash the input data and return the hash result as Vec<u8>.
pub fn hash_encode(&self, data: &[u8]) -> impl AsRef<[u8]> {
match self {
HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
HashAlgorithm::HighwayHash256 => {
let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
hasher.append(data);
HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()),
HashAlgorithm::HighwayHash256S => {
let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
hasher.append(data);
HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::BLAKE2b512 => HashEncoded::Blake2b512(blake3::hash(data)),
HashAlgorithm::None => HashEncoded::None,
}
}
/// Return the output size in bytes for the hash algorithm.
pub fn size(&self) -> usize {
match self {
HashAlgorithm::SHA256 => 32,
HashAlgorithm::HighwayHash256 => 32,
HashAlgorithm::HighwayHash256S => 32,
HashAlgorithm::BLAKE2b512 => 32, // blake3 outputs 32 bytes by default
HashAlgorithm::Md5 => 16,
HashAlgorithm::None => 0,
}
}
}
use crc32fast::Hasher;
use siphasher::sip::SipHasher;
pub fn sip_hash(key: &str, cardinality: usize, id: &[u8; 16]) -> usize {
// 你的密钥，必须是 16 字节
// 计算字符串的 SipHash 值
let result = SipHasher::new_with_key(id).hash(key.as_bytes());
result as usize % cardinality
}
pub fn crc_hash(key: &str, cardinality: usize) -> usize {
let mut hasher = Hasher::new(); // 创建一个新的哈希器
hasher.update(key.as_bytes()); // 更新哈希状态，添加数据
let checksum = hasher.finalize();
checksum as usize % cardinality
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_hash_algorithm_sizes() {
assert_eq!(HashAlgorithm::Md5.size(), 16);
assert_eq!(HashAlgorithm::HighwayHash256.size(), 32);
assert_eq!(HashAlgorithm::HighwayHash256S.size(), 32);
assert_eq!(HashAlgorithm::SHA256.size(), 32);
assert_eq!(HashAlgorithm::BLAKE2b512.size(), 32);
assert_eq!(HashAlgorithm::None.size(), 0);
}
#[test]
fn test_hash_encode_none() {
let data = b"test data";
let hash = HashAlgorithm::None.hash_encode(data);
let hash = hash.as_ref();
assert_eq!(hash.len(), 0);
}
#[test]
fn test_hash_encode_md5() {
let data = b"test data";
let hash = HashAlgorithm::Md5.hash_encode(data);
let hash = hash.as_ref();
assert_eq!(hash.len(), 16);
// MD5 should be deterministic
let hash2 = HashAlgorithm::Md5.hash_encode(data);
let hash2 = hash2.as_ref();
assert_eq!(hash, hash2);
}
#[test]
fn test_hash_encode_highway() {
let data = b"test data";
let hash = HashAlgorithm::HighwayHash256.hash_encode(data);
let hash = hash.as_ref();
assert_eq!(hash.len(), 32);
// HighwayHash should be deterministic
let hash2 = HashAlgorithm::HighwayHash256.hash_encode(data);
let hash2 = hash2.as_ref();
assert_eq!(hash, hash2);
}
#[test]
fn test_hash_encode_sha256() {
let data = b"test data";
let hash = HashAlgorithm::SHA256.hash_encode(data);
let hash = hash.as_ref();
assert_eq!(hash.len(), 32);
// SHA256 should be deterministic
let hash2 = HashAlgorithm::SHA256.hash_encode(data);
let hash2 = hash2.as_ref();
assert_eq!(hash, hash2);
}
#[test]
fn test_hash_encode_blake2b512() {
let data = b"test data";
let hash = HashAlgorithm::BLAKE2b512.hash_encode(data);
let hash = hash.as_ref();
assert_eq!(hash.len(), 32); // blake3 outputs 32 bytes by default
// BLAKE2b512 should be deterministic
let hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data);
let hash2 = hash2.as_ref();
assert_eq!(hash, hash2);
}
#[test]
fn test_different_data_different_hashes() {
let data1 = b"test data 1";
let data2 = b"test data 2";
let md5_hash1 = HashAlgorithm::Md5.hash_encode(data1);
let md5_hash2 = HashAlgorithm::Md5.hash_encode(data2);
assert_ne!(md5_hash1.as_ref(), md5_hash2.as_ref());
let highway_hash1 = HashAlgorithm::HighwayHash256.hash_encode(data1);
let highway_hash2 = HashAlgorithm::HighwayHash256.hash_encode(data2);
assert_ne!(highway_hash1.as_ref(), highway_hash2.as_ref());
let sha256_hash1 = HashAlgorithm::SHA256.hash_encode(data1);
let sha256_hash2 = HashAlgorithm::SHA256.hash_encode(data2);
assert_ne!(sha256_hash1.as_ref(), sha256_hash2.as_ref());
let blake_hash1 = HashAlgorithm::BLAKE2b512.hash_encode(data1);
let blake_hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data2);
assert_ne!(blake_hash1.as_ref(), blake_hash2.as_ref());
}
}
crates/utils/src/io.rs
+231
View File
@@ -0,0 +1,231 @@
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
/// Write all bytes from buf to writer, returning the total number of bytes written.
pub async fn write_all<W: AsyncWrite + Send + Sync + Unpin>(writer: &mut W, buf: &[u8]) -> std::io::Result<usize> {
let mut total = 0;
while total < buf.len() {
match writer.write(&buf[total..]).await {
Ok(0) => {
break;
}
Ok(n) => total += n,
Err(e) => return Err(e),
}
}
Ok(total)
}
/// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before.
/// Like Go's io.ReadFull.
#[allow(dead_code)]
pub async fn read_full<R: AsyncRead + Send + Sync + Unpin>(mut reader: R, mut buf: &mut [u8]) -> std::io::Result<usize> {
let mut total = 0;
while !buf.is_empty() {
let n = match reader.read(buf).await {
Ok(n) => n,
Err(e) => {
if total == 0 {
return Err(e);
}
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
format!("read {} bytes, error: {}", total, e),
));
}
};
if n == 0 {
if total > 0 {
return Ok(total);
}
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "early EOF"));
}
buf = &mut buf[n..];
total += n;
}
Ok(total)
}
/// Encodes a u64 into buf and returns the number of bytes written.
/// Panics if buf is too small.
pub fn put_uvarint(buf: &mut [u8], x: u64) -> usize {
let mut i = 0;
let mut x = x;
while x >= 0x80 {
buf[i] = (x as u8) | 0x80;
x >>= 7;
i += 1;
}
buf[i] = x as u8;
i + 1
}
pub fn put_uvarint_len(x: u64) -> usize {
let mut i = 0;
let mut x = x;
while x >= 0x80 {
x >>= 7;
i += 1;
}
i + 1
}
/// Decodes a u64 from buf and returns (value, number of bytes read).
/// If buf is too small, returns (0, 0).
/// If overflow, returns (0, -(n as isize)), where n is the number of bytes read.
pub fn uvarint(buf: &[u8]) -> (u64, isize) {
let mut x: u64 = 0;
let mut s: u32 = 0;
for (i, &b) in buf.iter().enumerate() {
if i == 10 {
// MaxVarintLen64 = 10
return (0, -((i + 1) as isize));
}
if b < 0x80 {
if i == 9 && b > 1 {
return (0, -((i + 1) as isize));
}
return (x | ((b as u64) << s), (i + 1) as isize);
}
x |= ((b & 0x7F) as u64) << s;
s += 7;
}
(0, 0)
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::BufReader;
#[tokio::test]
async fn test_read_full_exact() {
// let data = b"abcdef";
let data = b"channel async callback test data!";
let mut reader = BufReader::new(&data[..]);
let size = data.len();
let mut total = 0;
let mut rev = vec![0u8; size];
let mut count = 0;
while total < size {
let mut buf = [0u8; 8];
let n = read_full(&mut reader, &mut buf).await.unwrap();
total += n;
rev[total - n..total].copy_from_slice(&buf[..n]);
count += 1;
println!("count: {}, total: {}, n: {}", count, total, n);
}
assert_eq!(total, size);
assert_eq!(&rev, data);
}
#[tokio::test]
async fn test_read_full_short() {
let data = b"abc";
let mut reader = BufReader::new(&data[..]);
let mut buf = [0u8; 6];
let n = read_full(&mut reader, &mut buf).await.unwrap();
assert_eq!(n, 3);
assert_eq!(&buf[..n], data);
}
#[tokio::test]
async fn test_read_full_1m() {
let size = 1024 * 1024;
let data = vec![42u8; size];
let mut reader = BufReader::new(&data[..]);
let mut buf = vec![0u8; size / 3];
read_full(&mut reader, &mut buf).await.unwrap();
assert_eq!(buf, data[..size / 3]);
}
#[test]
fn test_put_uvarint_and_uvarint_zero() {
let mut buf = [0u8; 16];
let n = put_uvarint(&mut buf, 0);
let (decoded, m) = uvarint(&buf[..n]);
assert_eq!(decoded, 0);
assert_eq!(m as usize, n);
}
#[test]
fn test_put_uvarint_and_uvarint_max() {
let mut buf = [0u8; 16];
let n = put_uvarint(&mut buf, u64::MAX);
let (decoded, m) = uvarint(&buf[..n]);
assert_eq!(decoded, u64::MAX);
assert_eq!(m as usize, n);
}
#[test]
fn test_put_uvarint_and_uvarint_various() {
let mut buf = [0u8; 16];
for &v in &[1u64, 127, 128, 255, 300, 16384, u32::MAX as u64] {
let n = put_uvarint(&mut buf, v);
let (decoded, m) = uvarint(&buf[..n]);
assert_eq!(decoded, v, "decode mismatch for {}", v);
assert_eq!(m as usize, n, "length mismatch for {}", v);
}
}
#[test]
fn test_uvarint_incomplete() {
let buf = [0x80u8, 0x80, 0x80];
let (v, n) = uvarint(&buf);
assert_eq!(v, 0);
assert_eq!(n, 0);
}
#[test]
fn test_uvarint_overflow_case() {
let buf = [0xFFu8; 11];
let (v, n) = uvarint(&buf);
assert_eq!(v, 0);
assert!(n < 0);
}
#[tokio::test]
async fn test_write_all_basic() {
let data = b"hello world!";
let mut buf = Vec::new();
let n = write_all(&mut buf, data).await.unwrap();
assert_eq!(n, data.len());
assert_eq!(&buf, data);
}
#[tokio::test]
async fn test_write_all_partial() {
struct PartialWriter {
inner: Vec<u8>,
max_write: usize,
}
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::AsyncWrite;
impl AsyncWrite for PartialWriter {
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
let n = buf.len().min(self.max_write);
self.inner.extend_from_slice(&buf[..n]);
Poll::Ready(Ok(n))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
let data = b"abcdefghijklmnopqrstuvwxyz";
let mut writer = PartialWriter {
inner: Vec::new(),
max_write: 5,
};
let n = write_all(&mut writer, data).await.unwrap();
assert_eq!(n, data.len());
assert_eq!(&writer.inner, data);
}
}
crates/utils/src/lib.rs
+36 -3
View File
@@ -1,11 +1,44 @@
#[cfg(feature = "tls")]
mod certs;
pub mod certs;
#[cfg(feature = "ip")]
mod ip;
pub mod ip;
#[cfg(feature = "net")]
mod net;
pub mod net;
#[cfg(feature = "net")]
pub use net::*;
#[cfg(feature = "io")]
pub mod io;
#[cfg(feature = "hash")]
pub mod hash;
#[cfg(feature = "os")]
pub mod os;
#[cfg(feature = "path")]
pub mod path;
#[cfg(feature = "string")]
pub mod string;
#[cfg(feature = "crypto")]
pub mod crypto;
#[cfg(feature = "compress")]
pub mod compress;
#[cfg(feature = "tls")]
pub use certs::*;
#[cfg(feature = "hash")]
pub use hash::*;
#[cfg(feature = "io")]
pub use io::*;
#[cfg(feature = "ip")]
pub use ip::*;
#[cfg(feature = "crypto")]
pub use crypto::*;
#[cfg(feature = "compress")]
pub use compress::*;
crates/utils/src/net.rs
+498
View File
@@ -1 +1,499 @@
use lazy_static::lazy_static;
use std::{
collections::HashSet,
fmt::Display,
net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
};
use url::Host;
lazy_static! {
static ref LOCAL_IPS: Vec<IpAddr> = must_get_local_ips().unwrap();
}
/// helper for validating if the provided arg is an ip address.
pub fn is_socket_addr(addr: &str) -> bool {
// TODO IPv6 zone information?
addr.parse::<SocketAddr>().is_ok() || addr.parse::<IpAddr>().is_ok()
}
/// checks if server_addr is valid and local host.
pub fn check_local_server_addr(server_addr: &str) -> std::io::Result<SocketAddr> {
let addr: Vec<SocketAddr> = match server_addr.to_socket_addrs() {
Ok(addr) => addr.collect(),
Err(err) => return Err(std::io::Error::other(err)),
};
// 0.0.0.0 is a wildcard address and refers to local network
// addresses. I.e, 0.0.0.0:9000 like ":9000" refers to port
// 9000 on localhost.
for a in addr {
if a.ip().is_unspecified() {
return Ok(a);
}
let host = match a {
SocketAddr::V4(a) => Host::<&str>::Ipv4(*a.ip()),
SocketAddr::V6(a) => Host::Ipv6(*a.ip()),
};
if is_local_host(host, 0, 0)? {
return Ok(a);
}
}
Err(std::io::Error::other("host in server address should be this server"))
}
/// checks if the given parameter correspond to one of
/// the local IP of the current machine
pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> std::io::Result<bool> {
let local_set: HashSet<IpAddr> = LOCAL_IPS.iter().copied().collect();
let is_local_host = match host {
Host::Domain(domain) => {
let ips = match (domain, 0).to_socket_addrs().map(|v| v.map(|v| v.ip()).collect::<Vec<_>>()) {
Ok(ips) => ips,
Err(err) => return Err(std::io::Error::other(err)),
};
ips.iter().any(|ip| local_set.contains(ip))
}
Host::Ipv4(ip) => local_set.contains(&IpAddr::V4(ip)),
Host::Ipv6(ip) => local_set.contains(&IpAddr::V6(ip)),
};
if port > 0 {
return Ok(is_local_host && port == local_port);
}
Ok(is_local_host)
}
/// returns IP address of given host.
pub fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
match host {
Host::Domain(domain) => match (domain, 0)
.to_socket_addrs()
.map(|v| v.map(|v| v.ip()).collect::<HashSet<_>>())
{
Ok(ips) => Ok(ips),
Err(err) => Err(std::io::Error::other(err)),
},
Host::Ipv4(ip) => {
let mut set = HashSet::with_capacity(1);
set.insert(IpAddr::V4(ip));
Ok(set)
}
Host::Ipv6(ip) => {
let mut set = HashSet::with_capacity(1);
set.insert(IpAddr::V6(ip));
Ok(set)
}
}
}
pub fn get_available_port() -> u16 {
TcpListener::bind("0.0.0.0:0").unwrap().local_addr().unwrap().port()
}
/// returns IPs of local interface
pub fn must_get_local_ips() -> std::io::Result<Vec<IpAddr>> {
match netif::up() {
Ok(up) => Ok(up.map(|x| x.address().to_owned()).collect()),
Err(err) => Err(std::io::Error::other(format!("Unable to get IP addresses of this host: {}", err))),
}
}
#[derive(Debug, Clone)]
pub struct XHost {
pub name: String,
pub port: u16,
pub is_port_set: bool,
}
impl Display for XHost {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if !self.is_port_set {
write!(f, "{}", self.name)
} else if self.name.contains(':') {
write!(f, "[{}]:{}", self.name, self.port)
} else {
write!(f, "{}:{}", self.name, self.port)
}
}
}
impl TryFrom<String> for XHost {
type Error = std::io::Error;
fn try_from(value: String) -> std::result::Result<Self, Self::Error> {
if let Some(addr) = value.to_socket_addrs()?.next() {
Ok(Self {
name: addr.ip().to_string(),
port: addr.port(),
is_port_set: addr.port() > 0,
})
} else {
Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "value invalid"))
}
}
}
/// parses the address string, process the ":port" format for double-stack binding,
/// and resolve the host name or IP address. If the port is 0, an available port is assigned.
pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result<SocketAddr> {
let resolved_addr: SocketAddr = if let Some(port) = addr_str.strip_prefix(":") {
// Process the ":port" format for double stack binding
let port_str = port;
let port: u16 = port_str
.parse()
.map_err(|e| std::io::Error::other(format!("Invalid port format: {}, err:{:?}", addr_str, e)))?;
let final_port = if port == 0 {
get_available_port() // assume get_available_port is available here
} else {
port
};
// Using IPv6 without address specified [::], it should handle both IPv4 and IPv6
SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), final_port)
} else {
// Use existing logic to handle regular address formats
let mut addr = check_local_server_addr(addr_str)?; // assume check_local_server_addr is available here
if addr.port() == 0 {
addr.set_port(get_available_port());
}
addr
};
Ok(resolved_addr)
}
#[cfg(test)]
mod test {
use std::net::{Ipv4Addr, Ipv6Addr};
use super::*;
#[test]
fn test_is_socket_addr() {
let test_cases = [
// Valid IP addresses
("192.168.1.0", true),
("127.0.0.1", true),
("10.0.0.1", true),
("0.0.0.0", true),
("255.255.255.255", true),
// Valid IPv6 addresses
("2001:db8::1", true),
("::1", true),
("::", true),
("fe80::1", true),
// Valid socket addresses
("192.168.1.0:8080", true),
("127.0.0.1:9000", true),
("[2001:db8::1]:9000", true),
("[::1]:8080", true),
("0.0.0.0:0", true),
// Invalid addresses
("localhost", false),
("localhost:9000", false),
("example.com", false),
("example.com:8080", false),
("http://192.168.1.0", false),
("http://192.168.1.0:9000", false),
("256.256.256.256", false),
("192.168.1", false),
("192.168.1.0.1", false),
("", false),
(":", false),
(":::", false),
("invalid_ip", false),
];
for (addr, expected) in test_cases {
let result = is_socket_addr(addr);
assert_eq!(expected, result, "addr: '{}', expected: {}, got: {}", addr, expected, result);
}
}
#[test]
fn test_check_local_server_addr() {
// Test valid local addresses
let valid_cases = ["localhost:54321", "127.0.0.1:9000", "0.0.0.0:9000", "[::1]:8080", "::1:8080"];
for addr in valid_cases {
let result = check_local_server_addr(addr);
assert!(result.is_ok(), "Expected '{}' to be valid, but got error: {:?}", addr, result);
}
// Test invalid addresses
let invalid_cases = [
("localhost", "invalid socket address"),
("", "invalid socket address"),
("example.org:54321", "host in server address should be this server"),
("8.8.8.8:53", "host in server address should be this server"),
(":-10", "invalid port value"),
("invalid:port", "invalid port value"),
];
for (addr, expected_error_pattern) in invalid_cases {
let result = check_local_server_addr(addr);
assert!(result.is_err(), "Expected '{}' to be invalid, but it was accepted: {:?}", addr, result);
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains(expected_error_pattern) || error_msg.contains("invalid socket address"),
"Error message '{}' doesn't contain expected pattern '{}' for address '{}'",
error_msg,
expected_error_pattern,
addr
);
}
}
#[test]
fn test_is_local_host() {
// Test localhost domain
let localhost_host = Host::Domain("localhost");
assert!(is_local_host(localhost_host, 0, 0).unwrap());
// Test loopback IP addresses
let ipv4_loopback = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
assert!(is_local_host(ipv4_loopback, 0, 0).unwrap());
let ipv6_loopback = Host::Ipv6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1));
assert!(is_local_host(ipv6_loopback, 0, 0).unwrap());
// Test port matching
let localhost_with_port1 = Host::Domain("localhost");
assert!(is_local_host(localhost_with_port1, 8080, 8080).unwrap());
let localhost_with_port2 = Host::Domain("localhost");
assert!(!is_local_host(localhost_with_port2, 8080, 9000).unwrap());
// Test non-local host
let external_host = Host::Ipv4(Ipv4Addr::new(8, 8, 8, 8));
assert!(!is_local_host(external_host, 0, 0).unwrap());
// Test invalid domain should return error
let invalid_host = Host::Domain("invalid.nonexistent.domain.example");
assert!(is_local_host(invalid_host, 0, 0).is_err());
}
#[test]
fn test_get_host_ip() {
// Test IPv4 address
let ipv4_host = Host::Ipv4(Ipv4Addr::new(192, 168, 1, 1));
let ipv4_result = get_host_ip(ipv4_host).unwrap();
assert_eq!(ipv4_result.len(), 1);
assert!(ipv4_result.contains(&IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))));
// Test IPv6 address
let ipv6_host = Host::Ipv6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1));
let ipv6_result = get_host_ip(ipv6_host).unwrap();
assert_eq!(ipv6_result.len(), 1);
assert!(ipv6_result.contains(&IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1))));
// Test localhost domain
let localhost_host = Host::Domain("localhost");
let localhost_result = get_host_ip(localhost_host).unwrap();
assert!(!localhost_result.is_empty());
// Should contain at least loopback address
assert!(
localhost_result.contains(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)))
|| localhost_result.contains(&IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)))
);
// Test invalid domain
let invalid_host = Host::Domain("invalid.nonexistent.domain.example");
assert!(get_host_ip(invalid_host).is_err());
}
#[test]
fn test_get_available_port() {
let port1 = get_available_port();
let port2 = get_available_port();
// Port should be in valid range (u16 max is always <= 65535)
assert!(port1 > 0);
assert!(port2 > 0);
// Different calls should typically return different ports
assert_ne!(port1, port2);
}
#[test]
fn test_must_get_local_ips() {
let local_ips = must_get_local_ips().unwrap();
let local_set: HashSet<IpAddr> = local_ips.into_iter().collect();
// Should contain loopback addresses
assert!(local_set.contains(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))));
// Should not be empty
assert!(!local_set.is_empty());
// All IPs should be valid
for ip in &local_set {
match ip {
IpAddr::V4(_) | IpAddr::V6(_) => {} // Valid
}
}
}
#[test]
fn test_xhost_display() {
// Test without port
let host_no_port = XHost {
name: "example.com".to_string(),
port: 0,
is_port_set: false,
};
assert_eq!(host_no_port.to_string(), "example.com");
// Test with port (IPv4-like name)
let host_with_port = XHost {
name: "192.168.1.1".to_string(),
port: 8080,
is_port_set: true,
};
assert_eq!(host_with_port.to_string(), "192.168.1.1:8080");
// Test with port (IPv6-like name)
let host_ipv6_with_port = XHost {
name: "2001:db8::1".to_string(),
port: 9000,
is_port_set: true,
};
assert_eq!(host_ipv6_with_port.to_string(), "[2001:db8::1]:9000");
// Test domain name with port
let host_domain_with_port = XHost {
name: "example.com".to_string(),
port: 443,
is_port_set: true,
};
assert_eq!(host_domain_with_port.to_string(), "example.com:443");
}
#[test]
fn test_xhost_try_from() {
// Test valid IPv4 address with port
let result = XHost::try_from("192.168.1.1:8080".to_string()).unwrap();
assert_eq!(result.name, "192.168.1.1");
assert_eq!(result.port, 8080);
assert!(result.is_port_set);
// Test valid IPv4 address without port
let result = XHost::try_from("192.168.1.1:0".to_string()).unwrap();
assert_eq!(result.name, "192.168.1.1");
assert_eq!(result.port, 0);
assert!(!result.is_port_set);
// Test valid IPv6 address with port
let result = XHost::try_from("[2001:db8::1]:9000".to_string()).unwrap();
assert_eq!(result.name, "2001:db8::1");
assert_eq!(result.port, 9000);
assert!(result.is_port_set);
// Test localhost with port (localhost may resolve to either IPv4 or IPv6)
let result = XHost::try_from("localhost:3000".to_string()).unwrap();
// localhost can resolve to either 127.0.0.1 or ::1 depending on system configuration
assert!(result.name == "127.0.0.1" || result.name == "::1");
assert_eq!(result.port, 3000);
assert!(result.is_port_set);
// Test invalid format
let result = XHost::try_from("invalid_format".to_string());
assert!(result.is_err());
// Test empty string
let result = XHost::try_from("".to_string());
assert!(result.is_err());
}
#[test]
fn test_parse_and_resolve_address() {
// Test port-only format
let result = parse_and_resolve_address(":8080").unwrap();
assert_eq!(result.ip(), IpAddr::V6(Ipv6Addr::UNSPECIFIED));
assert_eq!(result.port(), 8080);
// Test port-only format with port 0 (should get available port)
let result = parse_and_resolve_address(":0").unwrap();
assert_eq!(result.ip(), IpAddr::V6(Ipv6Addr::UNSPECIFIED));
assert!(result.port() > 0);
// Test localhost with port
let result = parse_and_resolve_address("localhost:9000").unwrap();
assert_eq!(result.port(), 9000);
// Test localhost with port 0 (should get available port)
let result = parse_and_resolve_address("localhost:0").unwrap();
assert!(result.port() > 0);
// Test 0.0.0.0 with port
let result = parse_and_resolve_address("0.0.0.0:7000").unwrap();
assert_eq!(result.ip(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
assert_eq!(result.port(), 7000);
// Test invalid port format
let result = parse_and_resolve_address(":invalid_port");
assert!(result.is_err());
// Test invalid address
let result = parse_and_resolve_address("example.org:8080");
assert!(result.is_err());
}
#[test]
fn test_edge_cases() {
// Test empty string for is_socket_addr
assert!(!is_socket_addr(""));
// Test single colon for is_socket_addr
assert!(!is_socket_addr(":"));
// Test malformed IPv6 for is_socket_addr
assert!(!is_socket_addr("[::]"));
assert!(!is_socket_addr("[::1"));
// Test very long strings
let long_string = "a".repeat(1000);
assert!(!is_socket_addr(&long_string));
// Test unicode characters
assert!(!is_socket_addr("测试.example.com"));
// Test special characters
assert!(!is_socket_addr("test@example.com:8080"));
assert!(!is_socket_addr("http://example.com:8080"));
}
#[test]
fn test_boundary_values() {
// Test port boundaries
assert!(is_socket_addr("127.0.0.1:0"));
assert!(is_socket_addr("127.0.0.1:65535"));
assert!(!is_socket_addr("127.0.0.1:65536"));
// Test IPv4 boundaries
assert!(is_socket_addr("0.0.0.0"));
assert!(is_socket_addr("255.255.255.255"));
assert!(!is_socket_addr("256.0.0.0"));
assert!(!is_socket_addr("0.0.0.256"));
// Test XHost with boundary ports
let host_max_port = XHost {
name: "example.com".to_string(),
port: 65535,
is_port_set: true,
};
assert_eq!(host_max_port.to_string(), "example.com:65535");
let host_zero_port = XHost {
name: "example.com".to_string(),
port: 0,
is_port_set: true,
};
assert_eq!(host_zero_port.to_string(), "example.com:0");
}
}
crates/utils/src/os/linux.rs
+176
View File
@@ -0,0 +1,176 @@
use nix::sys::stat::{self, stat};
use nix::sys::statfs::{self, FsType, statfs};
use std::fs::File;
use std::io::{self, BufRead, Error, ErrorKind};
use std::path::Path;
use super::{DiskInfo, IOStats};
/// Returns total and free bytes available in a directory, e.g. `/`.
pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
let stat_fs = statfs(p.as_ref())?;
let bsize = stat_fs.block_size() as u64;
let bfree = stat_fs.blocks_free() as u64;
let bavail = stat_fs.blocks_available() as u64;
let blocks = stat_fs.blocks() as u64;
let reserved = match bfree.checked_sub(bavail) {
Some(reserved) => reserved,
None => {
return Err(Error::other(format!(
"detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run 'fsck'",
bavail,
bfree,
p.as_ref().display()
)));
}
};
let total = match blocks.checked_sub(reserved) {
Some(total) => total * bsize,
None => {
return Err(Error::other(format!(
"detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run 'fsck'",
reserved,
blocks,
p.as_ref().display()
)));
}
};
let free = bavail * bsize;
let used = match total.checked_sub(free) {
Some(used) => used,
None => {
return Err(Error::other(format!(
"detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'",
free,
total,
p.as_ref().display()
)));
}
};
let st = stat(p.as_ref())?;
Ok(DiskInfo {
total,
free,
used,
files: stat_fs.files(),
ffree: stat_fs.files_free(),
fstype: get_fs_type(stat_fs.filesystem_type()).to_string(),
major: stat::major(st.st_dev),
minor: stat::minor(st.st_dev),
..Default::default()
})
}
/// Returns the filesystem type of the underlying mounted filesystem
///
/// TODO The following mapping could not find the corresponding constant in `nix`:
///
/// "137d" => "EXT",
/// "4244" => "HFS",
/// "5346544e" => "NTFS",
/// "61756673" => "AUFS",
/// "ef51" => "EXT2OLD",
/// "2fc12fc1" => "zfs",
/// "ff534d42" => "cifs",
/// "53464846" => "wslfs",
fn get_fs_type(fs_type: FsType) -> &'static str {
match fs_type {
statfs::TMPFS_MAGIC => "TMPFS",
statfs::MSDOS_SUPER_MAGIC => "MSDOS",
// statfs::XFS_SUPER_MAGIC => "XFS",
statfs::NFS_SUPER_MAGIC => "NFS",
statfs::EXT4_SUPER_MAGIC => "EXT4",
statfs::ECRYPTFS_SUPER_MAGIC => "ecryptfs",
statfs::OVERLAYFS_SUPER_MAGIC => "overlayfs",
statfs::REISERFS_SUPER_MAGIC => "REISERFS",
_ => "UNKNOWN",
}
}
pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
let stat1 = stat(disk1)?;
let stat2 = stat(disk2)?;
Ok(stat1.st_dev == stat2.st_dev)
}
pub fn get_drive_stats(major: u32, minor: u32) -> std::io::Result<IOStats> {
read_drive_stats(&format!("/sys/dev/block/{}:{}/stat", major, minor))
}
fn read_drive_stats(stats_file: &str) -> std::io::Result<IOStats> {
let stats = read_stat(stats_file)?;
if stats.len() < 11 {
return Err(Error::new(
ErrorKind::InvalidData,
format!("found invalid format while reading {}", stats_file),
));
}
let mut io_stats = IOStats {
read_ios: stats[0],
read_merges: stats[1],
read_sectors: stats[2],
read_ticks: stats[3],
write_ios: stats[4],
write_merges: stats[5],
write_sectors: stats[6],
write_ticks: stats[7],
current_ios: stats[8],
total_ticks: stats[9],
req_ticks: stats[10],
..Default::default()
};
if stats.len() > 14 {
io_stats.discard_ios = stats[11];
io_stats.discard_merges = stats[12];
io_stats.discard_sectors = stats[13];
io_stats.discard_ticks = stats[14];
}
Ok(io_stats)
}
fn read_stat(file_name: &str) -> std::io::Result<Vec<u64>> {
// Open file
let path = Path::new(file_name);
let file = File::open(path)?;
// Create a BufReader
let reader = io::BufReader::new(file);
// Read first line
let mut stats = Vec::new();
if let Some(line) = reader.lines().next() {
let line = line?;
// Split line and parse as u64
// https://rust-lang.github.io/rust-clippy/master/index.html#trim_split_whitespace
for token in line.split_whitespace() {
let ui64: u64 = token
.parse()
.map_err(|e| Error::new(ErrorKind::InvalidData, format!("failed to parse '{}' as u64: {}", token, e)))?;
stats.push(ui64);
}
}
Ok(stats)
}
#[cfg(test)]
mod test {
use super::get_drive_stats;
#[ignore] // FIXME: failed in github actions
#[test]
fn test_stats() {
let major = 7;
let minor = 11;
let s = get_drive_stats(major, minor).unwrap();
println!("{:?}", s);
}
}
crates/utils/src/os/mod.rs
+111
View File
@@ -0,0 +1,111 @@
#[cfg(target_os = "linux")]
mod linux;
#[cfg(all(unix, not(target_os = "linux")))]
mod unix;
#[cfg(target_os = "windows")]
mod windows;
#[cfg(target_os = "linux")]
pub use linux::{get_drive_stats, get_info, same_disk};
// pub use linux::same_disk;
#[cfg(all(unix, not(target_os = "linux")))]
pub use unix::{get_drive_stats, get_info, same_disk};
#[cfg(target_os = "windows")]
pub use windows::{get_drive_stats, get_info, same_disk};
#[derive(Debug, Default, PartialEq)]
pub struct IOStats {
pub read_ios: u64,
pub read_merges: u64,
pub read_sectors: u64,
pub read_ticks: u64,
pub write_ios: u64,
pub write_merges: u64,
pub write_sectors: u64,
pub write_ticks: u64,
pub current_ios: u64,
pub total_ticks: u64,
pub req_ticks: u64,
pub discard_ios: u64,
pub discard_merges: u64,
pub discard_sectors: u64,
pub discard_ticks: u64,
pub flush_ios: u64,
pub flush_ticks: u64,
}
#[derive(Debug, Default, PartialEq)]
pub struct DiskInfo {
pub total: u64,
pub free: u64,
pub used: u64,
pub files: u64,
pub ffree: u64,
pub fstype: String,
pub major: u64,
pub minor: u64,
pub name: String,
pub rotational: bool,
pub nrrequests: u64,
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
#[test]
fn test_get_info_valid_path() {
let temp_dir = tempfile::tempdir().unwrap();
let info = get_info(temp_dir.path()).unwrap();
println!("Disk Info: {:?}", info);
assert!(info.total > 0);
assert!(info.free > 0);
assert!(info.used > 0);
assert!(info.files > 0);
assert!(info.ffree > 0);
assert!(!info.fstype.is_empty());
}
#[test]
fn test_get_info_invalid_path() {
let invalid_path = PathBuf::from("/invalid/path");
let result = get_info(&invalid_path);
assert!(result.is_err());
}
#[test]
fn test_same_disk_same_path() {
let temp_dir = tempfile::tempdir().unwrap();
let path = temp_dir.path().to_str().unwrap();
let result = same_disk(path, path).unwrap();
assert!(result);
}
#[test]
fn test_same_disk_different_paths() {
let temp_dir1 = tempfile::tempdir().unwrap();
let temp_dir2 = tempfile::tempdir().unwrap();
let path1 = temp_dir1.path().to_str().unwrap();
let path2 = temp_dir2.path().to_str().unwrap();
let result = same_disk(path1, path2).unwrap();
// Since both temporary directories are created in the same file system,
// they should be on the same disk in most cases
println!("Path1: {}, Path2: {}, Same disk: {}", path1, path2, result);
// Test passes if the function doesn't panic - the actual result depends on test environment
}
#[ignore] // FIXME: failed in github actions
#[test]
fn test_get_drive_stats_default() {
let stats = get_drive_stats(0, 0).unwrap();
assert_eq!(stats, IOStats::default());
}
}
crates/utils/src/os/unix.rs
+72
View File
@@ -0,0 +1,72 @@
use super::{DiskInfo, IOStats};
use nix::sys::{stat::stat, statfs::statfs};
use std::io::Error;
use std::path::Path;
/// Returns total and free bytes available in a directory, e.g. `/`.
pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
let stat = statfs(p.as_ref())?;
let bsize = stat.block_size() as u64;
let bfree = stat.blocks_free();
let bavail = stat.blocks_available();
let blocks = stat.blocks();
let reserved = match bfree.checked_sub(bavail) {
Some(reserved) => reserved,
None => {
return Err(Error::other(format!(
"detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run 'fsck'",
bavail,
bfree,
p.as_ref().display()
)));
}
};
let total = match blocks.checked_sub(reserved) {
Some(total) => total * bsize,
None => {
return Err(Error::other(format!(
"detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run 'fsck'",
reserved,
blocks,
p.as_ref().display()
)));
}
};
let free = bavail * bsize;
let used = match total.checked_sub(free) {
Some(used) => used,
None => {
return Err(Error::other(format!(
"detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'",
free,
total,
p.as_ref().display()
)));
}
};
Ok(DiskInfo {
total,
free,
used,
files: stat.files(),
ffree: stat.files_free(),
fstype: stat.filesystem_type_name().to_string(),
..Default::default()
})
}
pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
let stat1 = stat(disk1)?;
let stat2 = stat(disk2)?;
Ok(stat1.st_dev == stat2.st_dev)
}
pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
Ok(IOStats::default())
}
crates/utils/src/os/windows.rs
+142
View File
@@ -0,0 +1,142 @@
#![allow(unsafe_code)] // TODO: audit unsafe code
use super::{DiskInfo, IOStats};
use std::io::{Error, ErrorKind};
use std::mem;
use std::os::windows::ffi::OsStrExt;
use std::path::Path;
use winapi::shared::minwindef::{DWORD, MAX_PATH};
use winapi::shared::ntdef::ULARGE_INTEGER;
use winapi::um::fileapi::{GetDiskFreeSpaceExW, GetDiskFreeSpaceW, GetVolumeInformationW, GetVolumePathNameW};
use winapi::um::winnt::{LPCWSTR, WCHAR};
/// Returns total and free bytes available in a directory, e.g. `C:\`.
pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
let path_wide: Vec<WCHAR> = p
.as_ref()
.canonicalize()?
.into_os_string()
.encode_wide()
.chain(std::iter::once(0)) // Null-terminate the string
.collect();
let mut lp_free_bytes_available: ULARGE_INTEGER = unsafe { mem::zeroed() };
let mut lp_total_number_of_bytes: ULARGE_INTEGER = unsafe { mem::zeroed() };
let mut lp_total_number_of_free_bytes: ULARGE_INTEGER = unsafe { mem::zeroed() };
let success = unsafe {
GetDiskFreeSpaceExW(
path_wide.as_ptr(),
&mut lp_free_bytes_available,
&mut lp_total_number_of_bytes,
&mut lp_total_number_of_free_bytes,
)
};
if success == 0 {
return Err(Error::last_os_error());
}
let total = unsafe { *lp_total_number_of_bytes.QuadPart() };
let free = unsafe { *lp_total_number_of_free_bytes.QuadPart() };
if free > total {
return Err(Error::new(
ErrorKind::Other,
format!(
"detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'",
free,
total,
p.as_ref().display()
),
));
}
let mut lp_sectors_per_cluster: DWORD = 0;
let mut lp_bytes_per_sector: DWORD = 0;
let mut lp_number_of_free_clusters: DWORD = 0;
let mut lp_total_number_of_clusters: DWORD = 0;
let success = unsafe {
GetDiskFreeSpaceW(
path_wide.as_ptr(),
&mut lp_sectors_per_cluster,
&mut lp_bytes_per_sector,
&mut lp_number_of_free_clusters,
&mut lp_total_number_of_clusters,
)
};
if success == 0 {
return Err(Error::last_os_error());
}
Ok(DiskInfo {
total,
free,
used: total - free,
files: lp_total_number_of_clusters as u64,
ffree: lp_number_of_free_clusters as u64,
fstype: get_fs_type(&path_wide)?,
..Default::default()
})
}
/// Returns leading volume name.
fn get_volume_name(v: &[WCHAR]) -> std::io::Result<LPCWSTR> {
let volume_name_size: DWORD = MAX_PATH as _;
let mut lp_volume_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH];
let success = unsafe { GetVolumePathNameW(v.as_ptr(), lp_volume_name_buffer.as_mut_ptr(), volume_name_size) };
if success == 0 {
return Err(Error::last_os_error());
}
Ok(lp_volume_name_buffer.as_ptr())
}
fn utf16_to_string(v: &[WCHAR]) -> String {
let len = v.iter().position(|&x| x == 0).unwrap_or(v.len());
String::from_utf16_lossy(&v[..len])
}
/// Returns the filesystem type of the underlying mounted filesystem
fn get_fs_type(p: &[WCHAR]) -> std::io::Result<String> {
let path = get_volume_name(p)?;
let volume_name_size: DWORD = MAX_PATH as _;
let n_file_system_name_size: DWORD = MAX_PATH as _;
let mut lp_volume_serial_number: DWORD = 0;
let mut lp_maximum_component_length: DWORD = 0;
let mut lp_file_system_flags: DWORD = 0;
let mut lp_volume_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH];
let mut lp_file_system_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH];
let success = unsafe {
GetVolumeInformationW(
path,
lp_volume_name_buffer.as_mut_ptr(),
volume_name_size,
&mut lp_volume_serial_number,
&mut lp_maximum_component_length,
&mut lp_file_system_flags,
lp_file_system_name_buffer.as_mut_ptr(),
n_file_system_name_size,
)
};
if success == 0 {
return Err(Error::last_os_error());
}
Ok(utf16_to_string(&lp_file_system_name_buffer))
}
pub fn same_disk(_disk1: &str, _disk2: &str) -> std::io::Result<bool> {
Ok(false)
}
pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
Ok(IOStats::default())
}
crates/utils/src/path.rs
+308
View File
@@ -0,0 +1,308 @@
use std::path::Path;
use std::path::PathBuf;
pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
pub const SLASH_SEPARATOR: &str = "/";
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/";
pub fn has_suffix(s: &str, suffix: &str) -> bool {
if cfg!(target_os = "windows") {
s.to_lowercase().ends_with(&suffix.to_lowercase())
} else {
s.ends_with(suffix)
}
}
pub fn encode_dir_object(object: &str) -> String {
if has_suffix(object, SLASH_SEPARATOR) {
format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX)
} else {
object.to_string()
}
}
pub fn is_dir_object(object: &str) -> bool {
let obj = encode_dir_object(object);
obj.ends_with(GLOBAL_DIR_SUFFIX)
}
#[allow(dead_code)]
pub fn decode_dir_object(object: &str) -> String {
if has_suffix(object, GLOBAL_DIR_SUFFIX) {
format!("{}{}", object.trim_end_matches(GLOBAL_DIR_SUFFIX), SLASH_SEPARATOR)
} else {
object.to_string()
}
}
pub fn retain_slash(s: &str) -> String {
if s.is_empty() {
return s.to_string();
}
if s.ends_with(SLASH_SEPARATOR) {
s.to_string()
} else {
format!("{}{}", s, SLASH_SEPARATOR)
}
}
pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
}
pub fn has_prefix(s: &str, prefix: &str) -> bool {
if cfg!(target_os = "windows") {
return strings_has_prefix_fold(s, prefix);
}
s.starts_with(prefix)
}
pub fn path_join(elem: &[PathBuf]) -> PathBuf {
let mut joined_path = PathBuf::new();
for path in elem {
joined_path.push(path);
}
joined_path
}
pub fn path_join_buf(elements: &[&str]) -> String {
let trailing_slash = !elements.is_empty() && elements.last().unwrap().ends_with(SLASH_SEPARATOR);
let mut dst = String::new();
let mut added = 0;
for e in elements {
if added > 0 || !e.is_empty() {
if added > 0 {
dst.push_str(SLASH_SEPARATOR);
}
dst.push_str(e);
added += e.len();
}
}
let result = dst.to_string();
let cpath = Path::new(&result).components().collect::<PathBuf>();
let clean_path = cpath.to_string_lossy();
if trailing_slash {
return format!("{}{}", clean_path, SLASH_SEPARATOR);
}
clean_path.to_string()
}
pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) {
let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR);
if let Some(m) = path.find(SLASH_SEPARATOR) {
return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string());
}
(path.to_string(), "".to_string())
}
pub fn path_to_bucket_object(s: &str) -> (String, String) {
path_to_bucket_object_with_base_path("", s)
}
pub fn base_dir_from_prefix(prefix: &str) -> String {
let mut base_dir = dir(prefix).to_owned();
if base_dir == "." || base_dir == "./" || base_dir == "/" {
base_dir = "".to_owned();
}
if !prefix.contains('/') {
base_dir = "".to_owned();
}
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) {
base_dir.push_str(SLASH_SEPARATOR);
}
base_dir
}
pub struct LazyBuf {
s: String,
buf: Option<Vec<u8>>,
w: usize,
}
impl LazyBuf {
pub fn new(s: String) -> Self {
LazyBuf { s, buf: None, w: 0 }
}
pub fn index(&self, i: usize) -> u8 {
if let Some(ref buf) = self.buf {
buf[i]
} else {
self.s.as_bytes()[i]
}
}
pub fn append(&mut self, c: u8) {
if self.buf.is_none() {
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c {
self.w += 1;
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
pub fn clean(path: &str) -> String {
if path.is_empty() {
return ".".to_string();
}
let rooted = path.starts_with('/');
let n = path.len();
let mut out = LazyBuf::new(path.to_string());
let mut r = 0;
let mut dotdot = 0;
if rooted {
out.append(b'/');
r = 1;
dotdot = 1;
}
while r < n {
match path.as_bytes()[r] {
b'/' => {
// Empty path element
r += 1;
}
b'.' if r + 1 == n || path.as_bytes()[r + 1] == b'/' => {
// . element
r += 1;
}
b'.' if path.as_bytes()[r + 1] == b'.' && (r + 2 == n || path.as_bytes()[r + 2] == b'/') => {
// .. element: remove to last /
r += 2;
if out.w > dotdot {
// Can backtrack
out.w -= 1;
while out.w > dotdot && out.index(out.w) != b'/' {
out.w -= 1;
}
} else if !rooted {
// Cannot backtrack but not rooted, so append .. element.
if out.w > 0 {
out.append(b'/');
}
out.append(b'.');
out.append(b'.');
dotdot = out.w;
}
}
_ => {
// Real path element.
// Add slash if needed
if (rooted && out.w != 1) || (!rooted && out.w != 0) {
out.append(b'/');
}
// Copy element
while r < n && path.as_bytes()[r] != b'/' {
out.append(path.as_bytes()[r]);
r += 1;
}
}
}
}
// Turn empty string into "."
if out.w == 0 {
return ".".to_string();
}
out.string()
}
pub fn split(path: &str) -> (&str, &str) {
// Find the last occurrence of the '/' character
if let Some(i) = path.rfind('/') {
// Return the directory (up to and including the last '/') and the file name
return (&path[..i + 1], &path[i + 1..]);
}
// If no '/' is found, return an empty string for the directory and the whole path as the file name
(path, "")
}
pub fn dir(path: &str) -> String {
let (a, _) = split(path);
clean(a)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_base_dir_from_prefix() {
let a = "da/";
println!("---- in {}", a);
let a = base_dir_from_prefix(a);
println!("---- out {}", a);
}
#[test]
fn test_clean() {
assert_eq!(clean(""), ".");
assert_eq!(clean("abc"), "abc");
assert_eq!(clean("abc/def"), "abc/def");
assert_eq!(clean("a/b/c"), "a/b/c");
assert_eq!(clean("."), ".");
assert_eq!(clean(".."), "..");
assert_eq!(clean("../.."), "../..");
assert_eq!(clean("../../abc"), "../../abc");
assert_eq!(clean("/abc"), "/abc");
assert_eq!(clean("/"), "/");
assert_eq!(clean("abc/"), "abc");
assert_eq!(clean("abc/def/"), "abc/def");
assert_eq!(clean("a/b/c/"), "a/b/c");
assert_eq!(clean("./"), ".");
assert_eq!(clean("../"), "..");
assert_eq!(clean("../../"), "../..");
assert_eq!(clean("/abc/"), "/abc");
assert_eq!(clean("abc//def//ghi"), "abc/def/ghi");
assert_eq!(clean("//abc"), "/abc");
assert_eq!(clean("///abc"), "/abc");
assert_eq!(clean("//abc//"), "/abc");
assert_eq!(clean("abc//"), "abc");
assert_eq!(clean("abc/./def"), "abc/def");
assert_eq!(clean("/./abc/def"), "/abc/def");
assert_eq!(clean("abc/."), "abc");
assert_eq!(clean("abc/./../def"), "def");
assert_eq!(clean("abc//./../def"), "def");
assert_eq!(clean("abc/../../././../def"), "../../def");
assert_eq!(clean("abc/def/ghi/../jkl"), "abc/def/jkl");
assert_eq!(clean("abc/def/../ghi/../jkl"), "abc/jkl");
assert_eq!(clean("abc/def/.."), "abc");
assert_eq!(clean("abc/def/../.."), ".");
assert_eq!(clean("/abc/def/../.."), "/");
assert_eq!(clean("abc/def/../../.."), "..");
assert_eq!(clean("/abc/def/../../.."), "/");
assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno");
}
}
crates/utils/src/string.rs
+666
View File
@@ -0,0 +1,666 @@
use lazy_static::*;
use regex::Regex;
use std::io::{Error, Result};
pub fn parse_bool(str: &str) -> Result<bool> {
match str {
"1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Ok(true),
"0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Ok(false),
_ => Err(Error::other(format!("ParseBool: parsing {}", str))),
}
}
pub fn match_simple(pattern: &str, name: &str) -> bool {
if pattern.is_empty() {
return name == pattern;
}
if pattern == "*" {
return true;
}
// Do an extended wildcard '*' and '?' match.
deep_match_rune(name.as_bytes(), pattern.as_bytes(), true)
}
pub fn match_pattern(pattern: &str, name: &str) -> bool {
if pattern.is_empty() {
return name == pattern;
}
if pattern == "*" {
return true;
}
// Do an extended wildcard '*' and '?' match.
deep_match_rune(name.as_bytes(), pattern.as_bytes(), false)
}
pub fn has_pattern(patterns: &[&str], match_str: &str) -> bool {
for pattern in patterns {
if match_simple(pattern, match_str) {
return true;
}
}
false
}
pub fn has_string_suffix_in_slice(str: &str, list: &[&str]) -> bool {
let str = str.to_lowercase();
for v in list {
if *v == "*" {
return true;
}
if str.ends_with(&v.to_lowercase()) {
return true;
}
}
false
}
fn deep_match_rune(str_: &[u8], pattern: &[u8], simple: bool) -> bool {
let (mut str_, mut pattern) = (str_, pattern);
while !pattern.is_empty() {
match pattern[0] as char {
'*' => {
return if pattern.len() == 1 {
true
} else {
deep_match_rune(str_, &pattern[1..], simple)
|| (!str_.is_empty() && deep_match_rune(&str_[1..], pattern, simple))
};
}
'?' => {
if str_.is_empty() {
return simple;
}
}
_ => {
if str_.is_empty() || str_[0] != pattern[0] {
return false;
}
}
}
str_ = &str_[1..];
pattern = &pattern[1..];
}
str_.is_empty() && pattern.is_empty()
}
pub fn match_as_pattern_prefix(pattern: &str, text: &str) -> bool {
let mut i = 0;
while i < text.len() && i < pattern.len() {
match pattern.as_bytes()[i] as char {
'*' => return true,
'?' => i += 1,
_ => {
if pattern.as_bytes()[i] != text.as_bytes()[i] {
return false;
}
}
}
i += 1;
}
text.len() <= pattern.len()
}
lazy_static! {
static ref ELLIPSES_RE: Regex = Regex::new(r"(.*)(\{[0-9a-z]*\.\.\.[0-9a-z]*\})(.*)").unwrap();
}
/// Ellipses constants
const OPEN_BRACES: &str = "{";
const CLOSE_BRACES: &str = "}";
const ELLIPSES: &str = "...";
/// ellipses pattern, describes the range and also the
/// associated prefix and suffixes.
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Pattern {
pub prefix: String,
pub suffix: String,
pub seq: Vec<String>,
}
impl Pattern {
/// expands a ellipses pattern.
pub fn expand(&self) -> Vec<String> {
let mut ret = Vec::with_capacity(self.suffix.len());
for v in self.seq.iter() {
match (self.prefix.is_empty(), self.suffix.is_empty()) {
(false, true) => ret.push(format!("{}{}", self.prefix, v)),
(true, false) => ret.push(format!("{}{}", v, self.suffix)),
(true, true) => ret.push(v.to_string()),
(false, false) => ret.push(format!("{}{}{}", self.prefix, v, self.suffix)),
}
}
ret
}
pub fn len(&self) -> usize {
self.seq.len()
}
pub fn is_empty(&self) -> bool {
self.seq.is_empty()
}
}
/// contains a list of patterns provided in the input.
#[derive(Debug, PartialEq, Eq)]
pub struct ArgPattern {
inner: Vec<Pattern>,
}
impl AsRef<Vec<Pattern>> for ArgPattern {
fn as_ref(&self) -> &Vec<Pattern> {
&self.inner
}
}
impl AsMut<Vec<Pattern>> for ArgPattern {
fn as_mut(&mut self) -> &mut Vec<Pattern> {
&mut self.inner
}
}
impl ArgPattern {
pub fn new(inner: Vec<Pattern>) -> Self {
Self { inner }
}
/// expands all the ellipses patterns in the given argument.
pub fn expand(&self) -> Vec<Vec<String>> {
let ret: Vec<Vec<String>> = self.inner.iter().map(|v| v.expand()).collect();
Self::arg_expander(&ret)
}
/// recursively expands labels into its respective forms.
fn arg_expander(lbs: &[Vec<String>]) -> Vec<Vec<String>> {
if lbs.len() == 1 {
return lbs[0].iter().map(|v| vec![v.to_string()]).collect();
}
let mut ret = Vec::new();
let (first, others) = lbs.split_at(1);
for bs in first[0].iter() {
let ots = Self::arg_expander(others);
for mut obs in ots {
obs.push(bs.to_string());
ret.push(obs);
}
}
ret
}
/// returns the total number of sizes in the given patterns.
pub fn total_sizes(&self) -> usize {
self.inner.iter().fold(1, |acc, v| acc * v.seq.len())
}
}
/// finds all ellipses patterns, recursively and parses the ranges numerically.
pub fn find_ellipses_patterns(arg: &str) -> Result<ArgPattern> {
let mut parts = match ELLIPSES_RE.captures(arg) {
Some(caps) => caps,
None => {
return Err(Error::other(format!(
"Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4",
arg
)));
}
};
let mut patterns = Vec::new();
while let Some(prefix) = parts.get(1) {
let seq = parse_ellipses_range(parts[2].into())?;
match ELLIPSES_RE.captures(prefix.into()) {
Some(cs) => {
patterns.push(Pattern {
seq,
prefix: String::new(),
suffix: parts[3].into(),
});
parts = cs;
}
None => {
patterns.push(Pattern {
seq,
prefix: prefix.as_str().to_owned(),
suffix: parts[3].into(),
});
break;
}
};
}
// Check if any of the prefix or suffixes now have flower braces
// left over, in such a case we generally think that there is
// perhaps a typo in users input and error out accordingly.
for p in patterns.iter() {
if p.prefix.contains(OPEN_BRACES)
|| p.prefix.contains(CLOSE_BRACES)
|| p.suffix.contains(OPEN_BRACES)
|| p.suffix.contains(CLOSE_BRACES)
{
return Err(Error::other(format!(
"Invalid ellipsis format in ({}), Ellipsis range must be provided in format {{N...M}} where N and M are positive integers, M must be greater than N, with an allowed minimum range of 4",
arg
)));
}
}
Ok(ArgPattern::new(patterns))
}
/// returns true if input arg has ellipses type pattern.
pub fn has_ellipses<T: AsRef<str>>(s: &[T]) -> bool {
let pattern = [ELLIPSES, OPEN_BRACES, CLOSE_BRACES];
s.iter().any(|v| pattern.iter().any(|p| v.as_ref().contains(p)))
}
/// Parses an ellipses range pattern of following style
///
/// example:
/// {1...64}
/// {33...64}
pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
if !pattern.contains(OPEN_BRACES) {
return Err(Error::other("Invalid argument"));
}
if !pattern.contains(CLOSE_BRACES) {
return Err(Error::other("Invalid argument"));
}
let ellipses_range: Vec<&str> = pattern
.trim_start_matches(OPEN_BRACES)
.trim_end_matches(CLOSE_BRACES)
.split(ELLIPSES)
.collect();
if ellipses_range.len() != 2 {
return Err(Error::other("Invalid argument"));
}
// TODO: Add support for hexadecimals.
let start = ellipses_range[0].parse::<usize>().map_err(Error::other)?;
let end = ellipses_range[1].parse::<usize>().map_err(Error::other)?;
if start > end {
return Err(Error::other("Invalid argument:range start cannot be bigger than end"));
}
let mut ret: Vec<String> = Vec::with_capacity(end - start + 1);
for i in start..=end {
if ellipses_range[0].starts_with('0') && ellipses_range[0].len() > 1 {
ret.push(format!("{:0width$}", i, width = ellipses_range[1].len()));
} else {
ret.push(format!("{}", i));
}
}
Ok(ret)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_has_ellipses() {
// Tests for all args without ellipses.
let test_cases = [
(1, vec!["64"], false),
// Found flower braces, still attempt to parse and throw an error.
(2, vec!["{1..64}"], true),
(3, vec!["{1..2..}"], true),
// Test for valid input.
(4, vec!["1...64"], true),
(5, vec!["{1...2O}"], true),
(6, vec!["..."], true),
(7, vec!["{-1...1}"], true),
(8, vec!["{0...-1}"], true),
(9, vec!["{1....4}"], true),
(10, vec!["{1...64}"], true),
(11, vec!["{...}"], true),
(12, vec!["{1...64}", "{65...128}"], true),
(13, vec!["http://rustfs{2...3}/export/set{1...64}"], true),
(
14,
vec![
"http://rustfs{2...3}/export/set{1...64}",
"http://rustfs{2...3}/export/set{65...128}",
],
true,
),
(15, vec!["mydisk-{a...z}{1...20}"], true),
(16, vec!["mydisk-{1...4}{1..2.}"], true),
];
for (i, args, expected) in test_cases {
let ret = has_ellipses(&args);
assert_eq!(ret, expected, "Test{}: Expected {}, got {}", i, expected, ret);
}
}
#[test]
fn test_find_ellipses_patterns() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
pattern: &'a str,
success: bool,
want: Vec<Vec<&'a str>>,
}
let test_cases = [
TestCase {
num: 1,
pattern: "{1..64}",
..Default::default()
},
TestCase {
num: 2,
pattern: "1...64",
..Default::default()
},
TestCase {
num: 2,
pattern: "...",
..Default::default()
},
TestCase {
num: 3,
pattern: "{1...",
..Default::default()
},
TestCase {
num: 4,
pattern: "...64}",
..Default::default()
},
TestCase {
num: 5,
pattern: "{...}",
..Default::default()
},
TestCase {
num: 6,
pattern: "{-1...1}",
..Default::default()
},
TestCase {
num: 7,
pattern: "{0...-1}",
..Default::default()
},
TestCase {
num: 8,
pattern: "{1...2O}",
..Default::default()
},
TestCase {
num: 9,
pattern: "{64...1}",
..Default::default()
},
TestCase {
num: 10,
pattern: "{1....4}",
..Default::default()
},
TestCase {
num: 11,
pattern: "mydisk-{a...z}{1...20}",
..Default::default()
},
TestCase {
num: 12,
pattern: "mydisk-{1...4}{1..2.}",
..Default::default()
},
TestCase {
num: 13,
pattern: "{1..2.}-mydisk-{1...4}",
..Default::default()
},
TestCase {
num: 14,
pattern: "{{1...4}}",
..Default::default()
},
TestCase {
num: 16,
pattern: "{4...02}",
..Default::default()
},
TestCase {
num: 17,
pattern: "{f...z}",
..Default::default()
},
// Test for valid input.
TestCase {
num: 18,
pattern: "{1...64}",
success: true,
want: vec![
vec!["1"],
vec!["2"],
vec!["3"],
vec!["4"],
vec!["5"],
vec!["6"],
vec!["7"],
vec!["8"],
vec!["9"],
vec!["10"],
vec!["11"],
vec!["12"],
vec!["13"],
vec!["14"],
vec!["15"],
vec!["16"],
vec!["17"],
vec!["18"],
vec!["19"],
vec!["20"],
vec!["21"],
vec!["22"],
vec!["23"],
vec!["24"],
vec!["25"],
vec!["26"],
vec!["27"],
vec!["28"],
vec!["29"],
vec!["30"],
vec!["31"],
vec!["32"],
vec!["33"],
vec!["34"],
vec!["35"],
vec!["36"],
vec!["37"],
vec!["38"],
vec!["39"],
vec!["40"],
vec!["41"],
vec!["42"],
vec!["43"],
vec!["44"],
vec!["45"],
vec!["46"],
vec!["47"],
vec!["48"],
vec!["49"],
vec!["50"],
vec!["51"],
vec!["52"],
vec!["53"],
vec!["54"],
vec!["55"],
vec!["56"],
vec!["57"],
vec!["58"],
vec!["59"],
vec!["60"],
vec!["61"],
vec!["62"],
vec!["63"],
vec!["64"],
],
},
TestCase {
num: 19,
pattern: "{1...5} {65...70}",
success: true,
want: vec![
vec!["1 ", "65"],
vec!["2 ", "65"],
vec!["3 ", "65"],
vec!["4 ", "65"],
vec!["5 ", "65"],
vec!["1 ", "66"],
vec!["2 ", "66"],
vec!["3 ", "66"],
vec!["4 ", "66"],
vec!["5 ", "66"],
vec!["1 ", "67"],
vec!["2 ", "67"],
vec!["3 ", "67"],
vec!["4 ", "67"],
vec!["5 ", "67"],
vec!["1 ", "68"],
vec!["2 ", "68"],
vec!["3 ", "68"],
vec!["4 ", "68"],
vec!["5 ", "68"],
vec!["1 ", "69"],
vec!["2 ", "69"],
vec!["3 ", "69"],
vec!["4 ", "69"],
vec!["5 ", "69"],
vec!["1 ", "70"],
vec!["2 ", "70"],
vec!["3 ", "70"],
vec!["4 ", "70"],
vec!["5 ", "70"],
],
},
TestCase {
num: 20,
pattern: "{01...036}",
success: true,
want: vec![
vec!["001"],
vec!["002"],
vec!["003"],
vec!["004"],
vec!["005"],
vec!["006"],
vec!["007"],
vec!["008"],
vec!["009"],
vec!["010"],
vec!["011"],
vec!["012"],
vec!["013"],
vec!["014"],
vec!["015"],
vec!["016"],
vec!["017"],
vec!["018"],
vec!["019"],
vec!["020"],
vec!["021"],
vec!["022"],
vec!["023"],
vec!["024"],
vec!["025"],
vec!["026"],
vec!["027"],
vec!["028"],
vec!["029"],
vec!["030"],
vec!["031"],
vec!["032"],
vec!["033"],
vec!["034"],
vec!["035"],
vec!["036"],
],
},
TestCase {
num: 21,
pattern: "{001...036}",
success: true,
want: vec![
vec!["001"],
vec!["002"],
vec!["003"],
vec!["004"],
vec!["005"],
vec!["006"],
vec!["007"],
vec!["008"],
vec!["009"],
vec!["010"],
vec!["011"],
vec!["012"],
vec!["013"],
vec!["014"],
vec!["015"],
vec!["016"],
vec!["017"],
vec!["018"],
vec!["019"],
vec!["020"],
vec!["021"],
vec!["022"],
vec!["023"],
vec!["024"],
vec!["025"],
vec!["026"],
vec!["027"],
vec!["028"],
vec!["029"],
vec!["030"],
vec!["031"],
vec!["032"],
vec!["033"],
vec!["034"],
vec!["035"],
vec!["036"],
],
},
];
for test_case in test_cases {
let ret = find_ellipses_patterns(test_case.pattern);
match ret {
Ok(v) => {
if !test_case.success {
panic!("Test{}: Expected failure but passed instead", test_case.num);
}
let got = v.expand();
if got.len() != test_case.want.len() {
panic!("Test{}: Expected {}, got {}", test_case.num, test_case.want.len(), got.len());
}
assert_eq!(got, test_case.want, "Test{}: Expected {:?}, got {:?}", test_case.num, test_case.want, got);
}
Err(e) => {
if test_case.success {
panic!("Test{}: Expected success but failed instead {:?}", test_case.num, e);
}
}
}
}
}
}