Merge branch 'main' of https://github.com/rustfs/s3-rustfs into feature/ilm

# Conflicts:
#	Cargo.lock
#	Cargo.toml
#	crates/utils/Cargo.toml
#	crates/utils/src/net.rs
#	ecstore/Cargo.toml
#	ecstore/src/set_disk.rs
#	rustfs/src/storage/ecfs.rs

crates/utils/src/compress.rs

@@ -0,0 +1,318 @@
+use std::io::Write;
+use tokio::io;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub enum CompressionAlgorithm {
+    None,
+    Gzip,
+    Deflate,
+    Zstd,
+    #[default]
+    Lz4,
+    Brotli,
+    Snappy,
+}
+
+impl CompressionAlgorithm {
+    pub fn as_str(&self) -> &str {
+        match self {
+            CompressionAlgorithm::None => "none",
+            CompressionAlgorithm::Gzip => "gzip",
+            CompressionAlgorithm::Deflate => "deflate",
+            CompressionAlgorithm::Zstd => "zstd",
+            CompressionAlgorithm::Lz4 => "lz4",
+            CompressionAlgorithm::Brotli => "brotli",
+            CompressionAlgorithm::Snappy => "snappy",
+        }
+    }
+}
+
+impl std::fmt::Display for CompressionAlgorithm {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+impl std::str::FromStr for CompressionAlgorithm {
+    type Err = std::io::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "gzip" => Ok(CompressionAlgorithm::Gzip),
+            "deflate" => Ok(CompressionAlgorithm::Deflate),
+            "zstd" => Ok(CompressionAlgorithm::Zstd),
+            "lz4" => Ok(CompressionAlgorithm::Lz4),
+            "brotli" => Ok(CompressionAlgorithm::Brotli),
+            "snappy" => Ok(CompressionAlgorithm::Snappy),
+            "none" => Ok(CompressionAlgorithm::None),
+            _ => Err(std::io::Error::other(format!("Unsupported compression algorithm: {}", s))),
+        }
+    }
+}
+
+pub fn compress_block(input: &[u8], algorithm: CompressionAlgorithm) -> Vec<u8> {
+    match algorithm {
+        CompressionAlgorithm::Gzip => {
+            let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+            let _ = encoder.write_all(input);
+            let _ = encoder.flush();
+            encoder.finish().unwrap_or_default()
+        }
+        CompressionAlgorithm::Deflate => {
+            let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default());
+            let _ = encoder.write_all(input);
+            let _ = encoder.flush();
+            encoder.finish().unwrap_or_default()
+        }
+        CompressionAlgorithm::Zstd => {
+            let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder");
+            let _ = encoder.write_all(input);
+            encoder.finish().unwrap_or_default()
+        }
+        CompressionAlgorithm::Lz4 => {
+            let mut encoder = lz4::EncoderBuilder::new().build(Vec::new()).expect("lz4 encoder");
+            let _ = encoder.write_all(input);
+            let (out, result) = encoder.finish();
+            result.expect("lz4 finish");
+            out
+        }
+        CompressionAlgorithm::Brotli => {
+            let mut out = Vec::new();
+            brotli::CompressorWriter::new(&mut out, 4096, 5, 22)
+                .write_all(input)
+                .expect("brotli compress");
+            out
+        }
+        CompressionAlgorithm::Snappy => {
+            let mut encoder = snap::write::FrameEncoder::new(Vec::new());
+            let _ = encoder.write_all(input);
+            encoder.into_inner().unwrap_or_default()
+        }
+        CompressionAlgorithm::None => input.to_vec(),
+    }
+}
+
+pub fn decompress_block(compressed: &[u8], algorithm: CompressionAlgorithm) -> io::Result<Vec<u8>> {
+    match algorithm {
+        CompressionAlgorithm::Gzip => {
+            let mut decoder = flate2::read::GzDecoder::new(std::io::Cursor::new(compressed));
+            let mut out = Vec::new();
+            std::io::Read::read_to_end(&mut decoder, &mut out)?;
+            Ok(out)
+        }
+        CompressionAlgorithm::Deflate => {
+            let mut decoder = flate2::read::DeflateDecoder::new(std::io::Cursor::new(compressed));
+            let mut out = Vec::new();
+            std::io::Read::read_to_end(&mut decoder, &mut out)?;
+            Ok(out)
+        }
+        CompressionAlgorithm::Zstd => {
+            let mut decoder = zstd::Decoder::new(std::io::Cursor::new(compressed))?;
+            let mut out = Vec::new();
+            std::io::Read::read_to_end(&mut decoder, &mut out)?;
+            Ok(out)
+        }
+        CompressionAlgorithm::Lz4 => {
+            let mut decoder = lz4::Decoder::new(std::io::Cursor::new(compressed)).expect("lz4 decoder");
+            let mut out = Vec::new();
+            std::io::Read::read_to_end(&mut decoder, &mut out)?;
+            Ok(out)
+        }
+        CompressionAlgorithm::Brotli => {
+            let mut out = Vec::new();
+            let mut decoder = brotli::Decompressor::new(std::io::Cursor::new(compressed), 4096);
+            std::io::Read::read_to_end(&mut decoder, &mut out)?;
+            Ok(out)
+        }
+        CompressionAlgorithm::Snappy => {
+            let mut decoder = snap::read::FrameDecoder::new(std::io::Cursor::new(compressed));
+            let mut out = Vec::new();
+            std::io::Read::read_to_end(&mut decoder, &mut out)?;
+            Ok(out)
+        }
+        CompressionAlgorithm::None => Ok(Vec::new()),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::str::FromStr;
+    use std::time::Instant;
+
+    #[test]
+    fn test_compress_decompress_gzip() {
+        let data = b"hello gzip compress";
+        let compressed = compress_block(data, CompressionAlgorithm::Gzip);
+        let decompressed = decompress_block(&compressed, CompressionAlgorithm::Gzip).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_compress_decompress_deflate() {
+        let data = b"hello deflate compress";
+        let compressed = compress_block(data, CompressionAlgorithm::Deflate);
+        let decompressed = decompress_block(&compressed, CompressionAlgorithm::Deflate).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_compress_decompress_zstd() {
+        let data = b"hello zstd compress";
+        let compressed = compress_block(data, CompressionAlgorithm::Zstd);
+        let decompressed = decompress_block(&compressed, CompressionAlgorithm::Zstd).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_compress_decompress_lz4() {
+        let data = b"hello lz4 compress";
+        let compressed = compress_block(data, CompressionAlgorithm::Lz4);
+        let decompressed = decompress_block(&compressed, CompressionAlgorithm::Lz4).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_compress_decompress_brotli() {
+        let data = b"hello brotli compress";
+        let compressed = compress_block(data, CompressionAlgorithm::Brotli);
+        let decompressed = decompress_block(&compressed, CompressionAlgorithm::Brotli).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_compress_decompress_snappy() {
+        let data = b"hello snappy compress";
+        let compressed = compress_block(data, CompressionAlgorithm::Snappy);
+        let decompressed = decompress_block(&compressed, CompressionAlgorithm::Snappy).unwrap();
+        assert_eq!(decompressed, data);
+    }
+
+    #[test]
+    fn test_from_str() {
+        assert_eq!(CompressionAlgorithm::from_str("gzip").unwrap(), CompressionAlgorithm::Gzip);
+        assert_eq!(CompressionAlgorithm::from_str("deflate").unwrap(), CompressionAlgorithm::Deflate);
+        assert_eq!(CompressionAlgorithm::from_str("zstd").unwrap(), CompressionAlgorithm::Zstd);
+        assert_eq!(CompressionAlgorithm::from_str("lz4").unwrap(), CompressionAlgorithm::Lz4);
+        assert_eq!(CompressionAlgorithm::from_str("brotli").unwrap(), CompressionAlgorithm::Brotli);
+        assert_eq!(CompressionAlgorithm::from_str("snappy").unwrap(), CompressionAlgorithm::Snappy);
+        assert!(CompressionAlgorithm::from_str("unknown").is_err());
+    }
+
+    #[test]
+    fn test_compare_compression_algorithms() {
+        use std::time::Instant;
+        let data = vec![42u8; 1024 * 100]; // 100KB of repetitive data
+
+        // let mut data = vec![0u8; 1024 * 1024];
+        // rand::thread_rng().fill(&mut data[..]);
+
+        let start = Instant::now();
+
+        let mut times = Vec::new();
+        times.push(("original", start.elapsed(), data.len()));
+
+        let start = Instant::now();
+        let gzip = compress_block(&data, CompressionAlgorithm::Gzip);
+        let gzip_time = start.elapsed();
+        times.push(("gzip", gzip_time, gzip.len()));
+
+        let start = Instant::now();
+        let deflate = compress_block(&data, CompressionAlgorithm::Deflate);
+        let deflate_time = start.elapsed();
+        times.push(("deflate", deflate_time, deflate.len()));
+
+        let start = Instant::now();
+        let zstd = compress_block(&data, CompressionAlgorithm::Zstd);
+        let zstd_time = start.elapsed();
+        times.push(("zstd", zstd_time, zstd.len()));
+
+        let start = Instant::now();
+        let lz4 = compress_block(&data, CompressionAlgorithm::Lz4);
+        let lz4_time = start.elapsed();
+        times.push(("lz4", lz4_time, lz4.len()));
+
+        let start = Instant::now();
+        let brotli = compress_block(&data, CompressionAlgorithm::Brotli);
+        let brotli_time = start.elapsed();
+        times.push(("brotli", brotli_time, brotli.len()));
+
+        let start = Instant::now();
+        let snappy = compress_block(&data, CompressionAlgorithm::Snappy);
+        let snappy_time = start.elapsed();
+        times.push(("snappy", snappy_time, snappy.len()));
+
+        println!("Compression results:");
+        for (name, dur, size) in &times {
+            println!("{}: {} bytes, {:?}", name, size, dur);
+        }
+        // All should decompress to the original
+        assert_eq!(decompress_block(&gzip, CompressionAlgorithm::Gzip).unwrap(), data);
+        assert_eq!(decompress_block(&deflate, CompressionAlgorithm::Deflate).unwrap(), data);
+        assert_eq!(decompress_block(&zstd, CompressionAlgorithm::Zstd).unwrap(), data);
+        assert_eq!(decompress_block(&lz4, CompressionAlgorithm::Lz4).unwrap(), data);
+        assert_eq!(decompress_block(&brotli, CompressionAlgorithm::Brotli).unwrap(), data);
+        assert_eq!(decompress_block(&snappy, CompressionAlgorithm::Snappy).unwrap(), data);
+        // All compressed results should not be empty
+        assert!(
+            !gzip.is_empty()
+                && !deflate.is_empty()
+                && !zstd.is_empty()
+                && !lz4.is_empty()
+                && !brotli.is_empty()
+                && !snappy.is_empty()
+        );
+    }
+
+    #[test]
+    fn test_compression_benchmark() {
+        let sizes = [128 * 1024, 512 * 1024, 1024 * 1024];
+        let algorithms = [
+            CompressionAlgorithm::Gzip,
+            CompressionAlgorithm::Deflate,
+            CompressionAlgorithm::Zstd,
+            CompressionAlgorithm::Lz4,
+            CompressionAlgorithm::Brotli,
+            CompressionAlgorithm::Snappy,
+        ];
+
+        println!("\n压缩算法基准测试结果:");
+        println!(
+            "{:<10} {:<10} {:<15} {:<15} {:<15}",
+            "数据大小", "算法", "压缩时间(ms)", "压缩后大小", "压缩率"
+        );
+
+        for size in sizes {
+            // 生成可压缩的数据（重复的文本模式）
+            let pattern = b"Hello, this is a test pattern that will be repeated multiple times to create compressible data. ";
+            let data: Vec<u8> = pattern.iter().cycle().take(size).copied().collect();
+
+            for algo in algorithms {
+                // 压缩测试
+                let start = Instant::now();
+                let compressed = compress_block(&data, algo);
+                let compress_time = start.elapsed();
+
+                // 解压测试
+                let start = Instant::now();
+                let _decompressed = decompress_block(&compressed, algo).unwrap();
+                let _decompress_time = start.elapsed();
+
+                // 计算压缩率
+                let compression_ratio = (size as f64 / compressed.len() as f64) as f32;
+
+                println!(
+                    "{:<10} {:<10} {:<15.2} {:<15} {:<15.2}x",
+                    format!("{}KB", size / 1024),
+                    algo.as_str(),
+                    compress_time.as_secs_f64() * 1000.0,
+                    compressed.len(),
+                    compression_ratio
+                );
+
+                // 验证解压结果
+                assert_eq!(_decompressed, data);
+            }
+            println!(); // 添加空行分隔不同大小的结果
+        }
+    }
+}

crates/utils/src/dirs.rs

@@ -0,0 +1,60 @@
+use std::env;
+use std::path::{Path, PathBuf};
+
+/// Get the absolute path to the current project
+///
+/// This function will try the following method to get the project path:
+/// 1. Use the `CARGO_MANIFEST_DIR` environment variable to get the project root directory.
+/// 2. Use `std::env::current_exe()` to get the executable file path and deduce the project root directory.
+/// 3. Use `std::env::current_dir()` to get the current working directory and try to deduce the project root directory.
+///
+/// If all methods fail, an error is returned.
+///
+/// # Returns
+/// - `Ok(PathBuf)`: The absolute path of the project that was successfully obtained.
+/// - `Err(String)`: Error message for the failed path.
+pub fn get_project_root() -> Result<PathBuf, String> {
+    // Try to get the project root directory through the CARGO_MANIFEST_DIR environment variable
+    if let Ok(manifest_dir) = env::var("CARGO_MANIFEST_DIR") {
+        let project_root = Path::new(&manifest_dir).to_path_buf();
+        println!("Get the project root directory with CARGO_MANIFEST_DIR:{}", project_root.display());
+        return Ok(project_root);
+    }
+
+    // Try to deduce the project root directory through the current executable file path
+    if let Ok(current_exe) = env::current_exe() {
+        let mut project_root = current_exe;
+        // Assume that the project root directory is in the parent directory of the parent directory of the executable path (usually target/debug or target/release)
+        project_root.pop(); // Remove the executable file name
+        project_root.pop(); // Remove target/debug or target/release
+        println!("Deduce the project root directory through current_exe:{}", project_root.display());
+        return Ok(project_root);
+    }
+
+    // Try to deduce the project root directory from the current working directory
+    if let Ok(mut current_dir) = env::current_dir() {
+        // Assume that the project root directory is in the parent directory of the current working directory
+        current_dir.pop();
+        println!("Deduce the project root directory through current_dir:{}", current_dir.display());
+        return Ok(current_dir);
+    }
+
+    // If all methods fail, return an error
+    Err("The project root directory cannot be obtained. Please check the running environment and project structure.".to_string())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_project_root() {
+        match get_project_root() {
+            Ok(path) => {
+                assert!(path.exists(), "The project root directory does not exist:{}", path.display());
+                println!("The test is passed, the project root directory:{}", path.display());
+            }
+            Err(e) => panic!("Failed to get the project root directory:{}", e),
+        }
+    }
+}

crates/utils/src/hash.rs

@@ -24,24 +24,56 @@ pub enum HashAlgorithm {
     None,
 }
 
+enum HashEncoded {
+    Md5([u8; 16]),
+    Sha256([u8; 32]),
+    HighwayHash256([u8; 32]),
+    HighwayHash256S([u8; 32]),
+    Blake2b512(blake3::Hash),
+    None,
+}
+
+impl AsRef<[u8]> for HashEncoded {
+    #[inline]
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            HashEncoded::Md5(hash) => hash.as_ref(),
+            HashEncoded::Sha256(hash) => hash.as_ref(),
+            HashEncoded::HighwayHash256(hash) => hash.as_ref(),
+            HashEncoded::HighwayHash256S(hash) => hash.as_ref(),
+            HashEncoded::Blake2b512(hash) => hash.as_bytes(),
+            HashEncoded::None => &[],
+        }
+    }
+}
+
+#[inline]
+fn u8x32_from_u64x4(input: [u64; 4]) -> [u8; 32] {
+    let mut output = [0u8; 32];
+    for (i, &n) in input.iter().enumerate() {
+        output[i * 8..(i + 1) * 8].copy_from_slice(&n.to_le_bytes());
+    }
+    output
+}
+
 impl HashAlgorithm {
     /// Hash the input data and return the hash result as Vec<u8>.
-    pub fn hash_encode(&self, data: &[u8]) -> Vec<u8> {
+    pub fn hash_encode(&self, data: &[u8]) -> impl AsRef<[u8]> {
         match self {
-            HashAlgorithm::Md5 => Md5::digest(data).to_vec(),
+            HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
             HashAlgorithm::HighwayHash256 => {
                 let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
                 hasher.append(data);
-                hasher.finalize256().iter().flat_map(|&n| n.to_le_bytes()).collect()
+                HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256()))
             }
-            HashAlgorithm::SHA256 => Sha256::digest(data).to_vec(),
+            HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()),
             HashAlgorithm::HighwayHash256S => {
                 let mut hasher = HighwayHasher::new(Key(HIGHWAY_HASH256_KEY));
                 hasher.append(data);
-                hasher.finalize256().iter().flat_map(|&n| n.to_le_bytes()).collect()
+                HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256()))
             }
-            HashAlgorithm::BLAKE2b512 => blake3::hash(data).as_bytes().to_vec(),
-            HashAlgorithm::None => Vec::new(),
+            HashAlgorithm::BLAKE2b512 => HashEncoded::Blake2b512(blake3::hash(data)),
+            HashAlgorithm::None => HashEncoded::None,
         }
     }
 
@@ -100,6 +132,7 @@ mod tests {
     fn test_hash_encode_none() {
         let data = b"test data";
         let hash = HashAlgorithm::None.hash_encode(data);
+        let hash = hash.as_ref();
         assert_eq!(hash.len(), 0);
     }
 
@@ -107,9 +140,11 @@ mod tests {
     fn test_hash_encode_md5() {
         let data = b"test data";
         let hash = HashAlgorithm::Md5.hash_encode(data);
+        let hash = hash.as_ref();
         assert_eq!(hash.len(), 16);
         // MD5 should be deterministic
         let hash2 = HashAlgorithm::Md5.hash_encode(data);
+        let hash2 = hash2.as_ref();
         assert_eq!(hash, hash2);
     }
 
@@ -117,9 +152,11 @@ mod tests {
     fn test_hash_encode_highway() {
         let data = b"test data";
         let hash = HashAlgorithm::HighwayHash256.hash_encode(data);
+        let hash = hash.as_ref();
         assert_eq!(hash.len(), 32);
         // HighwayHash should be deterministic
         let hash2 = HashAlgorithm::HighwayHash256.hash_encode(data);
+        let hash2 = hash2.as_ref();
         assert_eq!(hash, hash2);
     }
 
@@ -127,9 +164,11 @@ mod tests {
     fn test_hash_encode_sha256() {
         let data = b"test data";
         let hash = HashAlgorithm::SHA256.hash_encode(data);
+        let hash = hash.as_ref();
         assert_eq!(hash.len(), 32);
         // SHA256 should be deterministic
         let hash2 = HashAlgorithm::SHA256.hash_encode(data);
+        let hash2 = hash2.as_ref();
         assert_eq!(hash, hash2);
     }
 
@@ -137,9 +176,11 @@ mod tests {
     fn test_hash_encode_blake2b512() {
         let data = b"test data";
         let hash = HashAlgorithm::BLAKE2b512.hash_encode(data);
+        let hash = hash.as_ref();
         assert_eq!(hash.len(), 32); // blake3 outputs 32 bytes by default
         // BLAKE2b512 should be deterministic
         let hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data);
+        let hash2 = hash2.as_ref();
         assert_eq!(hash, hash2);
     }
 
@@ -150,18 +191,18 @@ mod tests {
 
         let md5_hash1 = HashAlgorithm::Md5.hash_encode(data1);
         let md5_hash2 = HashAlgorithm::Md5.hash_encode(data2);
-        assert_ne!(md5_hash1, md5_hash2);
+        assert_ne!(md5_hash1.as_ref(), md5_hash2.as_ref());
 
         let highway_hash1 = HashAlgorithm::HighwayHash256.hash_encode(data1);
         let highway_hash2 = HashAlgorithm::HighwayHash256.hash_encode(data2);
-        assert_ne!(highway_hash1, highway_hash2);
+        assert_ne!(highway_hash1.as_ref(), highway_hash2.as_ref());
 
         let sha256_hash1 = HashAlgorithm::SHA256.hash_encode(data1);
         let sha256_hash2 = HashAlgorithm::SHA256.hash_encode(data2);
-        assert_ne!(sha256_hash1, sha256_hash2);
+        assert_ne!(sha256_hash1.as_ref(), sha256_hash2.as_ref());
 
         let blake_hash1 = HashAlgorithm::BLAKE2b512.hash_encode(data1);
         let blake_hash2 = HashAlgorithm::BLAKE2b512.hash_encode(data2);
-        assert_ne!(blake_hash1, blake_hash2);
+        assert_ne!(blake_hash1.as_ref(), blake_hash2.as_ref());
     }
 }

crates/utils/src/lib.rs

@@ -27,14 +27,29 @@ pub mod string;
 #[cfg(feature = "crypto")]
 pub mod crypto;
 
+#[cfg(feature = "compress")]
+pub mod compress;
+
+#[cfg(feature = "path")]
+pub mod dirs;
+
 #[cfg(feature = "tls")]
 pub use certs::*;
+
 #[cfg(feature = "hash")]
 pub use hash::*;
+
 #[cfg(feature = "io")]
 pub use io::*;
+
 #[cfg(feature = "ip")]
 pub use ip::*;
 
 #[cfg(feature = "crypto")]
 pub use crypto::*;
+
+#[cfg(feature = "compress")]
+pub use compress::*;
+
+#[cfg(feature = "sys")]
+pub mod sys;

crates/utils/src/net.rs

@@ -1,3 +1,6 @@
+use bytes::Bytes;
+use futures::pin_mut;
+use futures::{Stream, StreamExt};
 use hyper::client::conn::http2::Builder;
 use hyper_util::rt::TokioExecutor;
 use lazy_static::lazy_static;
@@ -6,6 +9,7 @@ use std::{
     fmt::Display,
     net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
 };
+use transform_stream::AsyncTryStream;
 use url::{Host, Url};
 //use hyper::{client::conn::http2::Builder, rt::Executor};
 //use tonic::{SharedExec, UserAgent};
@@ -278,6 +282,27 @@ pub fn parse_and_resolve_address(addr_str: &str) -> std::io::Result<SocketAddr>
     Ok(resolved_addr)
 }
 
+#[allow(dead_code)]
+pub fn bytes_stream<S, E>(stream: S, content_length: usize) -> impl Stream<Item = std::result::Result<Bytes, E>> + Send + 'static
+where
+    S: Stream<Item = std::result::Result<Bytes, E>> + Send + 'static,
+    E: Send + 'static,
+{
+    AsyncTryStream::<Bytes, E, _>::new(|mut y| async move {
+        pin_mut!(stream);
+        let mut remaining: usize = content_length;
+        while let Some(result) = stream.next().await {
+            let mut bytes = result?;
+            if bytes.len() > remaining {
+                bytes.truncate(remaining);
+            }
+            remaining -= bytes.len();
+            y.yield_ok(bytes).await;
+        }
+        Ok(())
+    })
+}
+
 #[cfg(test)]
 mod test {
     use std::net::{Ipv4Addr, Ipv6Addr};

crates/utils/src/os/mod.rs

@@ -102,6 +102,7 @@ mod tests {
         // Test passes if the function doesn't panic - the actual result depends on test environment
     }
 
+    #[ignore] // FIXME: failed in github actions
     #[test]
     fn test_get_drive_stats_default() {
         let stats = get_drive_stats(0, 0).unwrap();

crates/utils/src/os/unix.rs

@@ -8,9 +8,9 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
     let stat = statfs(p.as_ref())?;
 
     let bsize = stat.block_size() as u64;
-    let bfree = stat.blocks_free() as u64;
-    let bavail = stat.blocks_available() as u64;
-    let blocks = stat.blocks() as u64;
+    let bfree = stat.blocks_free();
+    let bavail = stat.blocks_available();
+    let blocks = stat.blocks();
 
     let reserved = match bfree.checked_sub(bavail) {
         Some(reserved) => reserved,

crates/utils/src/string.rs

@@ -1,4 +1,5 @@
 use lazy_static::*;
+use rand::{Rng, RngCore};
 use regex::Regex;
 use std::io::{Error, Result};
 
@@ -32,6 +33,29 @@ pub fn match_pattern(pattern: &str, name: &str) -> bool {
     deep_match_rune(name.as_bytes(), pattern.as_bytes(), false)
 }
 
+pub fn has_pattern(patterns: &[&str], match_str: &str) -> bool {
+    for pattern in patterns {
+        if match_simple(pattern, match_str) {
+            return true;
+        }
+    }
+    false
+}
+
+pub fn has_string_suffix_in_slice(str: &str, list: &[&str]) -> bool {
+    let str = str.to_lowercase();
+    for v in list {
+        if *v == "*" {
+            return true;
+        }
+
+        if str.ends_with(&v.to_lowercase()) {
+            return true;
+        }
+    }
+    false
+}
+
 fn deep_match_rune(str_: &[u8], pattern: &[u8], simple: bool) -> bool {
     let (mut str_, mut pattern) = (str_, pattern);
     while !pattern.is_empty() {
@@ -283,6 +307,43 @@ pub fn parse_ellipses_range(pattern: &str) -> Result<Vec<String>> {
     Ok(ret)
 }
 
+pub fn gen_access_key(length: usize) -> Result<String> {
+    const ALPHA_NUMERIC_TABLE: [char; 36] = [
+        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N',
+        'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    ];
+
+    if length < 3 {
+        return Err(Error::other("access key length is too short"));
+    }
+
+    let mut result = String::with_capacity(length);
+    let mut rng = rand::rng();
+
+    for _ in 0..length {
+        result.push(ALPHA_NUMERIC_TABLE[rng.random_range(0..ALPHA_NUMERIC_TABLE.len())]);
+    }
+
+    Ok(result)
+}
+
+pub fn gen_secret_key(length: usize) -> Result<String> {
+    use base64_simd::URL_SAFE_NO_PAD;
+
+    if length < 8 {
+        return Err(Error::other("secret key length is too short"));
+    }
+    let mut rng = rand::rng();
+
+    let mut key = vec![0u8; URL_SAFE_NO_PAD.estimated_decoded_length(length)];
+    rng.fill_bytes(&mut key);
+
+    let encoded = URL_SAFE_NO_PAD.encode_to_string(&key);
+    let key_str = encoded.replace("/", "+");
+
+    Ok(key_str)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/utils/src/sys/mod.rs

@@ -0,0 +1,4 @@
+mod user_agent;
+
+pub use user_agent::ServiceType;
+pub use user_agent::get_user_agent;

crates/utils/src/sys/user_agent.rs

@@ -0,0 +1,209 @@
+use rustfs_config::VERSION;
+use std::env;
+use std::fmt;
+use sysinfo::System;
+
+/// Business Type Enumeration
+#[derive(Debug, Clone, PartialEq)]
+pub enum ServiceType {
+    Basis,
+    Core,
+    Event,
+    Logger,
+    Custom(String),
+}
+
+impl ServiceType {
+    fn as_str(&self) -> &str {
+        match self {
+            ServiceType::Basis => "basis",
+            ServiceType::Core => "core",
+            ServiceType::Event => "event",
+            ServiceType::Logger => "logger",
+            ServiceType::Custom(s) => s.as_str(),
+        }
+    }
+}
+
+// UserAgent structure
+struct UserAgent {
+    os_platform: String,
+    arch: String,
+    version: String,
+    service: ServiceType,
+}
+
+impl UserAgent {
+    /// Create a new UserAgent instance and accept business type parameters
+    ///
+    /// # Arguments
+    /// * `service` - The type of service for which the User-Agent is being created.
+    /// # Returns
+    /// A new instance of `UserAgent` with the current OS platform, architecture, version, and service type.
+    fn new(service: ServiceType) -> Self {
+        let os_platform = Self::get_os_platform();
+        let arch = env::consts::ARCH.to_string();
+        let version = VERSION.to_string();
+
+        UserAgent {
+            os_platform,
+            arch,
+            version,
+            service,
+        }
+    }
+
+    /// Obtain operating system platform information
+    fn get_os_platform() -> String {
+        if cfg!(target_os = "windows") {
+            Self::get_windows_platform()
+        } else if cfg!(target_os = "macos") {
+            Self::get_macos_platform()
+        } else if cfg!(target_os = "linux") {
+            Self::get_linux_platform()
+        } else {
+            "Unknown".to_string()
+        }
+    }
+
+    /// Get Windows platform information
+    #[cfg(windows)]
+    fn get_windows_platform() -> String {
+        // Priority to using sysinfo to get versions
+        if let Some(version) = System::os_version() {
+            format!("Windows NT {}", version)
+        } else {
+            // Fallback to cmd /c ver
+            let output = std::process::Command::new("cmd")
+                .args(&["/C", "ver"])
+                .output()
+                .unwrap_or_default();
+            let version = String::from_utf8_lossy(&output.stdout);
+            let version = version
+                .lines()
+                .next()
+                .unwrap_or("Windows NT 10.0")
+                .replace("Microsoft Windows [Version ", "")
+                .replace("]", "");
+            format!("Windows NT {}", version.trim())
+        }
+    }
+
+    #[cfg(not(windows))]
+    fn get_windows_platform() -> String {
+        "N/A".to_string()
+    }
+
+    /// Get macOS platform information
+    #[cfg(target_os = "macos")]
+    fn get_macos_platform() -> String {
+        let binding = System::os_version().unwrap_or("14.5.0".to_string());
+        let version = binding.split('.').collect::<Vec<&str>>();
+        let major = version.first().unwrap_or(&"14").to_string();
+        let minor = version.get(1).unwrap_or(&"5").to_string();
+        let patch = version.get(2).unwrap_or(&"0").to_string();
+
+        let arch = env::consts::ARCH;
+        let cpu_info = if arch == "aarch64" { "Apple" } else { "Intel" };
+
+        // Convert to User-Agent format
+        format!("Macintosh; {} Mac OS X {}_{}_{}", cpu_info, major, minor, patch)
+    }
+
+    #[cfg(not(target_os = "macos"))]
+    fn get_macos_platform() -> String {
+        "N/A".to_string()
+    }
+
+    /// Get Linux platform information
+    #[cfg(target_os = "linux")]
+    fn get_linux_platform() -> String {
+        format!("X11; {}", System::long_os_version().unwrap_or("Linux Unknown".to_string()))
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn get_linux_platform() -> String {
+        "N/A".to_string()
+    }
+}
+
+/// Implement Display trait to format User-Agent
+impl fmt::Display for UserAgent {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if self.service == ServiceType::Basis {
+            return write!(f, "Mozilla/5.0 ({}; {}) Rustfs/{}", self.os_platform, self.arch, self.version);
+        }
+        write!(
+            f,
+            "Mozilla/5.0 ({}; {}) Rustfs/{} ({})",
+            self.os_platform,
+            self.arch,
+            self.version,
+            self.service.as_str()
+        )
+    }
+}
+
+// Get the User-Agent string and accept business type parameters
+pub fn get_user_agent(service: ServiceType) -> String {
+    UserAgent::new(service).to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_user_agent_format_basis() {
+        let ua = get_user_agent(ServiceType::Basis);
+        assert!(ua.starts_with("Mozilla/5.0"));
+        assert!(ua.contains("Rustfs/1.0.0"));
+        println!("User-Agent: {}", ua);
+    }
+
+    #[test]
+    fn test_user_agent_format_core() {
+        let ua = get_user_agent(ServiceType::Core);
+        assert!(ua.starts_with("Mozilla/5.0"));
+        assert!(ua.contains("Rustfs/1.0.0 (core)"));
+        println!("User-Agent: {}", ua);
+    }
+
+    #[test]
+    fn test_user_agent_format_event() {
+        let ua = get_user_agent(ServiceType::Event);
+        assert!(ua.starts_with("Mozilla/5.0"));
+        assert!(ua.contains("Rustfs/1.0.0 (event)"));
+        println!("User-Agent: {}", ua);
+    }
+
+    #[test]
+    fn test_user_agent_format_logger() {
+        let ua = get_user_agent(ServiceType::Logger);
+        assert!(ua.starts_with("Mozilla/5.0"));
+        assert!(ua.contains("Rustfs/1.0.0 (logger)"));
+        println!("User-Agent: {}", ua);
+    }
+
+    #[test]
+    fn test_user_agent_format_custom() {
+        let ua = get_user_agent(ServiceType::Custom("monitor".to_string()));
+        assert!(ua.starts_with("Mozilla/5.0"));
+        assert!(ua.contains("Rustfs/1.0.0 (monitor)"));
+        println!("User-Agent: {}", ua);
+    }
+
+    #[test]
+    fn test_all_service_type() {
+        // Example: Generate User-Agents of Different Business Types
+        let ua_core = get_user_agent(ServiceType::Core);
+        let ua_event = get_user_agent(ServiceType::Event);
+        let ua_logger = get_user_agent(ServiceType::Logger);
+        let ua_custom = get_user_agent(ServiceType::Custom("monitor".to_string()));
+
+        println!("Core User-Agent: {}", ua_core);
+        println!("Event User-Agent: {}", ua_event);
+        println!("Logger User-Agent: {}", ua_logger);
+        println!("Custom User-Agent: {}", ua_custom);
+    }
+}