Fix large file upload freeze with adaptive buffer sizing (#869)

* Initial plan * Fix large file upload freeze by increasing StreamReader buffer size Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Add comprehensive documentation for large file upload freeze fix Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * upgrade s3s version * Fix compilation error: use BufReader instead of non-existent StreamReader::with_capacity Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Update documentation with correct BufReader implementation Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * add tokio feature `io-util` * Implement adaptive buffer sizing based on file size Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * Constants are managed uniformly and fmt code * fix * Fix: Trigger self-heal on read when shards missing from rejoined nodes (#871) * Initial plan * Fix: Trigger self-heal when missing shards detected during read - Added proactive heal detection in get_object_with_fileinfo - When reading an object, now checks if any shards are missing even if read succeeds - Sends low-priority heal request to reconstruct missing shards on rejoined nodes - This fixes the issue where data written during node outage is not healed when node rejoins Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * fix * Unify CRC implementations to crc-fast (#873) * Initial plan * Replace CRC libraries with unified crc-fast implementation Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> * fix * fix: replace low to Normal --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
2026-08-09 14:49:25 +00:00 · 2025-11-17 23:15:20 +08:00
parent 1279baa72b
commit 601f3456bc
14 changed files with 380 additions and 78 deletions
@@ -15,7 +15,6 @@
 use crate::errors::ChecksumMismatch;
 use base64::{Engine as _, engine::general_purpose};
 use bytes::Bytes;
-use crc32fast::Hasher as Crc32Hasher;
 use http::HeaderMap;
 use sha1::Sha1;
 use sha2::{Digest, Sha256};
@@ -612,7 +611,7 @@ pub trait ChecksumHasher: Write + Send + Sync {

 /// CRC32 IEEE hasher
 pub struct Crc32IeeeHasher {
-    hasher: Crc32Hasher,
+    hasher: crc_fast::Digest,
 }

 impl Default for Crc32IeeeHasher {
@@ -624,7 +623,7 @@ impl Default for Crc32IeeeHasher {
 impl Crc32IeeeHasher {
    pub fn new() -> Self {
        Self {
-            hasher: Crc32Hasher::new(),
+            hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc),
        }
    }
 }
@@ -642,27 +641,36 @@ impl Write for Crc32IeeeHasher {

 impl ChecksumHasher for Crc32IeeeHasher {
    fn finalize(&mut self) -> Vec<u8> {
-        self.hasher.clone().finalize().to_be_bytes().to_vec()
+        (self.hasher.clone().finalize() as u32).to_be_bytes().to_vec()
    }

    fn reset(&mut self) {
-        self.hasher = Crc32Hasher::new();
+        self.hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
    }
 }

 /// CRC32 Castagnoli hasher
-#[derive(Default)]
-pub struct Crc32CastagnoliHasher(u32);
+pub struct Crc32CastagnoliHasher {
+    hasher: crc_fast::Digest,
+}
+
+impl Default for Crc32CastagnoliHasher {
+    fn default() -> Self {
+        Self::new()
+    }
+}

 impl Crc32CastagnoliHasher {
    pub fn new() -> Self {
-        Self::default()
+        Self {
+            hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi),
+        }
    }
 }

 impl Write for Crc32CastagnoliHasher {
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        self.0 = crc32c::crc32c_append(self.0, buf);
+        self.hasher.update(buf);
        Ok(buf.len())
    }

@@ -673,11 +681,11 @@ impl Write for Crc32CastagnoliHasher {

 impl ChecksumHasher for Crc32CastagnoliHasher {
    fn finalize(&mut self) -> Vec<u8> {
-        self.0.to_be_bytes().to_vec()
+        (self.hasher.clone().finalize() as u32).to_be_bytes().to_vec()
    }

    fn reset(&mut self) {
-        self.0 = 0;
+        self.hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi);
    }
 }

@@ -758,22 +766,27 @@ impl ChecksumHasher for Sha256Hasher {
 }

 /// CRC64 NVME hasher
-#[derive(Default)]
 pub struct Crc64NvmeHasher {
-    hasher: crc64fast_nvme::Digest,
+    hasher: crc_fast::Digest,
+}
+
+impl Default for Crc64NvmeHasher {
+    fn default() -> Self {
+        Self::new()
+    }
 }

 impl Crc64NvmeHasher {
    pub fn new() -> Self {
        Self {
-            hasher: Default::default(),
+            hasher: crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme),
        }
    }
 }

 impl Write for Crc64NvmeHasher {
    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        self.hasher.write(buf);
+        self.hasher.update(buf);
        Ok(buf.len())
    }

@@ -784,11 +797,11 @@ impl Write for Crc64NvmeHasher {

 impl ChecksumHasher for Crc64NvmeHasher {
    fn finalize(&mut self) -> Vec<u8> {
-        self.hasher.sum64().to_be_bytes().to_vec()
+        self.hasher.clone().finalize().to_be_bytes().to_vec()
    }

    fn reset(&mut self) {
-        self.hasher = Default::default();
+        self.hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme);
    }
 }

@@ -356,7 +356,11 @@ where
            *this.compressed_len = 0;
            return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "Decompressed length mismatch")));
        }
-        let actual_crc = crc32fast::hash(&decompressed);
+        let actual_crc = {
+            let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
+            hasher.update(&decompressed);
+            hasher.finalize() as u32
+        };
        if actual_crc != crc {
            // error!("DecompressReader CRC32 mismatch: actual {actual_crc} != expected {crc}");
            this.compressed_buf.take();
@@ -404,7 +408,11 @@ where

 /// Build compressed block with header + uvarint + compressed data
 fn build_compressed_block(uncompressed_data: &[u8], compression_algorithm: CompressionAlgorithm) -> Vec<u8> {
-    let crc = crc32fast::hash(uncompressed_data);
+    let crc = {
+        let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
+        hasher.update(uncompressed_data);
+        hasher.finalize() as u32
+    };
    let compressed_data = compress_block(uncompressed_data, compression_algorithm);
    let uncompressed_len = uncompressed_data.len();
    let mut uncompressed_len_buf = [0u8; 10];
@@ -102,7 +102,11 @@ where
                    let nonce = Nonce::try_from(this.nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?;
                    let plaintext = &temp_buf.filled()[..n];
                    let plaintext_len = plaintext.len();
-                    let crc = crc32fast::hash(plaintext);
+                    let crc = {
+                        let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
+                        hasher.update(plaintext);
+                        hasher.finalize() as u32
+                    };
                    let ciphertext = cipher
                        .encrypt(&nonce, plaintext)
                        .map_err(|e| Error::other(format!("encrypt error: {e}")))?;
@@ -409,7 +413,11 @@ where
                return Poll::Ready(Err(Error::other("Plaintext length mismatch")));
            }

-            let actual_crc = crc32fast::hash(&plaintext);
+            let actual_crc = {
+                let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
+                hasher.update(&plaintext);
+                hasher.finalize() as u32
+            };
            if actual_crc != crc {
                this.ciphertext_buf.take();
                *this.ciphertext_read = 0;