Merge pull request #480 from rustfs/feat/compress

feat: add object compression support

ecstore/src/erasure_coding/decode.rs

@@ -30,7 +30,7 @@ where
     // readers传入前应处理disk错误，确保每个reader达到可用数量的BitrotReader
     pub fn new(readers: Vec<Option<BitrotReader<R>>>, e: Erasure, offset: usize, total_length: usize) -> Self {
         let shard_size = e.shard_size();
-        let shard_file_size = e.shard_file_size(total_length);
+        let shard_file_size = e.shard_file_size(total_length as i64) as usize;
 
         let offset = (offset / e.block_size) * shard_size;
 
@@ -142,6 +142,7 @@ where
     W: tokio::io::AsyncWrite + Send + Sync + Unpin,
 {
     if get_data_block_len(en_blocks, data_blocks) < length {
+        error!("write_data_blocks get_data_block_len < length");
         return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
     }
 
@@ -150,6 +151,7 @@ where
 
     for block_op in &en_blocks[..data_blocks] {
         if block_op.is_none() {
+            error!("write_data_blocks block_op.is_none()");
             return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
         }
 
@@ -164,7 +166,10 @@ where
         offset = 0;
 
         if write_left < block.len() {
-            writer.write_all(&block_slice[..write_left]).await?;
+            writer.write_all(&block_slice[..write_left]).await.map_err(|e| {
+                error!("write_data_blocks write_all err: {}", e);
+                e
+            })?;
 
             total_written += write_left;
             break;
@@ -172,7 +177,10 @@ where
 
         let n = block_slice.len();
 
-        writer.write_all(block_slice).await?;
+        writer.write_all(block_slice).await.map_err(|e| {
+            error!("write_data_blocks write_all2 err: {}", e);
+            e
+        })?;
 
         write_left -= n;
 
@@ -228,6 +236,7 @@ impl Erasure {
             };
 
             if block_length == 0 {
+                // error!("erasure decode decode block_length == 0");
                 break;
             }
 

ecstore/src/erasure_coding/erasure.rs

@@ -469,22 +469,27 @@ impl Erasure {
     }
     /// Calculate the total erasure file size for a given original size.
     // Returns the final erasure size from the original size
-    pub fn shard_file_size(&self, total_length: usize) -> usize {
+    pub fn shard_file_size(&self, total_length: i64) -> i64 {
         if total_length == 0 {
             return 0;
         }
+        if total_length < 0 {
+            return total_length;
+        }
+
+        let total_length = total_length as usize;
 
         let num_shards = total_length / self.block_size;
         let last_block_size = total_length % self.block_size;
         let last_shard_size = calc_shard_size(last_block_size, self.data_shards);
-        num_shards * self.shard_size() + last_shard_size
+        (num_shards * self.shard_size() + last_shard_size) as i64
     }
 
     /// Calculate the offset in the erasure file where reading begins.
     // Returns the offset in the erasure file where reading begins
     pub fn shard_file_offset(&self, start_offset: usize, length: usize, total_length: usize) -> usize {
         let shard_size = self.shard_size();
-        let shard_file_size = self.shard_file_size(total_length);
+        let shard_file_size = self.shard_file_size(total_length as i64) as usize;
         let end_shard = (start_offset + length) / self.block_size;
         let mut till_offset = end_shard * shard_size + shard_size;
         if till_offset > shard_file_size {