update ec share size

update bitrot

crates/filemeta/src/fileinfo.rs

@@ -93,6 +93,10 @@ pub struct ErasureInfo {
     pub checksums: Vec<ChecksumInfo>,
 }
 
+pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize {
+    (block_size.div_ceil(data_shards) + 1) & !1
+}
+
 impl ErasureInfo {
     pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo {
         for sum in &self.checksums {
@@ -109,7 +113,7 @@ impl ErasureInfo {
 
     /// Calculate the size of each shard.
     pub fn shard_size(&self) -> usize {
-        self.block_size.div_ceil(self.data_blocks)
+        calc_shard_size(self.block_size, self.data_blocks)
     }
     /// Calculate the total erasure file size for a given original size.
     // Returns the final erasure size from the original size
@@ -120,7 +124,7 @@ impl ErasureInfo {
 
         let num_shards = total_length / self.block_size;
         let last_block_size = total_length % self.block_size;
-        let last_shard_size = last_block_size.div_ceil(self.data_blocks);
+        let last_shard_size = calc_shard_size(last_block_size, self.data_blocks);
         num_shards * self.shard_size() + last_shard_size
     }
 

crates/rio/src/bitrot.rs

@@ -1,13 +1,12 @@
-use crate::{Reader, Writer};
 use pin_project_lite::pin_project;
 use rustfs_utils::{HashAlgorithm, read_full, write_all};
-use tokio::io::{AsyncRead, AsyncReadExt};
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
 
 pin_project! {
     /// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity.
-    pub struct BitrotReader {
+    pub struct BitrotReader<R> {
         #[pin]
-        inner: Box<dyn Reader>,
+        inner: R,
         hash_algo: HashAlgorithm,
         shard_size: usize,
         buf: Vec<u8>,
@@ -19,14 +18,12 @@ pin_project! {
     }
 }
 
-impl BitrotReader {
-    /// Get a reference to the underlying reader.
-    pub fn get_ref(&self) -> &dyn Reader {
-        &*self.inner
-    }
-
+impl<R> BitrotReader<R>
+where
+    R: AsyncRead + Unpin + Send + Sync,
+{
     /// Create a new BitrotReader.
-    pub fn new(inner: Box<dyn Reader>, shard_size: usize, algo: HashAlgorithm) -> Self {
+    pub fn new(inner: R, shard_size: usize, algo: HashAlgorithm) -> Self {
         let hash_size = algo.size();
         Self {
             inner,
@@ -86,9 +83,9 @@ impl BitrotReader {
 
 pin_project! {
     /// BitrotWriter writes (hash+data) blocks to an async writer.
-    pub struct BitrotWriter {
+    pub struct BitrotWriter<W> {
         #[pin]
-        inner: Writer,
+        inner: W,
         hash_algo: HashAlgorithm,
         shard_size: usize,
         buf: Vec<u8>,
@@ -96,9 +93,12 @@ pin_project! {
     }
 }
 
-impl BitrotWriter {
+impl<W> BitrotWriter<W>
+where
+    W: AsyncWrite + Unpin + Send + Sync,
+{
     /// Create a new BitrotWriter.
-    pub fn new(inner: Writer, shard_size: usize, algo: HashAlgorithm) -> Self {
+    pub fn new(inner: W, shard_size: usize, algo: HashAlgorithm) -> Self {
         let hash_algo = algo;
         Self {
             inner,
@@ -109,7 +109,7 @@ impl BitrotWriter {
         }
     }
 
-    pub fn into_inner(self) -> Writer {
+    pub fn into_inner(self) -> W {
         self.inner
     }
 
@@ -209,7 +209,7 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
 #[cfg(test)]
 mod tests {
 
-    use crate::{BitrotReader, BitrotWriter, Writer};
+    use crate::{BitrotReader, BitrotWriter};
     use rustfs_utils::HashAlgorithm;
     use std::io::Cursor;
 
@@ -219,9 +219,9 @@ mod tests {
         let data_size = data.len();
         let shard_size = 8;
 
-        let buf = Vec::new();
+        let buf: Vec<u8> = Vec::new();
         let writer = Cursor::new(buf);
-        let mut bitrot_writer = BitrotWriter::new(Writer::from_cursor(writer), shard_size, HashAlgorithm::HighwayHash256);
+        let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
 
         let mut n = 0;
         for chunk in data.chunks(shard_size) {
@@ -230,8 +230,7 @@ mod tests {
         assert_eq!(n, data.len());
 
         // 读
-        let reader = Cursor::new(bitrot_writer.into_inner().into_cursor_inner().unwrap());
-        let reader = Box::new(reader);
+        let reader = bitrot_writer.into_inner();
         let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
         let mut out = Vec::new();
         let mut n = 0;
@@ -253,18 +252,17 @@ mod tests {
         let data = b"test data for bitrot";
         let data_size = data.len();
         let shard_size = 8;
-        let buf = Vec::new();
+        let buf: Vec<u8> = Vec::new();
         let writer = Cursor::new(buf);
-        let mut bitrot_writer = BitrotWriter::new(Writer::from_cursor(writer), shard_size, HashAlgorithm::HighwayHash256);
+        let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256);
         for chunk in data.chunks(shard_size) {
             let _ = bitrot_writer.write(chunk).await.unwrap();
         }
-        let mut written = bitrot_writer.into_inner().into_cursor_inner().unwrap();
+        let mut written = bitrot_writer.into_inner().into_inner();
         // change the last byte to make hash mismatch
         let pos = written.len() - 1;
         written[pos] ^= 0xFF;
         let reader = Cursor::new(written);
-        let reader = Box::new(reader);
         let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256);
 
         let count = data_size.div_ceil(shard_size);
@@ -297,9 +295,9 @@ mod tests {
         let data_size = data.len();
         let shard_size = 8;
 
-        let buf = Vec::new();
+        let buf: Vec<u8> = Vec::new();
         let writer = Cursor::new(buf);
-        let mut bitrot_writer = BitrotWriter::new(Writer::from_cursor(writer), shard_size, HashAlgorithm::None);
+        let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::None);
 
         let mut n = 0;
         for chunk in data.chunks(shard_size) {
@@ -307,8 +305,7 @@ mod tests {
         }
         assert_eq!(n, data.len());
 
-        let reader = Cursor::new(bitrot_writer.into_inner().into_cursor_inner().unwrap());
-        let reader = Box::new(reader);
+        let reader = bitrot_writer.into_inner();
         let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None);
         let mut out = Vec::new();
         let mut n = 0;

crates/rio/src/lib.rs

@@ -30,9 +30,6 @@ pub use writer::*;
 mod http_reader;
 pub use http_reader::*;
 
-mod bitrot;
-pub use bitrot::*;
-
 mod etag;
 
 pub trait Reader: tokio::io::AsyncRead + Unpin + Send + Sync + EtagResolvable + HashReaderDetector {}