feat(storage): add direct chunk GET fast path (#2351)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>

crates/ecstore/src/erasure_coding/bitrot.rs

@@ -172,6 +172,52 @@ where
     }
 }
 
+impl BitrotWriter<CustomWriter> {
+    fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        if buf.is_empty() {
+            return Ok(0);
+        }
+
+        if self.finished {
+            return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished"));
+        }
+
+        if buf.len() > self.shard_size {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                format!("data size {} exceeds shard size {}", buf.len(), self.shard_size),
+            ));
+        }
+
+        if buf.len() < self.shard_size {
+            self.finished = true;
+        }
+
+        match &mut self.inner {
+            CustomWriter::InlineBuffer(data) => {
+                if self.hash_algo.size() > 0 {
+                    let hash = self.hash_algo.hash_encode(buf);
+                    if hash.as_ref().is_empty() {
+                        error!("bitrot writer write hash error: hash is empty");
+                        return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "hash is empty"));
+                    }
+                    data.extend_from_slice(hash.as_ref());
+                }
+                data.extend_from_slice(buf);
+                Ok(buf.len())
+            }
+            CustomWriter::Other(_) => Err(std::io::Error::other("inline sync write requires inline buffer writer")),
+        }
+    }
+
+    fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
+        match self.inner {
+            CustomWriter::InlineBuffer(_) => Ok(()),
+            CustomWriter::Other(_) => Err(std::io::Error::other("inline sync shutdown requires inline buffer writer")),
+        }
+    }
+}
+
 async fn write_all_vectored<W>(writer: &mut W, hash: &[u8], data: &[u8]) -> std::io::Result<()>
 where
     W: AsyncWrite + Unpin,
@@ -280,6 +326,10 @@ impl CustomWriter {
             Self::Other(_) => None,
         }
     }
+
+    pub fn is_inline_buffer(&self) -> bool {
+        matches!(self, Self::InlineBuffer(_))
+    }
 }
 
 impl AsyncWrite for CustomWriter {
@@ -397,6 +447,24 @@ impl BitrotWriterWrapper {
         self.bitrot_writer.shutdown().await
     }
 
+    pub fn is_inline_buffer(&self) -> bool {
+        matches!(self.writer_type, WriterType::InlineBuffer)
+    }
+
+    pub fn write_inline_sync(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        if !self.is_inline_buffer() {
+            return Err(std::io::Error::other("inline sync write requires inline buffer writer"));
+        }
+        self.bitrot_writer.write_inline_sync(buf)
+    }
+
+    pub fn shutdown_inline_sync(&mut self) -> std::io::Result<()> {
+        if !self.is_inline_buffer() {
+            return Err(std::io::Error::other("inline sync shutdown requires inline buffer writer"));
+        }
+        self.bitrot_writer.shutdown_inline_sync()
+    }
+
     /// Extract the inline buffer data, consuming the wrapper
     pub fn into_inline_data(self) -> Option<Vec<u8>> {
         match self.writer_type {

crates/ecstore/src/erasure_coding/decode.rs

@@ -17,6 +17,7 @@ use crate::disk::error_reduce::reduce_errs;
 use crate::erasure_coding::{BitrotReader, Erasure};
 use futures::stream::{FuturesUnordered, StreamExt};
 use pin_project_lite::pin_project;
+use rustfs_io_core::{IoChunk, PooledChunk};
 use std::io;
 use std::io::ErrorKind;
 use tokio::io::AsyncRead;
@@ -155,6 +156,68 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
     size
 }
 
+fn block_window(
+    offset: usize,
+    length: usize,
+    block_size: usize,
+    block_index: usize,
+    start_block: usize,
+    end_block: usize,
+) -> (usize, usize) {
+    let end_remainder = offset.saturating_add(length) % block_size;
+    if start_block == end_block {
+        (offset % block_size, length)
+    } else if block_index == start_block {
+        (offset % block_size, block_size - (offset % block_size))
+    } else if block_index == end_block {
+        (0, if end_remainder == 0 { block_size } else { end_remainder })
+    } else {
+        (0, block_size)
+    }
+}
+
+fn take_data_blocks_as_chunks(
+    shards: &mut [Option<Vec<u8>>],
+    data_blocks: usize,
+    mut offset: usize,
+    length: usize,
+) -> io::Result<Vec<IoChunk>> {
+    if get_data_block_len(shards, data_blocks) < length {
+        error!("take_data_blocks_as_chunks get_data_block_len < length");
+        return Err(io::Error::new(ErrorKind::UnexpectedEof, "Not enough data blocks to write"));
+    }
+
+    let mut chunks = Vec::new();
+    let mut remaining = length;
+    for block_op in shards.iter_mut().take(data_blocks) {
+        let Some(block) = block_op.take() else {
+            error!("take_data_blocks_as_chunks block_op.is_none()");
+            return Err(io::Error::new(ErrorKind::UnexpectedEof, "Missing data block"));
+        };
+
+        if offset >= block.len() {
+            offset -= block.len();
+            continue;
+        }
+
+        let start = offset;
+        offset = 0;
+        let take = (block.len() - start).min(remaining);
+        let chunk = if start == 0 && take == block.len() {
+            IoChunk::Pooled(PooledChunk::from_vec(block))
+        } else {
+            IoChunk::Pooled(PooledChunk::from_vec(block).slice(start, take)?)
+        };
+        chunks.push(chunk);
+        remaining -= take;
+        if remaining == 0 {
+            break;
+        }
+    }
+
+    Ok(chunks)
+}
+
 /// Write data blocks from encoded blocks to target, supporting offset and length
 async fn write_data_blocks<W>(
     writer: &mut W,
@@ -213,6 +276,134 @@ where
     Ok(total_written)
 }
 
+pub(crate) struct ErasureChunkDecoder<R> {
+    erasure: Erasure,
+    reader: ParallelReader<R>,
+    offset: usize,
+    length: usize,
+    start_block: usize,
+    end_block: usize,
+    current_block: usize,
+    written: usize,
+    healable_error: Option<Error>,
+    finished: bool,
+}
+
+impl<R> ErasureChunkDecoder<R>
+where
+    R: AsyncRead + Unpin + Send + Sync,
+{
+    pub(crate) fn new(
+        erasure: Erasure,
+        readers: Vec<Option<BitrotReader<R>>>,
+        offset: usize,
+        length: usize,
+        total_length: usize,
+    ) -> io::Result<Self> {
+        if readers.len() != erasure.data_shards + erasure.parity_shards {
+            return Err(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"));
+        }
+
+        let end_offset = offset
+            .checked_add(length)
+            .ok_or_else(|| io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))?;
+        if end_offset > total_length {
+            return Err(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"));
+        }
+
+        let start_block = offset / erasure.block_size;
+        let end_block = if length == 0 {
+            start_block
+        } else {
+            end_offset.saturating_sub(1) / erasure.block_size
+        };
+        let reader = ParallelReader::new(readers, erasure.clone(), offset, total_length);
+
+        Ok(Self {
+            erasure,
+            reader,
+            offset,
+            length,
+            start_block,
+            end_block,
+            current_block: start_block,
+            written: 0,
+            healable_error: None,
+            finished: length == 0,
+        })
+    }
+
+    pub(crate) async fn next_chunks(&mut self) -> io::Result<Option<Vec<IoChunk>>> {
+        if self.finished {
+            return Ok(None);
+        }
+
+        if self.current_block > self.end_block {
+            self.finished = true;
+            return Ok(None);
+        }
+
+        let block_index = self.current_block;
+        self.current_block += 1;
+
+        let (block_offset, block_length) = block_window(
+            self.offset,
+            self.length,
+            self.erasure.block_size,
+            block_index,
+            self.start_block,
+            self.end_block,
+        );
+        if block_length == 0 {
+            self.finished = true;
+            return Ok(None);
+        }
+
+        let (mut shards, errs) = self.reader.read().await;
+
+        if self.healable_error.is_none()
+            && let (_, Some(err)) = reduce_errs(&errs, &[])
+            && (err == Error::FileNotFound || err == Error::FileCorrupt)
+        {
+            self.healable_error = Some(err);
+        }
+
+        if !self.reader.can_decode(&shards) {
+            self.finished = true;
+            error!("reconstructed chunk decoder can_decode errs: {:?}", &errs);
+            return Err(Error::ErasureReadQuorum.into());
+        }
+
+        if let Err(err) = self.erasure.decode_data(&mut shards) {
+            self.finished = true;
+            error!("reconstructed chunk decoder decode_data err: {:?}", err);
+            return Err(err);
+        }
+
+        let chunks = take_data_blocks_as_chunks(&mut shards, self.erasure.data_shards, block_offset, block_length)?;
+        self.written += chunks.iter().map(IoChunk::len).sum::<usize>();
+        Ok(Some(chunks))
+    }
+
+    pub(crate) fn written(&self) -> usize {
+        self.written
+    }
+
+    pub(crate) fn finish_error(&self) -> Option<io::Error> {
+        if self.written < self.length {
+            Some(Error::LessData.into())
+        } else {
+            None
+        }
+    }
+
+    pub(crate) fn take_healable_error(&mut self) -> Option<Error> {
+        self.healable_error.take()
+    }
+}
+
+pub(crate) type ReconstructedChunkDecoder<R> = ErasureChunkDecoder<R>;
+
 impl Erasure {
     pub async fn decode<W, R>(
         &self,
@@ -230,7 +421,10 @@ impl Erasure {
             return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
         }
 
-        if offset + length > total_length {
+        let Some(end_offset) = offset.checked_add(length) else {
+            return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
+        };
+        if end_offset > total_length {
             return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
         }
 
@@ -245,7 +439,7 @@ impl Erasure {
         let mut reader = ParallelReader::new(readers, self.clone(), offset, total_length);
 
         let start = offset / self.block_size;
-        let end = (offset + length) / self.block_size;
+        let end = end_offset.saturating_sub(1) / self.block_size;
 
         for i in start..=end {
             let (block_offset, block_length) = if start == end {
@@ -253,7 +447,8 @@ impl Erasure {
             } else if i == start {
                 (offset % self.block_size, self.block_size - (offset % self.block_size))
             } else if i == end {
-                (0, (offset + length) % self.block_size)
+                let end_remainder = end_offset % self.block_size;
+                (0, if end_remainder == 0 { self.block_size } else { end_remainder })
             } else {
                 (0, self.block_size)
             };
@@ -316,6 +511,7 @@ mod tests {
         disk::error::DiskError,
         erasure_coding::{BitrotReader, BitrotWriter},
     };
+    use bytes::Bytes;
     use rustfs_utils::HashAlgorithm;
     use std::io::Cursor;
 
@@ -456,4 +652,47 @@ mod tests {
         let reader_cursor = Cursor::new(buf);
         BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
     }
+
+    async fn create_bitrot_reader_from_shard(
+        shard: Bytes,
+        shard_size: usize,
+        hash_algo: &HashAlgorithm,
+    ) -> BitrotReader<Cursor<Vec<u8>>> {
+        let writer = Cursor::new(Vec::new());
+        let mut writer = BitrotWriter::new(writer, shard_size, hash_algo.clone());
+        writer.write(shard.as_ref()).await.unwrap();
+        let reader_cursor = Cursor::new(writer.into_inner().into_inner());
+        BitrotReader::new(reader_cursor, shard_size, hash_algo.clone(), false)
+    }
+
+    #[tokio::test]
+    async fn test_erasure_chunk_decoder_reconstructs_missing_data_shard_as_pooled_chunks() {
+        let erasure = Erasure::new(2, 1, 4);
+        let original = b"abcd";
+        let encoded = erasure.encode_data(original).unwrap();
+        let shard_size = erasure.shard_size();
+        let hash_algo = HashAlgorithm::None;
+
+        let readers = vec![
+            None,
+            Some(create_bitrot_reader_from_shard(encoded[1].clone(), shard_size, &hash_algo).await),
+            Some(create_bitrot_reader_from_shard(encoded[2].clone(), shard_size, &hash_algo).await),
+        ];
+
+        let mut decoder = ErasureChunkDecoder::new(erasure, readers, 0, original.len(), original.len()).unwrap();
+        let first_batch = decoder.next_chunks().await.unwrap().unwrap();
+        assert!(
+            first_batch.iter().all(|chunk| matches!(chunk, IoChunk::Pooled(_))),
+            "reconstructed decoder should produce pooled chunks"
+        );
+        let collected = first_batch
+            .into_iter()
+            .flat_map(|chunk| chunk.as_bytes().to_vec())
+            .collect::<Vec<_>>();
+
+        assert_eq!(collected, original);
+        assert!(decoder.next_chunks().await.unwrap().is_none());
+        assert_eq!(decoder.written(), original.len());
+        assert!(decoder.finish_error().is_none());
+    }
 }

crates/ecstore/src/erasure_coding/encode.rs

@@ -17,11 +17,12 @@ use crate::disk::error_reduce::count_errs;
 use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
 use crate::erasure_coding::BitrotWriterWrapper;
 use crate::erasure_coding::Erasure;
+use crate::erasure_coding::erasure::{EncodeBlockBuffer, EncodedShardBlock, EncodedShardBufferPool};
 use bytes::Bytes;
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
+use rustfs_rio::BlockReadable;
 use std::sync::Arc;
-use std::vec;
 use tokio::io::AsyncRead;
 use tokio::sync::mpsc;
 use tracing::error;
@@ -32,6 +33,164 @@ pub(crate) struct MultiWriter<'a> {
     errs: Vec<Option<Error>>,
 }
 
+pub(crate) struct BlockAssembler<R> {
+    reader: R,
+    block_buffer: EncodeBlockBuffer,
+    total_bytes: usize,
+}
+
+impl<R> BlockAssembler<R>
+where
+    R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static,
+{
+    pub(crate) fn new(reader: R, block_size: usize) -> Self {
+        Self {
+            reader,
+            block_buffer: EncodeBlockBuffer::new(block_size),
+            total_bytes: 0,
+        }
+    }
+
+    pub(crate) async fn next_block(&mut self) -> std::io::Result<Option<Vec<u8>>> {
+        match self.block_buffer.read_from_block(&mut self.reader).await {
+            Ok(n) if n > 0 => {
+                self.total_bytes += n;
+                Ok(Some(self.block_buffer.filled(n).to_vec()))
+            }
+            Ok(_) => Ok(None),
+            Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
+                if let Some(inner) = e.get_ref()
+                    && rustfs_rio::is_checksum_mismatch(inner)
+                {
+                    return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
+                }
+                Ok(None)
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    pub(crate) fn total_bytes(&self) -> usize {
+        self.total_bytes
+    }
+
+    pub(crate) fn into_inner(self) -> R {
+        self.reader
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct ErasureChunkEncoder {
+    erasure: Arc<Erasure>,
+    buffer_pool: EncodedShardBufferPool,
+}
+
+impl ErasureChunkEncoder {
+    pub(crate) async fn new(erasure: Arc<Erasure>) -> Self {
+        let reusable_capacity = erasure.shard_size() * erasure.total_shard_count();
+        Self {
+            erasure,
+            buffer_pool: EncodedShardBufferPool::with_prefill(reusable_capacity, 2).await,
+        }
+    }
+
+    pub(crate) async fn encode_block(&self, block: &[u8]) -> std::io::Result<EncodedShardBlock> {
+        let reusable_buffer = self.buffer_pool.acquire().await;
+        self.erasure.encode_data_block_with_buffer(block, reusable_buffer)
+    }
+
+    pub(crate) async fn release(&self, block: EncodedShardBlock) {
+        self.buffer_pool.release(block).await;
+    }
+}
+
+pub(crate) struct ErasureWritePipeline {
+    erasure: Arc<Erasure>,
+    write_quorum: usize,
+}
+
+impl ErasureWritePipeline {
+    pub(crate) fn new(erasure: Arc<Erasure>, write_quorum: usize) -> Self {
+        Self { erasure, write_quorum }
+    }
+
+    pub(crate) async fn run<R>(&self, reader: R, writers: &mut [Option<BitrotWriterWrapper>]) -> std::io::Result<(R, usize)>
+    where
+        R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static,
+    {
+        let (tx, mut rx) = mpsc::channel::<EncodedShardBlock>(8);
+        let producer = ErasureChunkEncoder::new(self.erasure.clone()).await;
+        let writer_pool = producer.clone();
+        let block_size = self.erasure.block_size;
+
+        let task = tokio::spawn(async move {
+            let mut assembler = BlockAssembler::new(reader, block_size);
+            while let Some(block) = assembler.next_block().await? {
+                let res = producer.encode_block(&block).await?;
+                if let Err(err) = tx.send(res).await {
+                    return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
+                }
+            }
+
+            let total = assembler.total_bytes();
+            Ok((assembler.into_inner(), total))
+        });
+
+        let mut writers = MultiWriter::new(writers, self.write_quorum);
+        let mut write_err = None;
+
+        while let Some(block) = rx.recv().await {
+            if block.is_empty() {
+                break;
+            }
+            let write_result = writers.write(&block).await;
+            writer_pool.release(block).await;
+            if let Err(err) = write_result {
+                write_err = Some(err);
+                break;
+            }
+        }
+
+        if let Some(err) = write_err {
+            task.abort();
+            let _ = task.await;
+            if let Err(shutdown_err) = writers.shutdown().await {
+                error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
+            }
+            return Err(err);
+        }
+
+        let (reader, total) = task.await??;
+        writers.shutdown().await?;
+        Ok((reader, total))
+    }
+}
+
+pub(crate) trait ShardSource {
+    fn shard_count(&self) -> usize;
+    fn shard(&self, idx: usize) -> Bytes;
+}
+
+impl ShardSource for EncodedShardBlock {
+    fn shard_count(&self) -> usize {
+        self.shard_count()
+    }
+
+    fn shard(&self, idx: usize) -> Bytes {
+        self.shard(idx)
+    }
+}
+
+impl ShardSource for Vec<Bytes> {
+    fn shard_count(&self) -> usize {
+        self.len()
+    }
+
+    fn shard(&self, idx: usize) -> Bytes {
+        self[idx].clone()
+    }
+}
+
 impl<'a> MultiWriter<'a> {
     pub fn new(writers: &'a mut [Option<BitrotWriterWrapper>], write_quorum: usize) -> Self {
         let length = writers.len();
@@ -42,10 +201,10 @@ impl<'a> MultiWriter<'a> {
         }
     }
 
-    async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: &Bytes) {
+    async fn write_shard(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: Bytes) {
         match writer_opt {
             Some(writer) => {
-                match writer.write(shard).await {
+                match writer.write(&shard).await {
                     Ok(n) => {
                         if n < shard.len() {
                             *err = Some(Error::ShortWrite);
@@ -65,16 +224,40 @@ impl<'a> MultiWriter<'a> {
         }
     }
 
-    pub async fn write(&mut self, data: Vec<Bytes>) -> std::io::Result<()> {
-        assert_eq!(data.len(), self.writers.len());
+    fn write_shard_inline(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>, shard: Bytes) {
+        match writer_opt {
+            Some(writer) => match writer.write_inline_sync(&shard) {
+                Ok(n) => {
+                    if n < shard.len() {
+                        *err = Some(Error::ShortWrite);
+                        *writer_opt = None;
+                    } else {
+                        *err = None;
+                    }
+                }
+                Err(e) => {
+                    *err = Some(Error::from(e));
+                }
+            },
+            None => {
+                *err = Some(Error::DiskNotFound);
+            }
+        }
+    }
+
+    pub async fn write<T>(&mut self, data: &T) -> std::io::Result<()>
+    where
+        T: ShardSource,
+    {
+        assert_eq!(data.shard_count(), self.writers.len());
 
         {
             let mut futures = FuturesUnordered::new();
-            for ((writer_opt, err), shard) in self.writers.iter_mut().zip(self.errs.iter_mut()).zip(data.iter()) {
+            for (idx, (writer_opt, err)) in self.writers.iter_mut().zip(self.errs.iter_mut()).enumerate() {
                 if err.is_some() {
                     continue; // Skip if we already have an error for this writer
                 }
-                futures.push(Self::write_shard(writer_opt, err, shard));
+                futures.push(Self::write_shard(writer_opt, err, data.shard(idx)));
             }
             while let Some(()) = futures.next().await {}
         }
@@ -112,6 +295,45 @@ impl<'a> MultiWriter<'a> {
         )))
     }
 
+    pub fn write_inline<T>(&mut self, data: &T) -> std::io::Result<()>
+    where
+        T: ShardSource,
+    {
+        assert_eq!(data.shard_count(), self.writers.len());
+
+        for (idx, (writer_opt, err)) in self.writers.iter_mut().zip(self.errs.iter_mut()).enumerate() {
+            if err.is_some() {
+                continue;
+            }
+            Self::write_shard_inline(writer_opt, err, data.shard(idx));
+        }
+
+        let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
+        if nil_count >= self.write_quorum {
+            return Ok(());
+        }
+
+        if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
+            return Err(std::io::Error::other(format!(
+                "Failed to write inline data: {} (offline-disks={}/{})",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len()
+            )));
+        }
+
+        Err(std::io::Error::other(format!(
+            "Failed to write inline data: (offline-disks={}/{}): {}",
+            count_errs(&self.errs, &Error::DiskNotFound),
+            self.writers.len(),
+            self.errs
+                .iter()
+                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )))
+    }
+
     async fn shutdown_writer(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
         match writer_opt {
             Some(writer) => match writer.shutdown().await {
@@ -129,6 +351,23 @@ impl<'a> MultiWriter<'a> {
         }
     }
 
+    fn shutdown_writer_inline(writer_opt: &mut Option<BitrotWriterWrapper>, err: &mut Option<Error>) {
+        match writer_opt {
+            Some(writer) => match writer.shutdown_inline_sync() {
+                Ok(()) => {
+                    *err = None;
+                }
+                Err(e) => {
+                    *err = Some(Error::from(e));
+                    *writer_opt = None;
+                }
+            },
+            None => {
+                *err = Some(Error::DiskNotFound);
+            }
+        }
+    }
+
     pub async fn shutdown(&mut self) -> std::io::Result<()> {
         {
             let mut futures = FuturesUnordered::new();
@@ -173,66 +412,53 @@ impl<'a> MultiWriter<'a> {
                 .join(", ")
         )))
     }
+
+    pub fn shutdown_inline(&mut self) -> std::io::Result<()> {
+        for (writer_opt, err) in self.writers.iter_mut().zip(self.errs.iter_mut()) {
+            if err.is_some() {
+                continue;
+            }
+            Self::shutdown_writer_inline(writer_opt, err);
+        }
+
+        let nil_count = self.errs.iter().filter(|&e| e.is_none()).count();
+        if nil_count >= self.write_quorum {
+            return Ok(());
+        }
+
+        if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
+            return Err(std::io::Error::other(format!(
+                "Failed to shutdown inline writers: {} (offline-disks={}/{})",
+                write_err,
+                count_errs(&self.errs, &Error::DiskNotFound),
+                self.writers.len()
+            )));
+        }
+
+        Err(std::io::Error::other(format!(
+            "Failed to shutdown inline writers: (offline-disks={}/{}): {}",
+            count_errs(&self.errs, &Error::DiskNotFound),
+            self.writers.len(),
+            self.errs
+                .iter()
+                .map(|e| e.as_ref().map_or("<nil>".to_string(), |e| e.to_string()))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )))
+    }
 }
 
 impl Erasure {
     pub async fn encode<R>(
         self: Arc<Self>,
-        mut reader: R,
+        reader: R,
         writers: &mut [Option<BitrotWriterWrapper>],
         quorum: usize,
     ) -> std::io::Result<(R, usize)>
     where
-        R: AsyncRead + Send + Sync + Unpin + 'static,
+        R: AsyncRead + BlockReadable + Send + Sync + Unpin + 'static,
     {
-        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(8);
-
-        let task = tokio::spawn(async move {
-            let block_size = self.block_size;
-            let mut total = 0;
-            let mut buf = vec![0u8; block_size];
-            loop {
-                match rustfs_utils::read_full(&mut reader, &mut buf).await {
-                    Ok(n) if n > 0 => {
-                        total += n;
-                        let res = self.encode_data(&buf[..n])?;
-                        if let Err(err) = tx.send(res).await {
-                            return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
-                        }
-                    }
-                    Ok(_) => {
-                        break;
-                    }
-                    Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
-                        // Check if the inner error is a checksum mismatch - if so, propagate it
-                        if let Some(inner) = e.get_ref()
-                            && rustfs_rio::is_checksum_mismatch(inner)
-                        {
-                            return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
-                        }
-                        break;
-                    }
-                    Err(e) => {
-                        return Err(e);
-                    }
-                }
-            }
-
-            Ok((reader, total))
-        });
-
-        let mut writers = MultiWriter::new(writers, quorum);
-
-        while let Some(block) = rx.recv().await {
-            if block.is_empty() {
-                break;
-            }
-            writers.write(block).await?;
-        }
-
-        let (reader, total) = task.await??;
-        writers.shutdown().await?;
-        Ok((reader, total))
+        ErasureWritePipeline::new(self, quorum).run(reader, writers).await
     }
 }
 
@@ -241,6 +467,7 @@ mod tests {
     use super::*;
     use crate::erasure_coding::{BitrotWriterWrapper, CustomWriter};
     use rustfs_utils::HashAlgorithm;
+    use std::io::Cursor;
     use std::pin::Pin;
     use std::sync::{Arc, Mutex};
     use std::task::{Context, Poll};
@@ -296,4 +523,28 @@ mod tests {
         assert_eq!(written, b"small payload".len());
         assert!(!committed.lock().unwrap().is_empty());
     }
+
+    #[tokio::test]
+    async fn block_assembler_splits_input_into_erasure_blocks() {
+        let reader = tokio::io::BufReader::new(Cursor::new(b"abcdefghijkl".to_vec()));
+        let mut assembler = BlockAssembler::new(reader, 4);
+
+        assert_eq!(assembler.next_block().await.unwrap(), Some(b"abcd".to_vec()));
+        assert_eq!(assembler.next_block().await.unwrap(), Some(b"efgh".to_vec()));
+        assert_eq!(assembler.next_block().await.unwrap(), Some(b"ijkl".to_vec()));
+        assert_eq!(assembler.next_block().await.unwrap(), None);
+        assert_eq!(assembler.total_bytes(), 12);
+    }
+
+    #[tokio::test]
+    async fn erasure_chunk_encoder_produces_full_shard_block() {
+        let erasure = Arc::new(Erasure::new(2, 1, 4));
+        let encoder = ErasureChunkEncoder::new(erasure.clone()).await;
+        let block = encoder.encode_block(b"abcd").await.unwrap();
+
+        assert_eq!(block.shard_count(), 3);
+        assert_eq!(block.shard(0).len(), erasure.shard_size());
+
+        encoder.release(block).await;
+    }
 }

crates/ecstore/src/erasure_coding/erasure.rs

@@ -19,12 +19,131 @@
 use bytes::{Bytes, BytesMut};
 use reed_solomon_erasure::galois_8::ReedSolomon;
 use reed_solomon_simd;
+use rustfs_rio::BlockReadable;
 use smallvec::SmallVec;
 use std::io;
+use std::sync::Arc;
 use tokio::io::AsyncRead;
+use tokio::sync::Mutex;
 use tracing::warn;
 use uuid::Uuid;
 
+pub(crate) struct EncodeBlockBuffer {
+    buf: Vec<u8>,
+}
+
+impl EncodeBlockBuffer {
+    pub(crate) fn new(block_size: usize) -> Self {
+        Self {
+            buf: vec![0u8; block_size],
+        }
+    }
+
+    pub(crate) async fn read_from<R>(&mut self, reader: &mut R) -> io::Result<usize>
+    where
+        R: AsyncRead + Send + Sync + Unpin,
+    {
+        rustfs_utils::read_full(&mut *reader, &mut self.buf).await
+    }
+
+    pub(crate) async fn read_from_block<R>(&mut self, reader: &mut R) -> io::Result<usize>
+    where
+        R: BlockReadable + Send + Sync + Unpin,
+    {
+        reader.read_block(&mut self.buf).await
+    }
+
+    pub(crate) fn filled(&self, len: usize) -> &[u8] {
+        &self.buf[..len]
+    }
+}
+
+pub struct EncodedShardBlock {
+    data: Bytes,
+    shard_size: usize,
+    shard_count: usize,
+}
+
+impl EncodedShardBlock {
+    pub(crate) fn new(data: Bytes, shard_size: usize, shard_count: usize) -> Self {
+        Self {
+            data,
+            shard_size,
+            shard_count,
+        }
+    }
+
+    pub fn shard_count(&self) -> usize {
+        self.shard_count
+    }
+
+    pub fn len(&self) -> usize {
+        self.shard_count
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.shard_count == 0
+    }
+
+    pub fn shard(&self, idx: usize) -> Bytes {
+        let start = idx * self.shard_size;
+        let end = start + self.shard_size;
+        self.data.slice(start..end)
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = Bytes> + '_ {
+        (0..self.shard_count).map(|idx| self.shard(idx))
+    }
+
+    pub fn into_vec(self) -> Vec<Bytes> {
+        (0..self.shard_count).map(|idx| self.shard(idx)).collect()
+    }
+
+    pub fn into_reusable_buffer(self) -> BytesMut {
+        match self.data.try_into_mut() {
+            Ok(mut buf) => {
+                buf.clear();
+                buf
+            }
+            Err(data) => BytesMut::with_capacity(data.len()),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub(crate) struct EncodedShardBufferPool {
+    capacity: usize,
+    free: Arc<Mutex<Vec<BytesMut>>>,
+}
+
+impl EncodedShardBufferPool {
+    pub(crate) async fn with_prefill(capacity: usize, initial: usize) -> Self {
+        let mut free = Vec::with_capacity(initial);
+        for _ in 0..initial {
+            free.push(BytesMut::with_capacity(capacity));
+        }
+
+        Self {
+            capacity,
+            free: Arc::new(Mutex::new(free)),
+        }
+    }
+
+    pub(crate) async fn acquire(&self) -> BytesMut {
+        let mut free = self.free.lock().await;
+        free.pop().unwrap_or_else(|| BytesMut::with_capacity(self.capacity))
+    }
+
+    pub(crate) async fn release(&self, block: EncodedShardBlock) {
+        let mut free = self.free.lock().await;
+        let mut buf = block.into_reusable_buffer();
+        if buf.capacity() < self.capacity {
+            buf.reserve(self.capacity - buf.capacity());
+        }
+        free.push(buf);
+    }
+}
+
 /// Legacy calc_shard_size formula: (block_size.div_ceil(data_shards) + 1) & !1
 /// Matches main branch and filemeta::ErasureInfo for old-version files.
 pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize {
@@ -351,6 +470,25 @@ impl Erasure {
     /// A vector of encoded shards as `Bytes`.
     #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
     pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
+        Ok(self.encode_data_block(data)?.into_vec())
+    }
+
+    /// Encode one logical block into an `EncodedShardBlock` using a caller-provided backing buffer.
+    ///
+    /// This is the explicit reuse-oriented variant for non-hot paths that want to
+    /// thread a reusable `BytesMut` across multiple encode calls.
+    #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
+    pub fn encode_data_with_buffer(&self, data: &[u8], data_buffer: BytesMut) -> io::Result<EncodedShardBlock> {
+        self.encode_data_block_with_buffer(data, data_buffer)
+    }
+
+    #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
+    pub(crate) fn encode_data_block(&self, data: &[u8]) -> io::Result<EncodedShardBlock> {
+        self.encode_data_block_with_buffer(data, BytesMut::with_capacity(self.shard_size() * self.total_shard_count()))
+    }
+
+    #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
+    pub(crate) fn encode_data_block_with_buffer(&self, data: &[u8], mut data_buffer: BytesMut) -> io::Result<EncodedShardBlock> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
         } else {
@@ -359,7 +497,10 @@ impl Erasure {
         let per_shard_size = shard_size_fn(data.len(), self.data_shards);
         let need_total_size = per_shard_size * self.total_shard_count();
 
-        let mut data_buffer = BytesMut::with_capacity(need_total_size);
+        data_buffer.clear();
+        if data_buffer.capacity() < need_total_size {
+            data_buffer.reserve(need_total_size - data_buffer.capacity());
+        }
         data_buffer.extend_from_slice(data);
         data_buffer.resize(need_total_size, 0u8);
 
@@ -382,14 +523,7 @@ impl Erasure {
         }
 
         // Zero-copy split, all shards reference data_buffer
-        let mut data_buffer = data_buffer.freeze();
-        let mut shards = Vec::with_capacity(self.total_shard_count());
-        for _ in 0..self.total_shard_count() {
-            let shard = data_buffer.split_to(per_shard_size);
-            shards.push(shard);
-        }
-
-        Ok(shards)
+        Ok(EncodedShardBlock::new(data_buffer.freeze(), per_shard_size, self.total_shard_count()))
     }
 
     /// Decode and reconstruct missing shards in-place.
@@ -478,8 +612,8 @@ impl Erasure {
     ///
     /// # Arguments
     /// * `reader` - An async reader implementing AsyncRead + Send + Sync + Unpin
-    /// * `mut on_block` - Async callback that receives encoded blocks and returns a Result
-    /// * `F` - Callback type: FnMut(Result<Vec<Bytes>, std::io::Error>) -> Future<Output=Result<(), E>> + Send
+    /// * `mut on_block` - Async callback that receives encoded blocks and returns the block for reuse
+    /// * `F` - Callback type: FnMut(Result<EncodedShardBlock, std::io::Error>) -> Future<Output=Result<Option<EncodedShardBlock>, E>> + Send
     /// * `Fut` - Future type returned by the callback
     /// * `E` - Error type returned by the callback
     /// * `R` - Reader type implementing AsyncRead + Send + Sync + Unpin
@@ -489,26 +623,31 @@ impl Erasure {
     ///
     /// # Errors
     /// Returns error if reading from reader fails or if callback returns error
-    pub async fn encode_stream_callback_async<F, Fut, E, R>(
+    pub(crate) async fn encode_stream_callback_async<F, Fut, E, R>(
         self: std::sync::Arc<Self>,
         reader: &mut R,
         mut on_block: F,
     ) -> Result<usize, E>
     where
         R: AsyncRead + Send + Sync + Unpin,
-        F: FnMut(std::io::Result<Vec<Bytes>>) -> Fut + Send,
-        Fut: std::future::Future<Output = Result<(), E>> + Send,
+        F: FnMut(std::io::Result<EncodedShardBlock>) -> Fut + Send,
+        Fut: std::future::Future<Output = Result<Option<EncodedShardBlock>, E>> + Send,
     {
         let block_size = self.block_size;
         let mut total = 0;
+        let mut block_buffer = EncodeBlockBuffer::new(block_size);
+        let reusable_capacity = self.shard_size() * self.total_shard_count();
+        let buffer_pool = EncodedShardBufferPool::with_prefill(reusable_capacity, 1).await;
         loop {
-            let mut buf = vec![0u8; block_size];
-            match rustfs_utils::read_full(&mut *reader, &mut buf).await {
+            match block_buffer.read_from(&mut *reader).await {
                 Ok(n) if n > 0 => {
                     warn!("encode_stream_callback_async read n={}", n);
                     total += n;
-                    let res = self.encode_data(&buf[..n]);
-                    on_block(res).await?
+                    let reusable_buffer = buffer_pool.acquire().await;
+                    let res = self.encode_data_block_with_buffer(block_buffer.filled(n), reusable_buffer);
+                    if let Some(block) = on_block(res).await? {
+                        buffer_pool.release(block).await;
+                    }
                 }
                 Ok(_) => {
                     warn!("encode_stream_callback_async read unexpected ok");
@@ -520,11 +659,10 @@ impl Erasure {
                 }
                 Err(e) => {
                     warn!("encode_stream_callback_async read error={:?}", e);
-                    on_block(Err(e)).await?;
+                    let _ = on_block(Err(e)).await?;
                     break;
                 }
             }
-            buf.clear();
         }
         Ok(total)
     }
@@ -747,8 +885,8 @@ mod tests {
                     let tx = tx.clone();
                     async move {
                         let shards = res.unwrap();
-                        tx.send(shards).await.unwrap();
-                        Ok(())
+                        tx.send(shards.iter().collect()).await.unwrap();
+                        Ok(Some(shards))
                     }
                 })
                 .await
@@ -760,6 +898,36 @@ mod tests {
         assert_eq!(collected_shards.len(), data_shards + parity_shards);
     }
 
+    #[test]
+    fn test_encode_data_with_buffer_supports_explicit_reuse() {
+        let erasure = Erasure::new(4, 2, 1024);
+        let reusable_capacity = erasure.shard_size() * erasure.total_shard_count();
+
+        let first_data = b"explicit reusable buffer path".repeat(32);
+        let first_block = erasure
+            .encode_data_with_buffer(&first_data, BytesMut::with_capacity(reusable_capacity))
+            .expect("first encode should succeed");
+        let reusable_buffer = first_block.into_reusable_buffer();
+        assert!(reusable_buffer.capacity() >= reusable_capacity);
+
+        let second_data = b"second encode through same reusable buffer".repeat(24);
+        let second_block = erasure
+            .encode_data_with_buffer(&second_data, reusable_buffer)
+            .expect("second encode should succeed");
+
+        let mut shards_opt: Vec<Option<Vec<u8>>> = second_block.iter().map(|shard| Some(shard.to_vec())).collect();
+        shards_opt[1] = None;
+        shards_opt[5] = None;
+        erasure.decode_data(&mut shards_opt).expect("decode should succeed");
+
+        let mut recovered = Vec::new();
+        for shard in shards_opt.iter().take(erasure.data_shards) {
+            recovered.extend_from_slice(shard.as_ref().expect("data shard should exist after decode"));
+        }
+        recovered.truncate(second_data.len());
+        assert_eq!(&recovered, &second_data);
+    }
+
     #[tokio::test]
     async fn test_encode_stream_callback_async_channel_decode() {
         use std::io::Cursor;
@@ -786,8 +954,8 @@ mod tests {
                     let tx = tx.clone();
                     async move {
                         let shards = res.unwrap();
-                        tx.send(shards).await.unwrap();
-                        Ok(())
+                        tx.send(shards.iter().collect()).await.unwrap();
+                        Ok(Some(shards))
                     }
                 })
                 .await
@@ -800,8 +968,8 @@ mod tests {
 
         // Test decode using the old API that operates in-place
         let mut decode_input: Vec<Option<Vec<u8>>> = vec![None; data_shards + parity_shards];
-        for i in 0..data_shards {
-            decode_input[i] = Some(shards[i].to_vec());
+        for (i, shard) in shards.iter().enumerate().take(data_shards) {
+            decode_input[i] = Some(shard.to_vec());
         }
         erasure.decode_data(&mut decode_input).unwrap();
 
@@ -1198,8 +1366,8 @@ mod tests {
                         let tx = tx.clone();
                         async move {
                             let shards = res.unwrap();
-                            tx.send(shards).await.unwrap();
-                            Ok(())
+                            tx.send(shards.iter().collect()).await.unwrap();
+                            Ok(Some(shards))
                         }
                     })
                     .await
@@ -1233,5 +1401,63 @@ mod tests {
             recovered.truncate(data_clone.len());
             assert_eq!(&recovered, &data_clone);
         }
+
+        #[tokio::test]
+        #[ignore]
+        async fn stress_simd_stream_callback_reuses_backing_buffers_across_many_blocks() {
+            use std::io::Cursor;
+            use std::sync::Arc;
+            use std::sync::atomic::{AtomicUsize, Ordering};
+            use tokio::sync::Mutex;
+
+            let data_shards = 4;
+            let parity_shards = 2;
+            let block_size = 1024;
+            let erasure = Arc::new(Erasure::new(data_shards, parity_shards, block_size));
+
+            let sample =
+                b"SIMD stress callback test payload that intentionally spans many blocks to exercise reusable backing buffers.";
+            let data = sample.repeat((4 * 1024 * 1024 / sample.len()).max(1));
+            let data_clone = data.clone();
+            let mut reader = Cursor::new(data);
+
+            let recovered = Arc::new(Mutex::new(Vec::with_capacity(data_clone.len())));
+            let block_count = Arc::new(AtomicUsize::new(0));
+            let erasure_for_callback = erasure.clone();
+            let recovered_for_callback = recovered.clone();
+            let block_count_for_callback = block_count.clone();
+
+            erasure
+                .clone()
+                .encode_stream_callback_async::<_, _, (), _>(&mut reader, move |res| {
+                    let erasure_for_callback = erasure_for_callback.clone();
+                    let recovered_for_callback = recovered_for_callback.clone();
+                    let block_count_for_callback = block_count_for_callback.clone();
+                    async move {
+                        let shards = res.unwrap();
+                        block_count_for_callback.fetch_add(1, Ordering::Relaxed);
+
+                        let mut shards_opt: Vec<Option<Vec<u8>>> = shards.iter().map(|b| Some(b.to_vec())).collect();
+                        shards_opt[1] = None;
+                        shards_opt[5] = None;
+                        erasure_for_callback.decode_data(&mut shards_opt).unwrap();
+
+                        let mut recovered = recovered_for_callback.lock().await;
+                        for shard in shards_opt.iter().take(data_shards) {
+                            recovered.extend_from_slice(shard.as_ref().unwrap());
+                        }
+
+                        Ok(Some(shards))
+                    }
+                })
+                .await
+                .unwrap();
+
+            assert!(block_count.load(Ordering::Relaxed) > 1024);
+
+            let mut recovered = recovered.lock().await;
+            recovered.truncate(data_clone.len());
+            assert_eq!(&*recovered, &data_clone);
+        }
     }
 }

crates/ecstore/src/erasure_coding/heal.rs

@@ -77,7 +77,7 @@ impl super::Erasure {
             let available_writers = writers.iter().filter(|w| w.is_some()).count();
             let write_quorum = available_writers.max(1); // At least 1 writer must succeed
             let mut writers = MultiWriter::new(writers, write_quorum);
-            writers.write(shards).await?;
+            writers.write(&shards).await?;
         }
 
         Ok(())