perf(ecstore): slice in-memory shards instead of copying them twice (#4687)

* perf(ecstore): slice in-memory shards instead of copying them twice (backlog#1159)

The GET path reads a shard out of the page cache into a `Bytes`, then
`open_disk_reader` erased it behind `Box<dyn AsyncRead>` by wrapping it in
a `Cursor`. Downstream, `BitrotReader` could only get it back by copying:
once out of the `Cursor` into its scratch buffer, and once from there into
the caller's buffer. CPU profiling of a cached 1 MiB GET (device reads = 0)
attributed 8.23% of the whole server to `Cursor::poll_read` alone — a copy
of data that was already sitting in memory.

Keep the source concrete instead of erasing it. `ShardReader` is an enum of
`InMemory(Cursor<Bytes>)` and `Stream(Box<dyn AsyncRead ...>)`, and the new
`ShardSource::try_take_block(n)` lets an in-memory source hand over the
`[hash][data]` block as a slice. `read_appending` uses it to verify the hash
on the slice and `extend_from_slice` the shard straight into the caller's
buffer: one copy instead of two.

`try_take_block` defaults to `None`, so a streaming source keeps the old
path byte for byte, along with its short-read and EOF semantics. A source
that cannot serve `n` bytes declines rather than truncating, so a partial
block still becomes UnexpectedEof rather than a short shard. The hash is
still checked before anything is appended, so a corrupt shard never reaches
the caller's buffer on either path. The deferred parity reader opens its
source lazily and stays on the streaming path; parity is only read when a
data shard fails.

Tests gate equivalence and non-vacuity:
  * `try_take_block` fires for `Cursor<Bytes>`, advances the position exactly
    as a read of the same length would, declines when fewer than `n` bytes
    remain, and returns `None` for a non-`Bytes` source — without this the
    equivalence test below would silently compare one path against itself;
  * both paths return identical bytes for the same shard;
  * a corrupt shard fails on the fast path too, appending nothing.

Verified: clippy --tests -D warnings clean; `erasure::` 215 passed, 0 failed;
`set_disk::core::io_primitives` 49 and `io_support::` 22 pass.

Stacked on #4681 (`read_appending`), which this builds on.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): implement ShardSource for Cursor<&[u8]> used by the erasure bench

`crates/ecstore/benches/erasure_benchmark.rs` builds
`BitrotReader<Cursor<&[u8]>>`, which the new `ShardSource` bound on
`ParallelReader`/`decode` does not accept. `cargo clippy --tests` does not
compile bench targets, so this only surfaced in CI's `--all-targets` run.

A borrowed slice carries no `Bytes` to hand out, so it takes the default
`try_take_block` and keeps the old streaming copy path — no behavior change.

Verified with the same target set CI uses:
`cargo clippy -p rustfs-ecstore --all-targets -- -D warnings` clean.

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/io_support/bitrot.rs

@@ -33,7 +33,36 @@ use std::time::Instant;
 use tokio::io::{AsyncRead, ReadBuf};
 use tracing::debug;
 
-type BoxedObjectReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
+/// A shard source for the bitrot reader.
+///
+/// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
+/// `dyn AsyncRead`, so `BitrotReader` can slice the `[hash][data]` block straight
+/// out of the page-cache copy rather than copying it into a scratch buffer first
+/// (rustfs/backlog#1159). Everything else is a stream and keeps the old path.
+pub enum ShardReader {
+    InMemory(Cursor<Bytes>),
+    Stream(Box<dyn AsyncRead + Send + Sync + Unpin>),
+}
+
+impl AsyncRead for ShardReader {
+    fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+        match self.get_mut() {
+            Self::InMemory(cursor) => Pin::new(cursor).poll_read(cx, buf),
+            Self::Stream(reader) => Pin::new(reader).poll_read(cx, buf),
+        }
+    }
+}
+
+impl crate::erasure::coding::ShardSource for ShardReader {
+    fn try_take_block(&mut self, n: usize) -> Option<Bytes> {
+        match self {
+            Self::InMemory(cursor) => cursor.try_take_block(n),
+            Self::Stream(_) => None,
+        }
+    }
+}
+
+type BoxedObjectReader = ShardReader;
 type OpenObjectReaderFuture = Pin<Box<dyn Future<Output = disk::error::Result<Option<BoxedObjectReader>>> + Send>>;
 
 #[derive(Clone, Copy)]
@@ -70,7 +99,7 @@ impl BitrotReaderSource {
             let mut rd = Cursor::new(data);
             let offset = u64::try_from(self.offset).map_err(|_| DiskError::FileCorrupt)?;
             rd.set_position(offset);
-            Ok(Some(Box::new(rd)))
+            Ok(Some(ShardReader::InMemory(rd)))
         } else if let Some(disk) = self.disk {
             open_disk_reader(
                 &disk,
@@ -272,7 +301,7 @@ async fn open_disk_reader(
     length: usize,
     use_mmap_read: bool,
     metrics_path: Option<&'static str>,
-) -> disk::error::Result<FileReader> {
+) -> disk::error::Result<ShardReader> {
     let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
     let stage_metrics_enabled = metrics_path.is_some();
 
@@ -309,7 +338,7 @@ async fn open_disk_reader(
                     "zero_copy_read_success"
                 );
 
-                return Ok(Box::new(Cursor::new(bytes)));
+                return Ok(ShardReader::InMemory(Cursor::new(bytes)));
             }
             Err(err) => {
                 if let Some(metrics_path) = metrics_path {
@@ -345,14 +374,18 @@ async fn open_disk_reader(
     Ok(wrap_first_read_metrics(reader, metrics_path))
 }
 
-fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str>) -> FileReader {
+fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str>) -> ShardReader {
     if let Some(metrics_path) = metrics_path
         && rustfs_io_metrics::get_stage_metrics_enabled()
     {
-        return Box::new(FirstReadMetricsReader::new(reader, metrics_path, GET_STAGE_READER_STREAM_FIRST_READ));
+        return ShardReader::Stream(Box::new(FirstReadMetricsReader::new(
+            reader,
+            metrics_path,
+            GET_STAGE_READER_STREAM_FIRST_READ,
+        )));
     }
 
-    reader
+    ShardReader::Stream(reader)
 }
 
 fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
@@ -387,7 +420,7 @@ pub async fn create_bitrot_reader(
     checksum_algo: HashAlgorithm,
     skip_verify: bool,
     use_mmap_read: bool,
-) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
+) -> disk::error::Result<Option<BitrotReader<ShardReader>>> {
     create_bitrot_reader_with_stage_metrics(
         inline_data,
         disk,
@@ -417,7 +450,7 @@ pub(crate) async fn create_bitrot_reader_with_stage_metrics(
     skip_verify: bool,
     use_mmap_read: bool,
     stage_metrics: Option<BitrotReaderStageMetrics>,
-) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
+) -> disk::error::Result<Option<BitrotReader<ShardReader>>> {
     create_bitrot_reader_from_bytes_with_stage_metrics(
         inline_data.map(Bytes::copy_from_slice),
         disk,
@@ -450,7 +483,7 @@ pub async fn create_bitrot_reader_from_bytes(
     checksum_algo: HashAlgorithm,
     skip_verify: bool,
     use_mmap_read: bool,
-) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
+) -> disk::error::Result<Option<BitrotReader<ShardReader>>> {
     create_bitrot_reader_from_bytes_with_stage_metrics(
         inline_data,
         disk,
@@ -480,7 +513,7 @@ async fn create_bitrot_reader_from_bytes_with_stage_metrics(
     skip_verify: bool,
     use_mmap_read: bool,
     stage_metrics: Option<BitrotReaderStageMetrics>,
-) -> disk::error::Result<Option<BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>>> {
+) -> disk::error::Result<Option<BitrotReader<ShardReader>>> {
     let stage_metrics = stage_metrics.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
     let stage_metrics_enabled = stage_metrics.is_some();
 
@@ -527,7 +560,7 @@ pub fn create_deferred_bitrot_reader(
     checksum_algo: HashAlgorithm,
     skip_verify: bool,
     use_mmap_read: bool,
-) -> BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>> {
+) -> BitrotReader<ShardReader> {
     create_deferred_bitrot_reader_with_stripe_handle(
         inline_data,
         disk,
@@ -558,7 +591,7 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
     checksum_algo: HashAlgorithm,
     skip_verify: bool,
     use_mmap_read: bool,
-) -> (BitrotReader<Box<dyn AsyncRead + Send + Sync + Unpin>>, DeferredReaderStripeHandle) {
+) -> (BitrotReader<ShardReader>, DeferredReaderStripeHandle) {
     let stripe_stride = shard_size + checksum_algo.size();
     let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
     let source = BitrotReaderSource {
@@ -574,12 +607,10 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
 
     let deferred = DeferredObjectReader::new(source);
     let handle = deferred.stripe_handle(stripe_stride);
-    let reader = BitrotReader::new(
-        Box::new(deferred) as Box<dyn AsyncRead + Send + Sync + Unpin>,
-        shard_size,
-        checksum_algo,
-        skip_verify,
-    );
+    // The deferred parity reader opens its source lazily, so it cannot hand out an
+    // in-memory block up front; it stays on the streaming path. Parity shards are
+    // only read when a data shard fails, so the fast path is not needed here.
+    let reader = BitrotReader::new(ShardReader::Stream(Box::new(deferred)), shard_size, checksum_algo, skip_verify);
     (reader, handle)
 }