feat(storage): add direct chunk GET fast path (#2351)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>

crates/io-core/src/adapter.rs

@@ -0,0 +1,124 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Compatibility adapter that exposes chunk streams as `AsyncRead`.
+
+use crate::chunk::BoxChunkStream;
+use bytes::Bytes;
+use std::io;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::io::{AsyncRead, ReadBuf};
+
+/// `AsyncRead` adapter for boxed chunk streams.
+pub struct ChunkStreamReader {
+    stream: BoxChunkStream,
+    current: Option<Bytes>,
+    offset: usize,
+}
+
+impl ChunkStreamReader {
+    #[must_use]
+    pub fn new(stream: BoxChunkStream) -> Self {
+        Self {
+            stream,
+            current: None,
+            offset: 0,
+        }
+    }
+}
+
+impl AsyncRead for ChunkStreamReader {
+    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
+        loop {
+            if let Some(current) = &self.current {
+                if self.offset < current.len() {
+                    let remaining = &current[self.offset..];
+                    let to_read = remaining.len().min(buf.remaining());
+                    buf.put_slice(&remaining[..to_read]);
+                    self.offset += to_read;
+                    return Poll::Ready(Ok(()));
+                }
+
+                self.current = None;
+                self.offset = 0;
+                continue;
+            }
+
+            match self.stream.as_mut().poll_next(cx) {
+                Poll::Pending => return Poll::Pending,
+                Poll::Ready(Some(Ok(chunk))) => {
+                    let next = chunk.as_bytes();
+                    if next.is_empty() {
+                        continue;
+                    }
+                    self.current = Some(next);
+                    self.offset = 0;
+                }
+                Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(err)),
+                Poll::Ready(None) => return Poll::Ready(Ok(())),
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chunk::{IoChunk, MappedChunk, PooledChunk};
+    use bytes::Bytes;
+    use futures_util::stream;
+    use tokio::io::AsyncReadExt;
+
+    #[tokio::test]
+    async fn test_chunk_stream_reader_reads_single_chunk() {
+        let stream: BoxChunkStream = Box::pin(stream::iter(vec![Ok(IoChunk::Shared(Bytes::from_static(b"hello")))]));
+        let mut reader = ChunkStreamReader::new(stream);
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out, b"hello");
+    }
+
+    #[tokio::test]
+    async fn test_chunk_stream_reader_reads_multiple_chunks() {
+        let stream: BoxChunkStream = Box::pin(stream::iter(vec![
+            Ok(IoChunk::Shared(Bytes::from_static(b"he"))),
+            Ok(IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"llo!"), 0, 4).unwrap())),
+            Ok(IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b" world")).unwrap())),
+        ]));
+        let mut reader = ChunkStreamReader::new(stream);
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert_eq!(out, b"hello! world");
+    }
+
+    #[tokio::test]
+    async fn test_chunk_stream_reader_handles_empty_stream() {
+        let stream: BoxChunkStream = Box::pin(stream::iter(Vec::<io::Result<IoChunk>>::new()));
+        let mut reader = ChunkStreamReader::new(stream);
+        let mut out = Vec::new();
+        reader.read_to_end(&mut out).await.unwrap();
+        assert!(out.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_chunk_stream_reader_propagates_stream_error() {
+        let stream: BoxChunkStream = Box::pin(stream::iter(vec![Err(io::Error::other("chunk stream failure"))]));
+        let mut reader = ChunkStreamReader::new(stream);
+        let mut out = Vec::new();
+        let err = reader.read_to_end(&mut out).await.unwrap_err();
+        assert_eq!(err.kind(), io::ErrorKind::Other);
+        assert!(err.to_string().contains("chunk stream failure"));
+    }
+}

crates/io-core/src/chunk.rs

@@ -0,0 +1,276 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Core chunk ownership abstractions for the zero-copy data plane.
+
+use crate::pool::PooledBuffer;
+use bytes::Bytes;
+use futures_core::Stream;
+use std::io;
+use std::pin::Pin;
+
+/// Boxed asynchronous stream of I/O chunks.
+pub type BoxChunkStream = Pin<Box<dyn Stream<Item = io::Result<IoChunk>> + Send + Sync + 'static>>;
+
+/// Source of chunked data.
+pub trait ChunkSource {
+    fn into_chunk_stream(self) -> BoxChunkStream
+    where
+        Self: Sized;
+}
+
+/// Owned chunk variants used by the zero-copy data plane.
+#[derive(Debug)]
+pub enum IoChunk {
+    Shared(Bytes),
+    Mapped(MappedChunk),
+    Pooled(PooledChunk),
+}
+
+impl IoChunk {
+    /// Returns the visible length of this chunk.
+    #[must_use]
+    pub fn len(&self) -> usize {
+        match self {
+            Self::Shared(bytes) => bytes.len(),
+            Self::Mapped(chunk) => chunk.len(),
+            Self::Pooled(chunk) => chunk.len(),
+        }
+    }
+
+    /// Returns true when the chunk has no visible data.
+    #[must_use]
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Returns a shared read-only view of the visible bytes.
+    #[must_use]
+    pub fn as_bytes(&self) -> Bytes {
+        match self {
+            Self::Shared(bytes) => bytes.clone(),
+            Self::Mapped(chunk) => chunk.as_bytes(),
+            Self::Pooled(chunk) => chunk.as_bytes(),
+        }
+    }
+
+    /// Returns a sliced view relative to the currently visible bytes.
+    pub fn slice(&self, offset: usize, len: usize) -> io::Result<Self> {
+        match self {
+            Self::Shared(bytes) => {
+                validate_slice_bounds(bytes.len(), offset, len)?;
+                Ok(Self::Shared(bytes.slice(offset..offset + len)))
+            }
+            Self::Mapped(chunk) => chunk.slice(offset, len).map(Self::Mapped),
+            Self::Pooled(chunk) => chunk.slice(offset, len).map(Self::Pooled),
+        }
+    }
+}
+
+/// Logical view into mapped file bytes.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MappedChunk {
+    bytes: Bytes,
+    logical_offset: usize,
+    logical_len: usize,
+}
+
+impl MappedChunk {
+    pub fn new(bytes: Bytes, logical_offset: usize, logical_len: usize) -> io::Result<Self> {
+        validate_slice_bounds(bytes.len(), logical_offset, logical_len)?;
+        Ok(Self {
+            bytes,
+            logical_offset,
+            logical_len,
+        })
+    }
+
+    /// Returns the visible length of this mapped chunk.
+    #[must_use]
+    pub const fn len(&self) -> usize {
+        self.logical_len
+    }
+
+    /// Returns true when the chunk has no visible data.
+    #[must_use]
+    pub const fn is_empty(&self) -> bool {
+        self.logical_len == 0
+    }
+
+    /// Returns the visible bytes for this logical view.
+    #[must_use]
+    pub fn as_bytes(&self) -> Bytes {
+        self.bytes
+            .slice(self.logical_offset..self.logical_offset.saturating_add(self.logical_len))
+    }
+
+    /// Returns a sliced logical view relative to the current logical view.
+    pub fn slice(&self, offset: usize, len: usize) -> io::Result<Self> {
+        validate_slice_bounds(self.logical_len, offset, len)?;
+        Self::new(self.bytes.clone(), self.logical_offset + offset, len)
+    }
+}
+
+/// Placeholder pooled chunk variant for commit 4.
+///
+/// This is backed by a `PooledBuffer` and exposes a visible read-only window.
+#[derive(Debug)]
+pub struct PooledChunk {
+    bytes: Bytes,
+}
+
+#[derive(Debug)]
+struct PooledChunkOwner {
+    buffer: PooledBuffer,
+    visible_len: usize,
+}
+
+impl AsRef<[u8]> for PooledChunkOwner {
+    fn as_ref(&self) -> &[u8] {
+        &self.buffer[..self.visible_len]
+    }
+}
+
+#[derive(Debug)]
+struct DetachedVecChunkOwner {
+    bytes: Vec<u8>,
+}
+
+impl AsRef<[u8]> for DetachedVecChunkOwner {
+    fn as_ref(&self) -> &[u8] {
+        &self.bytes
+    }
+}
+
+impl PooledChunk {
+    pub fn new(buffer: PooledBuffer, len: usize) -> io::Result<Self> {
+        validate_slice_bounds(buffer.len(), 0, len)?;
+        Ok(Self {
+            bytes: Bytes::from_owner(PooledChunkOwner {
+                buffer,
+                visible_len: len,
+            }),
+        })
+    }
+
+    /// Convenience constructor for detached test and compatibility values.
+    pub fn from_bytes(bytes: Bytes) -> io::Result<Self> {
+        let len = bytes.len();
+        Self::new(PooledBuffer::from_bytes(bytes), len)
+    }
+
+    /// Detached constructor that takes ownership of an existing `Vec<u8>`
+    /// without introducing an additional copy.
+    pub fn from_vec(bytes: Vec<u8>) -> Self {
+        Self {
+            bytes: Bytes::from_owner(DetachedVecChunkOwner { bytes }),
+        }
+    }
+
+    /// Returns the visible length of this pooled chunk.
+    #[must_use]
+    pub fn len(&self) -> usize {
+        self.bytes.len()
+    }
+
+    /// Returns true when the chunk has no visible data.
+    #[must_use]
+    pub fn is_empty(&self) -> bool {
+        self.bytes.is_empty()
+    }
+
+    /// Returns the visible bytes for this pooled chunk.
+    #[must_use]
+    pub fn as_bytes(&self) -> Bytes {
+        self.bytes.clone()
+    }
+
+    /// Returns a sliced pooled chunk relative to the current visible view.
+    pub fn slice(&self, offset: usize, len: usize) -> io::Result<Self> {
+        validate_slice_bounds(self.bytes.len(), offset, len)?;
+        Ok(Self {
+            bytes: self.bytes.slice(offset..offset + len),
+        })
+    }
+}
+
+fn validate_slice_bounds(visible_len: usize, offset: usize, len: usize) -> io::Result<()> {
+    let end = offset
+        .checked_add(len)
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "chunk slice overflows"))?;
+    if end > visible_len {
+        return Err(io::Error::new(io::ErrorKind::InvalidInput, "chunk slice exceeds visible length"));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::pool::BytesPool;
+
+    #[test]
+    fn test_shared_chunk_len_and_slice() {
+        let chunk = IoChunk::Shared(Bytes::from_static(b"abcdef"));
+        assert_eq!(chunk.len(), 6);
+        assert!(!chunk.is_empty());
+        assert_eq!(chunk.as_bytes(), Bytes::from_static(b"abcdef"));
+        assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"bcd"));
+    }
+
+    #[test]
+    fn test_mapped_chunk_len_and_slice() {
+        let chunk = MappedChunk::new(Bytes::from_static(b"abcdefgh"), 2, 4).unwrap();
+        assert_eq!(chunk.len(), 4);
+        assert_eq!(chunk.as_bytes(), Bytes::from_static(b"cdef"));
+        assert_eq!(chunk.slice(1, 2).unwrap().as_bytes(), Bytes::from_static(b"de"));
+    }
+
+    #[test]
+    fn test_pooled_chunk_len_and_as_bytes() {
+        let chunk = PooledChunk::from_bytes(Bytes::from_static(b"hello")).unwrap();
+        assert_eq!(chunk.len(), 5);
+        assert_eq!(chunk.as_bytes(), Bytes::from_static(b"hello"));
+        assert_eq!(chunk.slice(1, 3).unwrap().as_bytes(), Bytes::from_static(b"ell"));
+    }
+
+    #[test]
+    fn test_io_chunk_as_bytes_for_all_variants() {
+        let shared = IoChunk::Shared(Bytes::from_static(b"s"));
+        let mapped = IoChunk::Mapped(MappedChunk::new(Bytes::from_static(b"mapped"), 0, 6).unwrap());
+        let pooled = IoChunk::Pooled(PooledChunk::from_bytes(Bytes::from_static(b"p")).unwrap());
+
+        assert_eq!(shared.as_bytes(), Bytes::from_static(b"s"));
+        assert_eq!(mapped.as_bytes(), Bytes::from_static(b"mapped"));
+        assert_eq!(pooled.as_bytes(), Bytes::from_static(b"p"));
+    }
+
+    #[tokio::test]
+    async fn test_pooled_chunk_keeps_owner_alive_until_last_view_drops() {
+        let pool = BytesPool::new_tiered();
+        let mut buffer = pool.acquire_buffer(16).await;
+        buffer.extend_from_slice(b"pooled-bytes");
+
+        let chunk = PooledChunk::new(buffer, "pooled-bytes".len()).unwrap();
+        let bytes = chunk.as_bytes();
+
+        assert_eq!(pool.available_buffers(), 0);
+        drop(chunk);
+        assert_eq!(pool.available_buffers(), 0);
+        assert_eq!(bytes, Bytes::from_static(b"pooled-bytes"));
+
+        drop(bytes);
+        assert_eq!(pool.available_buffers(), 1);
+    }
+}

crates/io-core/src/lib.rs

@@ -46,8 +46,10 @@
 //! let mut buffer = pool.acquire_buffer(8192).await;
 //! ```
 
+pub mod adapter;
 pub mod backpressure;
 pub mod bufreader_optimizer;
+pub mod chunk;
 pub mod config;
 pub mod deadlock_detector;
 pub mod direct_io;
@@ -68,7 +70,9 @@ pub use reader::{ZeroCopyObjectReader, ZeroCopyReadError};
 pub use writer::{ZeroCopyObjectWriter, ZeroCopyWriteError};
 
 // BufReader optimizer exports
+pub use adapter::ChunkStreamReader;
 pub use bufreader_optimizer::{BufReaderConfig, BufReaderOptimizer, BufReaderStats, BufferedSource};
+pub use chunk::{BoxChunkStream, ChunkSource, IoChunk, MappedChunk, PooledChunk};
 
 // Shared memory exports
 pub use shared_memory::{ArcData, ArcMetadata, SharedMemoryConfig, SharedMemoryPool, SharedMemoryStats};

crates/io-core/src/pool.rs

@@ -17,7 +17,7 @@
 //! Migrated from rustfs-ecstore to provide unified buffer pooling
 //! across rustfs and rustfs-ecstore without cyclic dependencies.
 
-use bytes::BytesMut;
+use bytes::{Bytes, BytesMut};
 use std::mem::ManuallyDrop;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
@@ -108,6 +108,7 @@ pub struct BytesPoolMetrics {
 /// A buffer managed by the BytesPool.
 ///
 /// When dropped, the buffer is automatically returned to the pool for reuse.
+#[derive(Debug)]
 pub struct PooledBuffer {
     /// The underlying buffer (ManuallyDrop to allow taking on drop)
     pub buffer: ManuallyDrop<BytesMut>,
@@ -117,6 +118,45 @@ pub struct PooledBuffer {
     _permit: Option<OwnedSemaphorePermit>,
 }
 
+impl PooledBuffer {
+    /// Create a detached pooled buffer from bytes.
+    ///
+    /// This is primarily used for tests and transitional adapters where the
+    /// chunk abstraction needs a pool-shaped owner before a real pool-backed
+    /// producer exists.
+    #[must_use]
+    pub fn from_bytes(bytes: Bytes) -> Self {
+        Self {
+            buffer: ManuallyDrop::new(BytesMut::from(bytes.as_ref())),
+            tier: None,
+            _permit: None,
+        }
+    }
+
+    /// Current visible length of the underlying buffer.
+    #[must_use]
+    pub fn len(&self) -> usize {
+        self.buffer.len()
+    }
+
+    /// Total buffer capacity.
+    #[must_use]
+    pub fn capacity(&self) -> usize {
+        self.buffer.capacity()
+    }
+
+    /// Clear the visible contents while preserving capacity.
+    pub fn clear(&mut self) {
+        self.buffer.clear();
+    }
+
+    /// Returns true when the visible buffer is empty.
+    #[must_use]
+    pub fn is_empty(&self) -> bool {
+        self.buffer.is_empty()
+    }
+}
+
 /// BytesPool configuration.
 ///
 /// Allows customization of buffer sizes and limits for each tier.