feat(storage): add direct chunk GET fast path (#2351)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>

crates/ecstore/src/store_api/readers.rs

@@ -1,7 +1,101 @@
 use super::*;
+use rustfs_rio::TryGetIndex;
+
+pub struct ChunkNativePutData {
+    stream: Option<HashReader>,
+    size: i64,
+    actual_size: i64,
+}
+
+impl Debug for ChunkNativePutData {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ChunkNativePutData").finish()
+    }
+}
+
+impl ChunkNativePutData {
+    pub fn new(stream: HashReader) -> Self {
+        let size = stream.size();
+        let actual_size = stream.actual_size();
+        Self {
+            stream: Some(stream),
+            size,
+            actual_size,
+        }
+    }
+
+    pub fn from_vec(data: Vec<u8>) -> Self {
+        use sha2::{Digest, Sha256};
+
+        let content_length = data.len() as i64;
+        let sha256hex = if content_length > 0 {
+            Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
+        } else {
+            None
+        };
+        Self::new(HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false).unwrap())
+    }
+
+    pub fn take_stream(&mut self) -> std::io::Result<HashReader> {
+        self.stream
+            .take()
+            .ok_or_else(|| std::io::Error::other("ChunkNativePutData stream already taken"))
+    }
+
+    pub fn restore_stream(&mut self, stream: HashReader) {
+        self.size = stream.size();
+        self.actual_size = stream.actual_size();
+        self.stream = Some(stream);
+    }
+
+    pub fn as_hash_reader(&self) -> Option<&HashReader> {
+        self.stream.as_ref()
+    }
+
+    pub fn as_hash_reader_mut(&mut self) -> Option<&mut HashReader> {
+        self.stream.as_mut()
+    }
+
+    pub fn index_bytes(&self) -> Option<Bytes> {
+        self.as_hash_reader()
+            .and_then(|reader| reader.try_get_index().map(|index| index.clone().into_vec()))
+    }
+
+    pub fn resolve_etag(&mut self) -> Option<String> {
+        self.as_hash_reader_mut()
+            .and_then(rustfs_rio::EtagResolvable::try_resolve_etag)
+    }
+
+    pub fn content_hash_bytes(&mut self) -> std::io::Result<Option<Bytes>> {
+        let Some(reader) = self.as_hash_reader_mut() else {
+            return Ok(None);
+        };
+
+        Ok(reader
+            .finalize_content_hash()?
+            .as_ref()
+            .map(|checksum| checksum.to_bytes(&[])))
+    }
+
+    pub fn content_crc_type(&self) -> Option<rustfs_rio::ChecksumType> {
+        self.as_hash_reader().and_then(HashReader::content_crc_type)
+    }
+
+    pub fn content_crc(&self) -> HashMap<String, String> {
+        self.as_hash_reader().map_or_else(HashMap::new, HashReader::content_crc)
+    }
+
+    pub fn size(&self) -> i64 {
+        self.size
+    }
+
+    pub fn actual_size(&self) -> i64 {
+        self.actual_size
+    }
+}
 
 pub struct PutObjReader {
-    pub stream: HashReader,
+    data: ChunkNativePutData,
 }
 
 impl Debug for PutObjReader {
@@ -12,32 +106,81 @@ impl Debug for PutObjReader {
 
 impl PutObjReader {
     pub fn new(stream: HashReader) -> Self {
-        PutObjReader { stream }
+        Self {
+            data: ChunkNativePutData::new(stream),
+        }
     }
 
-    pub fn as_hash_reader(&self) -> &HashReader {
-        &self.stream
+    pub fn chunk_native_data(&self) -> &ChunkNativePutData {
+        &self.data
+    }
+
+    pub fn chunk_native_data_mut(&mut self) -> &mut ChunkNativePutData {
+        &mut self.data
+    }
+
+    pub fn take_stream(&mut self) -> std::io::Result<HashReader> {
+        self.data.take_stream()
+    }
+
+    pub fn restore_stream(&mut self, stream: HashReader) {
+        self.data.restore_stream(stream);
+    }
+
+    pub fn as_hash_reader(&self) -> Option<&HashReader> {
+        self.data.as_hash_reader()
+    }
+
+    pub fn as_hash_reader_mut(&mut self) -> Option<&mut HashReader> {
+        self.data.as_hash_reader_mut()
+    }
+
+    pub fn index_bytes(&self) -> Option<Bytes> {
+        self.data.index_bytes()
+    }
+
+    pub fn resolve_etag(&mut self) -> Option<String> {
+        self.data.resolve_etag()
+    }
+
+    pub fn content_hash_bytes(&mut self) -> std::io::Result<Option<Bytes>> {
+        self.data.content_hash_bytes()
+    }
+
+    pub fn content_crc_type(&self) -> Option<rustfs_rio::ChecksumType> {
+        self.data.content_crc_type()
+    }
+
+    pub fn content_crc(&self) -> HashMap<String, String> {
+        self.data.content_crc()
     }
 
     pub fn from_vec(data: Vec<u8>) -> Self {
-        use sha2::{Digest, Sha256};
-        let content_length = data.len() as i64;
-        let sha256hex = if content_length > 0 {
-            Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
-        } else {
-            None
-        };
-        PutObjReader {
-            stream: HashReader::from_stream(Cursor::new(data), content_length, content_length, None, sha256hex, false).unwrap(),
+        Self {
+            data: ChunkNativePutData::from_vec(data),
         }
     }
 
     pub fn size(&self) -> i64 {
-        self.stream.size()
+        self.data.size()
     }
 
     pub fn actual_size(&self) -> i64 {
-        self.stream.actual_size()
+        self.data.actual_size()
+    }
+}
+
+impl std::ops::Deref for PutObjReader {
+    type Target = ChunkNativePutData;
+
+    fn deref(&self) -> &Self::Target {
+        &self.data
+    }
+}
+
+impl std::ops::DerefMut for PutObjReader {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.data
     }
 }
 
@@ -46,6 +189,26 @@ pub struct GetObjectReader {
     pub object_info: ObjectInfo,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GetObjectChunkPath {
+    Direct,
+    Bridge,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GetObjectChunkCopyMode {
+    TrueZeroCopy,
+    SharedBytes,
+    SingleCopy,
+    Reconstructed,
+}
+
+pub struct GetObjectChunkResult {
+    pub stream: BoxChunkStream,
+    pub path: GetObjectChunkPath,
+    pub copy_mode: GetObjectChunkCopyMode,
+}
+
 impl GetObjectReader {
     #[tracing::instrument(level = "debug", skip(reader, rs, opts, _h))]
     pub fn new(
@@ -63,31 +226,34 @@ impl GetObjectReader {
             rs = HTTPRangeSpec::from_object_info(oi, part_number);
         }
 
-        // TODO:Encrypted
+        let logical_size = oi.get_actual_size()?;
+        let encrypted_object = oi.user_defined.contains_key("x-rustfs-encryption-key")
+            || oi
+                .user_defined
+                .contains_key("x-amz-server-side-encryption-customer-algorithm");
 
         let (algo, is_compressed) = oi.is_compressed_ok()?;
 
         // TODO: check TRANSITION
 
         if is_compressed {
-            let actual_size = oi.get_actual_size()?;
             let (off, length, dec_off, dec_length) = if let Some(rs) = rs {
                 // Support range requests for compressed objects
-                let (dec_off, dec_length) = rs.get_offset_length(actual_size)?;
+                let (dec_off, dec_length) = rs.get_offset_length(logical_size)?;
                 (0, oi.size, dec_off, dec_length)
             } else {
-                (0, oi.size, 0, actual_size)
+                (0, oi.size, 0, logical_size)
             };
 
             let dec_reader = DecompressReader::new(reader, algo);
 
-            let actual_size_usize = if actual_size > 0 {
-                actual_size as usize
+            let actual_size_usize = if logical_size > 0 {
+                logical_size as usize
             } else {
-                return Err(Error::other(format!("invalid decompressed size {actual_size}")));
+                return Err(Error::other(format!("invalid decompressed size {logical_size}")));
             };
 
-            let final_reader: Box<dyn AsyncRead + Unpin + Send + Sync> = if dec_off > 0 || dec_length != actual_size {
+            let final_reader: Box<dyn AsyncRead + Unpin + Send + Sync> = if dec_off > 0 || dec_length != logical_size {
                 // Use RangedDecompressReader for streaming range processing
                 // The new implementation supports any offset size by streaming and skipping data
                 match RangedDecompressReader::new(dec_reader, dec_off, dec_length, actual_size_usize) {
@@ -122,8 +288,19 @@ impl GetObjectReader {
             ));
         }
 
+        if encrypted_object && rs.is_none() {
+            return Ok((
+                GetObjectReader {
+                    stream: reader,
+                    object_info: oi.clone(),
+                },
+                0,
+                oi.size,
+            ));
+        }
+
         if let Some(rs) = rs {
-            let (off, length) = rs.get_offset_length(oi.size)?;
+            let (off, length) = rs.get_offset_length(logical_size)?;
 
             Ok((
                 GetObjectReader {
@@ -140,7 +317,7 @@ impl GetObjectReader {
                     object_info: oi.clone(),
                 },
                 0,
-                oi.size,
+                logical_size,
             ))
         }
     }

crates/ecstore/src/store_api/traits.rs

@@ -11,7 +11,13 @@ pub trait ObjectIO: Send + Sync + Debug + 'static {
         opts: &ObjectOptions,
     ) -> Result<GetObjectReader>;
 
-    async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo>;
+    async fn put_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        data: &mut ChunkNativePutData,
+        opts: &ObjectOptions,
+    ) -> Result<ObjectInfo>;
 }
 
 /// Bucket-level storage operations.
@@ -126,7 +132,7 @@ pub trait MultipartOperations: Send + Sync + Debug {
         object: &str,
         upload_id: &str,
         part_id: usize,
-        data: &mut PutObjReader,
+        data: &mut ChunkNativePutData,
         opts: &ObjectOptions,
     ) -> Result<PartInfo>;
     async fn get_multipart_info(

crates/ecstore/src/store_api/types.rs

@@ -70,6 +70,7 @@ pub struct ObjectOptions {
 
     pub eval_metadata: Option<HashMap<String, String>>,
 
+    pub resolved_checksum: Option<Bytes>,
     pub want_checksum: Option<Checksum>,
     pub skip_verify_bitrot: bool,
 }
@@ -283,7 +284,7 @@ pub struct ObjectInfo {
     pub expires: Option<OffsetDateTime>,
     pub num_versions: usize,
     pub successor_mod_time: Option<OffsetDateTime>,
-    pub put_object_reader: Option<PutObjReader>,
+    pub put_object_reader: Option<ChunkNativePutData>,
     pub etag: Option<String>,
     pub inlined: bool,
     pub metadata_only: bool,
@@ -509,6 +510,18 @@ impl ObjectInfo {
             })
             .collect();
 
+        let actual_size = fi
+            .parts
+            .iter()
+            .map(|part| {
+                if part.actual_size > 0 {
+                    part.actual_size
+                } else {
+                    i64::try_from(part.size).unwrap_or_default()
+                }
+            })
+            .sum();
+
         // TODO: part checksums
 
         ObjectInfo {
@@ -521,6 +534,7 @@ impl ObjectInfo {
             delete_marker: fi.deleted,
             mod_time: fi.mod_time,
             size: fi.size,
+            actual_size,
             parts,
             is_latest: fi.is_latest,
             user_tags,