feat(rio): rio_v2 is compatible with minio for storing data. (#3115)

* Set up a compatibility layer for replacing old Rio components with new ones.

* fix(rio). compress range

* feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level.

* feat(rio_v2): add sse-c test

* test compression component

* simple fix

* fix minlz encode

* fix metadata

* fix kms key cache error

* Update launch.json

* ci: set nix crate download user agent

* fix: gate obs pyroscope backend

* ignore minio test

* fix encrypt check

* fix

* fix

* fix

* Update object_usecase.rs

* Update ci.yml

* fix

* ci add rio-v2 test

* fix

* ci fix

* fix

* Reconstructed into a more reasonable compatibility mode

* fix

* fix

---------

Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: 唐小鸭 <tangtang1251@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>

crates/ecstore/Cargo.toml

@@ -32,11 +32,13 @@ workspace = true
 
 [features]
 default = []
+rio-v2 = ["dep:rustfs-rio-v2"]
 
 [dependencies]
 rustfs-filemeta.workspace = true
 rustfs-utils = { workspace = true, features = ["full"] }
 rustfs-rio.workspace = true
+rustfs-rio-v2 = { workspace = true, optional = true }
 rustfs-signer.workspace = true
 rustfs-tls-runtime.workspace = true
 rustfs-checksums.workspace = true

crates/ecstore/src/erasure_coding/decode.rs

@@ -433,6 +433,63 @@ mod tests {
         }
     }
 
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn test_erasure_decode_preserves_compressed_stream_near_block_boundary() {
+        const DATA_SHARDS: usize = 2;
+        const PARITY_SHARDS: usize = 2;
+        const BLOCK_SIZE: usize = 1024 * 1024;
+
+        use crate::rio::CompressReader;
+        use rustfs_utils::CompressionAlgorithm;
+        use tokio::io::AsyncReadExt;
+
+        let plaintext_size = 8 * BLOCK_SIZE + 123;
+        let plaintext = (0..plaintext_size)
+            .scan(0x9e37_79b9_7f4a_7c15u64, |state, _| {
+                *state ^= *state << 7;
+                *state ^= *state >> 9;
+                *state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9);
+                Some((*state >> 32) as u8)
+            })
+            .collect::<Vec<_>>();
+
+        let mut compressor = CompressReader::new(Cursor::new(plaintext), CompressionAlgorithm::default());
+        let mut compressed = Vec::new();
+        compressor.read_to_end(&mut compressed).await.unwrap();
+
+        let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
+        let total_shards = DATA_SHARDS + PARITY_SHARDS;
+        let shard_size = erasure.shard_size();
+        let hash_algo = HashAlgorithm::HighwayHash256;
+
+        let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
+            .map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
+            .collect();
+
+        for block in compressed.chunks(BLOCK_SIZE) {
+            let shards = erasure.encode_data(block).unwrap();
+            for (i, shard) in shards.iter().enumerate() {
+                shard_writers[i].write(shard).await.unwrap();
+            }
+        }
+
+        let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
+        let readers = shard_bufs
+            .iter()
+            .map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
+            .collect();
+
+        let mut decoded = Vec::new();
+        let (written, err) = erasure
+            .decode(&mut decoded, readers, 0, compressed.len(), compressed.len())
+            .await;
+
+        assert!(err.is_none(), "unexpected decode error: {err:?}");
+        assert_eq!(written, compressed.len());
+        assert_eq!(decoded, compressed);
+    }
+
     #[tokio::test]
     async fn test_parallel_reader_normal() {
         const BLOCK_SIZE: usize = 64;

crates/ecstore/src/lib.rs

@@ -34,6 +34,7 @@ pub mod metrics_realtime;
 pub mod notification_sys;
 pub mod pools;
 pub mod rebalance;
+pub mod rio;
 pub mod rpc;
 pub mod set_disk;
 mod sets;
@@ -60,3 +61,12 @@ pub use global::{get_global_lock_client, get_global_lock_clients, set_global_loc
 
 pub use global::GLOBAL_Endpoints;
 pub use store_api::StorageAPI;
+
+#[cfg(test)]
+mod rio_tests {
+    #[test]
+    fn uses_expected_rio_backend() {
+        let expected = if cfg!(feature = "rio-v2") { "rio-v2" } else { "legacy-rio" };
+        assert_eq!(crate::rio::backend_name(), expected);
+    }
+}

crates/ecstore/src/rio.rs

@@ -0,0 +1,767 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(feature = "rio-v2")]
+pub use rustfs_rio_v2::*;
+
+#[cfg(not(feature = "rio-v2"))]
+pub use rustfs_rio::*;
+
+use bytes::Bytes;
+use rustfs_utils::CompressionAlgorithm;
+use std::str::FromStr;
+use tokio::io::AsyncRead;
+
+#[cfg(feature = "rio-v2")]
+const MINIO_S2_COMPRESSION_SCHEME: &str = "klauspost/compress/s2";
+#[cfg(feature = "rio-v2")]
+const ENCRYPTED_S2_PADDING_MULTIPLE: usize = 256;
+
+pub const fn backend_name() -> &'static str {
+    #[cfg(feature = "rio-v2")]
+    {
+        "rio-v2"
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        "legacy-rio"
+    }
+}
+
+pub fn compression_metadata_value(algorithm: CompressionAlgorithm) -> String {
+    #[cfg(feature = "rio-v2")]
+    {
+        let _ = algorithm;
+        MINIO_S2_COMPRESSION_SCHEME.to_string()
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        algorithm.to_string()
+    }
+}
+
+pub fn compression_scheme_to_algorithm(scheme: &str) -> std::io::Result<CompressionAlgorithm> {
+    #[cfg(feature = "rio-v2")]
+    if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
+        // rio_v2 currently routes all compressed-object handling through the S2
+        // reader implementation, so the enum is only a placeholder token here.
+        return Ok(CompressionAlgorithm::default());
+    }
+
+    CompressionAlgorithm::from_str(scheme)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReadCompressionBackend {
+    Legacy,
+    V2,
+}
+
+pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(CompressionAlgorithm, ReadCompressionBackend)> {
+    #[cfg(feature = "rio-v2")]
+    if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
+        return Ok((CompressionAlgorithm::default(), ReadCompressionBackend::V2));
+    }
+
+    Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReadEncryptionBackend {
+    Legacy,
+    V2,
+}
+
+pub fn compression_index_storage_bytes(index: &Index) -> Bytes {
+    #[cfg(feature = "rio-v2")]
+    {
+        minio_index_storage_bytes(index)
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        index.clone().into_vec()
+    }
+}
+
+pub fn decode_compression_index_bytes(bytes: &Bytes) -> Option<Index> {
+    #[cfg(feature = "rio-v2")]
+    {
+        if let Some(decoded) = decode_minio_index_bytes(bytes) {
+            return Some(decoded);
+        }
+    }
+
+    let mut decoded = Index::new();
+    if decoded.load(bytes.as_ref()).is_ok() {
+        return Some(decoded);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    {
+        let restored = restore_legacy_index_headers(bytes.as_ref());
+        let mut decoded = Index::new();
+        if decoded.load(&restored).is_ok() {
+            return Some(decoded);
+        }
+    }
+
+    None
+}
+
+pub fn compression_reader<R>(reader: R, algorithm: CompressionAlgorithm, encrypted: bool) -> CompressReader<R>
+where
+    R: AsyncRead + Unpin + Send + Sync,
+{
+    #[cfg(feature = "rio-v2")]
+    {
+        if encrypted {
+            return CompressReader::with_encrypted_padding(reader, algorithm);
+        }
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    let _ = encrypted;
+
+    CompressReader::new(reader, algorithm)
+}
+
+pub fn decompression_reader<R>(
+    reader: R,
+    algorithm: CompressionAlgorithm,
+    backend: ReadCompressionBackend,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync>
+where
+    R: AsyncRead + Unpin + Send + Sync + 'static,
+{
+    #[cfg(feature = "rio-v2")]
+    {
+        match backend {
+            ReadCompressionBackend::Legacy => Box::new(rustfs_rio::DecompressReader::new(reader, algorithm)),
+            ReadCompressionBackend::V2 => Box::new(rustfs_rio_v2::DecompressReader::new(reader, algorithm)),
+        }
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        let _ = backend;
+        Box::new(rustfs_rio::DecompressReader::new(reader, algorithm))
+    }
+}
+
+pub fn decrypt_reader<R>(
+    reader: R,
+    key: [u8; 32],
+    base_nonce: [u8; 12],
+    backend: ReadEncryptionBackend,
+    sequence_number: u32,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync>
+where
+    R: AsyncRead + Unpin + Send + Sync + 'static,
+{
+    #[cfg(feature = "rio-v2")]
+    {
+        match backend {
+            ReadEncryptionBackend::Legacy => Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce)),
+            ReadEncryptionBackend::V2 => {
+                Box::new(rustfs_rio_v2::DecryptReader::new_with_sequence(reader, key, base_nonce, sequence_number))
+            }
+        }
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        let _ = (backend, sequence_number);
+        Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce))
+    }
+}
+
+pub fn decrypt_reader_with_object_key<R>(
+    reader: R,
+    object_key: [u8; 32],
+    sequence_number: u32,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync>
+where
+    R: AsyncRead + Unpin + Send + Sync + 'static,
+{
+    #[cfg(feature = "rio-v2")]
+    {
+        Box::new(rustfs_rio_v2::DecryptReader::new_with_object_key_and_sequence(
+            reader,
+            object_key,
+            sequence_number,
+        ))
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        let _ = sequence_number;
+        Box::new(rustfs_rio::DecryptReader::new(reader, object_key, [0u8; 12]))
+    }
+}
+
+pub fn decrypt_multipart_reader<R>(
+    reader: R,
+    key: [u8; 32],
+    base_nonce: [u8; 12],
+    multipart_parts: Vec<usize>,
+    backend: ReadEncryptionBackend,
+    sequence_number: u32,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync>
+where
+    R: AsyncRead + Unpin + Send + Sync + 'static,
+{
+    #[cfg(feature = "rio-v2")]
+    {
+        match backend {
+            ReadEncryptionBackend::Legacy => {
+                Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
+            }
+            ReadEncryptionBackend::V2 => Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_sequence(
+                reader,
+                key,
+                base_nonce,
+                multipart_parts,
+                sequence_number,
+            )),
+        }
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        let _ = (backend, sequence_number);
+        Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
+    }
+}
+
+pub fn decrypt_multipart_reader_with_object_key<R>(
+    reader: R,
+    object_key: [u8; 32],
+    multipart_parts: Vec<usize>,
+    sequence_number: u32,
+) -> Box<dyn AsyncRead + Unpin + Send + Sync>
+where
+    R: AsyncRead + Unpin + Send + Sync + 'static,
+{
+    #[cfg(feature = "rio-v2")]
+    {
+        Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_object_key_and_sequence(
+            reader,
+            object_key,
+            multipart_parts,
+            sequence_number,
+        ))
+    }
+
+    #[cfg(not(feature = "rio-v2"))]
+    {
+        let _ = sequence_number;
+        Box::new(rustfs_rio::DecryptReader::new_multipart(reader, object_key, [0u8; 12], multipart_parts))
+    }
+}
+
+#[cfg(feature = "rio-v2")]
+fn restore_legacy_index_headers(bytes: &[u8]) -> Vec<u8> {
+    if bytes.is_empty() {
+        return Vec::new();
+    }
+
+    const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
+    const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
+    let mut restored = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + bytes.len() + 4 + S2_INDEX_TRAILER.len());
+    restored.extend_from_slice(&[0x99, 0x2A, 0x4D, 0x18]);
+    restored.extend_from_slice(S2_INDEX_HEADER);
+    restored.extend_from_slice(bytes);
+
+    let total_size = (restored.len() + 4 + S2_INDEX_TRAILER.len()) as u32;
+    restored.extend_from_slice(&total_size.to_le_bytes());
+    restored.extend_from_slice(S2_INDEX_TRAILER);
+
+    let chunk_len = restored.len() - 4;
+    restored[1] = chunk_len as u8;
+    restored[2] = (chunk_len >> 8) as u8;
+    restored[3] = (chunk_len >> 16) as u8;
+    restored
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct WriteEncryption {
+    key_bytes: [u8; 32],
+    mode: WriteEncryptionMode,
+}
+
+#[derive(Debug, Clone, Copy)]
+enum WriteEncryptionMode {
+    SinglepartObjectKey,
+    Singlepart {
+        base_nonce: [u8; 12],
+    },
+    MultipartLegacy {
+        base_nonce: [u8; 12],
+        multipart_part_number: usize,
+    },
+    MultipartObjectKey {
+        multipart_part_number: u32,
+    },
+}
+
+impl WriteEncryption {
+    pub const fn singlepart_object_key(object_key: [u8; 32]) -> Self {
+        Self {
+            key_bytes: object_key,
+            mode: WriteEncryptionMode::SinglepartObjectKey,
+        }
+    }
+
+    pub const fn singlepart(key_bytes: [u8; 32], base_nonce: [u8; 12]) -> Self {
+        Self {
+            key_bytes,
+            mode: WriteEncryptionMode::Singlepart { base_nonce },
+        }
+    }
+
+    pub const fn multipart(key_bytes: [u8; 32], base_nonce: [u8; 12], multipart_part_number: usize) -> Self {
+        Self {
+            key_bytes,
+            mode: WriteEncryptionMode::MultipartLegacy {
+                base_nonce,
+                multipart_part_number,
+            },
+        }
+    }
+
+    pub const fn multipart_object_key(object_key: [u8; 32], multipart_part_number: u32) -> Self {
+        Self {
+            key_bytes: object_key,
+            mode: WriteEncryptionMode::MultipartObjectKey { multipart_part_number },
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct WritePlan {
+    compression: Option<CompressionAlgorithm>,
+    encryption: Option<WriteEncryption>,
+}
+
+impl WritePlan {
+    pub const fn new() -> Self {
+        Self {
+            compression: None,
+            encryption: None,
+        }
+    }
+
+    pub const fn with_compression(mut self, algorithm: CompressionAlgorithm) -> Self {
+        self.compression = Some(algorithm);
+        self
+    }
+
+    pub const fn with_encryption(mut self, encryption: WriteEncryption) -> Self {
+        self.encryption = Some(encryption);
+        self
+    }
+
+    pub const fn is_passthrough(&self) -> bool {
+        self.compression.is_none() && self.encryption.is_none()
+    }
+
+    pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result<HashReader> {
+        let encrypted = self.encryption.is_some();
+        if let Some(algorithm) = self.compression {
+            reader = HashReader::from_reader(
+                compression_reader(reader, algorithm, encrypted),
+                HashReader::SIZE_PRESERVE_LAYER,
+                actual_size,
+                None,
+                None,
+                false,
+            )?;
+        }
+
+        if let Some(encryption) = self.encryption {
+            reader = match encryption.mode {
+                WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
+                    #[cfg(feature = "rio-v2")]
+                    EncryptReader::new_with_object_key(reader, encryption.key_bytes),
+                    #[cfg(not(feature = "rio-v2"))]
+                    EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
+                    HashReader::SIZE_PRESERVE_LAYER,
+                    actual_size,
+                    None,
+                    None,
+                    false,
+                )?,
+                WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader(
+                    EncryptReader::new(reader, encryption.key_bytes, base_nonce),
+                    HashReader::SIZE_PRESERVE_LAYER,
+                    actual_size,
+                    None,
+                    None,
+                    false,
+                )?,
+                WriteEncryptionMode::MultipartLegacy {
+                    base_nonce,
+                    multipart_part_number,
+                } => HashReader::from_reader(
+                    EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
+                    HashReader::SIZE_PRESERVE_LAYER,
+                    actual_size,
+                    None,
+                    None,
+                    false,
+                )?,
+                WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
+                    #[cfg(feature = "rio-v2")]
+                    EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
+                    #[cfg(not(feature = "rio-v2"))]
+                    EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
+                    HashReader::SIZE_PRESERVE_LAYER,
+                    actual_size,
+                    None,
+                    None,
+                    false,
+                )?,
+            };
+        }
+
+        Ok(reader)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_utils::CompressionAlgorithm;
+    use std::io::Cursor;
+    use tokio::io::AsyncReadExt;
+
+    #[cfg(feature = "rio-v2")]
+    fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
+        let mut chunk_types = Vec::new();
+        let mut offset = 0usize;
+        while offset + 4 <= stream.len() {
+            let chunk_type = stream[offset];
+            let chunk_len =
+                (stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16);
+            chunk_types.push(chunk_type);
+            offset += 4 + chunk_len;
+        }
+        chunk_types
+    }
+
+    #[tokio::test]
+    async fn write_plan_passthrough_keeps_plaintext() {
+        let plaintext = b"write-plan-plain".to_vec();
+        let reader = HashReader::from_stream(
+            Cursor::new(plaintext.clone()),
+            plaintext.len() as i64,
+            plaintext.len() as i64,
+            None,
+            None,
+            false,
+        )
+        .expect("create hash reader");
+
+        let mut reader = WritePlan::new()
+            .apply(reader, plaintext.len() as i64)
+            .expect("apply passthrough plan");
+
+        let mut actual = Vec::new();
+        reader.read_to_end(&mut actual).await.expect("read passthrough stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[tokio::test]
+    async fn write_plan_compress_then_encrypt_multipart_roundtrip() {
+        let plaintext = b"abcdefghijklmnopqrstuvwxyz".repeat(128);
+        let actual_size = plaintext.len() as i64;
+        let key_bytes = [0x5Au8; 32];
+        let base_nonce = [0xA5u8; 12];
+        let part_number = 7;
+
+        let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+
+        let mut transformed = WritePlan::new()
+            .with_compression(CompressionAlgorithm::default())
+            .with_encryption(WriteEncryption::multipart(key_bytes, base_nonce, part_number))
+            .apply(reader, actual_size)
+            .expect("apply transform plan");
+
+        let mut ciphertext = Vec::new();
+        transformed
+            .read_to_end(&mut ciphertext)
+            .await
+            .expect("read transformed ciphertext");
+
+        let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
+        let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
+
+        let mut actual = Vec::new();
+        decompressed
+            .read_to_end(&mut actual)
+            .await
+            .expect("decrypt and decompress transformed stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn write_plan_supports_singlepart_object_key_encryption_roundtrip() {
+        let plaintext = b"singlepart-object-key".repeat(512);
+        let actual_size = plaintext.len() as i64;
+        let object_key = [0x7Cu8; 32];
+
+        let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+
+        let mut transformed = WritePlan::new()
+            .with_encryption(WriteEncryption::singlepart_object_key(object_key))
+            .apply(reader, actual_size)
+            .expect("apply singlepart object-key plan");
+
+        let mut encrypted = Vec::new();
+        transformed
+            .read_to_end(&mut encrypted)
+            .await
+            .expect("read encrypted object-key stream");
+
+        let mut decrypted = DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key);
+        let mut actual = Vec::new();
+        decrypted.read_to_end(&mut actual).await.expect("decrypt object-key stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn write_plan_supports_multipart_object_key_encryption_roundtrip() {
+        let plaintext = b"multipart-object-key-".repeat(4096);
+        let actual_size = plaintext.len() as i64;
+        let object_key = [0x2Du8; 32];
+        let part_number = 3u32;
+
+        let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+
+        let mut transformed = WritePlan::new()
+            .with_encryption(WriteEncryption::multipart_object_key(object_key, part_number))
+            .apply(reader, actual_size)
+            .expect("apply multipart object-key encryption");
+
+        let mut ciphertext = Vec::new();
+        transformed
+            .read_to_end(&mut ciphertext)
+            .await
+            .expect("read multipart object-key ciphertext");
+
+        let mut actual = Vec::new();
+        DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), object_key, vec![part_number as usize])
+            .read_to_end(&mut actual)
+            .await
+            .expect("decrypt multipart object-key ciphertext");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn write_plan_rio_v2_compression_emits_s2_stream_and_seekable_index() {
+        let plaintext = b"rustfs-rio-v2-s2-".repeat(600_000);
+        let actual_size = plaintext.len() as i64;
+        let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+
+        let mut transformed = WritePlan::new()
+            .with_compression(CompressionAlgorithm::default())
+            .apply(reader, actual_size)
+            .expect("apply compression plan");
+
+        let mut compressed = Vec::new();
+        transformed
+            .read_to_end(&mut compressed)
+            .await
+            .expect("read compressed stream");
+
+        assert!(
+            compressed.starts_with(b"\xff\x06\x00\x00S2sTwO"),
+            "rio_v2 compressed stream must start with the S2 stream identifier"
+        );
+
+        let index = transformed
+            .try_get_index()
+            .cloned()
+            .expect("rio_v2 compressed stream should expose a compression index");
+        let (compressed_offset, uncompressed_offset) = index.find(2 * 1024 * 1024).expect("seek into compression index");
+
+        assert!(compressed_offset > 0, "expected a non-zero compressed offset for the second block");
+        assert!(uncompressed_offset > 0, "expected a non-zero uncompressed offset for the second block");
+
+        let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
+        let mut actual = Vec::new();
+        decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn write_plan_rio_v2_small_compression_skips_index_below_minio_threshold() {
+        let plaintext = b"rustfs-rio-v2-s2-".repeat(32_768);
+        let actual_size = plaintext.len() as i64;
+        let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+
+        let mut transformed = WritePlan::new()
+            .with_compression(CompressionAlgorithm::default())
+            .apply(reader, actual_size)
+            .expect("apply compression plan");
+
+        let mut compressed = Vec::new();
+        transformed
+            .read_to_end(&mut compressed)
+            .await
+            .expect("read compressed stream");
+
+        assert!(
+            transformed.try_get_index().is_none(),
+            "rio_v2 should match MinIO and skip compression indexes for small objects"
+        );
+
+        let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
+        let mut actual = Vec::new();
+        decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn rio_v2_singlepart_encrypt_decrypt_roundtrip_preserves_small_compressed_stream() {
+        let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
+        let key_bytes = [0x33u8; 32];
+        let base_nonce = [0x55u8; 12];
+
+        let mut compressed = Vec::new();
+        CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
+            .read_to_end(&mut compressed)
+            .await
+            .expect("compress plaintext");
+
+        let mut encrypted = Vec::new();
+        EncryptReader::new(Cursor::new(compressed), key_bytes, base_nonce)
+            .read_to_end(&mut encrypted)
+            .await
+            .expect("encrypt compressed stream");
+
+        let decrypt_reader = DecryptReader::new(Cursor::new(encrypted), key_bytes, base_nonce);
+        let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
+        let mut actual = Vec::new();
+        decompressed
+            .read_to_end(&mut actual)
+            .await
+            .expect("decrypt and decompress small stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn rio_v2_compress_then_encrypt_adds_s2_padding_frames() {
+        let plaintext = b"padding-check-".repeat(4097);
+        let actual_size = plaintext.len() as i64;
+        let key_bytes = [0x1Bu8; 32];
+        let base_nonce = [0xC4u8; 12];
+
+        let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
+            .expect("create hash reader");
+
+        let mut transformed = WritePlan::new()
+            .with_compression(CompressionAlgorithm::default())
+            .with_encryption(WriteEncryption::singlepart(key_bytes, base_nonce))
+            .apply(reader, actual_size)
+            .expect("apply transform plan");
+
+        let mut ciphertext = Vec::new();
+        transformed
+            .read_to_end(&mut ciphertext)
+            .await
+            .expect("read transformed ciphertext");
+
+        let mut decrypted_compressed = Vec::new();
+        DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
+            .read_to_end(&mut decrypted_compressed)
+            .await
+            .expect("decrypt compressed stream");
+
+        assert_eq!(decrypted_compressed.len() % ENCRYPTED_S2_PADDING_MULTIPLE, 0);
+
+        let chunk_types = s2_chunk_types(&decrypted_compressed);
+        assert!(
+            chunk_types.contains(&0xfe),
+            "rio_v2 compressed+encrypted streams must include S2 padding frames before encryption"
+        );
+
+        let mut actual = Vec::new();
+        DecompressReader::new(Cursor::new(decrypted_compressed), CompressionAlgorithm::default())
+            .read_to_end(&mut actual)
+            .await
+            .expect("decompress padded stream");
+
+        assert_eq!(actual, plaintext);
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn rio_v2_decompress_reader_returns_bytes_on_first_read() {
+        let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
+
+        let mut compressed = Vec::new();
+        CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
+            .read_to_end(&mut compressed)
+            .await
+            .expect("compress plaintext");
+
+        let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
+        let mut buf = [0u8; 64];
+        let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
+
+        assert!(n > 0);
+        assert_eq!(&buf[..n], plaintext.as_slice());
+    }
+
+    #[cfg(feature = "rio-v2")]
+    #[tokio::test]
+    async fn rio_v2_decompress_reader_returns_bytes_on_first_large_read() {
+        let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
+
+        let mut compressed = Vec::new();
+        CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
+            .read_to_end(&mut compressed)
+            .await
+            .expect("compress plaintext");
+
+        let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
+        let mut buf = [0u8; 8192];
+        let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
+
+        assert!(n > 0);
+        assert_eq!(&buf[..n], plaintext.as_slice());
+    }
+}

crates/ecstore/src/set_disk.rs

@@ -85,7 +85,6 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
 use rustfs_object_capacity::capacity_scope::{
     CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
 };
-use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
 use rustfs_s3_types::EventName;
 use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
 use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
@@ -130,6 +129,8 @@ use tracing::error;
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
+use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
+
 pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
 pub const MAX_PARTS_COUNT: usize = 10000;
 pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
@@ -1146,7 +1147,7 @@ impl ObjectIO for SetDisks {
                 insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string());
             }
 
-            let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
+            let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes);
 
             //TODO: userDefined
 
@@ -3035,7 +3036,7 @@ impl MultipartOperations for SetDisks {
             )));
         }
 
-        let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
+        let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes);
 
         let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
 

crates/ecstore/src/store_api.rs

@@ -17,6 +17,7 @@ use crate::bucket::versioning::VersioningApi as _;
 use crate::config::storageclass;
 use crate::disk::DiskStore;
 use crate::error::{Error, Result};
+use crate::rio::{HashReader, LimitReader};
 use crate::store_utils::clean_metadata;
 use crate::{
     bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
@@ -34,7 +35,6 @@ use rustfs_filemeta::{
 use rustfs_lock::NamespaceLockWrapper;
 use rustfs_madmin::heal_commands::HealResultItem;
 use rustfs_rio::Checksum;
-use rustfs_rio::{DecompressReader, HashReader, LimitReader};
 use rustfs_utils::CompressionAlgorithm;
 use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
 use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
@@ -44,7 +44,6 @@ use std::collections::HashMap;
 use std::fmt::Debug;
 use std::io::Cursor;
 use std::pin::Pin;
-use std::str::FromStr as _;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 use time::OffsetDateTime;

crates/ecstore/src/store_api/types.rs

@@ -369,13 +369,18 @@ impl ObjectInfo {
     }
 
     pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> {
+        let (algorithm, _, compressed) = self.compression_read_plan()?;
+        Ok((algorithm, compressed))
+    }
+
+    pub fn compression_read_plan(&self) -> Result<(CompressionAlgorithm, crate::rio::ReadCompressionBackend, bool)> {
         let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
 
         if let Some(scheme) = scheme {
-            let algorithm = CompressionAlgorithm::from_str(&scheme)?;
-            Ok((algorithm, true))
+            let (algorithm, backend) = crate::rio::compression_scheme_to_read_plan(&scheme)?;
+            Ok((algorithm, backend, true))
         } else {
-            Ok((CompressionAlgorithm::None, false))
+            Ok((CompressionAlgorithm::None, crate::rio::ReadCompressionBackend::Legacy, false))
         }
     }
 
@@ -388,13 +393,18 @@ impl ObjectInfo {
         use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
 
         self.user_defined.keys().any(|key| {
-            let key = key.to_lowercase();
-            key.starts_with("x-minio-encryption-")
+            let lower = key.to_ascii_lowercase();
+            lower.starts_with("x-minio-encryption-")
+                || lower.starts_with("x-minio-internal-server-side-encryption-")
                 || matches!(
-                    key.as_str(),
-                    "x-rustfs-encryption-key"
+                    lower.as_str(),
+                    "x-minio-internal-encrypted-multipart"
+                        | "x-rustfs-encryption-key"
                         | "x-rustfs-encryption-algorithm"
                         | "x-rustfs-encryption-iv"
+                        | "x-rustfs-encryption-key-id"
+                        | "x-rustfs-encryption-context"
+                        | "x-rustfs-encryption-tag"
                         | "x-amz-server-side-encryption-aws-kms-key-id"
                         | SSEC_ALGORITHM_HEADER
                         | SSEC_KEY_HEADER
@@ -405,10 +415,17 @@ impl ObjectInfo {
     }
 
     pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
+        let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
         if let Some(size_str) = self
             .user_defined
             .get("x-rustfs-encryption-original-size")
-            .or_else(|| self.user_defined.get("x-amz-server-side-encryption-customer-original-size"))
+            .map(String::as_str)
+            .or_else(|| {
+                self.user_defined
+                    .get("x-amz-server-side-encryption-customer-original-size")
+                    .map(String::as_str)
+            })
+            .or(actual_size.as_deref())
             && !size_str.is_empty()
         {
             let size = size_str

crates/ecstore/tests/README.md

@@ -0,0 +1,24 @@
+# ecstore Integration Tests
+
+## MinIO-generated encrypted fixtures
+
+`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
+`.\rustfs\scripts\minio_fixture_lab\lab.py`.
+
+It currently covers multipart fixtures for:
+
+- `sse-s3-multipart-8m`
+- `sse-kms-multipart-8m`
+
+Required environment variables:
+
+- `RUSTFS_MINIO_FIXTURE_ROOT`
+- `RUSTFS_MINIO_STATIC_KMS_KEY_B64`
+
+Example:
+
+```powershell
+$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
+$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
+cargo +1.95.0 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
+```

crates/ecstore/tests/minio_generated_read_test.rs

@@ -0,0 +1,207 @@
+#![cfg(feature = "rio-v2")]
+
+use std::fs;
+use std::io::Cursor;
+use std::path::{Path, PathBuf};
+
+use rustfs_ecstore::bitrot::create_bitrot_reader;
+use rustfs_ecstore::disk::endpoint::Endpoint;
+use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
+use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
+use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
+use serde::Deserialize;
+use sha2::{Digest, Sha256};
+use temp_env::async_with_vars;
+use tokio::io::AsyncReadExt;
+
+#[derive(Debug, Deserialize)]
+struct ManifestRecord {
+    bucket: String,
+    object: String,
+    backend_files: Vec<String>,
+}
+
+fn fixture_root() -> PathBuf {
+    std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
+        .map(PathBuf::from)
+        .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
+}
+
+fn case_dir(case_id: &str) -> PathBuf {
+    fixture_root().join("cases").join(case_id)
+}
+
+fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
+    let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
+    serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
+}
+
+fn require_fixture_case(case_id: &str) -> PathBuf {
+    let path = case_dir(case_id);
+    assert!(
+        path.is_dir(),
+        "fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
+        path.display()
+    );
+    path
+}
+
+fn read_plaintext_sha256(case_dir: &Path) -> String {
+    fs::read_to_string(case_dir.join("plaintext.sha256"))
+        .unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
+        .trim()
+        .to_string()
+}
+
+fn minio_static_kms_key_b64() -> String {
+    std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
+        .unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
+}
+
+fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
+    let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object);
+    let relative = manifest
+        .backend_files
+        .iter()
+        .find(|entry| entry.as_str() == expected)
+        .unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}"));
+    case_dir.join("backend").join(relative)
+}
+
+fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo {
+    let xl_meta_path = object_xl_meta_path(case_dir, manifest);
+    let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display()));
+    get_file_info(
+        &xl_meta,
+        &manifest.bucket,
+        &manifest.object,
+        "",
+        FileInfoOpts {
+            data: true,
+            include_free_versions: true,
+        },
+    )
+    .unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display()))
+}
+
+fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo {
+    ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false)
+}
+
+fn sha256_hex(bytes: &[u8]) -> String {
+    hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
+}
+
+async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
+    let disk_root = case_dir.join("backend").join("disk1");
+    let disk_root_str = disk_root
+        .to_str()
+        .unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
+    let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(0);
+    let disk = new_disk(
+        &endpoint,
+        &DiskOption {
+            cleanup: false,
+            health_check: false,
+        },
+    )
+    .await
+    .expect("open fixture disk");
+    let data_dir = file_info
+        .data_dir
+        .as_ref()
+        .unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object));
+
+    let mut encrypted = Vec::new();
+    for part in &file_info.parts {
+        let checksum_info = file_info.erasure.get_checksum_info(part.number);
+        let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
+        let mut reader = create_bitrot_reader(
+            None,
+            Some(&disk),
+            &manifest.bucket,
+            &path,
+            0,
+            part.size,
+            file_info.erasure.shard_size(),
+            checksum_info.algorithm.clone(),
+            false,
+            false,
+        )
+        .await
+        .unwrap_or_else(|err| panic!("create bitrot reader for {path}: {err:?}"))
+        .unwrap_or_else(|| panic!("missing bitrot reader for {path}"));
+        let mut block = vec![0u8; file_info.erasure.shard_size()];
+        loop {
+            let n = reader
+                .read(&mut block)
+                .await
+                .unwrap_or_else(|err| panic!("read decoded encrypted bytes from {path}: {err}"));
+            if n == 0 {
+                break;
+            }
+            encrypted.extend_from_slice(&block[..n]);
+            if n < block.len() {
+                break;
+            }
+        }
+    }
+    disk.close().await.expect("close fixture disk");
+    encrypted
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_s3_multipart_fixture() {
+    assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
+}
+
+#[tokio::test]
+#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
+async fn reads_minio_generated_sse_kms_multipart_fixture() {
+    assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
+}
+
+async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
+    let case_dir = require_fixture_case(case_id);
+    let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
+    let expected_sha256 = read_plaintext_sha256(&case_dir);
+    let file_info = load_file_info(&case_dir, &manifest);
+    let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
+    let object_info = load_object_info(&file_info, &manifest);
+    let kms_key_b64 = minio_static_kms_key_b64();
+
+    async_with_vars(
+        [
+            ("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
+            ("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
+        ],
+        async {
+            let (mut reader, offset, length) = GetObjectReader::new(
+                Box::new(Cursor::new(encrypted)),
+                None,
+                &object_info,
+                &ObjectOptions::default(),
+                &http::HeaderMap::new(),
+            )
+            .await
+            .expect("construct GetObjectReader from MinIO raw fixture");
+
+            let mut plaintext = Vec::new();
+            reader
+                .read_to_end(&mut plaintext)
+                .await
+                .expect("read plaintext from MinIO raw fixture");
+
+            assert_eq!(offset, 0);
+            assert_eq!(length, object_info.size);
+            assert_eq!(reader.object_info.size, expected_size);
+            assert_eq!(plaintext.len(), expected_size as usize);
+            assert_eq!(sha256_hex(&plaintext), expected_sha256);
+        },
+    )
+    .await;
+}