From f7724d223bec3dd27c1fc615ef0874cabbc0905e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Mon, 8 Jun 2026 19:59:14 +0800 Subject: [PATCH] feat(rio): rio_v2 is compatible with minio for storing data. (#3115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Set up a compatibility layer for replacing old Rio components with new ones. * fix(rio). compress range * feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level. * feat(rio_v2): add sse-c test * test compression component * simple fix * fix minlz encode * fix metadata * fix kms key cache error * Update launch.json * ci: set nix crate download user agent * fix: gate obs pyroscope backend * ignore minio test * fix encrypt check * fix * fix * fix * Update object_usecase.rs * Update ci.yml * fix * ci add rio-v2 test * fix * ci fix * fix * Reconstructed into a more reasonable compatibility mode * fix * fix --------- Signed-off-by: houseme Signed-off-by: 唐小鸭 Co-authored-by: houseme Co-authored-by: cxymds Co-authored-by: 安正超 --- .github/AGENTS.md | 5 +- .github/workflows/ci.yml | 112 +- .github/workflows/nix.yml | 5 +- .gitignore | 1 + .vscode/launch.json | 43 +- AGENTS.md | 8 + Cargo.lock | 32 + Cargo.toml | 2 + crates/ecstore/Cargo.toml | 2 + crates/ecstore/src/erasure_coding/decode.rs | 57 + crates/ecstore/src/lib.rs | 10 + crates/ecstore/src/rio.rs | 767 ++++++ crates/ecstore/src/set_disk.rs | 7 +- crates/ecstore/src/store_api.rs | 3 +- crates/ecstore/src/store_api/readers.rs | 2081 +++++++++++++++-- crates/ecstore/src/store_api/types.rs | 33 +- crates/ecstore/tests/README.md | 24 + .../tests/minio_generated_read_test.rs | 207 ++ crates/filemeta/examples/dump_fileinfo.rs | 247 ++ crates/kms/src/cache.rs | 96 +- crates/kms/src/lib.rs | 8 + crates/kms/src/manager.rs | 87 +- crates/kms/src/service.rs | 67 +- crates/obs/Cargo.toml | 5 +- crates/obs/src/telemetry/guard.rs | 38 +- crates/obs/src/telemetry/local.rs | 4 - crates/obs/src/telemetry/otel.rs | 31 +- crates/rio-v2/Cargo.toml | 48 + crates/rio-v2/src/compress_reader.rs | 755 ++++++ crates/rio-v2/src/encrypt_reader.rs | 759 ++++++ crates/rio-v2/src/lib.rs | 91 + crates/rio-v2/src/s2_index.rs | 434 ++++ crates/rio-v2/tests/README.md | 55 + crates/rio-v2/tests/fixtures/.gitignore | 2 + .../rio-v2/tests/minio_fixture_lab/README.md | 150 ++ crates/rio-v2/tests/minio_fixture_lab/lab.py | 963 ++++++++ .../tests/minio_fixture_lab/test_lab.py | 207 ++ .../rio-v2/tests/minio_generated_fixtures.rs | 288 +++ crates/rio/src/compress_index.rs | 8 +- crates/s3select-api/src/object_store.rs | 38 +- crates/utils/src/http/metadata_compat.rs | 30 +- rustfs/Cargo.toml | 2 + rustfs/src/app/multipart_usecase.rs | 270 ++- rustfs/src/app/object_usecase.rs | 118 +- rustfs/src/storage/sse.rs | 1190 +++++++++- rustfs/src/storage/sse_test.rs | 17 +- scripts/s3-tests/run.sh | 17 + 47 files changed, 8742 insertions(+), 682 deletions(-) create mode 100644 crates/ecstore/src/rio.rs create mode 100644 crates/ecstore/tests/README.md create mode 100644 crates/ecstore/tests/minio_generated_read_test.rs create mode 100644 crates/rio-v2/Cargo.toml create mode 100644 crates/rio-v2/src/compress_reader.rs create mode 100644 crates/rio-v2/src/encrypt_reader.rs create mode 100644 crates/rio-v2/src/lib.rs create mode 100644 crates/rio-v2/src/s2_index.rs create mode 100644 crates/rio-v2/tests/README.md create mode 100644 crates/rio-v2/tests/fixtures/.gitignore create mode 100644 crates/rio-v2/tests/minio_fixture_lab/README.md create mode 100644 crates/rio-v2/tests/minio_fixture_lab/lab.py create mode 100644 crates/rio-v2/tests/minio_fixture_lab/test_lab.py create mode 100644 crates/rio-v2/tests/minio_generated_fixtures.rs diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 6c8195515..d97e8c9f3 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -32,6 +32,9 @@ Current `test-and-lint` gate includes: - `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes` - `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path` - `cargo fmt --all --check` -- `cargo clippy --all-targets --all-features -- -D warnings` +- `cargo clippy --all-targets -- -D warnings` +- `cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2` +- `cargo test -p rustfs --doc --features rio-v2` +- `cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings` - `./scripts/check_layer_dependencies.sh` - `./scripts/check_architecture_migration_rules.sh` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc05e246..664c21d6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,7 @@ jobs: run: ./scripts/check_unsafe_code_allowances.sh - name: Run clippy lints - run: cargo clippy --all-targets --all-features -- -D warnings + run: cargo clippy --all-targets -- -D warnings - name: Check layered dependencies run: ./scripts/check_layer_dependencies.sh @@ -142,6 +142,34 @@ jobs: - name: Check architecture migration rules run: ./scripts/check_architecture_migration_rules.sh + test-and-lint-rio-v2: + name: Test and Lint (rio-v2) + needs: skip-check + if: needs.skip-check.outputs.should_skip != 'true' + runs-on: ubicloud-standard-4 + timeout-minutes: 60 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-test-rio-v2-${{ hashFiles('**/Cargo.lock') }} + github-token: ${{ secrets.GITHUB_TOKEN }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Run rio-v2 feature tests + run: | + cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2 + cargo test -p rustfs --doc --features rio-v2 + + - name: Run rio-v2 clippy lints + run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings + build-rustfs-debug-binary: name: Build RustFS Debug Binary needs: skip-check @@ -175,6 +203,39 @@ jobs: if-no-files-found: error retention-days: 1 + build-rustfs-debug-binary-rio-v2: + name: Build RustFS Debug Binary (rio-v2) + needs: skip-check + if: needs.skip-check.outputs.should_skip != 'true' + runs-on: ubicloud-standard-4 + timeout-minutes: 30 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-rustfs-debug-binary-rio-v2-${{ hashFiles('**/Cargo.lock') }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build debug binary with rio-v2 + run: | + touch rustfs/build.rs + cargo build -p rustfs --bins --features rio-v2 --jobs 2 + + - name: Upload debug binary + uses: actions/upload-artifact@v6 + with: + name: rustfs-debug-binary-rio-v2 + path: target/debug/rustfs + if-no-files-found: error + retention-days: 1 + e2e-tests: name: End-to-End Tests needs: [ skip-check, build-rustfs-debug-binary ] @@ -222,9 +283,56 @@ jobs: path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log retention-days: 3 + e2e-tests-rio-v2: + name: End-to-End Tests (rio-v2) + needs: [ skip-check, build-rustfs-debug-binary-rio-v2 ] + if: needs.skip-check.outputs.should_skip != 'true' + runs-on: ubicloud-standard-2 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Clean up previous test run + run: | + rm -rf /tmp/rustfs + rm -f /tmp/rustfs.log + + - name: Download debug binary + uses: actions/download-artifact@v7 + with: + name: rustfs-debug-binary-rio-v2 + path: target/debug + + - name: Make binary executable + run: chmod +x ./target/debug/rustfs + + - name: Setup Rust toolchain for s3s-e2e installation + uses: dtolnay/rust-toolchain@stable + + - name: Install s3s-e2e test tool + uses: taiki-e/cache-cargo-install-action@v2 + with: + tool: s3s-e2e + git: https://github.com/s3s-project/s3s.git + rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52 + + - name: Run end-to-end tests + run: | + s3s-e2e --version + ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs + + - name: Upload test logs + if: failure() + uses: actions/upload-artifact@v6 + with: + name: e2e-test-logs-rio-v2-${{ github.run_number }} + path: /tmp/rustfs.log + retention-days: 3 + s3-implemented-tests: name: S3 Implemented Tests - needs: [ skip-check, build-rustfs-debug-binary ] + needs: [ skip-check, build-rustfs-debug-binary, e2e-tests ] if: needs.skip-check.outputs.should_skip != 'true' runs-on: ubicloud-standard-4 timeout-minutes: 60 diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 270170685..0ef1cabf6 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -48,6 +48,8 @@ jobs: permissions: contents: read id-token: write + env: + NIX_CURL_FLAGS: -A cargo/1.95.0 steps: - name: Checkout repository uses: actions/checkout@v6 @@ -57,7 +59,8 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} extra-conf: | - experimental-features = nix-command flakes + experimental-features = nix-command flakes configurable-impure-env + impure-env = NIX_CURL_FLAGS max-jobs = 1 - name: Cache Nix diff --git a/.gitignore b/.gitignore index d4acd5af1..89e16ca9c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ artifacts/ PR_DESCRIPTION.md scripts/s3-tests/selected_tests.txt docs +__pycache__/ !docs/ docs/* !docs/architecture/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 5f33c194a..e69583e4a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -224,6 +224,47 @@ "rust" ], }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'rustfs' with rio-v2 full matrix", + "cargo": { + "args": [ + "build", + "--bin=rustfs", + "--package=rustfs", + "--features=full,rio-v2" + ], + "filter": { + "name": "rustfs", + "kind": "bin" + } + }, + "env": { + "RUST_LOG": "rustfs=error,ecstore=error,s3s=error,iam=error", + "RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS": "true", + "RUST_BACKTRACE": "full", + "RUSTFS_ACCESS_KEY": "rustfsadmin", + "RUSTFS_SECRET_KEY": "rustfsadmin", + "RUSTFS_VOLUMES": "./target/rio-v2/test{1...4}", + "RUSTFS_SCANNER_ENABLED": "false", + "RUSTFS_HEAL_ENABLED": "false", + "RUSTFS_REGION": "us-east-1", + "RUSTFS_ADDRESS": ":9000", + "RUSTFS_CONSOLE_ENABLE": "true", + "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001", + "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs/rio-v2", + "RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true", + "RUSTFS_COMPRESSION_ENABLED": "true", + "RUSTFS_KMS_ENABLE": "true", + "RUSTFS_KMS_BACKEND": "local", + "RUSTFS_KMS_KEY_DIR": "./target/rio-v2/kms-key-dir", + "RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password + "RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key", + }, + "args": [], + "cwd": "${workspaceFolder}" + }, { "name": "E2E test executable target/debug/rustfs", "type": "lldb", @@ -256,4 +297,4 @@ ], }, ] -} \ No newline at end of file +} diff --git a/AGENTS.md b/AGENTS.md index bed363174..7951a4b89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,6 +103,14 @@ Do not open a PR with code changes when the required checks fail. - Use environment variables or vault tooling for sensitive configuration. - For localhost-sensitive tests, verify proxy settings to avoid traffic leakage. +## Tools + +### xl.meta decode tool Quick Use + +``` +cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta" +``` + ## Serde Safety - Add `#[serde(deny_unknown_fields)]` to structs deserialized from untrusted input (S3 API XML/JSON, lifecycle rules, bucket policies, replication configs). diff --git a/Cargo.lock b/Cargo.lock index 5114d9b46..95bc4555d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6255,6 +6255,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "minlz" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c19c645132db7e3664aabe291a16896f601053c6b5f567f4c5a196e6817827" +dependencies = [ + "crc", +] + [[package]] name = "mio" version = "1.2.1" @@ -9047,6 +9056,7 @@ dependencies = [ "futures-util", "hashbrown 0.17.1", "hex-simd", + "hmac 0.13.0", "http 1.4.1", "http-body 1.0.1", "http-body-util", @@ -9328,6 +9338,7 @@ dependencies = [ "rustfs-policy", "rustfs-protos", "rustfs-rio", + "rustfs-rio-v2", "rustfs-s3-types", "rustfs-signer", "rustfs-tls-runtime", @@ -9846,6 +9857,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "rustfs-rio-v2" +version = "1.0.0-beta.7" +dependencies = [ + "aes-gcm", + "bytes", + "hex", + "hmac 0.13.0", + "minlz", + "pin-project-lite", + "rand 0.10.1", + "rustfs-filemeta", + "rustfs-rio", + "rustfs-utils", + "serde", + "serde_json", + "sha2 0.11.0", + "tokio", + "walkdir", +] + [[package]] name = "rustfs-s3-ops" version = "1.0.0-beta.7" diff --git a/Cargo.toml b/Cargo.toml index 76f2ea86f..95714a8db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "crates/protocols", # Protocol implementations (FTPS, SFTP, etc.) "crates/protos", # Protocol buffer definitions "crates/rio", # Rust I/O utilities and abstractions + "crates/rio-v2", # Next-generation Rust I/O compatibility layer "crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling "crates/s3-types", # S3 event type definitions "crates/s3-ops", # S3 operation definitions and mapping @@ -104,6 +105,7 @@ rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.7" } rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.7" } rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.7" } rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.7" } +rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.7" } rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.7" } rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.7" } rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.7" } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index fe6ca2b9f..4f5de209e 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -32,11 +32,13 @@ workspace = true [features] default = [] +rio-v2 = ["dep:rustfs-rio-v2"] [dependencies] rustfs-filemeta.workspace = true rustfs-utils = { workspace = true, features = ["full"] } rustfs-rio.workspace = true +rustfs-rio-v2 = { workspace = true, optional = true } rustfs-signer.workspace = true rustfs-tls-runtime.workspace = true rustfs-checksums.workspace = true diff --git a/crates/ecstore/src/erasure_coding/decode.rs b/crates/ecstore/src/erasure_coding/decode.rs index 56668891e..225514f8a 100644 --- a/crates/ecstore/src/erasure_coding/decode.rs +++ b/crates/ecstore/src/erasure_coding/decode.rs @@ -433,6 +433,63 @@ mod tests { } } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_erasure_decode_preserves_compressed_stream_near_block_boundary() { + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + const BLOCK_SIZE: usize = 1024 * 1024; + + use crate::rio::CompressReader; + use rustfs_utils::CompressionAlgorithm; + use tokio::io::AsyncReadExt; + + let plaintext_size = 8 * BLOCK_SIZE + 123; + let plaintext = (0..plaintext_size) + .scan(0x9e37_79b9_7f4a_7c15u64, |state, _| { + *state ^= *state << 7; + *state ^= *state >> 9; + *state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9); + Some((*state >> 32) as u8) + }) + .collect::>(); + + let mut compressor = CompressReader::new(Cursor::new(plaintext), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compressor.read_to_end(&mut compressed).await.unwrap(); + + let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); + let total_shards = DATA_SHARDS + PARITY_SHARDS; + let shard_size = erasure.shard_size(); + let hash_algo = HashAlgorithm::HighwayHash256; + + let mut shard_writers: Vec>>> = (0..total_shards) + .map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone())) + .collect(); + + for block in compressed.chunks(BLOCK_SIZE) { + let shards = erasure.encode_data(block).unwrap(); + for (i, shard) in shards.iter().enumerate() { + shard_writers[i].write(shard).await.unwrap(); + } + } + + let shard_bufs: Vec> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect(); + let readers = shard_bufs + .iter() + .map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false))) + .collect(); + + let mut decoded = Vec::new(); + let (written, err) = erasure + .decode(&mut decoded, readers, 0, compressed.len(), compressed.len()) + .await; + + assert!(err.is_none(), "unexpected decode error: {err:?}"); + assert_eq!(written, compressed.len()); + assert_eq!(decoded, compressed); + } + #[tokio::test] async fn test_parallel_reader_normal() { const BLOCK_SIZE: usize = 64; diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 891318b32..693a7af91 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -34,6 +34,7 @@ pub mod metrics_realtime; pub mod notification_sys; pub mod pools; pub mod rebalance; +pub mod rio; pub mod rpc; pub mod set_disk; mod sets; @@ -60,3 +61,12 @@ pub use global::{get_global_lock_client, get_global_lock_clients, set_global_loc pub use global::GLOBAL_Endpoints; pub use store_api::StorageAPI; + +#[cfg(test)] +mod rio_tests { + #[test] + fn uses_expected_rio_backend() { + let expected = if cfg!(feature = "rio-v2") { "rio-v2" } else { "legacy-rio" }; + assert_eq!(crate::rio::backend_name(), expected); + } +} diff --git a/crates/ecstore/src/rio.rs b/crates/ecstore/src/rio.rs new file mode 100644 index 000000000..38773ccc1 --- /dev/null +++ b/crates/ecstore/src/rio.rs @@ -0,0 +1,767 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(feature = "rio-v2")] +pub use rustfs_rio_v2::*; + +#[cfg(not(feature = "rio-v2"))] +pub use rustfs_rio::*; + +use bytes::Bytes; +use rustfs_utils::CompressionAlgorithm; +use std::str::FromStr; +use tokio::io::AsyncRead; + +#[cfg(feature = "rio-v2")] +const MINIO_S2_COMPRESSION_SCHEME: &str = "klauspost/compress/s2"; +#[cfg(feature = "rio-v2")] +const ENCRYPTED_S2_PADDING_MULTIPLE: usize = 256; + +pub const fn backend_name() -> &'static str { + #[cfg(feature = "rio-v2")] + { + "rio-v2" + } + + #[cfg(not(feature = "rio-v2"))] + { + "legacy-rio" + } +} + +pub fn compression_metadata_value(algorithm: CompressionAlgorithm) -> String { + #[cfg(feature = "rio-v2")] + { + let _ = algorithm; + MINIO_S2_COMPRESSION_SCHEME.to_string() + } + + #[cfg(not(feature = "rio-v2"))] + { + algorithm.to_string() + } +} + +pub fn compression_scheme_to_algorithm(scheme: &str) -> std::io::Result { + #[cfg(feature = "rio-v2")] + if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) { + // rio_v2 currently routes all compressed-object handling through the S2 + // reader implementation, so the enum is only a placeholder token here. + return Ok(CompressionAlgorithm::default()); + } + + CompressionAlgorithm::from_str(scheme) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadCompressionBackend { + Legacy, + V2, +} + +pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(CompressionAlgorithm, ReadCompressionBackend)> { + #[cfg(feature = "rio-v2")] + if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) { + return Ok((CompressionAlgorithm::default(), ReadCompressionBackend::V2)); + } + + Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadEncryptionBackend { + Legacy, + V2, +} + +pub fn compression_index_storage_bytes(index: &Index) -> Bytes { + #[cfg(feature = "rio-v2")] + { + minio_index_storage_bytes(index) + } + + #[cfg(not(feature = "rio-v2"))] + { + index.clone().into_vec() + } +} + +pub fn decode_compression_index_bytes(bytes: &Bytes) -> Option { + #[cfg(feature = "rio-v2")] + { + if let Some(decoded) = decode_minio_index_bytes(bytes) { + return Some(decoded); + } + } + + let mut decoded = Index::new(); + if decoded.load(bytes.as_ref()).is_ok() { + return Some(decoded); + } + + #[cfg(feature = "rio-v2")] + { + let restored = restore_legacy_index_headers(bytes.as_ref()); + let mut decoded = Index::new(); + if decoded.load(&restored).is_ok() { + return Some(decoded); + } + } + + None +} + +pub fn compression_reader(reader: R, algorithm: CompressionAlgorithm, encrypted: bool) -> CompressReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + #[cfg(feature = "rio-v2")] + { + if encrypted { + return CompressReader::with_encrypted_padding(reader, algorithm); + } + } + + #[cfg(not(feature = "rio-v2"))] + let _ = encrypted; + + CompressReader::new(reader, algorithm) +} + +pub fn decompression_reader( + reader: R, + algorithm: CompressionAlgorithm, + backend: ReadCompressionBackend, +) -> Box +where + R: AsyncRead + Unpin + Send + Sync + 'static, +{ + #[cfg(feature = "rio-v2")] + { + match backend { + ReadCompressionBackend::Legacy => Box::new(rustfs_rio::DecompressReader::new(reader, algorithm)), + ReadCompressionBackend::V2 => Box::new(rustfs_rio_v2::DecompressReader::new(reader, algorithm)), + } + } + + #[cfg(not(feature = "rio-v2"))] + { + let _ = backend; + Box::new(rustfs_rio::DecompressReader::new(reader, algorithm)) + } +} + +pub fn decrypt_reader( + reader: R, + key: [u8; 32], + base_nonce: [u8; 12], + backend: ReadEncryptionBackend, + sequence_number: u32, +) -> Box +where + R: AsyncRead + Unpin + Send + Sync + 'static, +{ + #[cfg(feature = "rio-v2")] + { + match backend { + ReadEncryptionBackend::Legacy => Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce)), + ReadEncryptionBackend::V2 => { + Box::new(rustfs_rio_v2::DecryptReader::new_with_sequence(reader, key, base_nonce, sequence_number)) + } + } + } + + #[cfg(not(feature = "rio-v2"))] + { + let _ = (backend, sequence_number); + Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce)) + } +} + +pub fn decrypt_reader_with_object_key( + reader: R, + object_key: [u8; 32], + sequence_number: u32, +) -> Box +where + R: AsyncRead + Unpin + Send + Sync + 'static, +{ + #[cfg(feature = "rio-v2")] + { + Box::new(rustfs_rio_v2::DecryptReader::new_with_object_key_and_sequence( + reader, + object_key, + sequence_number, + )) + } + + #[cfg(not(feature = "rio-v2"))] + { + let _ = sequence_number; + Box::new(rustfs_rio::DecryptReader::new(reader, object_key, [0u8; 12])) + } +} + +pub fn decrypt_multipart_reader( + reader: R, + key: [u8; 32], + base_nonce: [u8; 12], + multipart_parts: Vec, + backend: ReadEncryptionBackend, + sequence_number: u32, +) -> Box +where + R: AsyncRead + Unpin + Send + Sync + 'static, +{ + #[cfg(feature = "rio-v2")] + { + match backend { + ReadEncryptionBackend::Legacy => { + Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts)) + } + ReadEncryptionBackend::V2 => Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_sequence( + reader, + key, + base_nonce, + multipart_parts, + sequence_number, + )), + } + } + + #[cfg(not(feature = "rio-v2"))] + { + let _ = (backend, sequence_number); + Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts)) + } +} + +pub fn decrypt_multipart_reader_with_object_key( + reader: R, + object_key: [u8; 32], + multipart_parts: Vec, + sequence_number: u32, +) -> Box +where + R: AsyncRead + Unpin + Send + Sync + 'static, +{ + #[cfg(feature = "rio-v2")] + { + Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_object_key_and_sequence( + reader, + object_key, + multipart_parts, + sequence_number, + )) + } + + #[cfg(not(feature = "rio-v2"))] + { + let _ = sequence_number; + Box::new(rustfs_rio::DecryptReader::new_multipart(reader, object_key, [0u8; 12], multipart_parts)) + } +} + +#[cfg(feature = "rio-v2")] +fn restore_legacy_index_headers(bytes: &[u8]) -> Vec { + if bytes.is_empty() { + return Vec::new(); + } + + const S2_INDEX_HEADER: &[u8] = b"s2idx\x00"; + const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s"; + let mut restored = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + bytes.len() + 4 + S2_INDEX_TRAILER.len()); + restored.extend_from_slice(&[0x99, 0x2A, 0x4D, 0x18]); + restored.extend_from_slice(S2_INDEX_HEADER); + restored.extend_from_slice(bytes); + + let total_size = (restored.len() + 4 + S2_INDEX_TRAILER.len()) as u32; + restored.extend_from_slice(&total_size.to_le_bytes()); + restored.extend_from_slice(S2_INDEX_TRAILER); + + let chunk_len = restored.len() - 4; + restored[1] = chunk_len as u8; + restored[2] = (chunk_len >> 8) as u8; + restored[3] = (chunk_len >> 16) as u8; + restored +} + +#[derive(Debug, Clone, Copy)] +pub struct WriteEncryption { + key_bytes: [u8; 32], + mode: WriteEncryptionMode, +} + +#[derive(Debug, Clone, Copy)] +enum WriteEncryptionMode { + SinglepartObjectKey, + Singlepart { + base_nonce: [u8; 12], + }, + MultipartLegacy { + base_nonce: [u8; 12], + multipart_part_number: usize, + }, + MultipartObjectKey { + multipart_part_number: u32, + }, +} + +impl WriteEncryption { + pub const fn singlepart_object_key(object_key: [u8; 32]) -> Self { + Self { + key_bytes: object_key, + mode: WriteEncryptionMode::SinglepartObjectKey, + } + } + + pub const fn singlepart(key_bytes: [u8; 32], base_nonce: [u8; 12]) -> Self { + Self { + key_bytes, + mode: WriteEncryptionMode::Singlepart { base_nonce }, + } + } + + pub const fn multipart(key_bytes: [u8; 32], base_nonce: [u8; 12], multipart_part_number: usize) -> Self { + Self { + key_bytes, + mode: WriteEncryptionMode::MultipartLegacy { + base_nonce, + multipart_part_number, + }, + } + } + + pub const fn multipart_object_key(object_key: [u8; 32], multipart_part_number: u32) -> Self { + Self { + key_bytes: object_key, + mode: WriteEncryptionMode::MultipartObjectKey { multipart_part_number }, + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct WritePlan { + compression: Option, + encryption: Option, +} + +impl WritePlan { + pub const fn new() -> Self { + Self { + compression: None, + encryption: None, + } + } + + pub const fn with_compression(mut self, algorithm: CompressionAlgorithm) -> Self { + self.compression = Some(algorithm); + self + } + + pub const fn with_encryption(mut self, encryption: WriteEncryption) -> Self { + self.encryption = Some(encryption); + self + } + + pub const fn is_passthrough(&self) -> bool { + self.compression.is_none() && self.encryption.is_none() + } + + pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result { + let encrypted = self.encryption.is_some(); + if let Some(algorithm) = self.compression { + reader = HashReader::from_reader( + compression_reader(reader, algorithm, encrypted), + HashReader::SIZE_PRESERVE_LAYER, + actual_size, + None, + None, + false, + )?; + } + + if let Some(encryption) = self.encryption { + reader = match encryption.mode { + WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader( + #[cfg(feature = "rio-v2")] + EncryptReader::new_with_object_key(reader, encryption.key_bytes), + #[cfg(not(feature = "rio-v2"))] + EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]), + HashReader::SIZE_PRESERVE_LAYER, + actual_size, + None, + None, + false, + )?, + WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader( + EncryptReader::new(reader, encryption.key_bytes, base_nonce), + HashReader::SIZE_PRESERVE_LAYER, + actual_size, + None, + None, + false, + )?, + WriteEncryptionMode::MultipartLegacy { + base_nonce, + multipart_part_number, + } => HashReader::from_reader( + EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number), + HashReader::SIZE_PRESERVE_LAYER, + actual_size, + None, + None, + false, + )?, + WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader( + #[cfg(feature = "rio-v2")] + EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number), + #[cfg(not(feature = "rio-v2"))] + EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize), + HashReader::SIZE_PRESERVE_LAYER, + actual_size, + None, + None, + false, + )?, + }; + } + + Ok(reader) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_utils::CompressionAlgorithm; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + #[cfg(feature = "rio-v2")] + fn s2_chunk_types(stream: &[u8]) -> Vec { + let mut chunk_types = Vec::new(); + let mut offset = 0usize; + while offset + 4 <= stream.len() { + let chunk_type = stream[offset]; + let chunk_len = + (stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16); + chunk_types.push(chunk_type); + offset += 4 + chunk_len; + } + chunk_types + } + + #[tokio::test] + async fn write_plan_passthrough_keeps_plaintext() { + let plaintext = b"write-plan-plain".to_vec(); + let reader = HashReader::from_stream( + Cursor::new(plaintext.clone()), + plaintext.len() as i64, + plaintext.len() as i64, + None, + None, + false, + ) + .expect("create hash reader"); + + let mut reader = WritePlan::new() + .apply(reader, plaintext.len() as i64) + .expect("apply passthrough plan"); + + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.expect("read passthrough stream"); + + assert_eq!(actual, plaintext); + } + + #[tokio::test] + async fn write_plan_compress_then_encrypt_multipart_roundtrip() { + let plaintext = b"abcdefghijklmnopqrstuvwxyz".repeat(128); + let actual_size = plaintext.len() as i64; + let key_bytes = [0x5Au8; 32]; + let base_nonce = [0xA5u8; 12]; + let part_number = 7; + + let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false) + .expect("create hash reader"); + + let mut transformed = WritePlan::new() + .with_compression(CompressionAlgorithm::default()) + .with_encryption(WriteEncryption::multipart(key_bytes, base_nonce, part_number)) + .apply(reader, actual_size) + .expect("apply transform plan"); + + let mut ciphertext = Vec::new(); + transformed + .read_to_end(&mut ciphertext) + .await + .expect("read transformed ciphertext"); + + let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]); + let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default()); + + let mut actual = Vec::new(); + decompressed + .read_to_end(&mut actual) + .await + .expect("decrypt and decompress transformed stream"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn write_plan_supports_singlepart_object_key_encryption_roundtrip() { + let plaintext = b"singlepart-object-key".repeat(512); + let actual_size = plaintext.len() as i64; + let object_key = [0x7Cu8; 32]; + + let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false) + .expect("create hash reader"); + + let mut transformed = WritePlan::new() + .with_encryption(WriteEncryption::singlepart_object_key(object_key)) + .apply(reader, actual_size) + .expect("apply singlepart object-key plan"); + + let mut encrypted = Vec::new(); + transformed + .read_to_end(&mut encrypted) + .await + .expect("read encrypted object-key stream"); + + let mut decrypted = DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key); + let mut actual = Vec::new(); + decrypted.read_to_end(&mut actual).await.expect("decrypt object-key stream"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn write_plan_supports_multipart_object_key_encryption_roundtrip() { + let plaintext = b"multipart-object-key-".repeat(4096); + let actual_size = plaintext.len() as i64; + let object_key = [0x2Du8; 32]; + let part_number = 3u32; + + let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false) + .expect("create hash reader"); + + let mut transformed = WritePlan::new() + .with_encryption(WriteEncryption::multipart_object_key(object_key, part_number)) + .apply(reader, actual_size) + .expect("apply multipart object-key encryption"); + + let mut ciphertext = Vec::new(); + transformed + .read_to_end(&mut ciphertext) + .await + .expect("read multipart object-key ciphertext"); + + let mut actual = Vec::new(); + DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), object_key, vec![part_number as usize]) + .read_to_end(&mut actual) + .await + .expect("decrypt multipart object-key ciphertext"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn write_plan_rio_v2_compression_emits_s2_stream_and_seekable_index() { + let plaintext = b"rustfs-rio-v2-s2-".repeat(600_000); + let actual_size = plaintext.len() as i64; + let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false) + .expect("create hash reader"); + + let mut transformed = WritePlan::new() + .with_compression(CompressionAlgorithm::default()) + .apply(reader, actual_size) + .expect("apply compression plan"); + + let mut compressed = Vec::new(); + transformed + .read_to_end(&mut compressed) + .await + .expect("read compressed stream"); + + assert!( + compressed.starts_with(b"\xff\x06\x00\x00S2sTwO"), + "rio_v2 compressed stream must start with the S2 stream identifier" + ); + + let index = transformed + .try_get_index() + .cloned() + .expect("rio_v2 compressed stream should expose a compression index"); + let (compressed_offset, uncompressed_offset) = index.find(2 * 1024 * 1024).expect("seek into compression index"); + + assert!(compressed_offset > 0, "expected a non-zero compressed offset for the second block"); + assert!(uncompressed_offset > 0, "expected a non-zero uncompressed offset for the second block"); + + let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn write_plan_rio_v2_small_compression_skips_index_below_minio_threshold() { + let plaintext = b"rustfs-rio-v2-s2-".repeat(32_768); + let actual_size = plaintext.len() as i64; + let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false) + .expect("create hash reader"); + + let mut transformed = WritePlan::new() + .with_compression(CompressionAlgorithm::default()) + .apply(reader, actual_size) + .expect("apply compression plan"); + + let mut compressed = Vec::new(); + transformed + .read_to_end(&mut compressed) + .await + .expect("read compressed stream"); + + assert!( + transformed.try_get_index().is_none(), + "rio_v2 should match MinIO and skip compression indexes for small objects" + ); + + let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn rio_v2_singlepart_encrypt_decrypt_roundtrip_preserves_small_compressed_stream() { + let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + let key_bytes = [0x33u8; 32]; + let base_nonce = [0x55u8; 12]; + + let mut compressed = Vec::new(); + CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut compressed) + .await + .expect("compress plaintext"); + + let mut encrypted = Vec::new(); + EncryptReader::new(Cursor::new(compressed), key_bytes, base_nonce) + .read_to_end(&mut encrypted) + .await + .expect("encrypt compressed stream"); + + let decrypt_reader = DecryptReader::new(Cursor::new(encrypted), key_bytes, base_nonce); + let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressed + .read_to_end(&mut actual) + .await + .expect("decrypt and decompress small stream"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn rio_v2_compress_then_encrypt_adds_s2_padding_frames() { + let plaintext = b"padding-check-".repeat(4097); + let actual_size = plaintext.len() as i64; + let key_bytes = [0x1Bu8; 32]; + let base_nonce = [0xC4u8; 12]; + + let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false) + .expect("create hash reader"); + + let mut transformed = WritePlan::new() + .with_compression(CompressionAlgorithm::default()) + .with_encryption(WriteEncryption::singlepart(key_bytes, base_nonce)) + .apply(reader, actual_size) + .expect("apply transform plan"); + + let mut ciphertext = Vec::new(); + transformed + .read_to_end(&mut ciphertext) + .await + .expect("read transformed ciphertext"); + + let mut decrypted_compressed = Vec::new(); + DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce) + .read_to_end(&mut decrypted_compressed) + .await + .expect("decrypt compressed stream"); + + assert_eq!(decrypted_compressed.len() % ENCRYPTED_S2_PADDING_MULTIPLE, 0); + + let chunk_types = s2_chunk_types(&decrypted_compressed); + assert!( + chunk_types.contains(&0xfe), + "rio_v2 compressed+encrypted streams must include S2 padding frames before encryption" + ); + + let mut actual = Vec::new(); + DecompressReader::new(Cursor::new(decrypted_compressed), CompressionAlgorithm::default()) + .read_to_end(&mut actual) + .await + .expect("decompress padded stream"); + + assert_eq!(actual, plaintext); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn rio_v2_decompress_reader_returns_bytes_on_first_read() { + let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + + let mut compressed = Vec::new(); + CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut compressed) + .await + .expect("compress plaintext"); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut buf = [0u8; 64]; + let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk"); + + assert!(n > 0); + assert_eq!(&buf[..n], plaintext.as_slice()); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn rio_v2_decompress_reader_returns_bytes_on_first_large_read() { + let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + + let mut compressed = Vec::new(); + CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut compressed) + .await + .expect("compress plaintext"); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut buf = [0u8; 8192]; + let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk"); + + assert!(n > 0); + assert_eq!(&buf[..n], plaintext.as_slice()); + } +} diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index c86c368d0..c9ec2d0c8 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -85,7 +85,6 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_object_capacity::capacity_scope::{ CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope, }; -use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; use rustfs_s3_types::EventName; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; @@ -130,6 +129,8 @@ use tracing::error; use tracing::{debug, info, warn}; use uuid::Uuid; +use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; + pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; pub const MAX_PARTS_COUNT: usize = 10000; pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket"; @@ -1146,7 +1147,7 @@ impl ObjectIO for SetDisks { insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string()); } - let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec()); + let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes); //TODO: userDefined @@ -3035,7 +3036,7 @@ impl MultipartOperations for SetDisks { ))); } - let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec()); + let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes); let mut etag = data.stream.try_resolve_etag().unwrap_or_default(); diff --git a/crates/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs index cdfde143b..8a99ea7ee 100644 --- a/crates/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -17,6 +17,7 @@ use crate::bucket::versioning::VersioningApi as _; use crate::config::storageclass; use crate::disk::DiskStore; use crate::error::{Error, Result}; +use crate::rio::{HashReader, LimitReader}; use crate::store_utils::clean_metadata; use crate::{ bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, @@ -34,7 +35,6 @@ use rustfs_filemeta::{ use rustfs_lock::NamespaceLockWrapper; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_rio::Checksum; -use rustfs_rio::{DecompressReader, HashReader, LimitReader}; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS}; @@ -44,7 +44,6 @@ use std::collections::HashMap; use std::fmt::Debug; use std::io::Cursor; use std::pin::Pin; -use std::str::FromStr as _; use std::sync::Arc; use std::task::{Context, Poll}; use time::OffsetDateTime; diff --git a/crates/ecstore/src/store_api/readers.rs b/crates/ecstore/src/store_api/readers.rs index c855288b9..4db385e7e 100644 --- a/crates/ecstore/src/store_api/readers.rs +++ b/crates/ecstore/src/store_api/readers.rs @@ -4,19 +4,90 @@ use aes_gcm::{ aead::{Aead, KeyInit}, }; use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; +#[cfg(feature = "rio-v2")] +use hmac::{Hmac, Mac}; use md5::{Digest, Md5}; use rustfs_kms::{service_manager::get_global_encryption_service, types::ObjectEncryptionContext}; -use rustfs_rio::DecryptReader; use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; +use rustfs_utils::path::path_join_buf; +#[cfg(feature = "rio-v2")] +use sha2::Sha256; use std::collections::HashMap; use std::env; +use crate::rio::Index; + const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key"; +const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context"; const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv"; const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size"; const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size"; const DEFAULT_SSE_ALGORITHM: &str = "AES256"; +#[cfg(feature = "rio-v2")] +const DARE_PAYLOAD_SIZE: i64 = 64 * 1024; +#[cfg(feature = "rio-v2")] +const DARE_PACKAGE_SIZE: i64 = DARE_PAYLOAD_SIZE + 32; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256"; +#[cfg(feature = "rio-v2")] +const DARE_VERSION_20: u8 = 0x20; +#[cfg(feature = "rio-v2")] +const DARE_CIPHER_AES_256_GCM: u8 = 0x00; +#[cfg(feature = "rio-v2")] +const DARE_HEADER_SIZE: usize = 16; +#[cfg(feature = "rio-v2")] +const DARE_TAG_SIZE: usize = 16; +#[cfg(feature = "rio-v2")] +const SEALED_KEY_IV_SIZE: usize = 32; +#[cfg(feature = "rio-v2")] +const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE; +#[cfg(feature = "rio-v2")] +const MINIO_SECRET_KEY_RANDOM_SIZE: usize = 28; +#[cfg(feature = "rio-v2")] +const MINIO_SECRET_KEY_IV_SIZE: usize = 16; +#[cfg(feature = "rio-v2")] +const MINIO_SECRET_KEY_NONCE_SIZE: usize = 12; + +#[cfg(feature = "rio-v2")] +type HmacSha256 = Hmac; + +fn canonical_kms_bucket_path(bucket: &str, object: &str) -> String { + path_join_buf(&[bucket, object]) +} + +fn build_object_encryption_context( + bucket: &str, + object: &str, + provided_context: Option<&HashMap>, +) -> ObjectEncryptionContext { + let mut context = provided_context.cloned().unwrap_or_default(); + context + .entry(bucket.to_string()) + .or_insert_with(|| canonical_kms_bucket_path(bucket, object)); + + let mut object_context = ObjectEncryptionContext::new(bucket.to_string(), object.to_string()); + for (ctx_key, ctx_value) in context { + object_context = object_context.with_encryption_context(ctx_key, ctx_value); + } + object_context +} fn part_plaintext_size(part: &ObjectPartInfo) -> i64 { if part.actual_size > 0 { @@ -31,14 +102,8 @@ fn restore_request_active(opts: &ObjectOptions) -> bool { restore.type_.is_some() || restore.days.is_some() || restore.output_location.is_some() || restore.select_parameters.is_some() } -fn decode_compression_index(index: Option<&bytes::Bytes>) -> Option { - let bytes = index?; - let mut decoded = rustfs_rio::Index::new(); - if decoded.load(bytes.as_ref()).is_ok() { - Some(decoded) - } else { - None - } +fn decode_compression_index(index: Option<&bytes::Bytes>) -> Option { + crate::rio::decode_compression_index_bytes(index?) } fn get_compressed_offsets(oi: &ObjectInfo, offset: i64) -> (i64, i64, usize, i64, u64) { @@ -59,8 +124,10 @@ fn get_compressed_offsets(oi: &ObjectInfo, offset: i64) -> (i64, i64, usize, i64 } let mut part_skip = offset - skip_length; - let decrypt_skip = 0_i64; - let seq_num = 0_u64; + #[cfg(feature = "rio-v2")] + let (mut decrypt_skip, mut seq_num) = (0_i64, 0_u64); + #[cfg(not(feature = "rio-v2"))] + let (decrypt_skip, seq_num) = (0_i64, 0_u64); if part_skip > 0 && let Some(part) = oi.parts.get(first_part_idx) @@ -68,13 +135,70 @@ fn get_compressed_offsets(oi: &ObjectInfo, offset: i64) -> (i64, i64, usize, i64 && let Ok((comp_off, uncomp_off)) = index.find(part_skip) && comp_off > 0 { - compressed_offset += comp_off; + #[cfg(feature = "rio-v2")] + if oi.is_encrypted() { + seq_num = (comp_off / DARE_PAYLOAD_SIZE) as u64; + decrypt_skip = comp_off % DARE_PAYLOAD_SIZE; + compressed_offset += (comp_off / DARE_PAYLOAD_SIZE) * DARE_PACKAGE_SIZE; + } else { + compressed_offset += comp_off; + } + + #[cfg(not(feature = "rio-v2"))] + { + compressed_offset += comp_off; + } + part_skip -= uncomp_off; } (compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num) } +#[cfg(feature = "rio-v2")] +fn get_encrypted_offsets(oi: &ObjectInfo, offset: i64) -> Result<(i64, usize, usize, u32, i64)> { + if oi.parts.is_empty() { + let plaintext_size = oi.decrypted_size()?; + let start_package_number = offset / DARE_PAYLOAD_SIZE; + let plaintext_skip = usize::try_from(offset % DARE_PAYLOAD_SIZE) + .map_err(|_| Error::other(format!("invalid DARE skip offset {offset}")))?; + let encrypted_offset = start_package_number * DARE_PACKAGE_SIZE; + let aligned_plaintext_offset = start_package_number * DARE_PAYLOAD_SIZE; + let remaining_plaintext_size = plaintext_size - aligned_plaintext_offset; + let sequence_number = u32::try_from(start_package_number) + .map_err(|_| Error::other(format!("invalid DARE sequence number {start_package_number}")))?; + return Ok((encrypted_offset, plaintext_skip, 0, sequence_number, remaining_plaintext_size)); + } + + let mut cumulative_plaintext_size = 0_i64; + let mut cumulative_encrypted_size = 0_i64; + + for (part_index, part) in oi.parts.iter().enumerate() { + let current_part_plaintext_size = part_plaintext_size(part); + if offset < cumulative_plaintext_size + current_part_plaintext_size { + let relative_offset = offset - cumulative_plaintext_size; + let start_package_number = relative_offset / DARE_PAYLOAD_SIZE; + let plaintext_skip = usize::try_from(relative_offset % DARE_PAYLOAD_SIZE) + .map_err(|_| Error::other(format!("invalid DARE skip offset {relative_offset}")))?; + let encrypted_offset = cumulative_encrypted_size + start_package_number * DARE_PACKAGE_SIZE; + let aligned_plaintext_offset = cumulative_plaintext_size + start_package_number * DARE_PAYLOAD_SIZE; + let remaining_plaintext_size = oi.decrypted_size()? - aligned_plaintext_offset; + let sequence_number = u32::try_from(start_package_number) + .map_err(|_| Error::other(format!("invalid DARE sequence number {start_package_number}")))?; + + return Ok((encrypted_offset, plaintext_skip, part_index, sequence_number, remaining_plaintext_size)); + } + + cumulative_plaintext_size += current_part_plaintext_size; + cumulative_encrypted_size += part.size as i64; + } + + Err(Error::other(format!( + "invalid encrypted offset {offset} for object with decrypted size {}", + oi.decrypted_size()? + ))) +} + pub struct PutObjReader { pub stream: HashReader, } @@ -125,6 +249,393 @@ pub struct GetObjectReader { struct EncryptionMaterial { key_bytes: [u8; 32], base_nonce: [u8; 12], + key_kind: EncryptionKeyKind, + reader_backend: crate::rio::ReadEncryptionBackend, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EncryptionKeyKind { + Direct, + Object, +} + +#[derive(Debug, Clone)] +enum ReadTransform { + Plain { + visible_offset: usize, + visible_length: i64, + }, + Compressed { + algorithm: CompressionAlgorithm, + backend: crate::rio::ReadCompressionBackend, + decompressed_offset: usize, + decompressed_length: i64, + total_plaintext_size: usize, + }, + Encrypted { + material: EncryptionMaterial, + is_multipart: bool, + part_numbers: Vec, + sequence_number: u32, + decrypt_skip: usize, + plaintext_offset: usize, + plaintext_length: i64, + total_plaintext_size: usize, + compression: Option<(CompressionAlgorithm, crate::rio::ReadCompressionBackend)>, + }, +} + +#[derive(Debug, Clone)] +struct ReadPlan { + storage_offset: usize, + storage_length: i64, + object_size: i64, + transform: ReadTransform, +} + +impl ReadPlan { + async fn build(rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result { + let mut rs = rs; + if let Some(part_number) = opts.part_number + && rs.is_none() + { + rs = HTTPRangeSpec::from_object_info(oi, part_number); + } + + let mut is_encrypted = oi.is_encrypted(); + let (algo, compression_backend, mut is_compressed) = oi.compression_read_plan()?; + + if restore_request_active(opts) { + is_encrypted = false; + is_compressed = false; + } + + if is_compressed && !is_encrypted { + let actual_size = oi.get_actual_size()?; + let (storage_offset, storage_length, decompressed_offset, decompressed_length) = if let Some(rs) = rs { + let (req_off, req_length) = rs.get_offset_length(actual_size)?; + let (physical_off, decompressed_skip, _, _, _) = get_compressed_offsets(oi, req_off as i64); + let storage_offset = usize::try_from(physical_off) + .map_err(|_| Error::other(format!("invalid compressed offset {physical_off}")))?; + (storage_offset, oi.size - physical_off, decompressed_skip as usize, req_length) + } else { + (0, oi.size, 0, actual_size) + }; + + let total_plaintext_size = + usize::try_from(actual_size).map_err(|_| Error::other(format!("invalid decompressed size {actual_size}")))?; + + return Ok(Self { + storage_offset, + storage_length, + object_size: decompressed_length, + transform: ReadTransform::Compressed { + algorithm: algo, + backend: compression_backend, + decompressed_offset, + decompressed_length, + total_plaintext_size, + }, + }); + } + + if is_encrypted { + let material = resolve_encryption_material(oi, h).await?; + #[cfg(feature = "rio-v2")] + let encryption_backend = material.reader_backend; + let is_multipart = is_multipart_encrypted_object(&oi.parts, oi.etag.as_deref()); + let plaintext_size = encrypted_plaintext_size(oi, is_multipart, is_compressed)?; + let full_plaintext_size = + usize::try_from(plaintext_size).map_err(|_| Error::other(format!("invalid decrypted size {plaintext_size}")))?; + let ( + storage_offset, + storage_length, + decrypt_skip, + plaintext_offset, + plaintext_length, + total_plaintext_size, + sequence_number, + part_numbers, + ) = if let Some(rs) = rs { + let (requested_offset, requested_length) = rs.get_offset_length(plaintext_size)?; + #[cfg(feature = "rio-v2")] + { + if encryption_backend == crate::rio::ReadEncryptionBackend::Legacy { + ( + 0, + oi.size, + 0, + requested_offset, + requested_length, + full_plaintext_size, + 0, + multipart_part_numbers(&oi.parts), + ) + } else if is_compressed { + let (physical_off, decompressed_skip, first_part_idx, decrypt_skip, seq_num) = + get_compressed_offsets(oi, requested_offset as i64); + ( + usize::try_from(physical_off) + .map_err(|_| Error::other(format!("invalid encrypted compressed offset {physical_off}")))?, + oi.size - physical_off, + usize::try_from(decrypt_skip) + .map_err(|_| Error::other(format!("invalid decrypt skip {decrypt_skip}")))?, + usize::try_from(decompressed_skip) + .map_err(|_| Error::other(format!("invalid decompressed skip {decompressed_skip}")))?, + requested_length, + full_plaintext_size, + u32::try_from(seq_num) + .map_err(|_| Error::other(format!("invalid DARE sequence number {seq_num}")))?, + multipart_part_numbers(&oi.parts[first_part_idx..]), + ) + } else { + let (encrypted_offset, plaintext_skip, part_start_index, sequence_number, remaining_plaintext_size) = + get_encrypted_offsets(oi, requested_offset as i64)?; + let total_plaintext_size = usize::try_from(remaining_plaintext_size) + .map_err(|_| Error::other(format!("invalid remaining decrypted size {remaining_plaintext_size}")))?; + ( + usize::try_from(encrypted_offset) + .map_err(|_| Error::other(format!("invalid encrypted offset {encrypted_offset}")))?, + oi.size - encrypted_offset, + 0, + plaintext_skip, + requested_length, + total_plaintext_size, + sequence_number, + multipart_part_numbers(&oi.parts[part_start_index..]), + ) + } + } + #[cfg(not(feature = "rio-v2"))] + { + ( + 0, + oi.size, + 0, + requested_offset, + requested_length, + full_plaintext_size, + 0, + multipart_part_numbers(&oi.parts), + ) + } + } else { + ( + 0, + oi.size, + 0, + 0, + plaintext_size, + full_plaintext_size, + 0, + multipart_part_numbers(&oi.parts), + ) + }; + + return Ok(Self { + storage_offset, + storage_length, + object_size: plaintext_length, + transform: ReadTransform::Encrypted { + material, + is_multipart, + part_numbers, + sequence_number, + decrypt_skip, + plaintext_offset, + plaintext_length, + total_plaintext_size, + compression: is_compressed.then_some((algo, compression_backend)), + }, + }); + } + + let (visible_offset, visible_length) = if let Some(rs) = rs { + rs.get_offset_length(oi.size)? + } else { + (0, oi.size) + }; + + Ok(Self { + storage_offset: visible_offset, + storage_length: visible_length, + object_size: oi.size, + transform: ReadTransform::Plain { + visible_offset, + visible_length, + }, + }) + } + + fn into_reader( + self, + reader: Box, + oi: &ObjectInfo, + ) -> Result<(GetObjectReader, usize, i64)> { + match self.transform { + ReadTransform::Plain { .. } => Ok(( + GetObjectReader { + stream: reader, + object_info: oi.clone(), + }, + self.storage_offset, + self.storage_length, + )), + ReadTransform::Compressed { + algorithm, + backend, + decompressed_offset, + decompressed_length, + total_plaintext_size, + } => { + let dec_reader = crate::rio::decompression_reader(reader, algorithm, backend); + #[cfg(feature = "rio-v2")] + let dec_reader = StreamConsumer::new(dec_reader); + let final_reader: Box = if decompressed_offset > 0 + || decompressed_length != total_plaintext_size as i64 + { + #[cfg(feature = "rio-v2")] + let ranged_result = RangedDecompressReader::new_draining( + dec_reader, + decompressed_offset, + decompressed_length, + total_plaintext_size, + ); + #[cfg(not(feature = "rio-v2"))] + let ranged_result = + RangedDecompressReader::new(dec_reader, decompressed_offset, decompressed_length, total_plaintext_size); + + match ranged_result { + Ok(ranged_reader) => { + tracing::debug!( + "Successfully created RangedDecompressReader for offset={}, length={}", + decompressed_offset, + decompressed_length + ); + Box::new(ranged_reader) + } + Err(e) => { + tracing::error!("RangedDecompressReader failed with invalid range parameters: {}", e); + return Err(e); + } + } + } else { + Box::new(LimitReader::new(dec_reader, total_plaintext_size)) + }; + + let mut object_info = oi.clone(); + object_info.size = self.object_size; + + Ok(( + GetObjectReader { + stream: final_reader, + object_info, + }, + self.storage_offset, + self.storage_length, + )) + } + ReadTransform::Encrypted { + material, + is_multipart, + part_numbers, + sequence_number, + decrypt_skip, + plaintext_offset, + plaintext_length, + total_plaintext_size, + compression, + } => { + #[cfg(not(feature = "rio-v2"))] + let _ = sequence_number; + let decrypted_reader: Box = if is_multipart { + match material.key_kind { + EncryptionKeyKind::Object => crate::rio::decrypt_multipart_reader_with_object_key( + reader, + material.key_bytes, + part_numbers, + sequence_number, + ), + EncryptionKeyKind::Direct => crate::rio::decrypt_multipart_reader( + reader, + material.key_bytes, + material.base_nonce, + part_numbers, + material.reader_backend, + sequence_number, + ), + } + } else { + match material.key_kind { + EncryptionKeyKind::Object => { + crate::rio::decrypt_reader_with_object_key(reader, material.key_bytes, sequence_number) + } + EncryptionKeyKind::Direct => crate::rio::decrypt_reader( + reader, + material.key_bytes, + material.base_nonce, + material.reader_backend, + sequence_number, + ), + } + }; + let decrypted_reader: Box = if decrypt_skip > 0 { + Box::new(SkipReader::new(decrypted_reader, decrypt_skip)) + } else { + decrypted_reader + }; + let total_plaintext_size_i64 = i64::try_from(total_plaintext_size) + .map_err(|_| Error::other(format!("invalid plaintext size {total_plaintext_size}")))?; + + let final_reader: Box = + if let Some((algo, compression_backend)) = compression { + let decompressed_reader = crate::rio::decompression_reader(decrypted_reader, algo, compression_backend); + #[cfg(feature = "rio-v2")] + let decompressed_reader = StreamConsumer::new(decompressed_reader); + if plaintext_offset > 0 || plaintext_length != total_plaintext_size_i64 { + #[cfg(feature = "rio-v2")] + let ranged_reader = RangedDecompressReader::new_draining( + decompressed_reader, + plaintext_offset, + plaintext_length, + total_plaintext_size, + )?; + #[cfg(not(feature = "rio-v2"))] + let ranged_reader = RangedDecompressReader::new( + decompressed_reader, + plaintext_offset, + plaintext_length, + total_plaintext_size, + )?; + Box::new(ranged_reader) + } else { + Box::new(LimitReader::new(decompressed_reader, total_plaintext_size)) + } + } else if plaintext_offset > 0 || plaintext_length != total_plaintext_size_i64 { + Box::new(RangedDecompressReader::new( + decrypted_reader, + plaintext_offset, + plaintext_length, + total_plaintext_size, + )?) + } else { + Box::new(LimitReader::new(decrypted_reader, total_plaintext_size)) + }; + + let mut object_info = oi.clone(); + object_info.size = self.object_size; + + Ok(( + GetObjectReader { + stream: final_reader, + object_info, + }, + self.storage_offset, + self.storage_length, + )) + } + } + } } impl GetObjectReader { @@ -135,155 +646,7 @@ impl GetObjectReader { opts: &ObjectOptions, h: &HeaderMap, ) -> Result<(Self, usize, i64)> { - let mut rs = rs; - - if let Some(part_number) = opts.part_number - && rs.is_none() - { - rs = HTTPRangeSpec::from_object_info(oi, part_number); - } - - let mut is_encrypted = oi.is_encrypted(); - let (algo, mut is_compressed) = oi.is_compressed_ok()?; - - if restore_request_active(opts) { - is_encrypted = false; - is_compressed = false; - } - - if is_compressed && !is_encrypted { - let actual_size = oi.get_actual_size()?; - let (off, length, dec_off, dec_length) = if let Some(rs) = rs { - let (req_off, req_length) = rs.get_offset_length(actual_size)?; - let (physical_off, decompressed_skip, _, _, _) = get_compressed_offsets(oi, req_off as i64); - (physical_off as usize, oi.size - physical_off, decompressed_skip as usize, req_length) - } else { - (0, oi.size, 0, actual_size) - }; - - let dec_reader = DecompressReader::new(reader, algo); - - let actual_size_usize = if actual_size >= 0 { - actual_size as usize - } else { - return Err(Error::other(format!("invalid decompressed size {actual_size}"))); - }; - - let final_reader: Box = if dec_off > 0 || dec_length != actual_size { - // Use RangedDecompressReader for streaming range processing - // The new implementation supports any offset size by streaming and skipping data - match RangedDecompressReader::new(dec_reader, dec_off, dec_length, actual_size_usize) { - Ok(ranged_reader) => { - tracing::debug!( - "Successfully created RangedDecompressReader for offset={}, length={}", - dec_off, - dec_length - ); - Box::new(ranged_reader) - } - Err(e) => { - // Only fail if the range parameters are fundamentally invalid (e.g., offset >= file size) - tracing::error!("RangedDecompressReader failed with invalid range parameters: {}", e); - return Err(e); - } - } - } else { - Box::new(LimitReader::new(dec_reader, actual_size_usize)) - }; - - let mut oi = oi.clone(); - oi.size = dec_length; - - return Ok(( - GetObjectReader { - stream: final_reader, - object_info: oi, - }, - off, - length, - )); - } - - if is_encrypted { - let material = resolve_encryption_material(oi, h).await?; - let is_multipart = is_multipart_encrypted_object(&oi.parts, oi.etag.as_deref()); - let plaintext_size = encrypted_plaintext_size(oi, is_multipart, is_compressed)?; - let plaintext_size_usize = - usize::try_from(plaintext_size).map_err(|_| Error::other(format!("invalid decrypted size {plaintext_size}")))?; - let (plain_offset, plain_length) = if let Some(rs) = rs { - rs.get_offset_length(plaintext_size)? - } else { - (0, plaintext_size) - }; - - let decrypted_reader: Box = if is_multipart { - Box::new(DecryptReader::new_multipart( - reader, - material.key_bytes, - material.base_nonce, - multipart_part_numbers(&oi.parts), - )) - } else { - Box::new(DecryptReader::new(reader, material.key_bytes, material.base_nonce)) - }; - - let final_reader: Box = if is_compressed { - let decompressed_reader = DecompressReader::new(decrypted_reader, algo); - if plain_offset > 0 || plain_length != plaintext_size { - Box::new(RangedDecompressReader::new( - decompressed_reader, - plain_offset, - plain_length, - plaintext_size_usize, - )?) - } else { - Box::new(LimitReader::new(decompressed_reader, plaintext_size_usize)) - } - } else if plain_offset > 0 || plain_length != plaintext_size { - Box::new(RangedDecompressReader::new( - decrypted_reader, - plain_offset, - plain_length, - plaintext_size_usize, - )?) - } else { - Box::new(LimitReader::new(decrypted_reader, plaintext_size_usize)) - }; - - let mut object_info = oi.clone(); - object_info.size = plain_length; - - return Ok(( - GetObjectReader { - stream: final_reader, - object_info, - }, - 0, - oi.size, - )); - } - - if let Some(rs) = rs { - let (off, length) = rs.get_offset_length(oi.size)?; - - Ok(( - GetObjectReader { - stream: reader, - object_info: oi.clone(), - }, - off, - length, - )) - } else { - Ok(( - GetObjectReader { - stream: reader, - object_info: oi.clone(), - }, - 0, - oi.size, - )) - } + ReadPlan::build(rs, oi, opts, h).await?.into_reader(reader, oi) } pub async fn read_all(&mut self) -> Result> { let mut data = Vec::new(); @@ -307,6 +670,53 @@ impl AsyncRead for GetObjectReader { } } +#[derive(Debug)] +struct SkipReader { + inner: R, + bytes_to_skip: usize, + bytes_skipped: usize, + scratch: Vec, +} + +impl SkipReader { + fn new(inner: R, bytes_to_skip: usize) -> Self { + Self { + inner, + bytes_to_skip, + bytes_skipped: 0, + scratch: vec![0u8; 8192], + } + } +} + +impl AsyncRead for SkipReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let this = self.as_mut().get_mut(); + + while this.bytes_skipped < this.bytes_to_skip { + let remaining = this.bytes_to_skip - this.bytes_skipped; + let scratch_len = remaining.min(this.scratch.len()); + let mut scratch_buf = ReadBuf::new(&mut this.scratch[..scratch_len]); + match Pin::new(&mut this.inner).poll_read(cx, &mut scratch_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Ok(())) => { + let n = scratch_buf.filled().len(); + if n == 0 { + return Poll::Ready(Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("unexpected EOF while skipping {} bytes from decrypted stream", this.bytes_to_skip), + ))); + } + this.bytes_skipped += n; + } + } + } + + Pin::new(&mut this.inner).poll_read(cx, buf) + } +} + #[derive(Debug, Clone)] pub struct HTTPRangeSpec { pub is_suffix_length: bool, @@ -410,16 +820,27 @@ impl HTTPRangeSpec { /// This implementation acknowledges that compressed streams (like LZ4) must be decompressed sequentially /// from the beginning, so it streams and discards data until reaching the target offset. #[derive(Debug)] -pub struct RangedDecompressReader { - inner: R, +pub struct RangedDecompressReader { + inner: Option, target_offset: usize, target_length: usize, current_offset: usize, bytes_returned: usize, + scratch: Vec, + drain_on_done: bool, + drain_task: Option>, } -impl RangedDecompressReader { +impl RangedDecompressReader { pub fn new(inner: R, offset: usize, length: i64, total_size: usize) -> Result { + Self::new_with_drain(inner, offset, length, total_size, false) + } + + pub fn new_draining(inner: R, offset: usize, length: i64, total_size: usize) -> Result { + Self::new_with_drain(inner, offset, length, total_size, true) + } + + fn new_with_drain(inner: R, offset: usize, length: i64, total_size: usize, drain_on_done: bool) -> Result { // Validate the range request if offset >= total_size { tracing::debug!("Range offset {} exceeds total size {}", offset, total_size); @@ -438,16 +859,40 @@ impl RangedDecompressReader { ); Ok(Self { - inner, + inner: Some(inner), target_offset: offset, target_length: actual_length, current_offset: 0, bytes_returned: 0, + scratch: vec![0u8; 8192], + drain_on_done, + drain_task: None, }) } + + fn start_drain(&mut self) { + if !self.drain_on_done || self.drain_task.is_some() { + return; + } + + let Some(mut inner) = self.inner.take() else { + return; + }; + + self.drain_task = Some(tokio::spawn(async move { + let mut buf = [0u8; 8192]; + loop { + match inner.read(&mut buf).await { + Ok(0) => break, + Ok(_) => continue, + Err(_) => break, + } + } + })); + } } -impl AsyncRead for RangedDecompressReader { +impl AsyncRead for RangedDecompressReader { fn poll_read( mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>, @@ -457,9 +902,12 @@ impl AsyncRead for RangedDecompressReader use std::task::Poll; use tokio::io::ReadBuf; + let this = self.as_mut().get_mut(); + loop { // If we've returned all the bytes we need, return EOF - if self.bytes_returned >= self.target_length { + if this.bytes_returned >= this.target_length { + this.start_drain(); return Poll::Ready(Ok(())); } @@ -469,24 +917,27 @@ impl AsyncRead for RangedDecompressReader return Poll::Ready(Ok(())); } - // Prepare a temporary buffer for reading - let mut temp_buf = vec![0u8; std::cmp::min(buf_capacity, 8192)]; - let mut temp_read_buf = ReadBuf::new(&mut temp_buf); + let scratch_len = std::cmp::min(this.scratch.len(), std::cmp::max(buf_capacity, 1)); + let mut temp_read_buf = ReadBuf::new(&mut this.scratch[..scratch_len]); - match Pin::new(&mut self.inner).poll_read(cx, &mut temp_read_buf) { + let Some(inner) = this.inner.as_mut() else { + return Poll::Ready(Ok(())); + }; + + match Pin::new(inner).poll_read(cx, &mut temp_read_buf) { Poll::Pending => return Poll::Pending, Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), Poll::Ready(Ok(())) => { let n = temp_read_buf.filled().len(); if n == 0 { // EOF from inner stream - if self.current_offset < self.target_offset { + if this.current_offset < this.target_offset { // We haven't reached the target offset yet - this is an error return Poll::Ready(Err(std::io::Error::new( std::io::ErrorKind::UnexpectedEof, format!( "Unexpected EOF: only read {} bytes, target offset is {}", - self.current_offset, self.target_offset + this.current_offset, this.target_offset ), ))); } @@ -495,16 +946,16 @@ impl AsyncRead for RangedDecompressReader } // Update current position - let old_offset = self.current_offset; - self.current_offset += n; + let old_offset = this.current_offset; + this.current_offset += n; // Check if we're still in the skip phase - if old_offset < self.target_offset { + if old_offset < this.target_offset { // We're still skipping data - let skip_end = std::cmp::min(self.current_offset, self.target_offset); + let skip_end = std::cmp::min(this.current_offset, this.target_offset); let bytes_to_skip_in_this_read = skip_end - old_offset; - if self.current_offset <= self.target_offset { + if this.current_offset <= this.target_offset { // All data in this read should be skipped tracing::trace!("Skipping {} bytes at offset {}", n, old_offset); // Continue reading in the loop instead of recursive call @@ -515,14 +966,13 @@ impl AsyncRead for RangedDecompressReader let available_data = n - data_start_in_buffer; let bytes_to_return = std::cmp::min( available_data, - std::cmp::min(buf.remaining(), self.target_length - self.bytes_returned), + std::cmp::min(buf.remaining(), this.target_length - this.bytes_returned), ); if bytes_to_return > 0 { - let data_slice = - &temp_read_buf.filled()[data_start_in_buffer..data_start_in_buffer + bytes_to_return]; + let data_slice = &this.scratch[data_start_in_buffer..data_start_in_buffer + bytes_to_return]; buf.put_slice(data_slice); - self.bytes_returned += bytes_to_return; + this.bytes_returned += bytes_to_return; tracing::trace!( "Skipped {} bytes, returned {} bytes at offset {}", @@ -536,11 +986,11 @@ impl AsyncRead for RangedDecompressReader } else { // We're in the data return phase let bytes_to_return = - std::cmp::min(n, std::cmp::min(buf.remaining(), self.target_length - self.bytes_returned)); + std::cmp::min(n, std::cmp::min(buf.remaining(), this.target_length - this.bytes_returned)); if bytes_to_return > 0 { - buf.put_slice(&temp_read_buf.filled()[..bytes_to_return]); - self.bytes_returned += bytes_to_return; + buf.put_slice(&this.scratch[..bytes_to_return]); + this.bytes_returned += bytes_to_return; tracing::trace!("Returned {} bytes at offset {}", bytes_to_return, old_offset); } @@ -552,6 +1002,14 @@ impl AsyncRead for RangedDecompressReader } } +impl Drop for RangedDecompressReader { + fn drop(&mut self) { + if self.bytes_returned >= self.target_length { + self.start_drain(); + } + } +} + /// A wrapper that ensures the inner stream is fully consumed even if the outer reader stops early. /// This prevents broken pipe errors in erasure coding scenarios where the writer expects /// the full stream to be consumed. @@ -665,13 +1123,156 @@ async fn resolve_encryption_material(oi: &ObjectInfo, headers: &HeaderMap
) -> bool { + if metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some() { + return true; + } + + #[cfg(feature = "rio-v2")] + { + metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_some() + || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_some() + || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER).is_some() + } + + #[cfg(not(feature = "rio-v2"))] + { + false + } +} + +#[cfg(feature = "rio-v2")] +fn canonical_sse_path(bucket: &str, object: &str) -> String { + let bucket = bucket.trim_matches('/'); + let object = object.trim_matches('/'); + if object.is_empty() { + bucket.to_string() + } else if bucket.is_empty() { + object.to_string() + } else { + format!("{bucket}/{object}") + } +} + +#[cfg(feature = "rio-v2")] +fn managed_sse_domain(metadata: &HashMap) -> &'static str { + if metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_some() + || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER).is_some() + || matches!(metadata_get(metadata, "x-amz-server-side-encryption"), Some("aws:kms")) + { + "SSE-KMS" + } else if metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER).is_some() { + "SSE-C" + } else { + "SSE-S3" + } +} + +#[cfg(feature = "rio-v2")] +fn derive_sealing_key( + external_key: [u8; 32], + iv: [u8; SEALED_KEY_IV_SIZE], + domain: &str, + bucket: &str, + object: &str, +) -> [u8; 32] { + let mut mac = HmacSha256::new_from_slice(&external_key).expect("32-byte HMAC key"); + mac.update(&iv); + mac.update(domain.as_bytes()); + mac.update(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.as_bytes()); + mac.update(canonical_sse_path(bucket, object).as_bytes()); + + let mut sealing_key = [0u8; 32]; + sealing_key.copy_from_slice(mac.finalize().into_bytes().as_slice()); + sealing_key +} + +#[cfg(feature = "rio-v2")] +fn try_decode_minio_sealed_key(bytes: &str) -> Result> { + let decoded = BASE64_STANDARD + .decode(bytes) + .map_err(|e| Error::other(format!("failed to decode sealed object key: {e}")))?; + match decoded.as_slice().try_into() { + Ok(sealed_key) => Ok(Some(sealed_key)), + Err(_) => Ok(None), + } +} + +#[cfg(feature = "rio-v2")] +fn try_decode_minio_sealing_iv(bytes: &str) -> Result> { + let decoded = BASE64_STANDARD + .decode(bytes) + .map_err(|e| Error::other(format!("failed to decode sealing IV: {e}")))?; + match decoded.as_slice().try_into() { + Ok(iv) => Ok(Some(iv)), + Err(_) => Ok(None), + } +} + +#[cfg(feature = "rio-v2")] +fn try_unseal_minio_object_key( + metadata: &HashMap, + bucket: &str, + object: &str, + external_key: [u8; 32], +) -> Result> { + let Some(algorithm) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) else { + return Ok(None); + }; + if algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM { + return Ok(None); + } + + let Some(iv_b64) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER) else { + return Ok(None); + }; + let Some(iv) = try_decode_minio_sealing_iv(iv_b64)? else { + return Ok(None); + }; + + let sealed_key_b64 = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) + .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) + .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)); + let Some(sealed_key_b64) = sealed_key_b64 else { + return Ok(None); + }; + let Some(sealed_key) = try_decode_minio_sealed_key(sealed_key_b64)? else { + return Ok(None); + }; + let header = &sealed_key[..DARE_HEADER_SIZE]; + if header[0] != DARE_VERSION_20 || header[1] != DARE_CIPHER_AES_256_GCM { + return Err(Error::other("unsupported sealed object-key DARE header")); + } + if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 { + return Err(Error::other("invalid sealed object-key payload header")); + } + + let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object); + let cipher = Aes256Gcm::new_from_slice(&sealing_key).map_err(|err| Error::other(format!("invalid sealing key: {err}")))?; + let nonce = Nonce::try_from(&header[4..16]).map_err(|_| Error::other("invalid sealed object-key package nonce"))?; + let plaintext = cipher + .decrypt( + &nonce, + aes_gcm::aead::Payload { + msg: &sealed_key[DARE_HEADER_SIZE..], + aad: &header[..4], + }, + ) + .map_err(|err| Error::other(format!("failed to unseal object key: {err}")))?; + let object_key: [u8; 32] = plaintext + .as_slice() + .try_into() + .map_err(|_| Error::other("sealed object key must decrypt to 32 bytes"))?; + Ok(Some(object_key)) +} + fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap) -> Result { let algorithm = headers .get(SSEC_ALGORITHM_HEADER) @@ -711,21 +1312,66 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap) -> R return Err(Error::other("SSE-C key does not match object metadata")); } + #[cfg(feature = "rio-v2")] + if let Some(object_key) = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)? { + return Ok(EncryptionMaterial { + key_bytes: object_key, + base_nonce: [0u8; 12], + key_kind: EncryptionKeyKind::Object, + reader_backend: crate::rio::ReadEncryptionBackend::V2, + }); + } + Ok(EncryptionMaterial { key_bytes, base_nonce: generate_ssec_nonce(&oi.bucket, &oi.name), + key_kind: EncryptionKeyKind::Direct, + reader_backend: crate::rio::ReadEncryptionBackend::Legacy, }) } -async fn resolve_managed_material(metadata: &HashMap) -> Result { - let encrypted_dek = - metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).ok_or_else(|| Error::other("missing managed encrypted DEK"))?; +async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap) -> Result { + let normalized_metadata = normalize_managed_metadata(metadata); + let encrypted_dek = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_HEADER) + .ok_or_else(|| Error::other("missing managed encrypted DEK"))?; let encrypted_dek = BASE64_STANDARD .decode(encrypted_dek) .map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?; - let iv_b64 = - metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER).ok_or_else(|| Error::other("missing managed encryption IV"))?; + let kms_key_id = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).unwrap_or("default"); + #[cfg(feature = "rio-v2")] + let kms_context = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_CONTEXT_HEADER) + .map(|value| { + serde_json::from_str::>(value) + .map_err(|e| Error::other(format!("failed to parse managed KMS context: {e}"))) + }) + .transpose()?; + #[cfg(not(feature = "rio-v2"))] + let kms_context: Option> = None; + let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref()); + + let decrypted_key = if let Some(service) = get_global_encryption_service().await { + service + .decrypt_data_key(&encrypted_dek, &object_context) + .await + .map_err(|e| Error::other(format!("failed to decrypt managed data key: {e}")))? + .plaintext_key + } else { + decrypt_local_sse_dek(&encrypted_dek, kms_key_id, &object_context)? + }; + + #[cfg(feature = "rio-v2")] + if let Some(object_key) = try_unseal_minio_object_key(&normalized_metadata, bucket, object, decrypted_key)? { + return Ok(EncryptionMaterial { + key_bytes: object_key, + base_nonce: [0u8; 12], + key_kind: EncryptionKeyKind::Object, + reader_backend: crate::rio::ReadEncryptionBackend::V2, + }); + } + + let iv_b64 = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_IV_HEADER) + .ok_or_else(|| Error::other("missing managed encryption IV"))?; let iv = BASE64_STANDARD .decode(iv_b64) .map_err(|e| Error::other(format!("failed to decode managed encryption IV: {e}")))?; @@ -734,22 +1380,74 @@ async fn resolve_managed_material(metadata: &HashMap) -> Result< .try_into() .map_err(|_| Error::other("managed encryption IV must be 12 bytes"))?; - let kms_key_id = metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).unwrap_or("default"); - - let key_bytes = if let Some(service) = get_global_encryption_service().await { - service - .decrypt_data_key(&encrypted_dek, &ObjectEncryptionContext::new(String::new(), String::new())) - .await - .map_err(|e| Error::other(format!("failed to decrypt managed data key: {e}")))? - .plaintext_key - } else { - decrypt_local_sse_dek(&encrypted_dek, kms_key_id)? - }; - - Ok(EncryptionMaterial { key_bytes, base_nonce }) + Ok(EncryptionMaterial { + key_bytes: decrypted_key, + base_nonce, + key_kind: EncryptionKeyKind::Direct, + reader_backend: crate::rio::ReadEncryptionBackend::Legacy, + }) } -fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str) -> Result<[u8; 32]> { +fn normalize_managed_metadata(metadata: &HashMap) -> HashMap { + #[cfg(feature = "rio-v2")] + { + let mut normalized = metadata.clone(); + if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_HEADER).is_none() + && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) + .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)) + .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) + { + normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.to_string()); + } + + if metadata_get(&normalized, INTERNAL_ENCRYPTION_IV_HEADER).is_none() + && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.to_string()); + } + + if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_ID_HEADER).is_none() + && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.to_string()); + } + + if metadata_get(&normalized, INTERNAL_ENCRYPTION_CONTEXT_HEADER).is_none() + && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) + && let Ok(decoded) = BASE64_STANDARD.decode(value) + && let Ok(context) = serde_json::from_slice::>(&decoded) + && let Ok(encoded) = serde_json::to_string(&context) + { + normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded); + } + + normalized + } + + #[cfg(not(feature = "rio-v2"))] + { + metadata.clone() + } +} + +fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> { + if let Ok(plaintext) = decrypt_rustfs_local_sse_dek(encrypted_dek) { + return Ok(plaintext); + } + + #[cfg(feature = "rio-v2")] + { + decrypt_minio_secret_key_dek(encrypted_dek, object_context) + } + + #[cfg(not(feature = "rio-v2"))] + { + let _ = object_context; + Err(Error::other("invalid managed DEK format")) + } +} + +fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> { let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?; let parts: Vec<&str> = encrypted_dek.split(':').collect(); if parts.len() != 2 { @@ -780,6 +1478,103 @@ fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str) -> Result<[u8; .map_err(|_| Error::other("managed DEK has invalid plaintext length")) } +#[cfg(feature = "rio-v2")] +#[derive(Deserialize)] +struct MinioLegacyCiphertext { + #[serde(rename = "aead")] + algorithm: String, + iv: Vec, + nonce: Vec, + bytes: Vec, +} + +#[cfg(feature = "rio-v2")] +fn decrypt_minio_secret_key_dek(encrypted_dek: &[u8], object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> { + let key = local_sse_master_key()?; + let (ciphertext, iv, nonce) = parse_minio_secret_key_ciphertext(encrypted_dek)?; + let associated_data = marshal_minio_kms_context(&object_context.encryption_context); + + let mut mac = HmacSha256::new_from_slice(&key).map_err(|err| Error::other(format!("invalid local SSE master key: {err}")))?; + mac.update(&iv); + let sealing_key = mac.finalize().into_bytes(); + let cipher = Aes256Gcm::new_from_slice(sealing_key.as_slice()) + .map_err(|err| Error::other(format!("invalid MinIO sealing key: {err}")))?; + let nonce = Nonce::try_from(&nonce[..]).map_err(|_| Error::other("invalid MinIO managed DEK nonce"))?; + let plaintext = cipher + .decrypt( + &nonce, + aes_gcm::aead::Payload { + msg: &ciphertext, + aad: &associated_data, + }, + ) + .map_err(|err| Error::other(format!("failed to decrypt MinIO managed DEK: {err}")))?; + + plaintext + .as_slice() + .try_into() + .map_err(|_| Error::other("MinIO managed DEK has invalid plaintext length")) +} + +#[cfg(feature = "rio-v2")] +fn parse_minio_secret_key_ciphertext( + encrypted_dek: &[u8], +) -> Result<(Vec, [u8; MINIO_SECRET_KEY_IV_SIZE], [u8; MINIO_SECRET_KEY_NONCE_SIZE])> { + if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') { + let legacy: MinioLegacyCiphertext = serde_json::from_slice(encrypted_dek) + .map_err(|err| Error::other(format!("failed to parse MinIO legacy managed DEK: {err}")))?; + if legacy.algorithm != "AES-256-GCM-HMAC-SHA-256" { + return Err(Error::other(format!( + "unsupported MinIO legacy managed DEK algorithm {}", + legacy.algorithm + ))); + } + let iv = legacy + .iv + .as_slice() + .try_into() + .map_err(|_| Error::other("invalid MinIO legacy managed DEK IV length"))?; + let nonce = legacy + .nonce + .as_slice() + .try_into() + .map_err(|_| Error::other("invalid MinIO legacy managed DEK nonce length"))?; + return Ok((legacy.bytes, iv, nonce)); + } + + if encrypted_dek.len() <= MINIO_SECRET_KEY_RANDOM_SIZE { + return Err(Error::other("invalid MinIO managed DEK length")); + } + + let split_at = encrypted_dek.len() - MINIO_SECRET_KEY_RANDOM_SIZE; + let (ciphertext, random) = encrypted_dek.split_at(split_at); + let iv = random[..MINIO_SECRET_KEY_IV_SIZE] + .try_into() + .map_err(|_| Error::other("invalid MinIO managed DEK IV length"))?; + let nonce = random[MINIO_SECRET_KEY_IV_SIZE..] + .try_into() + .map_err(|_| Error::other("invalid MinIO managed DEK nonce length"))?; + Ok((ciphertext.to_vec(), iv, nonce)) +} + +#[cfg(feature = "rio-v2")] +fn marshal_minio_kms_context(context: &HashMap) -> Vec { + let mut entries: Vec<_> = context.iter().collect(); + entries.sort_by_key(|(left, _)| *left); + + let mut json = String::from("{"); + for (index, (key, value)) in entries.into_iter().enumerate() { + if index > 0 { + json.push(','); + } + json.push_str(&serde_json::to_string(key).expect("string key serializes")); + json.push(':'); + json.push_str(&serde_json::to_string(value).expect("string value serializes")); + } + json.push('}'); + json.into_bytes() +} + fn local_sse_master_key() -> Result<[u8; 32]> { if let Some(key) = decode_master_key_env("__RUSTFS_SSE_SIMPLE_CMK")? { return Ok(key); @@ -856,6 +1651,40 @@ mod tests { headers } + #[cfg(feature = "rio-v2")] + fn seal_ssec_object_key_for_test( + bucket: &str, + object: &str, + customer_key: [u8; 32], + object_key: [u8; 32], + ) -> ([u8; 32], Vec) { + let iv = [0x23u8; SEALED_KEY_IV_SIZE]; + let sealing_key = derive_sealing_key(customer_key, iv, "SSE-C", bucket, object); + let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key"); + + let mut header = [0u8; DARE_HEADER_SIZE]; + header[0] = DARE_VERSION_20; + header[1] = DARE_CIPHER_AES_256_GCM; + header[2..4].copy_from_slice(&31u16.to_le_bytes()); + header[4] = 0x80; + header[5..16].copy_from_slice(&[0x45u8; 11]); + + let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce"); + let mut sealed = header.to_vec(); + sealed.extend_from_slice( + &cipher + .encrypt( + &nonce, + aes_gcm::aead::Payload { + msg: &object_key, + aad: &header[..4], + }, + ) + .expect("seal object key"), + ); + (iv, sealed) + } + #[tokio::test] async fn test_ranged_decompress_reader() { // Create test data @@ -1115,6 +1944,29 @@ mod tests { assert_eq!(&buf2[..1], b"e"); } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_ranged_decompress_reader_with_rio_v2_s2_stream() { + let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + let mut compressed = Vec::new(); + crate::rio::CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut compressed) + .await + .expect("compress plaintext into rio_v2 stream"); + + let decompress_reader = crate::rio::DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut ranged_reader = + RangedDecompressReader::new(decompress_reader, 5, 7, plaintext.len()).expect("create ranged reader"); + + let mut actual = Vec::new(); + ranged_reader + .read_to_end(&mut actual) + .await + .expect("read ranged decompressed plaintext"); + + assert_eq!(actual, b"fghijkl"); + } + fn encrypt_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String { let key = Key::::from(master_key); let cipher = Aes256Gcm::new(&key); @@ -1123,6 +1975,40 @@ mod tests { format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext)) } + #[cfg(feature = "rio-v2")] + fn seal_managed_s3_object_key_for_test( + bucket: &str, + object: &str, + data_key: [u8; 32], + object_key: [u8; 32], + ) -> ([u8; 32], Vec) { + let iv = [0x24u8; SEALED_KEY_IV_SIZE]; + let sealing_key = derive_sealing_key(data_key, iv, "SSE-S3", bucket, object); + let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key"); + + let mut header = [0u8; DARE_HEADER_SIZE]; + header[0] = DARE_VERSION_20; + header[1] = DARE_CIPHER_AES_256_GCM; + header[2..4].copy_from_slice(&31u16.to_le_bytes()); + header[4] = 0x80; + header[5..16].copy_from_slice(&[0x46u8; 11]); + + let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce"); + let mut sealed = header.to_vec(); + sealed.extend_from_slice( + &cipher + .encrypt( + &nonce, + aes_gcm::aead::Payload { + msg: &object_key, + aad: &header[..4], + }, + ) + .expect("seal managed object key"), + ); + (iv, sealed) + } + #[tokio::test] async fn resolve_managed_material_accepts_case_insensitive_metadata_keys() { async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { @@ -1134,7 +2020,7 @@ mod tests { ("X-Rustfs-Encryption-IV".to_string(), BASE64_STANDARD.encode(base_nonce)), ]); - let material = resolve_managed_material(&metadata) + let material = resolve_managed_material("", "", &metadata) .await .expect("managed material should resolve mixed-case metadata keys"); @@ -1231,28 +2117,95 @@ mod tests { assert_eq!(length, 4); } + #[tokio::test] + async fn test_read_plan_restore_request_uses_plain_range() { + let object_info = ObjectInfo { + size: 10, + user_defined: Arc::new(HashMap::from([ + ("x-rustfs-encryption-key".to_string(), "encrypted-key".to_string()), + ("x-rustfs-encryption-original-size".to_string(), "20".to_string()), + ])), + ..Default::default() + }; + + let range = HTTPRangeSpec { + is_suffix_length: true, + start: 4, + end: -1, + }; + + let mut opts = ObjectOptions::default(); + opts.transition.restore_request.days = Some(1); + + let plan = ReadPlan::build(Some(range), &object_info, &opts, &HeaderMap::new()) + .await + .expect("restore requests should bypass rio transforms"); + + assert_eq!(plan.storage_offset, 6); + assert_eq!(plan.storage_length, 4); + assert_eq!(plan.object_size, 10); + assert!(matches!( + plan.transform, + ReadTransform::Plain { + visible_offset: 6, + visible_length: 4 + } + )); + } + #[tokio::test] async fn test_get_object_reader_allows_encrypted_full_object_passthrough() { async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { let plaintext = b"managed-full-object".to_vec(); let data_key = [0x21; 32]; - let base_nonce = [0x11; 12]; let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); + let bucket = "bucket"; + let object = "managed-full-object"; let mut encrypted = Vec::new(); - rustfs_rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - - let object_info = ObjectInfo { - size: encrypted.len() as i64, - user_defined: Arc::new(HashMap::from([ + #[cfg(feature = "rio-v2")] + let user_defined = { + let object_key = [0x41; 32]; + let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed object"); + HashMap::from([ + ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), + ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), + ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + ]) + }; + #[cfg(not(feature = "rio-v2"))] + let user_defined = { + let base_nonce = [0x11; 12]; + crate::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed object"); + HashMap::from([ ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ])), + ]) + }; + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + user_defined: Arc::new(user_defined), ..Default::default() }; @@ -1282,23 +2235,54 @@ mod tests { async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { let plaintext = b"0123456789abcdefghijklmnopqrstuvwxyz".to_vec(); let data_key = [0x23; 32]; - let base_nonce = [0x13; 12]; let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); + let bucket = "bucket"; + let object = "managed-range-object"; let mut encrypted = Vec::new(); - rustfs_rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed ranged object"); - - let object_info = ObjectInfo { - size: encrypted.len() as i64, - user_defined: Arc::new(HashMap::from([ + #[cfg(feature = "rio-v2")] + let user_defined = { + let object_key = [0x43; 32]; + let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed ranged object"); + HashMap::from([ + ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), + ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), + ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + ]) + }; + #[cfg(not(feature = "rio-v2"))] + let user_defined = { + let base_nonce = [0x13; 12]; + crate::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed ranged object"); + HashMap::from([ ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ])), + ]) + }; + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + user_defined: Arc::new(user_defined), ..Default::default() }; let range = HTTPRangeSpec { @@ -1338,23 +2322,54 @@ mod tests { async { let plaintext = b"managed-local-fallback".to_vec(); let data_key = [0x22; 32]; - let base_nonce = [0x12; 12]; let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); + let bucket = "bucket"; + let object = "managed-local-fallback"; let mut encrypted = Vec::new(); - rustfs_rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object with local fallback key"); - - let object_info = ObjectInfo { - size: encrypted.len() as i64, - user_defined: Arc::new(HashMap::from([ + #[cfg(feature = "rio-v2")] + let user_defined = { + let object_key = [0x42; 32]; + let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed object with local fallback key"); + HashMap::from([ + ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), + ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), + ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + ]) + }; + #[cfg(not(feature = "rio-v2"))] + let user_defined = { + let base_nonce = [0x12; 12]; + crate::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed object with local fallback key"); + HashMap::from([ ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ])), + ]) + }; + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + user_defined: Arc::new(user_defined), ..Default::default() }; @@ -1378,9 +2393,127 @@ mod tests { .await; } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_get_object_reader_accepts_minio_only_managed_metadata() { + async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { + let plaintext = b"managed-minio-metadata".to_vec(); + let data_key = [0x23; 32]; + let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); + let bucket = "bucket"; + let object = "managed-minio-metadata"; + let object_key = [0x44; 32]; + let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); + + let mut encrypted = Vec::new(); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt managed object"); + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + user_defined: Arc::new(HashMap::from([ + ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), + ( + MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(encrypted_dek.as_bytes()), + ), + ( + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), + ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()), + ])), + ..Default::default() + }; + + let (mut reader, offset, length) = GetObjectReader::new( + Box::new(Cursor::new(encrypted.clone())), + None, + &object_info, + &ObjectOptions::default(), + &HeaderMap::new(), + ) + .await + .expect("managed encrypted reads should accept MinIO-style metadata"); + + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.expect("read managed plaintext"); + + assert_eq!(offset, 0); + assert_eq!(length, object_info.size); + assert_eq!(reader.object_info.size, plaintext.len() as i64); + assert_eq!(actual, plaintext); + }) + .await; + } + #[tokio::test] async fn test_get_object_reader_compressed_range_returns_physical_offset_from_index() { - let mut index = rustfs_rio::Index::new(); + let mut index = crate::rio::Index::new(); + index.add(0, 0).unwrap(); + index.add(1_048_576, 2_097_152).unwrap(); + + let object_info = ObjectInfo { + size: 3_000_000, + parts: Arc::new(vec![ObjectPartInfo { + etag: String::new(), + number: 1, + size: 3_000_000, + actual_size: 4_194_304, + index: Some(index.into_vec()), + ..Default::default() + }]), + user_defined: Arc::new(HashMap::from([ + ("x-minio-internal-compression".to_string(), "gzip".to_string()), + ("x-minio-internal-actual-size".to_string(), "4194304".to_string()), + ])), + ..Default::default() + }; + + let decoded = crate::rio::decode_compression_index_bytes(object_info.parts[0].index.as_ref().unwrap()) + .expect("headerless MinIO-style compression index should decode"); + let (compressed_offset, uncompressed_offset) = decoded.find(2_097_152).expect("seek into decoded index"); + assert!(compressed_offset > 0); + assert_eq!(uncompressed_offset, 2_097_152); + let (physical_offset, decompressed_skip, _, _, _) = get_compressed_offsets(&object_info, 2_097_152); + assert!(physical_offset > 0); + assert_eq!(decompressed_skip, 0); + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 2_097_152, + end: 2_097_161, + }; + + let (reader, offset, length) = GetObjectReader::new( + Box::new(Cursor::new(Vec::::new())), + Some(range), + &object_info, + &ObjectOptions::default(), + &HeaderMap::new(), + ) + .await + .unwrap(); + + assert!(offset > 0); + assert!(offset < 2_097_152); + assert_eq!(length, object_info.size - offset as i64); + assert_eq!(reader.object_info.size, 10); + } + + #[tokio::test] + async fn test_read_plan_compressed_range_tracks_storage_and_visible_offsets() { + let mut index = crate::rio::Index::new(); index.add(0, 0).unwrap(); index.add(1_048_576, 2_097_152).unwrap(); @@ -1407,20 +2540,149 @@ mod tests { end: 2_097_161, }; - let (reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(Vec::::new())), - Some(range), - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .unwrap(); + let plan = ReadPlan::build(Some(range), &object_info, &ObjectOptions::default(), &HeaderMap::new()) + .await + .expect("compressed range should plan physical offset and visible range"); - assert!(offset > 0); - assert!(offset < 2_097_152); - assert_eq!(length, object_info.size - offset as i64); - assert_eq!(reader.object_info.size, 10); + assert!(plan.storage_offset > 0); + assert!(plan.storage_offset < 2_097_152); + assert_eq!(plan.storage_length, object_info.size - plan.storage_offset as i64); + assert_eq!(plan.object_size, 10); + + assert!(matches!( + plan.transform, + ReadTransform::Compressed { + decompressed_offset, + decompressed_length: 10, + .. + } if decompressed_offset < 2_097_152 + )); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_read_plan_accepts_minio_s2_compression_scheme() { + let object_info = ObjectInfo { + size: 3_000_000, + user_defined: Arc::new(HashMap::from([ + ("x-minio-internal-compression".to_string(), "klauspost/compress/s2".to_string()), + ("x-minio-internal-actual-size".to_string(), "4194304".to_string()), + ])), + ..Default::default() + }; + + let plan = ReadPlan::build(None, &object_info, &ObjectOptions::default(), &HeaderMap::new()) + .await + .expect("MinIO S2 compression scheme should be accepted"); + + assert!(matches!( + plan.transform, + ReadTransform::Compressed { + decompressed_offset: 0, + decompressed_length: 4_194_304, + .. + } + )); + assert_eq!(plan.object_size, 4_194_304); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_read_plan_accepts_minio_headerless_compression_index() { + let mut index = crate::rio::Index::new(); + index.add(0, 0).unwrap(); + index.add(1_048_576, 2_097_152).unwrap(); + let headerless_index = crate::rio::compression_index_storage_bytes(&index); + assert!( + !headerless_index.starts_with(&[0x50, 0x2A, 0x4D, 0x18]), + "rio_v2 should store MinIO-style headerless compression indexes" + ); + + let object_info = ObjectInfo { + size: 3_000_000, + parts: Arc::new(vec![ObjectPartInfo { + etag: String::new(), + number: 1, + size: 3_000_000, + actual_size: 4_194_304, + index: Some(headerless_index), + ..Default::default() + }]), + user_defined: Arc::new(HashMap::from([ + ("x-minio-internal-compression".to_string(), "klauspost/compress/s2".to_string()), + ("x-minio-internal-actual-size".to_string(), "4194304".to_string()), + ])), + ..Default::default() + }; + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 2_097_152, + end: 2_097_161, + }; + + let decoded = crate::rio::decode_compression_index_bytes(object_info.parts[0].index.as_ref().unwrap()) + .expect("headerless MinIO-style compression index should decode"); + let (compressed_offset, uncompressed_offset) = decoded.find(range.start).expect("seek into decoded index"); + assert!(compressed_offset > 0); + assert_eq!(uncompressed_offset, range.start); + + let (physical_offset, decompressed_skip, _, _, _) = get_compressed_offsets(&object_info, range.start); + assert!(physical_offset > 0); + assert_eq!(decompressed_skip, 0); + + let plan = ReadPlan::build(Some(range), &object_info, &ObjectOptions::default(), &HeaderMap::new()) + .await + .expect("MinIO headerless compression index should be decoded"); + + assert_eq!(plan.storage_offset as i64, physical_offset); + assert_eq!(plan.storage_length, object_info.size - plan.storage_offset as i64); + assert_eq!(plan.object_size, 10); + } + + #[cfg(feature = "rio-v2")] + #[test] + fn test_get_compressed_offsets_aligns_encrypted_ranges_to_dare_packages() { + let mut index = crate::rio::Index::new(); + index.add(0, 0).unwrap(); + index.add(200_000, 2_097_152).unwrap(); + let stored_index = crate::rio::compression_index_storage_bytes(&index); + let expected_comp_off = crate::rio::decode_compression_index_bytes(&stored_index) + .expect("decode stored index") + .find(2_097_152) + .expect("find offset in stored index") + .0; + + let object_info = ObjectInfo { + size: 400_000, + parts: Arc::new(vec![ObjectPartInfo { + etag: String::new(), + number: 1, + size: 400_000, + actual_size: 4_194_304, + index: Some(stored_index), + ..Default::default() + }]), + user_defined: Arc::new(HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ("x-amz-server-side-encryption-customer-original-size".to_string(), "4194304".to_string()), + ( + "x-minio-internal-compression".to_string(), + crate::rio::compression_metadata_value(CompressionAlgorithm::default()), + ), + ("x-minio-internal-actual-size".to_string(), "4194304".to_string()), + ])), + ..Default::default() + }; + + let (physical_offset, decompressed_skip, first_part_idx, decrypt_skip, seq_num) = + get_compressed_offsets(&object_info, 2_097_152); + + assert_eq!(first_part_idx, 0); + assert_eq!(physical_offset, (expected_comp_off / DARE_PAYLOAD_SIZE) * DARE_PACKAGE_SIZE); + assert_eq!(decompressed_skip, 0); + assert_eq!(decrypt_skip, expected_comp_off % DARE_PAYLOAD_SIZE); + assert_eq!(seq_num, (expected_comp_off / DARE_PAYLOAD_SIZE) as u64); } #[tokio::test] @@ -1476,6 +2738,71 @@ mod tests { assert_eq!(actual, plaintext); } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_get_object_reader_decrypts_ssec_sealed_object_key_full_object() { + let plaintext = b"ecstore-rio-v2-ssec-sealed-object-key".repeat(4096); + let customer_key = [0x31; 32]; + let object_key = [0x67; 32]; + let bucket = "bucket"; + let object = "sealed-object"; + let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); + + let mut encrypted = Vec::new(); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt object with rio-v2 object key"); + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + user_defined: Arc::new(HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ( + "x-amz-server-side-encryption-customer-key-md5".to_string(), + BASE64_STANDARD.encode(md5_bytes(customer_key)), + ), + ( + "x-amz-server-side-encryption-customer-original-size".to_string(), + plaintext.len().to_string(), + ), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + ])), + ..Default::default() + }; + + let (mut reader, offset, length) = GetObjectReader::new( + Box::new(Cursor::new(encrypted.clone())), + None, + &object_info, + &ObjectOptions::default(), + &ssec_headers_from_key(customer_key), + ) + .await + .expect("rio-v2 ssec sealed-object-key read should be supported"); + + let mut actual = Vec::new(); + reader + .read_to_end(&mut actual) + .await + .expect("read decrypted rio-v2 ssec object"); + + assert_eq!(offset, 0); + assert_eq!(length, encrypted.len() as i64); + assert_eq!(reader.object_info.size, plaintext.len() as i64); + assert_eq!(actual, plaintext); + } + #[tokio::test] async fn test_get_object_reader_decrypts_ssec_range_on_plaintext_semantics() { let plaintext = b"0123456789abcdefghijklmnopqrstuvwxyz".to_vec(); @@ -1534,6 +2861,77 @@ mod tests { assert_eq!(actual, b"56789ab"); } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_get_object_reader_uses_dare_package_offset_for_large_ssec_ranges() { + const DARE_PACKAGE_SIZE: usize = 64 * 1024 + 32; + + let plaintext = vec![0x7Bu8; 2 * 64 * 1024 + 97]; + let customer_key = [0x61; 32]; + let object_key = [0x68; 32]; + let bucket = "bucket"; + let object = "large-range-object"; + let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); + + let mut encrypted = Vec::new(); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt large ranged object"); + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + user_defined: Arc::new(HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ( + "x-amz-server-side-encryption-customer-key-md5".to_string(), + BASE64_STANDARD.encode(md5_bytes(customer_key)), + ), + ( + "x-amz-server-side-encryption-customer-original-size".to_string(), + plaintext.len().to_string(), + ), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + ])), + ..Default::default() + }; + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 70_000, + end: 70_063, + }; + + let (mut reader, offset, _length) = GetObjectReader::new( + Box::new(Cursor::new(encrypted[DARE_PACKAGE_SIZE..].to_vec())), + Some(range), + &object_info, + &ObjectOptions::default(), + &ssec_headers_from_key(customer_key), + ) + .await + .expect("large ssec range read should be supported"); + + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.expect("read ranged decrypted object"); + + assert_eq!( + offset, DARE_PACKAGE_SIZE, + "rio_v2 encrypted ranges should start from the second DARE package" + ); + assert_eq!(reader.object_info.size, 64); + assert_eq!(actual, plaintext[70_000..70_064]); + } + #[tokio::test] async fn test_get_object_reader_decrypts_then_decompresses_before_applying_range() { let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); @@ -1551,7 +2949,7 @@ mod tests { .expect("compress plaintext"); let mut encrypted = Vec::new(); - rustfs_rio::EncryptReader::new(Cursor::new(compressed), key_bytes, base_nonce) + rustfs_rio::EncryptReader::new(Cursor::new(compressed.clone()), key_bytes, base_nonce) .read_to_end(&mut encrypted) .await .expect("encrypt compressed plaintext"); @@ -1602,4 +3000,183 @@ mod tests { assert_eq!(reader.object_info.size, 7); assert_eq!(actual, b"fghijkl"); } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_get_object_reader_uses_compression_index_for_encrypted_ranges() { + use crate::rio::TryGetIndex; + + let plaintext: Vec = (0..(10 * 1024 * 1024 + 123_456)) + .map(|i| (((i as u64).wrapping_mul(1_103_515_245).wrapping_add(12_345) >> 16) & 0xFF) as u8) + .collect(); + let customer_key = [0x73; 32]; + let object_key = [0x74; 32]; + let bucket = "bucket"; + let object = "compressed-large-object"; + let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); + let mut compressor = + crate::rio::CompressReader::with_encrypted_padding(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + compressor + .read_to_end(&mut compressed) + .await + .expect("compress large plaintext"); + + let index = compressor + .try_get_index() + .cloned() + .expect("large rio_v2 encrypted+compressed object should expose a compression index"); + let stored_index = crate::rio::compression_index_storage_bytes(&index); + let decoded_index = + crate::rio::decode_compression_index_bytes(&stored_index).expect("decode stored encrypted compression index"); + + let range = HTTPRangeSpec { + is_suffix_length: false, + start: 5 * 1024 * 1024, + end: 5 * 1024 * 1024 + 63, + }; + let original_offsets = index + .find(range.start) + .expect("find large-range compression block in original index"); + let (comp_off, uncomp_off) = decoded_index + .find(range.start) + .expect("find large-range compression block in stored index"); + assert_eq!(original_offsets, (comp_off, uncomp_off)); + assert!(comp_off > DARE_PAYLOAD_SIZE); + assert!(uncomp_off <= range.start); + + let expected_storage_offset = ((comp_off / DARE_PAYLOAD_SIZE) * DARE_PACKAGE_SIZE) as usize; + let expected_decrypt_skip = comp_off % DARE_PAYLOAD_SIZE; + assert!(expected_storage_offset > 0); + assert!(expected_decrypt_skip >= 0); + + let mut encrypted = Vec::new(); + crate::rio::EncryptReader::new_with_object_key(Cursor::new(compressed.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt compressed plaintext"); + + let expected_sequence_number = u32::try_from(comp_off / DARE_PAYLOAD_SIZE).expect("sequence number fits in u32"); + let chunk_offset = comp_off as usize; + let chunk_type = compressed[chunk_offset]; + let chunk_len = (compressed[chunk_offset + 1] as usize) + | ((compressed[chunk_offset + 2] as usize) << 8) + | ((compressed[chunk_offset + 3] as usize) << 16); + assert!(matches!(chunk_type, 0x00 | 0x01 | 0xff | 0xfe)); + assert!(chunk_offset + 4 + chunk_len <= compressed.len()); + + let mut decrypted_tail = Vec::new(); + crate::rio::DecryptReader::new_with_object_key_and_sequence( + Cursor::new(encrypted[expected_storage_offset..].to_vec()), + object_key, + expected_sequence_number, + ) + .read_to_end(&mut decrypted_tail) + .await + .expect("decrypt package-aligned ciphertext tail"); + assert_eq!( + &decrypted_tail[expected_decrypt_skip as usize..], + &compressed[comp_off as usize..], + "package-aligned decryption plus decrypt_skip must land on the indexed S2 chunk boundary" + ); + + let mut direct_reader = crate::rio::DecompressReader::new( + Cursor::new(decrypted_tail[expected_decrypt_skip as usize..].to_vec()), + CompressionAlgorithm::default(), + ); + let mut direct_plaintext = Vec::new(); + direct_reader + .read_to_end(&mut direct_plaintext) + .await + .expect("decompress tail starting at indexed S2 chunk boundary"); + assert_eq!(direct_plaintext, plaintext[uncomp_off as usize..]); + + let object_info = ObjectInfo { + bucket: bucket.to_string(), + name: object.to_string(), + size: encrypted.len() as i64, + parts: Arc::new(vec![ObjectPartInfo { + etag: String::new(), + number: 1, + size: encrypted.len(), + actual_size: plaintext.len() as i64, + index: Some(stored_index), + ..Default::default() + }]), + user_defined: Arc::new(HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ( + "x-amz-server-side-encryption-customer-key-md5".to_string(), + BASE64_STANDARD.encode(md5_bytes(customer_key)), + ), + ( + "x-amz-server-side-encryption-customer-original-size".to_string(), + plaintext.len().to_string(), + ), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), + ( + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed_key), + ), + ( + "x-minio-internal-compression".to_string(), + crate::rio::compression_metadata_value(CompressionAlgorithm::default()), + ), + ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()), + ])), + ..Default::default() + }; + + let plan = ReadPlan::build( + Some(range.clone()), + &object_info, + &ObjectOptions::default(), + &ssec_headers_from_key(customer_key), + ) + .await + .expect("build encrypted+compressed read plan"); + assert_eq!(plan.storage_offset, expected_storage_offset); + assert_eq!(plan.storage_length, encrypted.len() as i64 - expected_storage_offset as i64); + assert_eq!(plan.object_size, 64); + match plan.transform { + ReadTransform::Encrypted { + sequence_number, + decrypt_skip, + plaintext_offset, + plaintext_length, + .. + } => { + assert_eq!(sequence_number, expected_sequence_number); + assert_eq!(decrypt_skip, expected_decrypt_skip as usize); + assert_eq!(plaintext_offset as i64, range.start - uncomp_off); + assert_eq!(plaintext_length, 64); + } + other => panic!("expected encrypted read plan, got {other:?}"), + } + + let (mut reader, offset, length) = GetObjectReader::new( + Box::new(Cursor::new(encrypted[expected_storage_offset..].to_vec())), + Some(range.clone()), + &object_info, + &ObjectOptions::default(), + &ssec_headers_from_key(customer_key), + ) + .await + .expect("encrypted+compressed indexed range read should be supported"); + + let mut actual = Vec::new(); + reader + .read_to_end(&mut actual) + .await + .expect("read indexed encrypted+compressed range"); + + assert_eq!(offset, expected_storage_offset); + assert_eq!(length, encrypted.len() as i64 - expected_storage_offset as i64); + assert_eq!(reader.object_info.size, 64); + assert_eq!(actual, plaintext[range.start as usize..range.start as usize + 64]); + } } diff --git a/crates/ecstore/src/store_api/types.rs b/crates/ecstore/src/store_api/types.rs index 5b69f1be9..697fa69f4 100644 --- a/crates/ecstore/src/store_api/types.rs +++ b/crates/ecstore/src/store_api/types.rs @@ -369,13 +369,18 @@ impl ObjectInfo { } pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> { + let (algorithm, _, compressed) = self.compression_read_plan()?; + Ok((algorithm, compressed)) + } + + pub fn compression_read_plan(&self) -> Result<(CompressionAlgorithm, crate::rio::ReadCompressionBackend, bool)> { let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); if let Some(scheme) = scheme { - let algorithm = CompressionAlgorithm::from_str(&scheme)?; - Ok((algorithm, true)) + let (algorithm, backend) = crate::rio::compression_scheme_to_read_plan(&scheme)?; + Ok((algorithm, backend, true)) } else { - Ok((CompressionAlgorithm::None, false)) + Ok((CompressionAlgorithm::None, crate::rio::ReadCompressionBackend::Legacy, false)) } } @@ -388,13 +393,18 @@ impl ObjectInfo { use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; self.user_defined.keys().any(|key| { - let key = key.to_lowercase(); - key.starts_with("x-minio-encryption-") + let lower = key.to_ascii_lowercase(); + lower.starts_with("x-minio-encryption-") + || lower.starts_with("x-minio-internal-server-side-encryption-") || matches!( - key.as_str(), - "x-rustfs-encryption-key" + lower.as_str(), + "x-minio-internal-encrypted-multipart" + | "x-rustfs-encryption-key" | "x-rustfs-encryption-algorithm" | "x-rustfs-encryption-iv" + | "x-rustfs-encryption-key-id" + | "x-rustfs-encryption-context" + | "x-rustfs-encryption-tag" | "x-amz-server-side-encryption-aws-kms-key-id" | SSEC_ALGORITHM_HEADER | SSEC_KEY_HEADER @@ -405,10 +415,17 @@ impl ObjectInfo { } pub fn encryption_original_size(&self) -> std::io::Result> { + let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE); if let Some(size_str) = self .user_defined .get("x-rustfs-encryption-original-size") - .or_else(|| self.user_defined.get("x-amz-server-side-encryption-customer-original-size")) + .map(String::as_str) + .or_else(|| { + self.user_defined + .get("x-amz-server-side-encryption-customer-original-size") + .map(String::as_str) + }) + .or(actual_size.as_deref()) && !size_str.is_empty() { let size = size_str diff --git a/crates/ecstore/tests/README.md b/crates/ecstore/tests/README.md new file mode 100644 index 000000000..51b776e2a --- /dev/null +++ b/crates/ecstore/tests/README.md @@ -0,0 +1,24 @@ +# ecstore Integration Tests + +## MinIO-generated encrypted fixtures + +`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by +`.\rustfs\scripts\minio_fixture_lab\lab.py`. + +It currently covers multipart fixtures for: + +- `sse-s3-multipart-8m` +- `sse-kms-multipart-8m` + +Required environment variables: + +- `RUSTFS_MINIO_FIXTURE_ROOT` +- `RUSTFS_MINIO_STATIC_KMS_KEY_B64` + +Example: + +```powershell +$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key' +$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '' +cargo +1.95.0 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored +``` diff --git a/crates/ecstore/tests/minio_generated_read_test.rs b/crates/ecstore/tests/minio_generated_read_test.rs new file mode 100644 index 000000000..42bd5f1e7 --- /dev/null +++ b/crates/ecstore/tests/minio_generated_read_test.rs @@ -0,0 +1,207 @@ +#![cfg(feature = "rio-v2")] + +use std::fs; +use std::io::Cursor; +use std::path::{Path, PathBuf}; + +use rustfs_ecstore::bitrot::create_bitrot_reader; +use rustfs_ecstore::disk::endpoint::Endpoint; +use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk}; +use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions}; +use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use temp_env::async_with_vars; +use tokio::io::AsyncReadExt; + +#[derive(Debug, Deserialize)] +struct ManifestRecord { + bucket: String, + object: String, + backend_files: Vec, +} + +fn fixture_root() -> PathBuf { + std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated")) +} + +fn case_dir(case_id: &str) -> PathBuf { + fixture_root().join("cases").join(case_id) +} + +fn read_json Deserialize<'de>>(path: &Path) -> T { + let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display())) +} + +fn require_fixture_case(case_id: &str) -> PathBuf { + let path = case_dir(case_id); + assert!( + path.is_dir(), + "fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.", + path.display() + ); + path +} + +fn read_plaintext_sha256(case_dir: &Path) -> String { + fs::read_to_string(case_dir.join("plaintext.sha256")) + .unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display())) + .trim() + .to_string() +} + +fn minio_static_kms_key_b64() -> String { + std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64") + .unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key")) +} + +fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf { + let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object); + let relative = manifest + .backend_files + .iter() + .find(|entry| entry.as_str() == expected) + .unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}")); + case_dir.join("backend").join(relative) +} + +fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo { + let xl_meta_path = object_xl_meta_path(case_dir, manifest); + let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display())); + get_file_info( + &xl_meta, + &manifest.bucket, + &manifest.object, + "", + FileInfoOpts { + data: true, + include_free_versions: true, + }, + ) + .unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display())) +} + +fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo { + ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false) +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower) +} + +async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec { + let disk_root = case_dir.join("backend").join("disk1"); + let disk_root_str = disk_root + .to_str() + .unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display())); + let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open fixture disk"); + let data_dir = file_info + .data_dir + .as_ref() + .unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object)); + + let mut encrypted = Vec::new(); + for part in &file_info.parts { + let checksum_info = file_info.erasure.get_checksum_info(part.number); + let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number); + let mut reader = create_bitrot_reader( + None, + Some(&disk), + &manifest.bucket, + &path, + 0, + part.size, + file_info.erasure.shard_size(), + checksum_info.algorithm.clone(), + false, + false, + ) + .await + .unwrap_or_else(|err| panic!("create bitrot reader for {path}: {err:?}")) + .unwrap_or_else(|| panic!("missing bitrot reader for {path}")); + let mut block = vec![0u8; file_info.erasure.shard_size()]; + loop { + let n = reader + .read(&mut block) + .await + .unwrap_or_else(|err| panic!("read decoded encrypted bytes from {path}: {err}")); + if n == 0 { + break; + } + encrypted.extend_from_slice(&block[..n]); + if n < block.len() { + break; + } + } + } + disk.close().await.expect("close fixture disk"); + encrypted +} + +#[tokio::test] +#[ignore = "requires generated MinIO fixture data and a local static KMS key"] +async fn reads_minio_generated_sse_s3_multipart_fixture() { + assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await; +} + +#[tokio::test] +#[ignore = "requires generated MinIO fixture data and a local static KMS key"] +async fn reads_minio_generated_sse_kms_multipart_fixture() { + assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await; +} + +async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) { + let case_dir = require_fixture_case(case_id); + let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json")); + let expected_sha256 = read_plaintext_sha256(&case_dir); + let file_info = load_file_info(&case_dir, &manifest); + let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await; + let object_info = load_object_info(&file_info, &manifest); + let kms_key_b64 = minio_static_kms_key_b64(); + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)), + ("RUSTFS_SSE_S3_MASTER_KEY", None::), + ], + async { + let (mut reader, offset, length) = GetObjectReader::new( + Box::new(Cursor::new(encrypted)), + None, + &object_info, + &ObjectOptions::default(), + &http::HeaderMap::new(), + ) + .await + .expect("construct GetObjectReader from MinIO raw fixture"); + + let mut plaintext = Vec::new(); + reader + .read_to_end(&mut plaintext) + .await + .expect("read plaintext from MinIO raw fixture"); + + assert_eq!(offset, 0); + assert_eq!(length, object_info.size); + assert_eq!(reader.object_info.size, expected_size); + assert_eq!(plaintext.len(), expected_size as usize); + assert_eq!(sha256_hex(&plaintext), expected_sha256); + }, + ) + .await; +} diff --git a/crates/filemeta/examples/dump_fileinfo.rs b/crates/filemeta/examples/dump_fileinfo.rs index 35b58f308..41d819710 100644 --- a/crates/filemeta/examples/dump_fileinfo.rs +++ b/crates/filemeta/examples/dump_fileinfo.rs @@ -14,6 +14,30 @@ use rustfs_filemeta::{FileInfoOpts, get_file_info}; use std::{env, fs, path::PathBuf}; + +const S2_INDEX_HEADER: &[u8] = b"s2idx\x00"; +const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s"; +const CHUNK_TYPE_LEGACY_INDEX: u8 = 0x50; +const CHUNK_TYPE_MINIO_INDEX: u8 = 0x99; +const SKIPPABLE_FRAME_HEADER: usize = 4; +const MAX_INDEX_ENTRIES: i64 = 1 << 16; + +#[derive(Debug)] +struct CompressionIndex { + format: &'static str, + storage: &'static str, + total_uncompressed: i64, + total_compressed: i64, + est_block_uncompressed: i64, + offsets: Vec, +} + +#[derive(Debug)] +struct CompressionIndexOffset { + compressed: i64, + uncompressed: i64, +} + fn main() { let path = env::args() .nth(1) @@ -40,6 +64,10 @@ fn main() { "part#{idx}: number={} size={} actual_size={} etag={}", part.number, part.size, part.actual_size, part.etag ); + match part.index.as_deref() { + Some(index_bytes) => print_compression_index(idx, index_bytes), + None => println!("part#{idx}.index: none"), + } } // Tier / transition fields if !fi.transition_status.is_empty() { @@ -61,3 +89,222 @@ fn main() { println!("meta[{key}]={}", fi.metadata.get(&key).unwrap()); } } + +fn print_compression_index(part_idx: usize, bytes: &[u8]) { + match decode_compression_index(bytes) { + Ok(index) => { + println!( + "part#{part_idx}.index: bytes={} format={} storage={} entries={} total_uncompressed={} total_compressed={} est_block_uncompressed={}", + bytes.len(), + index.format, + index.storage, + index.offsets.len(), + index.total_uncompressed, + index.total_compressed, + index.est_block_uncompressed + ); + for (idx, offset) in index.offsets.iter().take(5).enumerate() { + println!( + "part#{part_idx}.index.offset#{idx}: compressed={} uncompressed={}", + offset.compressed, offset.uncompressed + ); + } + if index.offsets.len() > 5 + && let Some(offset) = index.offsets.last() + { + println!( + "part#{part_idx}.index.offset#last: compressed={} uncompressed={}", + offset.compressed, offset.uncompressed + ); + } + } + Err(err) => { + println!("part#{part_idx}.index: bytes={} decode_error={err}", bytes.len()); + } + } +} + +fn decode_compression_index(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err("empty index".to_string()); + } + + let mut errors = Vec::new(); + for candidate in index_candidates(bytes) { + match parse_index(&candidate.bytes, candidate.format, candidate.storage, candidate.signed_varint) { + Ok(index) => return Ok(index), + Err(err) => errors.push(format!("{} {}: {err}", candidate.format, candidate.storage)), + } + } + + Err(errors.join("; ")) +} + +struct IndexCandidate { + bytes: Vec, + format: &'static str, + storage: &'static str, + signed_varint: bool, +} + +fn index_candidates(bytes: &[u8]) -> Vec { + let mut candidates = Vec::new(); + match bytes[0] { + CHUNK_TYPE_MINIO_INDEX => candidates.push(IndexCandidate { + bytes: bytes.to_vec(), + format: "minio-s2", + storage: "full", + signed_varint: true, + }), + CHUNK_TYPE_LEGACY_INDEX => candidates.push(IndexCandidate { + bytes: bytes.to_vec(), + format: "legacy-rustfs", + storage: "full", + signed_varint: false, + }), + _ => { + candidates.push(IndexCandidate { + bytes: restore_index_headers(bytes, CHUNK_TYPE_MINIO_INDEX), + format: "minio-s2", + storage: "headerless", + signed_varint: true, + }); + candidates.push(IndexCandidate { + bytes: restore_index_headers(bytes, CHUNK_TYPE_LEGACY_INDEX), + format: "legacy-rustfs", + storage: "headerless", + signed_varint: false, + }); + } + } + candidates +} + +fn parse_index( + bytes: &[u8], + format: &'static str, + storage: &'static str, + signed_varint: bool, +) -> Result { + if bytes.len() <= SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() { + return Err("buffer too small".to_string()); + } + if bytes[0] != CHUNK_TYPE_MINIO_INDEX && bytes[0] != CHUNK_TYPE_LEGACY_INDEX { + return Err(format!("invalid chunk type 0x{:02x}", bytes[0])); + } + + let chunk_len = bytes[1] as usize | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16); + let chunk = bytes + .get(SKIPPABLE_FRAME_HEADER..SKIPPABLE_FRAME_HEADER + chunk_len) + .ok_or_else(|| "chunk length exceeds buffer".to_string())?; + let mut body = chunk + .strip_prefix(S2_INDEX_HEADER) + .ok_or_else(|| "invalid index header".to_string())?; + + let (total_uncompressed, used) = read_varint(body, signed_varint)?; + body = &body[used..]; + let (total_compressed, used) = read_varint(body, signed_varint)?; + body = &body[used..]; + let (est_block_uncompressed, used) = read_varint(body, signed_varint)?; + body = &body[used..]; + let (entries, used) = read_varint(body, signed_varint)?; + body = &body[used..]; + + if total_uncompressed < 0 || est_block_uncompressed < 0 || !(0..=MAX_INDEX_ENTRIES).contains(&entries) { + return Err("invalid index totals".to_string()); + } + + let has_uncompressed = *body.first().ok_or_else(|| "missing uncompressed-offset flag".to_string())?; + if has_uncompressed & 1 != has_uncompressed { + return Err("invalid uncompressed-offset flag".to_string()); + } + body = &body[1..]; + + let mut offsets = (0..entries) + .map(|_| CompressionIndexOffset { + compressed: 0, + uncompressed: 0, + }) + .collect::>(); + + for idx in 0..offsets.len() { + let mut uncompressed = 0; + if has_uncompressed != 0 { + let (value, used) = read_varint(body, signed_varint)?; + uncompressed = value; + body = &body[used..]; + } + if idx > 0 { + let prev = offsets[idx - 1].uncompressed; + uncompressed += prev + est_block_uncompressed; + } + offsets[idx].uncompressed = uncompressed; + } + + let mut compressed_predict = est_block_uncompressed / 2; + for idx in 0..offsets.len() { + let (mut compressed, used) = read_varint(body, signed_varint)?; + body = &body[used..]; + if idx > 0 { + let next_compressed_predict = compressed_predict + compressed / 2; + compressed += offsets[idx - 1].compressed + compressed_predict; + compressed_predict = next_compressed_predict; + } + offsets[idx].compressed = compressed; + } + + if body.len() < 4 + S2_INDEX_TRAILER.len() { + return Err("missing index trailer".to_string()); + } + body = &body[4..]; + if !body.starts_with(S2_INDEX_TRAILER) { + return Err("invalid index trailer".to_string()); + } + + Ok(CompressionIndex { + format, + storage, + total_uncompressed, + total_compressed, + est_block_uncompressed, + offsets, + }) +} + +fn restore_index_headers(input: &[u8], chunk_type: u8) -> Vec { + let mut bytes = Vec::with_capacity(SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + input.len() + 4 + S2_INDEX_TRAILER.len()); + bytes.extend_from_slice(&[chunk_type, 0, 0, 0]); + bytes.extend_from_slice(S2_INDEX_HEADER); + bytes.extend_from_slice(input); + bytes.extend_from_slice(&((bytes.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes()); + bytes.extend_from_slice(S2_INDEX_TRAILER); + + let chunk_len = bytes.len() - SKIPPABLE_FRAME_HEADER; + bytes[1] = chunk_len as u8; + bytes[2] = (chunk_len >> 8) as u8; + bytes[3] = (chunk_len >> 16) as u8; + bytes +} + +fn read_varint(bytes: &[u8], signed: bool) -> Result<(i64, usize), String> { + let mut value = 0_u64; + let mut shift = 0_u32; + + for (idx, byte) in bytes.iter().copied().enumerate() { + value |= u64::from(byte & 0x7f) << shift; + if byte < 0x80 { + let value = if signed { + ((value >> 1) as i64) ^ (-((value & 1) as i64)) + } else { + value as i64 + }; + return Ok((value, idx + 1)); + } + shift += 7; + if shift >= 64 { + return Err("varint overflow".to_string()); + } + } + + Err("unexpected EOF while reading varint".to_string()) +} diff --git a/crates/kms/src/cache.rs b/crates/kms/src/cache.rs index 858d7a6e4..cbf9b35b2 100644 --- a/crates/kms/src/cache.rs +++ b/crates/kms/src/cache.rs @@ -14,22 +14,13 @@ //! Caching layer for KMS operations to improve performance -use crate::types::{KeyMetadata, KeySpec}; +use crate::types::KeyMetadata; use moka::future::Cache; use std::time::Duration; -/// Cached data key entry -#[derive(Clone, Debug)] -pub struct CachedDataKey { - pub plaintext: Vec, - pub ciphertext: Vec, - pub key_spec: KeySpec, -} - /// KMS cache for storing frequently accessed keys and metadata pub struct KmsCache { key_metadata_cache: Cache, - data_key_cache: Cache, } impl KmsCache { @@ -44,13 +35,9 @@ impl KmsCache { pub fn new(capacity: u64) -> Self { Self { key_metadata_cache: Cache::builder() - .max_capacity(capacity / 2) + .max_capacity(capacity) .time_to_live(Duration::from_secs(300)) // 5 minutes default TTL .build(), - data_key_cache: Cache::builder() - .max_capacity(capacity / 2) - .time_to_live(Duration::from_secs(60)) // 1 minute for data keys (shorter for security) - .build(), } } @@ -77,35 +64,6 @@ impl KmsCache { self.key_metadata_cache.run_pending_tasks().await; } - /// Get data key from cache - /// - /// # Arguments - /// * `key_id` - The ID of the key to retrieve the data key for - /// - /// # Returns - /// An `Option` containing the `CachedDataKey` if found, or `None` if not found - /// - pub async fn get_data_key(&self, key_id: &str) -> Option { - self.data_key_cache.get(key_id).await - } - - /// Put data key into cache - /// - /// # Arguments - /// * `key_id` - The ID of the key to store the data key for - /// * `plaintext` - The plaintext data key bytes - /// * `ciphertext` - The ciphertext data key bytes - /// - pub async fn put_data_key(&mut self, key_id: &str, plaintext: &[u8], ciphertext: &[u8]) { - let cached_key = CachedDataKey { - plaintext: plaintext.to_vec(), - ciphertext: ciphertext.to_vec(), - key_spec: KeySpec::Aes256, // Default to AES-256 - }; - self.data_key_cache.insert(key_id.to_string(), cached_key).await; - self.data_key_cache.run_pending_tasks().await; - } - /// Remove key metadata from cache /// /// # Arguments @@ -115,23 +73,12 @@ impl KmsCache { self.key_metadata_cache.remove(key_id).await; } - /// Remove data key from cache - /// - /// # Arguments - /// * `key_id` - The ID of the key to remove the data key for - /// - pub async fn remove_data_key(&mut self, key_id: &str) { - self.data_key_cache.remove(key_id).await; - } - /// Clear all cached entries pub async fn clear(&mut self) { self.key_metadata_cache.invalidate_all(); - self.data_key_cache.invalidate_all(); // Wait for invalidation to complete self.key_metadata_cache.run_pending_tasks().await; - self.data_key_cache.run_pending_tasks().await; } /// Get cache statistics (hit count, miss count) @@ -140,13 +87,10 @@ impl KmsCache { /// A tuple containing total entries and total misses /// pub fn stats(&self) -> (u64, u64) { - let metadata_stats = ( + ( self.key_metadata_cache.entry_count(), 0u64, // moka doesn't provide miss count directly - ); - let data_key_stats = (self.data_key_cache.entry_count(), 0u64); - - (metadata_stats.0 + data_key_stats.0, metadata_stats.1 + data_key_stats.1) + ) } } @@ -160,37 +104,30 @@ mod tests { #[derive(Debug, Clone)] struct CacheInfo { key_metadata_count: u64, - data_key_count: u64, } impl CacheInfo { fn total_entries(&self) -> u64 { - self.key_metadata_count + self.data_key_count + self.key_metadata_count } } impl KmsCache { - fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration, data_key_ttl: Duration) -> Self { + fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self { Self { - key_metadata_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(metadata_ttl).build(), - data_key_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(data_key_ttl).build(), + key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(), } } fn info_for_tests(&self) -> CacheInfo { CacheInfo { key_metadata_count: self.key_metadata_cache.entry_count(), - data_key_count: self.data_key_cache.entry_count(), } } fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool { self.key_metadata_cache.contains_key(key_id) } - - fn contains_data_key_for_tests(&self, key_id: &str) -> bool { - self.data_key_cache.contains_key(key_id) - } } #[tokio::test] @@ -216,23 +153,10 @@ mod tests { assert!(retrieved.is_some()); assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1"); - // Test data key caching - let plaintext = vec![1, 2, 3, 4]; - let ciphertext = vec![5, 6, 7, 8]; - cache.put_data_key("test-key-1", &plaintext, &ciphertext).await; - - let cached_data_key = cache.get_data_key("test-key-1").await; - assert!(cached_data_key.is_some()); - let cached_data_key = cached_data_key.expect("data key should be cached"); - assert_eq!(cached_data_key.plaintext, plaintext); - assert_eq!(cached_data_key.ciphertext, ciphertext); - assert_eq!(cached_data_key.key_spec, KeySpec::Aes256); - // Test cache info let info = cache.info_for_tests(); assert_eq!(info.key_metadata_count, 1); - assert_eq!(info.data_key_count, 1); - assert_eq!(info.total_entries(), 2); + assert_eq!(info.total_entries(), 1); // Test cache clearing cache.clear().await; @@ -245,7 +169,6 @@ mod tests { let mut cache = KmsCache::with_ttl_for_tests( 100, Duration::from_millis(100), // Short TTL for testing - Duration::from_millis(50), ); let metadata = KeyMetadata { @@ -277,7 +200,6 @@ mod tests { let mut cache = KmsCache::new(100); assert!(!cache.contains_key_metadata_for_tests("nonexistent")); - assert!(!cache.contains_data_key_for_tests("nonexistent")); let metadata = KeyMetadata { key_id: "contains-test".to_string(), @@ -292,9 +214,7 @@ mod tests { }; cache.put_key_metadata("contains-test", &metadata).await; - cache.put_data_key("contains-test", &[1, 2, 3], &[4, 5, 6]).await; assert!(cache.contains_key_metadata_for_tests("contains-test")); - assert!(cache.contains_data_key_for_tests("contains-test")); } } diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index 8de6bc19d..3c968d07e 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -33,6 +33,14 @@ //! - **Data Encryption Keys (DEK)**: Generated per object, encrypted by master keys //! - **Object Data**: Encrypted using DEKs with AES-256-GCM or ChaCha20-Poly1305 //! +//! ## Caching Discipline +//! +//! KMS may cache stable master-key metadata, but it must not cache or reuse generated +//! data encryption keys by master key id alone. A generated DEK and its encrypted +//! ciphertext can be bound to the object encryption context, such as the bucket and +//! object path. Reusing it for another object can break context validation and would +//! also violate the expected per-object DEK model for SSE-S3 and SSE-KMS. +//! //! ## Example //! //! ```rust,no_run diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 31c1a4d8b..fec6dfb49 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -71,32 +71,7 @@ impl KmsManager { /// Generate a data encryption key pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result { - // Check cache first if enabled - if self.config.enable_cache { - let cache = self.cache.read().await; - if let Some(cached_key) = cache.get_data_key(&request.key_id).await - && cached_key.key_spec == request.key_spec - { - return Ok(GenerateDataKeyResponse { - key_id: request.key_id.clone(), - plaintext_key: cached_key.plaintext.clone(), - ciphertext_blob: cached_key.ciphertext, - }); - } - } - - // Generate new data key from backend - let response = self.backend.generate_data_key(request).await?; - - // Cache the data key if enabled - if self.config.enable_cache { - let mut cache = self.cache.write().await; - cache - .put_data_key(&response.key_id, &response.plaintext_key, &response.ciphertext_blob) - .await; - } - - Ok(response) + self.backend.generate_data_key(request).await } /// Describe a key @@ -156,7 +131,6 @@ impl KmsManager { if self.config.enable_cache { let mut cache = self.cache.write().await; cache.remove_key_metadata(&response.key_id).await; - cache.remove_data_key(&response.key_id).await; } Ok(response) @@ -186,6 +160,7 @@ mod tests { use super::*; use crate::backends::local::LocalKmsBackend; use crate::types::{KeySpec, KeyState, KeyUsage}; + use std::collections::HashMap; use tempfile::tempdir; #[tokio::test] @@ -237,4 +212,62 @@ mod tests { let health = manager.health_check().await.expect("Health check failed"); assert!(health); } + + #[tokio::test] + async fn generate_data_key_does_not_reuse_context_bound_ciphertext() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let config = KmsConfig::local(temp_dir.path().to_path_buf()); + + let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend")); + let manager = KmsManager::new(backend, config); + + let create_response = manager + .create_key(CreateKeyRequest { + key_usage: KeyUsage::EncryptDecrypt, + description: Some("Context-bound data key test".to_string()), + ..Default::default() + }) + .await + .expect("Failed to create key"); + + let first_context = HashMap::from([ + ("bucket".to_string(), "sse-smoke".to_string()), + ("object".to_string(), "first.bin".to_string()), + ]); + let second_context = HashMap::from([ + ("bucket".to_string(), "sse-smoke".to_string()), + ("object".to_string(), "second.bin".to_string()), + ]); + + let first = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: create_response.key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: first_context.clone(), + }) + .await + .expect("Failed to generate first data key"); + let second = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: create_response.key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: second_context.clone(), + }) + .await + .expect("Failed to generate second data key"); + + assert_ne!( + first.ciphertext_blob, second.ciphertext_blob, + "data keys must not be cached only by KMS key id because ciphertext is bound to object context" + ); + + manager + .decrypt(DecryptRequest { + ciphertext: second.ciphertext_blob, + encryption_context: second_context, + grant_tokens: Vec::new(), + }) + .await + .expect("second data key should decrypt with its own context"); + } } diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index c405c9992..03f893090 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -49,6 +49,26 @@ pub struct ObjectEncryptionService { kms_manager: KmsManager, } +fn canonical_bucket_path(bucket: &str, object_key: &str) -> String { + let bucket = bucket.trim_matches('/'); + let object_key = object_key.trim_matches('/'); + if object_key.is_empty() { + bucket.to_string() + } else if bucket.is_empty() { + object_key.to_string() + } else { + format!("{bucket}/{object_key}") + } +} + +fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap { + let mut enc_context = context.encryption_context.clone(); + enc_context + .entry(context.bucket.clone()) + .or_insert_with(|| canonical_bucket_path(&context.bucket, &context.object_key)); + enc_context +} + const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; /// Result of object encryption @@ -178,15 +198,10 @@ impl ObjectEncryptionService { .or_else(|| self.kms_manager.get_default_key_id().map(|s| s.as_str())) .ok_or_else(|| KmsError::configuration_error("No KMS key ID specified and no default configured"))?; - // Build encryption context - let mut enc_context = context.encryption_context.clone(); - enc_context.insert("bucket".to_string(), context.bucket.clone()); - enc_context.insert("object_key".to_string(), context.object_key.clone()); - let request = GenerateDataKeyRequest { key_id: actual_key_id.to_string(), key_spec: KeySpec::Aes256, - encryption_context: enc_context, + encryption_context: request_encryption_context(context), }; let data_key_response = self.kms_manager.generate_data_key(request).await?; @@ -216,10 +231,10 @@ impl ObjectEncryptionService { /// # Returns /// DataKey with decrypted key /// - pub async fn decrypt_data_key(&self, encrypted_key: &[u8], _context: &ObjectEncryptionContext) -> Result { + pub async fn decrypt_data_key(&self, encrypted_key: &[u8], context: &ObjectEncryptionContext) -> Result { let decrypt_request = DecryptRequest { ciphertext: encrypted_key.to_vec(), - encryption_context: HashMap::new(), + encryption_context: request_encryption_context(context), grant_tokens: Vec::new(), }; @@ -864,4 +879,40 @@ mod tests { .is_err() ); } + + #[tokio::test] + async fn test_decrypt_data_key_uses_object_encryption_context() { + let (service, _temp_dir) = create_test_service().await; + service + .create_key(CreateKeyRequest { + key_name: Some("test-key".to_string()), + key_usage: KeyUsage::EncryptDecrypt, + description: None, + policy: None, + tags: HashMap::new(), + origin: None, + }) + .await + .expect("test key should be created"); + let create_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string()) + .with_encryption_context("tenant".to_string(), "alpha".to_string()); + let kms_key = Some("test-key".to_string()); + let (_data_key, encrypted_key) = service + .create_data_key(&kms_key, &create_context) + .await + .expect("create data key should succeed"); + + let wrong_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string()) + .with_encryption_context("tenant".to_string(), "beta".to_string()); + assert!( + service.decrypt_data_key(&encrypted_key, &wrong_context).await.is_err(), + "decrypt should reject mismatched KMS context" + ); + + let decrypted = service + .decrypt_data_key(&encrypted_key, &create_context) + .await + .expect("decrypt should accept matching KMS context"); + assert_ne!(decrypted.plaintext_key, [0u8; 32]); + } } diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 856de8723..c32167c5b 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -28,6 +28,7 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/" [features] default = [] gpu = ["dep:nvml-wrapper"] +pyroscope = ["dep:jemalloc_pprof", "dep:pyroscope"] [lints] workspace = true @@ -72,8 +73,8 @@ sysinfo = { workspace = true } nvml-wrapper = { workspace = true, optional = true } [target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies] -pyroscope = { workspace = true } -jemalloc_pprof = { workspace = true } +pyroscope = { workspace = true, features = ["backend-pprof-rs"], optional = true } +jemalloc_pprof = { workspace = true, optional = true } [dev-dependencies] diff --git a/crates/obs/src/telemetry/guard.rs b/crates/obs/src/telemetry/guard.rs index 033103ba3..9d5aab941 100644 --- a/crates/obs/src/telemetry/guard.rs +++ b/crates/obs/src/telemetry/guard.rs @@ -28,6 +28,21 @@ use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider}; +#[cfg(all( + feature = "pyroscope", + any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) +))] +pub(crate) type ProfilingAgent = pyroscope::PyroscopeAgent; +#[cfg(not(all( + feature = "pyroscope", + any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) +)))] +pub(crate) type ProfilingAgent = (); +#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] +pub(crate) type MemoryProfilingAgent = pyroscope::PyroscopeAgent; +#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] +pub(crate) type MemoryProfilingAgent = (); + /// RAII guard that owns all active OpenTelemetry providers and the /// `tracing_appender` worker guard. /// @@ -41,10 +56,8 @@ pub struct OtelGuard { pub(crate) meter_provider: Option, /// Optional logger provider for OTLP log export. pub(crate) logger_provider: Option, - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] - pub(crate) profiling_agent: Option>, - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] - pub(crate) memory_profiling_agent: Option>, + pub(crate) profiling_agent: Option, + pub(crate) memory_profiling_agent: Option, /// Handle to the background log-cleanup task; aborted on drop. pub(crate) cleanup_handle: Option>, /// Worker guard that keeps the non-blocking `tracing_appender` thread @@ -59,12 +72,10 @@ impl std::fmt::Debug for OtelGuard { let mut s = f.debug_struct("OtelGuard"); s.field("tracer_provider", &self.tracer_provider.is_some()) .field("meter_provider", &self.meter_provider.is_some()) - .field("logger_provider", &self.logger_provider.is_some()); - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] - s.field("profiling_agent", &self.profiling_agent.is_some()); - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] - s.field("memory_profiling_agent", &self.memory_profiling_agent.is_some()); - s.field("cleanup_handle", &self.cleanup_handle.is_some()) + .field("logger_provider", &self.logger_provider.is_some()) + .field("profiling_agent", &self.profiling_agent.is_some()) + .field("memory_profiling_agent", &self.memory_profiling_agent.is_some()) + .field("cleanup_handle", &self.cleanup_handle.is_some()) .field("tracing_guard", &self.tracing_guard.is_some()) .field("stdout_guard", &self.stdout_guard.is_some()) .finish() @@ -95,7 +106,10 @@ impl Drop for OtelGuard { eprintln!("Logger shutdown error: {err:?}"); } - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] + #[cfg(all( + feature = "pyroscope", + any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) + ))] if let Some(agent) = self.profiling_agent.take() { match agent.stop() { Err(err) => eprintln!("Profiling agent stop error: {err:?}"), @@ -105,7 +119,7 @@ impl Drop for OtelGuard { } } - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] + #[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] if let Some(agent) = self.memory_profiling_agent.take() { match agent.stop() { Err(err) => eprintln!("Memory profiling agent stop error: {err:?}"), diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index f5bc20042..88c7256ff 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -155,9 +155,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo tracer_provider: None, meter_provider: None, logger_provider: None, - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] profiling_agent: None, - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] memory_profiling_agent: None, tracing_guard: Some(guard), stdout_guard: None, @@ -291,9 +289,7 @@ fn init_file_logging_internal( tracer_provider: None, meter_provider: None, logger_provider: None, - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] profiling_agent: None, - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] memory_profiling_agent: None, tracing_guard: Some(guard), stdout_guard, diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index 083b99602..7686dd831 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -40,7 +40,7 @@ use crate::cleaner::types::FileMatchMode; use crate::config::OtelConfig; use crate::global::set_observability_metric_enabled; use crate::telemetry::filter::build_env_filter; -use crate::telemetry::guard::OtelGuard; +use crate::telemetry::guard::{MemoryProfilingAgent, OtelGuard, ProfilingAgent}; use crate::telemetry::local::spawn_cleanup_task; use crate::telemetry::recorder::Recorder; use crate::telemetry::resource::build_resource; @@ -164,10 +164,7 @@ pub(super) fn init_observability_http( // ── Meter provider (HTTP) ───────────────────────────────────────────────── let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?; - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] let profiling_agent = init_profiler(config); - - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] let memory_profiling_agent = init_memory_profiler(config); // ── Logger Logic ────────────────────────────────────────────────────────── @@ -317,9 +314,7 @@ pub(super) fn init_observability_http( tracer_provider, meter_provider, logger_provider, - #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] profiling_agent, - #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] memory_profiling_agent, tracing_guard, stdout_guard, @@ -496,8 +491,11 @@ fn build_logger_provider( /// Returns `None` when profiling export is disabled, when no usable /// profiling endpoint is configured, or when building or starting the agent /// fails. -#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] -fn init_profiler(config: &OtelConfig) -> Option> { +#[cfg(all( + feature = "pyroscope", + any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) +))] +fn init_profiler(config: &OtelConfig) -> Option { use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend}; use pyroscope::pyroscope::PyroscopeAgentBuilder; use rustfs_config::VERSION; @@ -534,6 +532,14 @@ fn init_profiler(config: &OtelConfig) -> Option Option { + None +} + /// Initialise a Pyroscope agent for continuous **memory** profiling via jemalloc. /// /// This is only available on `linux + gnu + x86_64` where tikv-jemallocator @@ -541,8 +547,8 @@ fn init_profiler(config: &OtelConfig) -> Option Option> { +#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] +fn init_memory_profiler(config: &OtelConfig) -> Option { use pyroscope::backend::jemalloc_backend; use pyroscope::pyroscope::PyroscopeAgentBuilder; use rustfs_config::VERSION; @@ -588,6 +594,11 @@ fn init_memory_profiler(config: &OtelConfig) -> Option Option { + None +} + /// Create a stdout periodic metrics reader for the given interval. /// /// This helper is primarily used for local development and diagnostics when diff --git a/crates/rio-v2/Cargo.toml b/crates/rio-v2/Cargo.toml new file mode 100644 index 000000000..8e29db973 --- /dev/null +++ b/crates/rio-v2/Cargo.toml @@ -0,0 +1,48 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "rustfs-rio-v2" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true +homepage.workspace = true +description = "Feature-gated rio_v2 compatibility facade for incremental RustFS migration." +keywords = ["asynchronous", "io", "compatibility", "rustfs", "minio"] +categories = ["web-programming", "development-tools", "asynchronous"] +documentation = "https://docs.rs/rustfs-rio-v2/latest/rustfs_rio_v2/" + +[lints] +workspace = true + +[dependencies] +aes-gcm.workspace = true +bytes.workspace = true +hex.workspace = true +hmac.workspace = true +minlz = "1.1.0" +pin-project-lite.workspace = true +rand.workspace = true +rustfs-rio.workspace = true +rustfs-utils.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +tokio.workspace = true + +[dev-dependencies] +rustfs-filemeta.workspace = true +walkdir.workspace = true diff --git a/crates/rio-v2/src/compress_reader.rs b/crates/rio-v2/src/compress_reader.rs new file mode 100644 index 000000000..caafd156f --- /dev/null +++ b/crates/rio-v2/src/compress_reader.rs @@ -0,0 +1,755 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use minlz::{Encoder as MinlzEncoder, crc::crc, decode}; +use pin_project_lite::pin_project; +use rand::RngExt; +use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex}; +use rustfs_utils::CompressionAlgorithm; +use std::cmp::min; +use std::fmt; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, ReadBuf}; + +const MAGIC_CHUNK: &[u8] = b"\xff\x06\x00\x00S2sTwO"; +const MAGIC_CHUNK_SNAPPY: &[u8] = b"\xff\x06\x00\x00sNaPpY"; +const CHUNK_TYPE_COMPRESSED_DATA: u8 = 0x00; +const CHUNK_TYPE_UNCOMPRESSED_DATA: u8 = 0x01; +const CHUNK_TYPE_INDEX: u8 = 0x99; +const CHUNK_TYPE_PADDING: u8 = 0xfe; +const CHUNK_TYPE_STREAM_IDENTIFIER: u8 = 0xff; +const DEFAULT_BLOCK_SIZE: usize = 1 << 20; +const MAX_CHUNK_SIZE: usize = (1 << 24) - 1; +const CHECKSUM_SIZE: usize = 4; +const CHUNK_HEADER_LEN: usize = 4; +const ENCRYPTED_PADDING_MULTIPLE: usize = 256; +const MIN_INDEX_SIZE: usize = 8 << 20; + +pin_project! { + #[derive(Debug)] + pub struct CompressReader { + #[pin] + inner: R, + buffer: Vec, + pos: usize, + done: bool, + block_size: usize, + index: Index, + written: usize, + uncompressed_written: usize, + temp_buffer: Vec, + read_buffer: Vec, + wrote_stream_header: bool, + padding_multiple: Option, + block_encoder: S2BlockEncoder, + } +} + +struct S2BlockEncoder { + inner: MinlzEncoder, +} + +impl S2BlockEncoder { + fn new() -> Self { + Self { + inner: MinlzEncoder::new(), + } + } + + fn encode(&mut self, uncompressed: &[u8]) -> Vec { + self.inner.encode(uncompressed) + } +} + +impl fmt::Debug for S2BlockEncoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("S2BlockEncoder").finish_non_exhaustive() + } +} + +impl CompressReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self { + Self::with_block_size(inner, DEFAULT_BLOCK_SIZE, CompressionAlgorithm::default()) + } + + pub fn with_block_size(inner: R, block_size: usize, _compression_algorithm: CompressionAlgorithm) -> Self { + Self { + inner, + buffer: Vec::new(), + pos: 0, + done: false, + block_size, + index: Index::new(), + written: 0, + uncompressed_written: 0, + temp_buffer: Vec::with_capacity(block_size), + read_buffer: vec![0u8; block_size], + wrote_stream_header: false, + padding_multiple: None, + block_encoder: S2BlockEncoder::new(), + } + } + + pub fn with_encrypted_padding(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self { + let mut reader = Self::new(inner, CompressionAlgorithm::default()); + reader.padding_multiple = Some(ENCRYPTED_PADDING_MULTIPLE); + reader + } +} + +impl TryGetIndex for CompressReader { + fn try_get_index(&self) -> Option<&Index> { + (self.uncompressed_written > MIN_INDEX_SIZE).then_some(&self.index) + } +} + +impl AsyncRead for CompressReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let mut this = self.project(); + + if *this.pos < this.buffer.len() { + let to_copy = min(buf.remaining(), this.buffer.len() - *this.pos); + buf.put_slice(&this.buffer[*this.pos..*this.pos + to_copy]); + *this.pos += to_copy; + if *this.pos == this.buffer.len() { + this.buffer.clear(); + *this.pos = 0; + } + return Poll::Ready(Ok(())); + } + + if *this.done { + return Poll::Ready(Ok(())); + } + + while this.temp_buffer.len() < *this.block_size { + let remaining = *this.block_size - this.temp_buffer.len(); + let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]); + match this.inner.as_mut().poll_read(cx, &mut read_buf) { + Poll::Pending => { + return Poll::Pending; + } + Poll::Ready(Ok(())) => { + let n = read_buf.filled().len(); + if n == 0 { + break; + } + this.temp_buffer.extend_from_slice(read_buf.filled()); + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + if this.temp_buffer.is_empty() { + if let Some(padding_multiple) = *this.padding_multiple + && let Some(padding_chunk) = build_padding_chunk(*this.written, padding_multiple)? + { + *this.written += padding_chunk.len(); + this.index.total_compressed = *this.written as i64; + *this.buffer = padding_chunk; + *this.pos = 0; + *this.done = true; + + let to_copy = min(buf.remaining(), this.buffer.len()); + buf.put_slice(&this.buffer[..to_copy]); + *this.pos += to_copy; + if *this.pos == this.buffer.len() { + this.buffer.clear(); + *this.pos = 0; + } + return Poll::Ready(Ok(())); + } + + *this.done = true; + return Poll::Ready(Ok(())); + } + + let mut out = Vec::new(); + if !*this.wrote_stream_header { + out.extend_from_slice(MAGIC_CHUNK); + *this.written += MAGIC_CHUNK.len(); + *this.wrote_stream_header = true; + } + + if let Err(err) = this.index.add(*this.written as i64, *this.uncompressed_written as i64) { + return Poll::Ready(Err(err)); + } + + let block = build_s2_chunk(this.temp_buffer.as_slice(), this.block_encoder)?; + *this.uncompressed_written += this.temp_buffer.len(); + *this.written += block.len(); + this.index.total_uncompressed = *this.uncompressed_written as i64; + this.index.total_compressed = *this.written as i64; + + out.extend_from_slice(&block); + this.temp_buffer.clear(); + *this.buffer = out; + *this.pos = 0; + + let to_copy = min(buf.remaining(), this.buffer.len()); + buf.put_slice(&this.buffer[..to_copy]); + *this.pos += to_copy; + if *this.pos == this.buffer.len() { + this.buffer.clear(); + *this.pos = 0; + } + + Poll::Ready(Ok(())) + } +} + +impl EtagResolvable for CompressReader +where + R: EtagResolvable, +{ + fn try_resolve_etag(&mut self) -> Option { + self.inner.try_resolve_etag() + } +} + +impl HashReaderDetector for CompressReader +where + R: HashReaderDetector, +{ + fn is_hash_reader(&self) -> bool { + self.inner.is_hash_reader() + } + + fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> { + self.inner.as_hash_reader_mut() + } +} + +pin_project! { + #[derive(Debug)] + pub struct DecompressReader { + #[pin] + inner: R, + buffer: Vec, + buffer_pos: usize, + finished: bool, + header_buf: [u8; CHUNK_HEADER_LEN], + header_read: usize, + chunk_type: u8, + chunk_buf: Vec, + chunk_len: usize, + chunk_read: usize, + reading_chunk: bool, + stream_initialized: bool, + } +} + +impl DecompressReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self { + Self { + inner, + buffer: Vec::new(), + buffer_pos: 0, + finished: false, + header_buf: [0u8; CHUNK_HEADER_LEN], + header_read: 0, + chunk_type: 0, + chunk_buf: Vec::new(), + chunk_len: 0, + chunk_read: 0, + reading_chunk: false, + stream_initialized: false, + } + } +} + +impl AsyncRead for DecompressReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let mut this = self.project(); + + if *this.buffer_pos < this.buffer.len() { + let to_copy = min(buf.remaining(), this.buffer.len() - *this.buffer_pos); + buf.put_slice(&this.buffer[*this.buffer_pos..*this.buffer_pos + to_copy]); + *this.buffer_pos += to_copy; + if *this.buffer_pos == this.buffer.len() { + this.buffer.clear(); + *this.buffer_pos = 0; + } + return Poll::Ready(Ok(())); + } + + loop { + if *this.finished { + return Poll::Ready(Ok(())); + } + + if !*this.reading_chunk { + while *this.header_read < CHUNK_HEADER_LEN { + let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]); + match this.inner.as_mut().poll_read(cx, &mut read_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => { + let n = read_buf.filled().len(); + if n == 0 { + if *this.header_read == 0 { + *this.finished = true; + return Poll::Ready(Ok(())); + } + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading S2 chunk header", + ))); + } + *this.header_read += n; + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + *this.chunk_type = this.header_buf[0]; + *this.chunk_len = + (this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16); + *this.header_read = 0; + + if this.chunk_buf.len() < *this.chunk_len { + this.chunk_buf.resize(*this.chunk_len, 0); + } + *this.chunk_read = 0; + *this.reading_chunk = true; + } + + while *this.chunk_read < *this.chunk_len { + let mut read_buf = ReadBuf::new(&mut this.chunk_buf[*this.chunk_read..*this.chunk_len]); + match this.inner.as_mut().poll_read(cx, &mut read_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => { + let n = read_buf.filled().len(); + if n == 0 { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading S2 chunk body", + ))); + } + *this.chunk_read += n; + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + let chunk = &this.chunk_buf[..*this.chunk_len]; + *this.reading_chunk = false; + match *this.chunk_type { + CHUNK_TYPE_STREAM_IDENTIFIER => { + if chunk != &MAGIC_CHUNK[CHUNK_HEADER_LEN..] && chunk != &MAGIC_CHUNK_SNAPPY[CHUNK_HEADER_LEN..] { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid S2 stream identifier"))); + } + *this.stream_initialized = true; + continue; + } + CHUNK_TYPE_COMPRESSED_DATA => { + *this.stream_initialized = true; + let decompressed = decode_chunk(chunk, true)?; + *this.buffer = decompressed; + } + CHUNK_TYPE_UNCOMPRESSED_DATA => { + *this.stream_initialized = true; + let decompressed = decode_chunk(chunk, false)?; + *this.buffer = decompressed; + } + CHUNK_TYPE_INDEX | CHUNK_TYPE_PADDING | 0x80..=0xfd => { + *this.stream_initialized = true; + continue; + } + _ => { + if !*this.stream_initialized && *this.chunk_type != CHUNK_TYPE_COMPRESSED_DATA { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type), + ))); + } + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type), + ))); + } + } + + *this.buffer_pos = 0; + let to_copy = min(buf.remaining(), this.buffer.len()); + buf.put_slice(&this.buffer[..to_copy]); + *this.buffer_pos += to_copy; + if *this.buffer_pos == this.buffer.len() { + this.buffer.clear(); + *this.buffer_pos = 0; + } + return Poll::Ready(Ok(())); + } + } +} + +impl EtagResolvable for DecompressReader +where + R: EtagResolvable, +{ + fn try_resolve_etag(&mut self) -> Option { + self.inner.try_resolve_etag() + } +} + +impl HashReaderDetector for DecompressReader +where + R: HashReaderDetector, +{ + fn is_hash_reader(&self) -> bool { + self.inner.is_hash_reader() + } + + fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> { + self.inner.as_hash_reader_mut() + } +} + +fn build_s2_chunk(uncompressed: &[u8], encoder: &mut S2BlockEncoder) -> io::Result> { + let compressed = encode_block(uncompressed, encoder); + let checksum = crc(uncompressed); + let dst_limit = uncompressed.len().saturating_sub(uncompressed.len() / 32).saturating_sub(5); + let (chunk_type, payload) = if compressed.len() <= dst_limit { + (CHUNK_TYPE_COMPRESSED_DATA, compressed) + } else { + (CHUNK_TYPE_UNCOMPRESSED_DATA, uncompressed.to_vec()) + }; + + let chunk_len = payload.len() + CHECKSUM_SIZE; + if chunk_len > MAX_CHUNK_SIZE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk exceeds 24-bit framing limit")); + } + + let mut out = Vec::with_capacity(CHUNK_HEADER_LEN + chunk_len); + out.push(chunk_type); + out.push((chunk_len & 0xff) as u8); + out.push(((chunk_len >> 8) & 0xff) as u8); + out.push(((chunk_len >> 16) & 0xff) as u8); + out.extend_from_slice(&checksum.to_le_bytes()); + out.extend_from_slice(&payload); + Ok(out) +} + +fn encode_block(uncompressed: &[u8], encoder: &mut S2BlockEncoder) -> Vec { + encoder.encode(uncompressed) +} + +fn build_padding_chunk(current_size: usize, padding_multiple: usize) -> io::Result>> { + if padding_multiple == 0 || current_size.is_multiple_of(padding_multiple) { + return Ok(None); + } + + let padding_len = (padding_multiple - ((current_size + CHUNK_HEADER_LEN) % padding_multiple)) % padding_multiple; + if padding_len > MAX_CHUNK_SIZE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 padding exceeds 24-bit framing limit")); + } + + let mut out = Vec::with_capacity(CHUNK_HEADER_LEN + padding_len); + out.push(CHUNK_TYPE_PADDING); + out.push((padding_len & 0xff) as u8); + out.push(((padding_len >> 8) & 0xff) as u8); + out.push(((padding_len >> 16) & 0xff) as u8); + + if padding_len > 0 { + let mut padding = vec![0u8; padding_len]; + rand::rng().fill(padding.as_mut_slice()); + out.extend_from_slice(&padding); + } + + Ok(Some(out)) +} + +fn decode_chunk(chunk: &[u8], compressed: bool) -> io::Result> { + if chunk.len() < CHECKSUM_SIZE { + return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk smaller than checksum header")); + } + + let expected_crc = u32::from_le_bytes(chunk[..CHECKSUM_SIZE].try_into().expect("checksum header")); + let payload = &chunk[CHECKSUM_SIZE..]; + let decompressed = if compressed { + decode(payload).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 decode error: {err}")))? + } else { + payload.to_vec() + }; + + let actual_crc = crc(&decompressed); + if actual_crc != expected_crc { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "S2 CRC mismatch: expected={expected_crc:08x} actual={actual_crc:08x} compressed={compressed} payload_len={} decompressed_len={}", + payload.len(), + decompressed.len() + ), + )); + } + + Ok(decompressed) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::io::AsyncReadExt; + + struct PendingAfterBytes { + inner: R, + max_chunk: usize, + pending_next: bool, + } + + impl PendingAfterBytes { + fn new(inner: R, max_chunk: usize) -> Self { + Self { + inner, + max_chunk, + pending_next: false, + } + } + } + + impl AsyncRead for PendingAfterBytes { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.pending_next { + self.pending_next = false; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + + let allowed = self.max_chunk.min(buf.remaining()); + if allowed == 0 { + return Poll::Ready(Ok(())); + } + + let mut scratch = vec![0u8; allowed]; + let mut limited = ReadBuf::new(&mut scratch); + match Pin::new(&mut self.inner).poll_read(cx, &mut limited) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(err)) => Poll::Ready(Err(err)), + Poll::Ready(Ok(())) => { + let filled = limited.filled(); + if !filled.is_empty() { + buf.put_slice(filled); + self.pending_next = true; + } + Poll::Ready(Ok(())) + } + } + } + } + + fn s2_chunk_types(stream: &[u8]) -> Vec { + let mut chunk_types = Vec::new(); + let mut offset = 0usize; + while offset + CHUNK_HEADER_LEN <= stream.len() { + let chunk_type = stream[offset]; + let chunk_len = + (stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16); + chunk_types.push(chunk_type); + offset += CHUNK_HEADER_LEN + chunk_len; + } + chunk_types + } + + #[test] + fn minlz_encoded_payload_decodes_with_minlz() { + let plaintext = b"compressible-rio-v2-block-".repeat(4096); + let mut encoder = S2BlockEncoder::new(); + let compressed = encode_block(&plaintext, &mut encoder); + let decoded = decode(&compressed).expect("decode payload"); + + assert_eq!(decoded, plaintext); + } + + #[tokio::test] + async fn s2_compress_reader_roundtrip() { + let plaintext = b"hello-rio-v2-s2-".repeat(32_768); + let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("read compressed data"); + + assert!(compressed.starts_with(MAGIC_CHUNK)); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } + + #[tokio::test] + async fn s2_compress_reader_roundtrip_near_erasure_boundary() { + let size = 4 * 1024 * 1024 - 97; + let plaintext = pseudo_random_bytes(size); + let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("read compressed data"); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } + + fn pseudo_random_bytes(size: usize) -> Vec { + (0..size) + .scan(0x9e37_79b9_7f4a_7c15u64, |state, _| { + *state ^= *state << 7; + *state ^= *state >> 9; + *state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9); + Some((*state >> 32) as u8) + }) + .collect() + } + + #[tokio::test] + async fn s2_compress_reader_roundtrip_large_random() { + let plaintext = pseudo_random_bytes(8 * 1024 * 1024 + 123); + let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("read compressed data"); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } + + #[tokio::test] + async fn s2_compress_reader_roundtrip_with_pending_source() { + let plaintext = pseudo_random_bytes(2 * 1024 * 1024 + 17); + let pending_reader = PendingAfterBytes::new(Cursor::new(plaintext.clone()), 257); + let mut reader = CompressReader::new(pending_reader, CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("read compressed data"); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } + + #[tokio::test] + async fn s2_decompress_reader_returns_bytes_on_first_read() { + let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec(); + let mut compressed = Vec::new(); + CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut compressed) + .await + .expect("compress plaintext"); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut buf = [0u8; 64]; + let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk"); + + assert!(n > 0); + assert_eq!(&buf[..n], plaintext.as_slice()); + } + + #[tokio::test] + async fn s2_decompress_reader_resumes_chunk_body_after_pending() { + let plaintext = pseudo_random_bytes(1024 * 1024 + 123); + let mut compressed = Vec::new(); + CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut compressed) + .await + .expect("compress plaintext"); + + let pending_reader = PendingAfterBytes::new(Cursor::new(compressed), 257); + let mut decompressor = DecompressReader::new(pending_reader, CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } + + #[tokio::test] + async fn s2_decompress_reader_handles_concatenated_streams_after_pending() { + let first = pseudo_random_bytes(16 * 1024 * 1024); + let second = pseudo_random_bytes(12 * 1024 * 1024 + 123); + let mut first_compressed = Vec::new(); + CompressReader::new(Cursor::new(first.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut first_compressed) + .await + .expect("compress first stream"); + let mut second_compressed = Vec::new(); + CompressReader::new(Cursor::new(second.clone()), CompressionAlgorithm::default()) + .read_to_end(&mut second_compressed) + .await + .expect("compress second stream"); + + first_compressed.extend_from_slice(&second_compressed); + + let pending_reader = PendingAfterBytes::new(Cursor::new(first_compressed), 4096); + let mut decompressor = DecompressReader::new(pending_reader, CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + let mut expected = first; + expected.extend_from_slice(&second); + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn s2_compress_reader_with_encrypted_padding_emits_padding_frame() { + let plaintext = b"encrypted-padding-check-".repeat(8192); + let mut reader = CompressReader::with_encrypted_padding(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("read compressed data"); + + assert_eq!(compressed.len() % ENCRYPTED_PADDING_MULTIPLE, 0); + assert!(s2_chunk_types(&compressed).contains(&CHUNK_TYPE_PADDING)); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } + + #[tokio::test] + async fn s2_compress_reader_skips_index_for_small_streams() { + let plaintext = b"index-threshold-check-".repeat(16_384); + let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default()); + let mut compressed = Vec::new(); + reader.read_to_end(&mut compressed).await.expect("read compressed data"); + + assert!(reader.try_get_index().is_none()); + + let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default()); + let mut actual = Vec::new(); + decompressor.read_to_end(&mut actual).await.expect("read decompressed data"); + + assert_eq!(actual, plaintext); + } +} diff --git a/crates/rio-v2/src/encrypt_reader.rs b/crates/rio-v2/src/encrypt_reader.rs new file mode 100644 index 000000000..fbf568949 --- /dev/null +++ b/crates/rio-v2/src/encrypt_reader.rs @@ -0,0 +1,759 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use aes_gcm::aead::Aead; +use aes_gcm::{Aes256Gcm, KeyInit, Nonce}; +use hmac::{Hmac, Mac}; +use pin_project_lite::pin_project; +use rand::RngExt; +use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex, multipart_part_nonce}; +use sha2::Sha256; +use std::cmp::min; +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::io::{AsyncRead, ReadBuf}; + +const DARE_VERSION_20: u8 = 0x20; +const DARE_CIPHER_AES_256_GCM: u8 = 0x00; +const DARE_HEADER_SIZE: usize = 16; +const DARE_TAG_SIZE: usize = 16; +const DARE_PAYLOAD_SIZE: usize = 64 * 1024; + +type HmacSha256 = Hmac; + +#[derive(Clone, Copy)] +enum MultipartKeySource { + LegacyNonce { base_nonce: [u8; 12] }, + ObjectKey { object_key: [u8; 32] }, +} + +pin_project! { + pub struct EncryptReader { + #[pin] + inner: R, + cipher: Aes256Gcm, + base_nonce: [u8; 12], + sequence_number: u32, + temp_buffer: Vec, + read_buffer: Vec, + output_buffer: Vec, + output_pos: usize, + finished: bool, + } +} + +impl EncryptReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + pub fn new_with_object_key(inner: R, object_key: [u8; 32]) -> Self { + Self::new(inner, object_key, random_stream_nonce()) + } + + pub fn new(inner: R, key: [u8; 32], nonce: [u8; 12]) -> Self { + Self::new_with_sequence(inner, key, nonce, 0) + } + + pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self { + Self { + inner, + cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"), + base_nonce: nonce, + sequence_number, + temp_buffer: Vec::with_capacity(DARE_PAYLOAD_SIZE + 1), + read_buffer: vec![0u8; DARE_PAYLOAD_SIZE + 1], + output_buffer: Vec::new(), + output_pos: 0, + finished: false, + } + } + + pub fn new_multipart(inner: R, key: [u8; 32], base_nonce: [u8; 12], part_number: usize) -> Self { + Self::new(inner, key, multipart_part_nonce(base_nonce, part_number)) + } + + pub fn new_multipart_with_object_key(inner: R, object_key: [u8; 32], part_number: u32) -> Self { + Self::new(inner, derive_part_key(object_key, part_number), random_stream_nonce()) + } + + pub fn new_multipart_with_sequence( + inner: R, + key: [u8; 32], + base_nonce: [u8; 12], + part_number: usize, + sequence_number: u32, + ) -> Self { + Self::new_with_sequence(inner, key, multipart_part_nonce(base_nonce, part_number), sequence_number) + } +} + +impl AsyncRead for EncryptReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let mut this = self.project(); + + if *this.output_pos < this.output_buffer.len() { + let to_copy = min(buf.remaining(), this.output_buffer.len() - *this.output_pos); + buf.put_slice(&this.output_buffer[*this.output_pos..*this.output_pos + to_copy]); + *this.output_pos += to_copy; + if *this.output_pos == this.output_buffer.len() { + this.output_buffer.clear(); + *this.output_pos = 0; + } + return Poll::Ready(Ok(())); + } + + if *this.finished { + return Poll::Ready(Ok(())); + } + + loop { + if this.temp_buffer.len() > DARE_PAYLOAD_SIZE { + let package = build_dare_package( + this.cipher, + *this.sequence_number, + *this.base_nonce, + &this.temp_buffer[..DARE_PAYLOAD_SIZE], + false, + )?; + let carry = this.temp_buffer.split_off(DARE_PAYLOAD_SIZE); + *this.temp_buffer = carry; + *this.sequence_number = this.sequence_number.wrapping_add(1); + *this.output_buffer = package; + break; + } + + let remaining = DARE_PAYLOAD_SIZE + 1 - this.temp_buffer.len(); + let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]); + match this.inner.as_mut().poll_read(cx, &mut read_buf) { + Poll::Pending => { + if this.temp_buffer.len() > DARE_PAYLOAD_SIZE { + continue; + } + return Poll::Pending; + } + Poll::Ready(Ok(())) => { + let n = read_buf.filled().len(); + if n == 0 { + if this.temp_buffer.is_empty() { + *this.finished = true; + return Poll::Ready(Ok(())); + } + + let package = + build_dare_package(this.cipher, *this.sequence_number, *this.base_nonce, this.temp_buffer, true)?; + this.temp_buffer.clear(); + *this.sequence_number = this.sequence_number.wrapping_add(1); + *this.output_buffer = package; + *this.finished = true; + break; + } + this.temp_buffer.extend_from_slice(read_buf.filled()); + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + let to_copy = min(buf.remaining(), this.output_buffer.len()); + buf.put_slice(&this.output_buffer[..to_copy]); + *this.output_pos += to_copy; + if *this.output_pos == this.output_buffer.len() { + this.output_buffer.clear(); + *this.output_pos = 0; + } + + Poll::Ready(Ok(())) + } +} + +impl EtagResolvable for EncryptReader +where + R: EtagResolvable, +{ + fn try_resolve_etag(&mut self) -> Option { + self.inner.try_resolve_etag() + } +} + +impl HashReaderDetector for EncryptReader +where + R: HashReaderDetector, +{ + fn is_hash_reader(&self) -> bool { + self.inner.is_hash_reader() + } + + fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> { + self.inner.as_hash_reader_mut() + } +} + +impl TryGetIndex for EncryptReader +where + R: TryGetIndex, +{ + fn try_get_index(&self) -> Option<&Index> { + self.inner.try_get_index() + } +} + +pin_project! { + pub struct DecryptReader { + #[pin] + inner: R, + cipher: Aes256Gcm, + expected_base_nonce: Option<[u8; 12]>, + sequence_number: u32, + multipart_parts: Vec, + current_part_index: usize, + multipart_mode: bool, + multipart_key_source: Option, + header_buf: [u8; DARE_HEADER_SIZE], + header_read: usize, + ciphertext_buf: Vec, + ciphertext_len: usize, + ciphertext_read: usize, + ref_nonce: Option<[u8; 12]>, + finalized: bool, + finished: bool, + output_buffer: Vec, + output_pos: usize, + } +} + +impl DecryptReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + pub fn new_with_object_key(inner: R, object_key: [u8; 32]) -> Self { + Self::new_with_object_key_and_sequence(inner, object_key, 0) + } + + pub fn new(inner: R, key: [u8; 32], nonce: [u8; 12]) -> Self { + Self::new_with_sequence(inner, key, nonce, 0) + } + + pub fn new_with_object_key_and_sequence(inner: R, object_key: [u8; 32], sequence_number: u32) -> Self { + Self { + inner, + cipher: Aes256Gcm::new_from_slice(&object_key).expect("valid AES-256-GCM key"), + expected_base_nonce: None, + sequence_number, + multipart_parts: Vec::new(), + current_part_index: 0, + multipart_mode: false, + multipart_key_source: None, + header_buf: [0u8; DARE_HEADER_SIZE], + header_read: 0, + ciphertext_buf: Vec::new(), + ciphertext_len: 0, + ciphertext_read: 0, + ref_nonce: None, + finalized: false, + finished: false, + output_buffer: Vec::new(), + output_pos: 0, + } + } + + pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self { + Self { + inner, + cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"), + expected_base_nonce: Some(nonce), + sequence_number, + multipart_parts: Vec::new(), + current_part_index: 0, + multipart_mode: false, + multipart_key_source: None, + header_buf: [0u8; DARE_HEADER_SIZE], + header_read: 0, + ciphertext_buf: Vec::new(), + ciphertext_len: 0, + ciphertext_read: 0, + ref_nonce: None, + finalized: false, + finished: false, + output_buffer: Vec::new(), + output_pos: 0, + } + } + + pub fn new_multipart(inner: R, key: [u8; 32], base_nonce: [u8; 12], multipart_parts: Vec) -> Self { + Self::new_multipart_with_sequence(inner, key, base_nonce, multipart_parts, 0) + } + + pub fn new_multipart_with_object_key(inner: R, object_key: [u8; 32], multipart_parts: Vec) -> Self { + Self::new_multipart_with_object_key_and_sequence(inner, object_key, multipart_parts, 0) + } + + pub fn new_multipart_with_sequence( + inner: R, + key: [u8; 32], + base_nonce: [u8; 12], + multipart_parts: Vec, + sequence_number: u32, + ) -> Self { + let first_part = multipart_parts.first().copied().unwrap_or(1); + Self { + inner, + cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"), + expected_base_nonce: Some(multipart_part_nonce(base_nonce, first_part)), + sequence_number, + multipart_parts, + current_part_index: 0, + multipart_mode: true, + multipart_key_source: Some(MultipartKeySource::LegacyNonce { base_nonce }), + header_buf: [0u8; DARE_HEADER_SIZE], + header_read: 0, + ciphertext_buf: Vec::new(), + ciphertext_len: 0, + ciphertext_read: 0, + ref_nonce: None, + finalized: false, + finished: false, + output_buffer: Vec::new(), + output_pos: 0, + } + } + + pub fn new_multipart_with_object_key_and_sequence( + inner: R, + object_key: [u8; 32], + multipart_parts: Vec, + sequence_number: u32, + ) -> Self { + let first_part = multipart_parts.first().copied().unwrap_or(1); + let first_key = derive_part_key(object_key, first_part as u32); + Self { + inner, + cipher: Aes256Gcm::new_from_slice(&first_key).expect("valid AES-256-GCM key"), + expected_base_nonce: None, + sequence_number, + multipart_parts, + current_part_index: 0, + multipart_mode: true, + multipart_key_source: Some(MultipartKeySource::ObjectKey { object_key }), + header_buf: [0u8; DARE_HEADER_SIZE], + header_read: 0, + ciphertext_buf: Vec::new(), + ciphertext_len: 0, + ciphertext_read: 0, + ref_nonce: None, + finalized: false, + finished: false, + output_buffer: Vec::new(), + output_pos: 0, + } + } +} + +impl AsyncRead for DecryptReader +where + R: AsyncRead + Unpin + Send + Sync, +{ + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let mut this = self.project(); + + if *this.output_pos < this.output_buffer.len() { + let to_copy = min(buf.remaining(), this.output_buffer.len() - *this.output_pos); + buf.put_slice(&this.output_buffer[*this.output_pos..*this.output_pos + to_copy]); + *this.output_pos += to_copy; + if *this.output_pos == this.output_buffer.len() { + this.output_buffer.clear(); + *this.output_pos = 0; + } + return Poll::Ready(Ok(())); + } + + loop { + if *this.finished { + return Poll::Ready(Ok(())); + } + + if *this.finalized { + if !this.multipart_parts.is_empty() && *this.current_part_index + 1 < this.multipart_parts.len() { + let next_part = this.multipart_parts[*this.current_part_index + 1]; + *this.current_part_index += 1; + match this.multipart_key_source.as_ref().copied() { + Some(MultipartKeySource::LegacyNonce { base_nonce }) => { + *this.expected_base_nonce = Some(multipart_part_nonce(base_nonce, next_part)); + } + Some(MultipartKeySource::ObjectKey { object_key }) => { + let part_key = derive_part_key(object_key, next_part as u32); + *this.cipher = Aes256Gcm::new_from_slice(&part_key).expect("valid AES-256-GCM key"); + *this.expected_base_nonce = None; + } + None => {} + } + *this.sequence_number = 0; + *this.ref_nonce = None; + *this.finalized = false; + continue; + } + + *this.finished = true; + return Poll::Ready(Ok(())); + } + + while *this.header_read < DARE_HEADER_SIZE { + let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]); + match this.inner.as_mut().poll_read(cx, &mut read_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => { + let n = read_buf.filled().len(); + if n == 0 { + if *this.header_read == 0 { + *this.finished = true; + return Poll::Ready(Ok(())); + } + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading DARE header", + ))); + } + *this.header_read += n; + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + let header = this.header_buf; + let payload_len = usize::from(u16::from_le_bytes([header[2], header[3]])) + 1; + let package_len = payload_len + DARE_TAG_SIZE; + if payload_len == 0 || payload_len > DARE_PAYLOAD_SIZE { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid DARE payload size"))); + } + if !is_final_header(*header) && payload_len != DARE_PAYLOAD_SIZE { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::InvalidData, + "non-final DARE package must carry a full 64KiB payload", + ))); + } + if this.ciphertext_buf.len() < package_len { + this.ciphertext_buf.resize(package_len, 0); + } + *this.ciphertext_len = package_len; + *this.ciphertext_read = 0; + + while *this.ciphertext_read < *this.ciphertext_len { + let mut read_buf = ReadBuf::new(&mut this.ciphertext_buf[*this.ciphertext_read..*this.ciphertext_len]); + match this.inner.as_mut().poll_read(cx, &mut read_buf) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(())) => { + let n = read_buf.filled().len(); + if n == 0 { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF while reading DARE ciphertext", + ))); + } + *this.ciphertext_read += n; + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + match open_dare_package( + this.cipher, + *this.sequence_number, + *this.expected_base_nonce, + *header, + &this.ciphertext_buf[..*this.ciphertext_len], + this.ref_nonce, + ) { + Ok((plaintext, ref_nonce, finalized)) => { + *this.ref_nonce = Some(ref_nonce); + *this.finalized = finalized; + *this.header_read = 0; + *this.ciphertext_len = 0; + *this.ciphertext_read = 0; + *this.sequence_number = this.sequence_number.wrapping_add(1); + *this.output_buffer = plaintext; + let to_copy = min(buf.remaining(), this.output_buffer.len()); + buf.put_slice(&this.output_buffer[..to_copy]); + *this.output_pos += to_copy; + if *this.output_pos == this.output_buffer.len() { + this.output_buffer.clear(); + *this.output_pos = 0; + } + return Poll::Ready(Ok(())); + } + Err(err) => return Poll::Ready(Err(err)), + } + } + } +} + +impl EtagResolvable for DecryptReader +where + R: EtagResolvable, +{ + fn try_resolve_etag(&mut self) -> Option { + self.inner.try_resolve_etag() + } +} + +impl HashReaderDetector for DecryptReader +where + R: HashReaderDetector, +{ + fn is_hash_reader(&self) -> bool { + self.inner.is_hash_reader() + } + + fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> { + self.inner.as_hash_reader_mut() + } +} + +impl TryGetIndex for DecryptReader +where + R: TryGetIndex, +{ + fn try_get_index(&self) -> Option<&Index> { + self.inner.try_get_index() + } +} + +fn build_dare_package( + cipher: &Aes256Gcm, + sequence_number: u32, + base_nonce: [u8; 12], + plaintext: &[u8], + final_package: bool, +) -> io::Result> { + let mut header = [0u8; DARE_HEADER_SIZE]; + header[0] = DARE_VERSION_20; + header[1] = DARE_CIPHER_AES_256_GCM; + header[2..4].copy_from_slice( + &u16::try_from(plaintext.len() - 1) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "DARE payload too large for Version 2.0 framing"))? + .to_le_bytes(), + ); + header[4..16].copy_from_slice(&base_nonce); + if final_package { + header[4] |= 0x80; + } else { + header[4] &= 0x7F; + } + + let mut package_nonce = header[4..16].try_into().expect("nonce slice"); + xor_sequence_into_nonce(&mut package_nonce, sequence_number); + let nonce = Nonce::try_from(package_nonce.as_slice()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid DARE nonce length"))?; + let ciphertext = cipher + .encrypt( + &nonce, + aes_gcm::aead::Payload { + msg: plaintext, + aad: &header[..4], + }, + ) + .map_err(|err| io::Error::other(format!("failed to encrypt DARE package: {err}")))?; + + let mut package = Vec::with_capacity(DARE_HEADER_SIZE + ciphertext.len()); + package.extend_from_slice(&header); + package.extend_from_slice(&ciphertext); + Ok(package) +} + +fn open_dare_package( + cipher: &Aes256Gcm, + sequence_number: u32, + expected_base_nonce: Option<[u8; 12]>, + header: [u8; DARE_HEADER_SIZE], + ciphertext: &[u8], + ref_nonce: &mut Option<[u8; 12]>, +) -> io::Result<(Vec, [u8; 12], bool)> { + if header[0] != DARE_VERSION_20 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE version")); + } + if header[1] != DARE_CIPHER_AES_256_GCM { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE cipher suite")); + } + let header_nonce: [u8; 12] = header[4..16].try_into().expect("nonce slice"); + if let Some(expected_base_nonce) = expected_base_nonce { + let masked_expected = apply_final_flag(expected_base_nonce, is_final_header(header)); + if header_nonce != masked_expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "DARE package nonce does not match the configured stream nonce", + )); + } + } + + let current_ref = ref_nonce.get_or_insert(header_nonce); + let expected_ref = apply_final_flag(*current_ref, is_final_header(header)); + if header_nonce != expected_ref { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "DARE package nonce does not match the stream reference nonce", + )); + } + + let mut package_nonce = header_nonce; + xor_sequence_into_nonce(&mut package_nonce, sequence_number); + let nonce = Nonce::try_from(package_nonce.as_slice()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid DARE nonce length"))?; + let plaintext = cipher + .decrypt( + &nonce, + aes_gcm::aead::Payload { + msg: ciphertext, + aad: &header[..4], + }, + ) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("DARE authentication failed: {err}")))?; + + Ok((plaintext, *current_ref, is_final_header(header))) +} + +fn xor_sequence_into_nonce(nonce: &mut [u8; 12], sequence_number: u32) { + let last = u32::from_le_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]) ^ sequence_number; + nonce[8..12].copy_from_slice(&last.to_le_bytes()); +} + +fn apply_final_flag(mut nonce: [u8; 12], final_package: bool) -> [u8; 12] { + if final_package { + nonce[0] |= 0x80; + } else { + nonce[0] &= 0x7F; + } + nonce +} + +fn is_final_header(header: [u8; DARE_HEADER_SIZE]) -> bool { + header[4] & 0x80 != 0 +} + +pub fn derive_part_key(object_key: [u8; 32], part_number: u32) -> [u8; 32] { + let mut mac = HmacSha256::new_from_slice(&object_key).expect("HMAC-SHA256 accepts 32-byte object keys"); + mac.update(&part_number.to_le_bytes()); + + let mut part_key = [0u8; 32]; + part_key.copy_from_slice(mac.finalize().into_bytes().as_slice()); + part_key +} + +fn random_stream_nonce() -> [u8; 12] { + let mut nonce = [0u8; 12]; + rand::rng().fill(&mut nonce); + nonce +} + +#[cfg(test)] +mod tests { + use super::*; + use hex::encode as hex_encode; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE; + + #[tokio::test] + async fn decrypt_reader_can_start_from_non_zero_sequence_number() { + let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19]; + let key_bytes = [0x44u8; 32]; + let base_nonce = [0x66u8; 12]; + + let mut encrypted = Vec::new(); + EncryptReader::new(Cursor::new(plaintext.clone()), key_bytes, base_nonce) + .read_to_end(&mut encrypted) + .await + .expect("encrypt plaintext"); + + let tail = encrypted[DARE_PACKAGE_SIZE..].to_vec(); + let mut decrypted = Vec::new(); + DecryptReader::new_with_sequence(Cursor::new(tail), key_bytes, base_nonce, 1) + .read_to_end(&mut decrypted) + .await + .expect("decrypt tail packages with non-zero sequence"); + + assert_eq!(decrypted, plaintext[DARE_PAYLOAD_SIZE..]); + } + + #[tokio::test] + async fn singlepart_object_key_roundtrip_uses_header_nonce() { + let object_key = [0x51u8; 32]; + let plaintext = vec![0xAC; DARE_PAYLOAD_SIZE + 17]; + + let mut encrypted = Vec::new(); + EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) + .read_to_end(&mut encrypted) + .await + .expect("encrypt object-key singlepart stream"); + + let mut decrypted = Vec::new(); + DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key) + .read_to_end(&mut decrypted) + .await + .expect("decrypt object-key singlepart stream"); + + assert_eq!(decrypted, plaintext); + } + + #[test] + fn derive_part_key_matches_minio_test_vectors() { + assert_eq!( + hex_encode(derive_part_key([0u8; 32], 0)), + "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af" + ); + assert_eq!( + hex_encode(derive_part_key([0u8; 32], 1)), + "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8" + ); + assert_eq!( + hex_encode(derive_part_key([0u8; 32], 10_000)), + "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115" + ); + } + + #[tokio::test] + async fn multipart_object_key_roundtrip_resets_sequence_per_part() { + let object_key = [0x19u8; 32]; + let part_one_plaintext = vec![0xA1; DARE_PAYLOAD_SIZE + 31]; + let part_two_plaintext = vec![0xB2; DARE_PAYLOAD_SIZE * 2 + 7]; + + let mut encrypted_one = Vec::new(); + EncryptReader::new_multipart_with_object_key(Cursor::new(part_one_plaintext.clone()), object_key, 1) + .read_to_end(&mut encrypted_one) + .await + .expect("encrypt multipart part one with object key"); + + let mut encrypted_two = Vec::new(); + EncryptReader::new_multipart_with_object_key(Cursor::new(part_two_plaintext.clone()), object_key, 2) + .read_to_end(&mut encrypted_two) + .await + .expect("encrypt multipart part two with object key"); + + let mut encrypted = encrypted_one.clone(); + encrypted.extend_from_slice(&encrypted_two); + + let mut decrypted = Vec::new(); + DecryptReader::new_multipart_with_object_key(Cursor::new(encrypted), object_key, vec![1, 2]) + .read_to_end(&mut decrypted) + .await + .expect("decrypt multipart object-key stream"); + + let mut expected = part_one_plaintext; + expected.extend_from_slice(&part_two_plaintext); + + assert_eq!(decrypted, expected); + assert_ne!(encrypted_one, encrypted_two[..encrypted_one.len()]); + } +} diff --git a/crates/rio-v2/src/lib.rs b/crates/rio-v2/src/lib.rs new file mode 100644 index 000000000..e2bb25325 --- /dev/null +++ b/crates/rio-v2/src/lib.rs @@ -0,0 +1,91 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! rio_v2 selectively replaces legacy rio components while keeping the +//! remaining API surface stable for feature-gated integration. + +mod compress_reader; +mod encrypt_reader; +mod s2_index; + +pub use compress_reader::{CompressReader, DecompressReader}; +pub use encrypt_reader::{DecryptReader, EncryptReader, derive_part_key}; +pub use rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE; +pub use rustfs_rio::DynReader; +pub use rustfs_rio::EtagReader; +pub use rustfs_rio::EtagResolvable; +pub use rustfs_rio::HardLimitReader; +pub use rustfs_rio::HashReader; +pub use rustfs_rio::HashReaderDetector; +pub use rustfs_rio::HashReaderMut; +pub use rustfs_rio::Index; +pub use rustfs_rio::LimitReader; +pub use rustfs_rio::ReadStream; +pub use rustfs_rio::Reader; +pub use rustfs_rio::ReaderCapabilities; +pub use rustfs_rio::TryGetIndex; +pub use rustfs_rio::WarpReader; +pub use rustfs_rio::boxed_reader; +pub use rustfs_rio::read_checksums; +pub use rustfs_rio::resolve_etag_generic; +pub use rustfs_rio::wrap_reader; +pub use s2_index::{decode_minio_index_bytes, minio_index_storage_bytes}; + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + use tokio::io::AsyncReadExt; + + const DARE_VERSION_20: u8 = 0x20; + const DARE_HEADER_SIZE: usize = 16; + const DARE_TAG_SIZE: usize = 16; + const DARE_PAYLOAD_SIZE: usize = 64 * 1024; + const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE; + + #[tokio::test] + async fn encrypt_reader_emits_dare_v2_packages() { + let plaintext = vec![0x5Au8; DARE_PAYLOAD_SIZE + 17]; + let key_bytes = [0x11u8; 32]; + let base_nonce = [0x22u8; 12]; + + let mut encrypted = Vec::new(); + EncryptReader::new(Cursor::new(plaintext), key_bytes, base_nonce) + .read_to_end(&mut encrypted) + .await + .expect("encrypt plaintext into rio_v2 stream"); + + assert!( + encrypted.len() > DARE_PACKAGE_SIZE, + "expected at least one full DARE package and one final package" + ); + assert_eq!(encrypted[0], DARE_VERSION_20, "rio_v2 encrypted streams must start with a DARE V2 header"); + assert_eq!( + &encrypted[4..16], + &base_nonce, + "rio_v2 should preserve the configured nonce in the first DARE header" + ); + + let second_header_offset = DARE_PACKAGE_SIZE; + assert_eq!( + encrypted[second_header_offset], DARE_VERSION_20, + "rio_v2 should emit subsequent DARE V2 package headers at 64KiB boundaries" + ); + assert_ne!( + encrypted[second_header_offset + 4] & 0x80, + 0, + "the final DARE package must set the final flag" + ); + } +} diff --git a/crates/rio-v2/src/s2_index.rs b/crates/rio-v2/src/s2_index.rs new file mode 100644 index 000000000..fc3c0bad3 --- /dev/null +++ b/crates/rio-v2/src/s2_index.rs @@ -0,0 +1,434 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use rustfs_rio::Index; +use serde::Deserialize; +use std::io; + +const S2_INDEX_HEADER: &[u8] = b"s2idx\x00"; +const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s"; +const CHUNK_TYPE_INDEX: u8 = 0x99; +const SKIPPABLE_FRAME_HEADER: usize = 4; +const MAX_INDEX_ENTRIES: usize = 1 << 16; + +#[derive(Debug, Deserialize)] +struct LegacyIndexJson { + total_uncompressed: i64, + total_compressed: i64, + offsets: Vec, + est_block_uncompressed: i64, +} + +#[derive(Debug, Deserialize)] +struct LegacyIndexOffset { + compressed: i64, + uncompressed: i64, +} + +#[derive(Debug, Clone)] +struct S2IndexInfo { + compressed_offset: i64, + uncompressed_offset: i64, +} + +#[derive(Debug, Clone)] +struct S2Index { + total_uncompressed: i64, + total_compressed: i64, + est_block_uncompressed: i64, + info: Vec, +} + +pub fn minio_index_storage_bytes(index: &Index) -> Bytes { + let decoded = legacy_index_to_s2_index(index).unwrap_or_else(|_| S2Index { + total_uncompressed: index.total_uncompressed, + total_compressed: index.total_compressed, + est_block_uncompressed: 0, + info: Vec::new(), + }); + + let encoded = decoded.into_full_bytes(); + remove_index_headers(encoded.as_ref()) + .map(Bytes::copy_from_slice) + .unwrap_or(encoded) +} + +pub fn decode_minio_index_bytes(bytes: &Bytes) -> Option { + let decoded = S2Index::load(bytes.as_ref()) + .or_else(|_| S2Index::load(&restore_index_headers(bytes.as_ref()))) + .ok()?; + + let mut index = Index::new(); + for info in decoded.info { + index.add(info.compressed_offset, info.uncompressed_offset).ok()?; + } + index.total_uncompressed = decoded.total_uncompressed; + index.total_compressed = decoded.total_compressed; + Some(index) +} + +fn legacy_index_to_s2_index(index: &Index) -> io::Result { + let json = index + .to_json() + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + let decoded: LegacyIndexJson = + serde_json::from_slice(&json).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + + Ok(S2Index { + total_uncompressed: decoded.total_uncompressed, + total_compressed: decoded.total_compressed, + est_block_uncompressed: decoded.est_block_uncompressed, + info: decoded + .offsets + .into_iter() + .map(|offset| S2IndexInfo { + compressed_offset: offset.compressed, + uncompressed_offset: offset.uncompressed, + }) + .collect(), + }) +} + +impl S2Index { + fn into_full_bytes(self) -> Bytes { + let mut out = Vec::new(); + out.extend_from_slice(&[CHUNK_TYPE_INDEX, 0, 0, 0]); + out.extend_from_slice(S2_INDEX_HEADER); + + write_varint(&mut out, self.total_uncompressed); + write_varint(&mut out, self.total_compressed); + write_varint(&mut out, self.est_block_uncompressed); + write_varint(&mut out, self.info.len() as i64); + + let has_uncompressed = self.has_explicit_uncompressed_offsets(); + out.push(u8::from(has_uncompressed)); + + if has_uncompressed { + for (idx, info) in self.info.iter().enumerate() { + let mut offset = info.uncompressed_offset; + if idx > 0 { + let prev = &self.info[idx - 1]; + offset -= prev.uncompressed_offset + self.est_block_uncompressed; + } + write_varint(&mut out, offset); + } + } + + let mut compressed_predict = self.est_block_uncompressed / 2; + for (idx, info) in self.info.iter().enumerate() { + let mut offset = info.compressed_offset; + if idx > 0 { + let prev = &self.info[idx - 1]; + offset -= prev.compressed_offset + compressed_predict; + compressed_predict += offset / 2; + } + write_varint(&mut out, offset); + } + + let mut total_size = [0u8; 4]; + total_size.copy_from_slice(&((out.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes()); + out.extend_from_slice(&total_size); + out.extend_from_slice(S2_INDEX_TRAILER); + + let chunk_len = out.len() - SKIPPABLE_FRAME_HEADER; + out[1] = chunk_len as u8; + out[2] = (chunk_len >> 8) as u8; + out[3] = (chunk_len >> 16) as u8; + + Bytes::from(out) + } + + fn has_explicit_uncompressed_offsets(&self) -> bool { + for (idx, info) in self.info.iter().enumerate() { + if idx == 0 { + if info.uncompressed_offset != 0 { + return true; + } + continue; + } + if info.uncompressed_offset != self.info[idx - 1].uncompressed_offset + self.est_block_uncompressed { + return true; + } + } + false + } + + fn load(mut bytes: &[u8]) -> io::Result { + if bytes.len() <= SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small")); + } + if bytes[0] != CHUNK_TYPE_INDEX { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index chunk type")); + } + + let chunk_len = (bytes[1] as usize) | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16); + bytes = &bytes[SKIPPABLE_FRAME_HEADER..]; + if bytes.len() < chunk_len { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small")); + } + bytes = &bytes[..chunk_len]; + + if !bytes.starts_with(S2_INDEX_HEADER) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index header")); + } + bytes = &bytes[S2_INDEX_HEADER.len()..]; + + let (total_uncompressed, used) = read_varint(bytes)?; + if total_uncompressed < 0 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed size")); + } + bytes = &bytes[used..]; + + let (total_compressed, used) = read_varint(bytes)?; + bytes = &bytes[used..]; + + let (est_block_uncompressed, used) = read_varint(bytes)?; + if est_block_uncompressed < 0 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid block size")); + } + bytes = &bytes[used..]; + + let (entries, used) = read_varint(bytes)?; + if entries < 0 || entries > MAX_INDEX_ENTRIES as i64 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid number of entries")); + } + bytes = &bytes[used..]; + + if bytes.is_empty() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small")); + } + + let has_uncompressed = bytes[0]; + if has_uncompressed & 1 != has_uncompressed { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed flag")); + } + bytes = &bytes[1..]; + + let mut info = vec![ + S2IndexInfo { + compressed_offset: 0, + uncompressed_offset: 0, + }; + entries as usize + ]; + + for idx in 0..info.len() { + let mut uncompressed_offset = 0_i64; + if has_uncompressed != 0 { + let (value, used) = read_varint(bytes)?; + uncompressed_offset = value; + bytes = &bytes[used..]; + } + + if idx > 0 { + let prev = info[idx - 1].uncompressed_offset; + uncompressed_offset += prev + est_block_uncompressed; + if uncompressed_offset <= prev { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed offset")); + } + } + if uncompressed_offset < 0 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "negative uncompressed offset")); + } + info[idx].uncompressed_offset = uncompressed_offset; + } + + let mut compressed_predict = est_block_uncompressed / 2; + for idx in 0..info.len() { + let (mut compressed_offset, used) = read_varint(bytes)?; + bytes = &bytes[used..]; + + if idx > 0 { + let next_compressed_predict = compressed_predict + compressed_offset / 2; + let prev = info[idx - 1].compressed_offset; + compressed_offset += prev + compressed_predict; + if compressed_offset <= prev { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressed offset")); + } + compressed_predict = next_compressed_predict; + } + if compressed_offset < 0 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "negative compressed offset")); + } + info[idx].compressed_offset = compressed_offset; + } + + if bytes.len() < 4 + S2_INDEX_TRAILER.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small")); + } + bytes = &bytes[4..]; + if !bytes.starts_with(S2_INDEX_TRAILER) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index trailer")); + } + + Ok(Self { + total_uncompressed, + total_compressed, + est_block_uncompressed, + info, + }) + } +} + +fn remove_index_headers(bytes: &[u8]) -> Option<&[u8]> { + let save = SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() + 4; + if bytes.len() <= save || bytes[0] != CHUNK_TYPE_INDEX { + return None; + } + + let chunk_len = (bytes[1] as usize) | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16); + let bytes = &bytes[SKIPPABLE_FRAME_HEADER..]; + if bytes.len() < chunk_len { + return None; + } + let bytes = &bytes[..chunk_len]; + + let bytes = bytes.strip_prefix(S2_INDEX_HEADER)?; + let bytes = bytes.strip_suffix(S2_INDEX_TRAILER)?; + if bytes.len() < 4 { + return None; + } + Some(&bytes[..bytes.len() - 4]) +} + +fn restore_index_headers(input: &[u8]) -> Vec { + if input.is_empty() { + return Vec::new(); + } + + let mut bytes = Vec::with_capacity(SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + input.len() + 4 + S2_INDEX_TRAILER.len()); + bytes.extend_from_slice(&[CHUNK_TYPE_INDEX, 0, 0, 0]); + bytes.extend_from_slice(S2_INDEX_HEADER); + bytes.extend_from_slice(input); + bytes.extend_from_slice(&((bytes.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes()); + bytes.extend_from_slice(S2_INDEX_TRAILER); + + let chunk_len = bytes.len() - SKIPPABLE_FRAME_HEADER; + bytes[1] = chunk_len as u8; + bytes[2] = (chunk_len >> 8) as u8; + bytes[3] = (chunk_len >> 16) as u8; + bytes +} + +fn write_varint(out: &mut Vec, value: i64) { + let mut unsigned = ((value as u64) << 1) ^ ((value >> 63) as u64); + while unsigned >= 0x80 { + out.push((unsigned as u8) | 0x80); + unsigned >>= 7; + } + out.push(unsigned as u8); +} + +fn read_varint(bytes: &[u8]) -> io::Result<(i64, usize)> { + let (unsigned, used) = read_uvarint(bytes)?; + let value = ((unsigned >> 1) as i64) ^ (-((unsigned & 1) as i64)); + Ok((value, used)) +} + +fn read_uvarint(bytes: &[u8]) -> io::Result<(u64, usize)> { + let mut value = 0_u64; + let mut shift = 0_u32; + + for (idx, byte) in bytes.iter().copied().enumerate() { + if byte < 0x80 { + if idx > 9 || (idx == 9 && byte > 1) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "varint overflow")); + } + return Ok((value | ((byte as u64) << shift), idx + 1)); + } + + value |= ((byte & 0x7f) as u64) << shift; + shift += 7; + } + + Err(io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected EOF while reading varint")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn signed_varint_matches_go_binary_varint_examples() { + let cases = [ + (0, vec![0x00]), + (-1, vec![0x01]), + (1, vec![0x02]), + (-2, vec![0x03]), + (64, vec![0x80, 0x01]), + (-64, vec![0x7f]), + ]; + + for (value, expected) in cases { + let mut encoded = Vec::new(); + write_varint(&mut encoded, value); + assert_eq!(encoded, expected); + assert_eq!(read_varint(&encoded).unwrap(), (value, encoded.len())); + } + } + + #[test] + fn minio_storage_bytes_round_trip_through_headerless_form() { + let mut source = Index::new(); + source.add(0, 0).unwrap(); + source.add(1_048_576, 2_097_152).unwrap(); + source.total_uncompressed = 4_194_304; + source.total_compressed = 3_000_000; + + let stored = minio_index_storage_bytes(&source); + assert!(!stored.starts_with(&[CHUNK_TYPE_INDEX, 0x2a, 0x4d, 0x18])); + assert_eq!( + S2Index::load(&restore_index_headers(&stored)) + .expect("restore full index") + .info + .len(), + 2 + ); + + let decoded = decode_minio_index_bytes(&stored).expect("decode headerless MinIO index"); + assert_eq!(decoded.total_uncompressed, source.total_uncompressed); + assert_eq!(decoded.total_compressed, source.total_compressed); + assert_eq!(decoded.find(2_097_152).unwrap(), (1_048_576, 2_097_152)); + } + + #[test] + fn minio_index_allows_unknown_total_compressed_size() { + let index = S2Index { + total_uncompressed: 4_194_304, + total_compressed: -1, + est_block_uncompressed: 0, + info: vec![ + S2IndexInfo { + compressed_offset: 0, + uncompressed_offset: 0, + }, + S2IndexInfo { + compressed_offset: 1_048_576, + uncompressed_offset: 2_097_152, + }, + ], + }; + let full = index.into_full_bytes(); + assert_eq!(full[0], CHUNK_TYPE_INDEX); + let headerless = Bytes::copy_from_slice(remove_index_headers(full.as_ref()).expect("strip index headers")); + let restored = restore_index_headers(&headerless); + assert_eq!(restored[0], CHUNK_TYPE_INDEX); + + let decoded = decode_minio_index_bytes(&headerless).expect("decode index with unknown compressed total"); + assert_eq!(decoded.total_uncompressed, 4_194_304); + assert_eq!(decoded.total_compressed, -1); + assert_eq!(decoded.find(2_097_152).unwrap(), (1_048_576, 2_097_152)); + } +} diff --git a/crates/rio-v2/tests/README.md b/crates/rio-v2/tests/README.md new file mode 100644 index 000000000..7bbc39599 --- /dev/null +++ b/crates/rio-v2/tests/README.md @@ -0,0 +1,55 @@ +# rio-v2 Fixture Tests + +This directory holds interoperability tests for `rustfs-rio-v2`. + +## Generated MinIO fixtures + +The integration test `minio_generated_fixtures.rs` reads raw MinIO backend data generated by +`crates/rio-v2/tests/minio_fixture_lab/lab.py`. + +Default fixture root: + +```text +crates/rio-v2/tests/fixtures/minio-generated +``` + +The generated MinIO fixture data is intentionally not checked in and these tests are +ignored by default, so they are not part of automated CI runs. + +To generate or expand fixtures locally, use: + +```powershell +uv run python .\minio_fixture_lab\lab.py capture-matrix ` + --root .\rustfs\crates\rio-v2\tests\fixtures\minio-generated ` + --minio-binary .\rustfs\tmp\minio.darwin-arm64.RELEASE.2025-09-07T16-13-09Z ` + --endpoint https://127.0.0.1:19000 ` + --disk-count 1 +``` + +Or point the tests at another generated root: + +```powershell +$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-smoke' +cargo +1.95.0 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored +``` + +## Scope + +The current fixture test covers the full default capture matrix: + +- `sse-s3-singlepart-64k` +- `sse-s3-multipart-8m` +- `sse-kms-singlepart-64k` +- `sse-kms-multipart-8m` +- `sse-c-singlepart-64k` +- `sse-c-multipart-8m` + +It checks that: + +- raw `xl.meta` can be parsed through `rustfs-filemeta` +- singlepart and multipart fixture structures are recognized +- SSE-S3, SSE-KMS, and SSE-C metadata markers survive capture +- KMS key ids are derived from each fixture's own `manifest.json`, so local static-KMS runs are not tied to one hard-coded key name +- SSE-C `HEAD` responses round-trip the expected customer algorithm and customer-key MD5 + +These tests do not yet validate full plaintext reconstruction from MinIO-written encrypted data. diff --git a/crates/rio-v2/tests/fixtures/.gitignore b/crates/rio-v2/tests/fixtures/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/crates/rio-v2/tests/fixtures/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/crates/rio-v2/tests/minio_fixture_lab/README.md b/crates/rio-v2/tests/minio_fixture_lab/README.md new file mode 100644 index 000000000..4f5f96243 --- /dev/null +++ b/crates/rio-v2/tests/minio_fixture_lab/README.md @@ -0,0 +1,150 @@ +# MinIO Fixture Lab + +This lab captures real MinIO backend artifacts into a repeatable local layout for RustFS compatibility tests. + +It now supports two workflows: + +- manual capture from an already exported backend tree +- automated capture from a disposable local MinIO run + +## Scope + +Use the manual path after you already have: + +- a running MinIO instance +- one or more uploaded objects you want to preserve as fixtures +- the backend object tree or object-version directory you want to export + +Use the automated path when you want the lab to: + +- locate a local `minio` binary +- start a disposable MinIO instance +- upload a predefined SSE fixture case +- export the generated backend tree into the lab layout + +## Layout + +The default root is `artifacts/minio-fixture-lab`, which is already ignored by the repository. + +Each case is stored under: + +```text +artifacts/minio-fixture-lab/ + cases/ + / + backend/ + request.json + head.json + plaintext.sha256 + manifest.json +``` + +`manifest.json` is the source of truth for the captured case. + +## Commands + +Initialize the lab root: + +```powershell +uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py init +``` + +Capture one case from an existing MinIO backend tree: + +```powershell +uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py add-case ` + --case-id sse-kms-singlepart-64k ` + --bucket demo ` + --object dir/object.bin ` + --source-tree D:\minio-data-export\case-tree ` + --head-json D:\minio-data-export\head.json ` + --request-json D:\minio-data-export\request.json ` + --plaintext-sha256 D:\minio-data-export\plaintext.sha256 +``` + +Capture the default automated matrix: + +```powershell +uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix ` + --root D:\Github\rustfs\artifacts\minio-fixture-lab ` + --minio-binary D:\go\bin\minio.exe ` + --endpoint https://127.0.0.1:19000 +``` + +Capture only one automated case: + +```powershell +uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix ` + --root D:\Github\rustfs\artifacts\minio-fixture-lab ` + --minio-binary D:\Github\rustfs\tmp\minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe ` + --endpoint https://127.0.0.1:19000 ` + --case-id sse-s3-singlepart-64k +``` + +The automated default matrix is intentionally small: + +- `sse-s3-singlepart-64k` +- `sse-kms-singlepart-64k` +- `sse-c-singlepart-64k` +- `sse-s3-multipart-8m` +- `sse-kms-multipart-8m` +- `sse-c-multipart-8m` + +`64 KiB multipart` is intentionally excluded because S3 multipart semantics require a larger non-final part size. + +## Automated Runner Prerequisites + +The automated runner expects: + +- either `minio` in `PATH`, the bundled `D:\Github\rustfs\tmp\minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe`, `--minio-binary`, or `--minio-root` pointing at a directory containing `minio.exe` +- a free local endpoint port + +When the selected matrix includes SSE-C cases, use an `https://` endpoint. The lab will mint a short-lived local self-signed certificate and use its built-in SigV4 S3 client with certificate verification disabled for the disposable local MinIO run. + +The runner provisions backend directories under `--work-root` and exports the resulting backend tree into the lab. + +For local static-KMS runs, pass `--kms-secret-key` or set +`MINIO_FIXTURE_LAB_KMS_SECRET_KEY` using MinIO's +`:` format. The runner derives the SSE-KMS request +key id from that configured key name automatically. + +On some Windows MinIO builds, a multi-disk backend may not come online when all disk directories live on the same volume. If you only want a local smoke run of the upload/export pipeline, try: + +```powershell +uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix ` + --root D:\Github\rustfs\artifacts\minio-fixture-lab ` + --minio-binary D:\go\bin\minio.exe ` + --endpoint https://127.0.0.1:19000 ` + --disk-count 1 ` + --case-id sse-s3-singlepart-64k +``` + +That is a runner smoke path only. Real compatibility fixtures should still prefer the intended multi-disk backend layout when the local environment can support it. + +## Capture Guidance + +For each case, preserve these inputs when possible: + +- the exact backend tree that contains `xl.meta` and part files +- the request shape used to create the object +- the API metadata returned by `HEAD Object` +- the plaintext digest used for byte-for-byte read verification + +Recommended early matrix: + +- singlepart SSE-S3 +- singlepart SSE-KMS +- multipart SSE-S3 +- multipart SSE-KMS +- compressed + encrypted +- range-sensitive sizes around `64 KiB` and `8 MiB` + +## Next Stage + +The current runner covers launch, upload, HEAD capture, and backend export. + +The next iteration should focus on: + +- proving the multi-disk backend path in the target local environment +- adding compressed fixtures to the automated matrix +- tightening exported tree selection if a narrower object-level slice becomes practical diff --git a/crates/rio-v2/tests/minio_fixture_lab/lab.py b/crates/rio-v2/tests/minio_fixture_lab/lab.py new file mode 100644 index 000000000..ab5d56ca4 --- /dev/null +++ b/crates/rio-v2/tests/minio_fixture_lab/lab.py @@ -0,0 +1,963 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import base64 +import hashlib +import hmac +import ipaddress +import json +import os +import shutil +import ssl +import subprocess +import sys +import time +import urllib.parse +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +DEFAULT_ROOT = Path("artifacts/minio-fixture-lab") +DEFAULT_WORK_ROOT = DEFAULT_ROOT / "_runner" +DEFAULT_MINIO_BINARY = ( + Path(__file__).resolve().parents[4] + / "tmp" + / "minio.darwin-arm64.RELEASE.2025-09-07T16-13-09Z" +) +DEFAULT_KMS_KEY_ID = "minio-default-key" +DEFAULT_KMS_SECRET_KEY = ( + "minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=" +) +LAB_KMS_SECRET_KEY_ENV = "MINIO_FIXTURE_LAB_KMS_SECRET_KEY" +DEFAULT_ENDPOINT = "http://127.0.0.1:9000" +DEFAULT_DISK_COUNT = 4 +DEFAULT_ACCESS_KEY = "minioadmin" +DEFAULT_SECRET_KEY = "minioadmin" +DEFAULT_REGION = "us-east-1" +MULTIPART_CHUNK_SIZE = 5 * 1024 * 1024 +DEFAULT_BUCKET = "demo" +DEFAULT_OBJECT = "dir/object.bin" +DEFAULT_SSE_C_KEY_BYTES = bytes(range(32)) +DEFAULT_SSE_C_KEY_B64 = base64.b64encode(DEFAULT_SSE_C_KEY_BYTES).decode("ascii") +DEFAULT_SSE_C_KEY_MD5_B64 = base64.b64encode(hashlib.md5(DEFAULT_SSE_C_KEY_BYTES).digest()).decode("ascii") + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def ensure_dir(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def copy_optional_file(source: Path | None, destination: Path) -> str | None: + if source is None: + return None + if not source.is_file(): + raise FileNotFoundError(f"expected file: {source}") + shutil.copy2(source, destination) + return destination.name + + +def copy_tree(source: Path, destination: Path) -> list[str]: + if not source.exists(): + raise FileNotFoundError(f"source tree does not exist: {source}") + if source.is_file(): + raise ValueError(f"source tree must be a directory: {source}") + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(source, destination) + return sorted( + str(path.relative_to(destination)).replace("\\", "/") + for path in destination.rglob("*") + if path.is_file() + ) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def read_optional_json(path: Path | None) -> dict[str, Any] | None: + if path is None: + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def format_context_header(context: dict[str, str] | None) -> str | None: + if not context: + return None + json_bytes = json.dumps(context, sort_keys=True, separators=(",", ":")).encode("utf-8") + return base64.b64encode(json_bytes).decode("ascii") + + +def format_size_label(size_bytes: int) -> str: + if size_bytes == 64 * 1024: + return "64k" + if size_bytes == 8 * 1024 * 1024: + return "8m" + return f"{size_bytes}b" + + +@dataclass +class LabPaths: + root: Path + cases: Path + + +@dataclass +class MinioLauncher: + kind: str + command: list[str] + workdir: Path + + +@dataclass +class FixtureCase: + case_id: str + bucket: str + object_name: str + encryption: str + size_bytes: int + multipart: bool + kms_key_id: str | None = None + kms_context: dict[str, str] | None = None + + +@dataclass(frozen=True) +class KmsSecretKeyConfig: + secret_key: str + key_id: str + + +def resolve_lab_paths(root: Path) -> LabPaths: + root = root.resolve() + return LabPaths(root=root, cases=root / "cases") + + +def resolve_existing_command(name: str) -> Path: + resolved = shutil.which(name) + if resolved is None: + raise FileNotFoundError(f"required command not found in PATH: {name}") + return Path(resolved).resolve() + + +def discover_minio_launcher( + explicit_binary: Path | None, minio_root: Path | None +) -> MinioLauncher: + if explicit_binary is not None: + binary = explicit_binary.resolve() + if not binary.is_file(): + raise FileNotFoundError(f"minio binary not found: {binary}") + return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent) + + resolved_root = minio_root.resolve() if minio_root is not None else None + if resolved_root is not None: + candidate_binary = resolved_root / "minio.exe" + if candidate_binary.is_file(): + return MinioLauncher( + kind="binary", + command=[str(candidate_binary.resolve())], + workdir=candidate_binary.resolve().parent, + ) + + if DEFAULT_MINIO_BINARY.is_file(): + binary = DEFAULT_MINIO_BINARY.resolve() + return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent) + + path_binary = shutil.which("minio") + if path_binary is not None: + binary = Path(path_binary).resolve() + return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent) + + raise FileNotFoundError( + "unable to locate MinIO. Provide --minio-binary, point --minio-root at a directory " + "containing minio.exe, or ensure minio is in PATH." + ) + + +def parse_kms_secret_key(secret_key: str) -> KmsSecretKeyConfig: + key_id, separator, raw_key = secret_key.partition(":") + if not separator or not key_id or not raw_key: + raise ValueError( + "kms secret key must be in the form ':'" + ) + return KmsSecretKeyConfig(secret_key=secret_key, key_id=key_id) + + +def resolve_kms_secret_key(explicit_secret_key: str | None) -> KmsSecretKeyConfig: + secret_key = ( + explicit_secret_key + or os.environ.get(LAB_KMS_SECRET_KEY_ENV) + or os.environ.get("MINIO_KMS_SECRET_KEY") + or DEFAULT_KMS_SECRET_KEY + ) + return parse_kms_secret_key(secret_key) + + +def build_default_cases(kms_key_id: str = DEFAULT_KMS_KEY_ID) -> list[FixtureCase]: + kms_context = {"project": "rio-v2", "stage": "fixture"} + cases = [] + for encryption in ("SSE-S3", "SSE-KMS", "SSE-C"): + for multipart, size_bytes in ((False, 64 * 1024), (True, 8 * 1024 * 1024)): + shape = "multipart" if multipart else "singlepart" + case = FixtureCase( + case_id=f"{encryption.lower()}-{shape}-{format_size_label(size_bytes)}", + bucket=DEFAULT_BUCKET, + object_name=DEFAULT_OBJECT, + encryption=encryption, + size_bytes=size_bytes, + multipart=multipart, + kms_key_id=kms_key_id if encryption == "SSE-KMS" else None, + kms_context=kms_context if encryption == "SSE-KMS" else None, + ) + cases.append(case) + return cases + + +def build_request_record(case: FixtureCase) -> dict[str, Any]: + headers: dict[str, str] = {} + if case.encryption == "SSE-S3": + headers["x-amz-server-side-encryption"] = "AES256" + elif case.encryption == "SSE-KMS": + headers["x-amz-server-side-encryption"] = "aws:kms" + if case.kms_key_id is not None: + headers["x-amz-server-side-encryption-aws-kms-key-id"] = case.kms_key_id + context_header = format_context_header(case.kms_context) + if context_header is not None: + headers["x-amz-server-side-encryption-context"] = context_header + elif case.encryption == "SSE-C": + headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256" + headers["x-amz-server-side-encryption-customer-key"] = DEFAULT_SSE_C_KEY_B64 + headers["x-amz-server-side-encryption-customer-key-md5"] = DEFAULT_SSE_C_KEY_MD5_B64 + else: + raise ValueError(f"unsupported encryption mode: {case.encryption}") + + record: dict[str, Any] = { + "case_id": case.case_id, + "bucket": case.bucket, + "object": case.object_name, + "encryption": case.encryption, + "multipart": case.multipart, + "size_bytes": case.size_bytes, + "headers": headers, + } + if case.multipart: + record["multipart_chunk_size_bytes"] = MULTIPART_CHUNK_SIZE + return record + + +def cmd_init(args: argparse.Namespace) -> int: + paths = resolve_lab_paths(args.root) + ensure_layout(paths) + print(str(paths.root)) + return 0 + + +def ensure_layout(paths: LabPaths) -> None: + ensure_dir(paths.root) + ensure_dir(paths.cases) + write_json( + paths.root / "layout.json", + { + "schema_version": 1, + "created_at_utc": utc_now(), + "root": ".", + "description": "Captured MinIO fixture cases for RustFS compatibility validation.", + }, + ) + + +def store_case_artifacts( + *, + paths: LabPaths, + case_id: str, + bucket: str, + object_name: str, + source_tree: Path, + version_id: str | None, + request_payload: dict[str, Any] | None, + head_payload: dict[str, Any] | None, + plaintext_sha256: str | None, + notes: str | None, + capture_payload: dict[str, Any] | None = None, +) -> Path: + ensure_dir(paths.cases) + case_dir = paths.cases / case_id + backend_dir = case_dir / "backend" + if case_dir.exists(): + shutil.rmtree(case_dir) + ensure_dir(case_dir) + + exported_files = copy_tree(source_tree.resolve(), backend_dir) + request_file = None + if request_payload is not None: + write_json(case_dir / "request.json", request_payload) + request_file = "request.json" + + head_file = None + if head_payload is not None: + write_json(case_dir / "head.json", head_payload) + head_file = "head.json" + + plaintext_file = None + if plaintext_sha256 is not None: + (case_dir / "plaintext.sha256").write_text(plaintext_sha256 + "\n", encoding="utf-8") + plaintext_file = "plaintext.sha256" + + manifest = { + "schema_version": 1, + "case_id": case_id, + "created_at_utc": utc_now(), + "bucket": bucket, + "object": object_name, + "version_id": version_id, + "source_tree": str(source_tree).replace("\\", "/"), + "notes": notes, + "artifacts": { + "backend_dir": "backend", + "request_json": request_file, + "head_json": head_file, + "plaintext_sha256": plaintext_file, + }, + "backend_files": exported_files, + } + if capture_payload is not None: + manifest["capture"] = capture_payload + write_json(case_dir / "manifest.json", manifest) + return case_dir + + +def cmd_add_case(args: argparse.Namespace) -> int: + paths = resolve_lab_paths(args.root) + case_dir = store_case_artifacts( + paths=paths, + case_id=args.case_id, + bucket=args.bucket, + object_name=args.object, + source_tree=args.source_tree, + version_id=args.version_id, + request_payload=read_optional_json(args.request_json.resolve() if args.request_json else None), + head_payload=read_optional_json(args.head_json.resolve() if args.head_json else None), + plaintext_sha256=( + args.plaintext_sha256.resolve().read_text(encoding="utf-8").strip() + if args.plaintext_sha256 + else None + ), + notes=args.notes, + ) + print(str(case_dir)) + return 0 + + +def parse_endpoint(endpoint: str) -> tuple[str, int]: + parsed = urlparse(endpoint) + if parsed.scheme not in {"http", "https"}: + raise ValueError(f"unsupported endpoint scheme: {endpoint}") + if parsed.hostname is None or parsed.port is None: + raise ValueError(f"endpoint must include host and port: {endpoint}") + return parsed.hostname, parsed.port + + +def is_https_endpoint(endpoint: str) -> bool: + return urlparse(endpoint).scheme == "https" + + +def ensure_local_tls_certificates(certs_dir: Path, hostname: str) -> None: + public_crt = certs_dir / "public.crt" + private_key = certs_dir / "private.key" + if public_crt.is_file() and private_key.is_file(): + return + + ensure_dir(certs_dir) + openssl = resolve_existing_command("openssl") + san_entries = ["DNS:localhost", "IP:127.0.0.1"] + try: + ipaddress.ip_address(hostname) + san_entries.append(f"IP:{hostname}") + except ValueError: + san_entries.append(f"DNS:{hostname}") + config_path = certs_dir / "openssl.cnf" + config_path.write_text( + "\n".join( + [ + "[req]", + "distinguished_name = req_distinguished_name", + "x509_extensions = v3_req", + "prompt = no", + "[req_distinguished_name]", + f"CN = {hostname}", + "[v3_req]", + f"subjectAltName = {','.join(san_entries)}", + "", + ] + ), + encoding="utf-8", + ) + subprocess.run( + [ + str(openssl), + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "30", + "-keyout", + str(private_key), + "-out", + str(public_crt), + "-config", + str(config_path), + "-extensions", + "v3_req", + ], + check=True, + capture_output=True, + text=True, + ) + + +def build_server_command( + launcher: MinioLauncher, endpoint: str, disk_paths: list[Path], certs_dir: Path | None = None +) -> list[str]: + host, port = parse_endpoint(endpoint) + normalized_disk_paths = [str(path.resolve()).replace("\\", "/") for path in disk_paths] + command = launcher.command + [ + "server", + "--address", + f"{host}:{port}", + "--console-address", + f"{host}:{port + 1}", + ] + if certs_dir is not None: + command += ["--certs-dir", str(certs_dir.resolve())] + command += normalized_disk_paths + return command + + +def create_disk_paths(workdir: Path, disk_count: int) -> list[Path]: + disks = [] + for index in range(1, disk_count + 1): + disk = ensure_dir(workdir / "backend" / f"disk{index}") + disks.append(disk) + return disks + + +def build_payload_file(path: Path, size_bytes: int) -> str: + hasher = hashlib.sha256() + remaining = size_bytes + pattern = bytes(range(251)) + with path.open("wb") as handle: + while remaining > 0: + chunk = pattern[: min(len(pattern), remaining)] + handle.write(chunk) + hasher.update(chunk) + remaining -= len(chunk) + return hasher.hexdigest() + + +def minio_env(kms_secret_key: KmsSecretKeyConfig) -> dict[str, str]: + env = dict(os.environ) + env["CI"] = "on" + env["MINIO_CI_CD"] = "1" + env["MINIO_ROOT_USER"] = "minioadmin" + env["MINIO_ROOT_PASSWORD"] = "minioadmin" + env["MINIO_KMS_SECRET_KEY"] = kms_secret_key.secret_key + env["MINIO_UPDATE"] = "off" + return env + + +class S3Client: + def __init__( + self, + endpoint: str, + access_key: str = DEFAULT_ACCESS_KEY, + secret_key: str = DEFAULT_SECRET_KEY, + region: str = DEFAULT_REGION, + ) -> None: + self.endpoint = endpoint.rstrip("/") + self.access_key = access_key + self.secret_key = secret_key + self.region = region + self.parsed_endpoint = urlparse(self.endpoint) + self.ssl_context = ssl._create_unverified_context() if is_https_endpoint(endpoint) else None + + def request( + self, + method: str, + bucket: str | None = None, + key: str | None = None, + *, + query: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + body: bytes = b"", + ) -> tuple[int, dict[str, str], bytes]: + path = "/" + if bucket is not None: + path += urllib.parse.quote(bucket, safe="") + if key is not None: + path += "/" + urllib.parse.quote(key, safe="/") + query = query or {} + headers = dict(headers or {}) + signed_headers = self.sign_headers(method, path, query, headers, body) + url = self.endpoint + path + if query: + url += "?" + urllib.parse.urlencode(sorted(query.items()), quote_via=urllib.parse.quote) + request = urllib.request.Request(url, data=body if method != "HEAD" else None, method=method) + for name, value in signed_headers.items(): + request.add_header(name, value) + try: + with urllib.request.urlopen(request, timeout=30, context=self.ssl_context) as response: + return response.status, {k.lower(): v for k, v in response.headers.items()}, response.read() + except urllib.error.HTTPError as err: + payload = err.read() + raise RuntimeError( + f"S3 request failed: {method} {url}\nstatus: {err.code}\nbody:\n{payload.decode('utf-8', errors='replace')}" + ) from err + + def sign_headers( + self, + method: str, + canonical_uri: str, + query: dict[str, str], + headers: dict[str, str], + body: bytes, + ) -> dict[str, str]: + now = datetime.now(timezone.utc) + amz_date = now.strftime("%Y%m%dT%H%M%SZ") + date_stamp = now.strftime("%Y%m%d") + payload_hash = hashlib.sha256(body).hexdigest() + host = self.parsed_endpoint.netloc + signed_headers = {k.lower(): v.strip() for k, v in headers.items()} + signed_headers["host"] = host + signed_headers["x-amz-content-sha256"] = payload_hash + signed_headers["x-amz-date"] = amz_date + + canonical_query = "&".join( + f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}" + for k, v in sorted(query.items()) + ) + canonical_headers = "".join(f"{k}:{signed_headers[k]}\n" for k in sorted(signed_headers)) + signed_header_names = ";".join(sorted(signed_headers)) + canonical_request = "\n".join( + [ + method, + canonical_uri, + canonical_query, + canonical_headers, + signed_header_names, + payload_hash, + ] + ) + scope = f"{date_stamp}/{self.region}/s3/aws4_request" + string_to_sign = "\n".join( + [ + "AWS4-HMAC-SHA256", + amz_date, + scope, + hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(), + ] + ) + signature = hmac.new( + self.signing_key(date_stamp), + string_to_sign.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + signed_headers["authorization"] = ( + f"AWS4-HMAC-SHA256 Credential={self.access_key}/{scope}, " + f"SignedHeaders={signed_header_names}, Signature={signature}" + ) + return signed_headers + + def signing_key(self, date_stamp: str) -> bytes: + key = ("AWS4" + self.secret_key).encode("utf-8") + for value in (date_stamp, self.region, "s3", "aws4_request"): + key = hmac.new(key, value.encode("utf-8"), hashlib.sha256).digest() + return key + + def list_buckets(self) -> None: + self.request("GET") + + def create_bucket(self, bucket: str) -> None: + self.request("PUT", bucket=bucket) + + def put_object( + self, + bucket: str, + key: str, + body: bytes, + headers: dict[str, str], + ) -> None: + self.request("PUT", bucket=bucket, key=key, headers=headers, body=body) + + def create_multipart_upload(self, bucket: str, key: str, headers: dict[str, str]) -> str: + _, _, body = self.request("POST", bucket=bucket, key=key, query={"uploads": ""}, headers=headers) + root = ET.fromstring(body) + upload_id = root.findtext("{*}UploadId") + if upload_id is None: + raise RuntimeError(f"missing UploadId in create multipart response: {body!r}") + return upload_id + + def upload_part( + self, + bucket: str, + key: str, + upload_id: str, + part_number: int, + body: bytes, + headers: dict[str, str], + ) -> str: + _, response_headers, _ = self.request( + "PUT", + bucket=bucket, + key=key, + query={"partNumber": str(part_number), "uploadId": upload_id}, + headers=headers, + body=body, + ) + etag = response_headers.get("etag") + if etag is None: + raise RuntimeError("missing ETag in upload part response") + return etag + + def complete_multipart_upload( + self, + bucket: str, + key: str, + upload_id: str, + parts: list[dict[str, Any]], + ) -> None: + body = build_complete_multipart_xml(parts) + self.request( + "POST", + bucket=bucket, + key=key, + query={"uploadId": upload_id}, + headers={"content-type": "application/xml"}, + body=body, + ) + + def head_object(self, bucket: str, key: str, headers: dict[str, str]) -> dict[str, Any]: + _, response_headers, _ = self.request("HEAD", bucket=bucket, key=key, headers=headers) + return { + "ContentLength": int(response_headers.get("content-length", "0")), + "ServerSideEncryption": response_headers.get("x-amz-server-side-encryption"), + "SSECustomerAlgorithm": response_headers.get("x-amz-server-side-encryption-customer-algorithm"), + "SSECustomerKeyMD5": response_headers.get("x-amz-server-side-encryption-customer-key-md5"), + "SSEKMSKeyId": response_headers.get("x-amz-server-side-encryption-aws-kms-key-id"), + "VersionId": response_headers.get("x-amz-version-id"), + } + + +def wait_for_minio(endpoint: str, process: subprocess.Popen[str], timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + health_url = endpoint.rstrip("/") + "/minio/health/live" + ssl_context = ssl._create_unverified_context() if is_https_endpoint(endpoint) else None + while time.time() < deadline: + if process.poll() is not None: + raise RuntimeError("MinIO exited before becoming healthy") + try: + with urllib.request.urlopen(health_url, timeout=2, context=ssl_context) as response: + if response.status == 200: + return + except (urllib.error.URLError, TimeoutError): + time.sleep(1) + raise TimeoutError(f"timed out waiting for MinIO health endpoint: {health_url}") + + +def wait_for_s3_ready(client: S3Client, timeout_seconds: int) -> None: + deadline = time.time() + timeout_seconds + while time.time() < deadline: + try: + client.list_buckets() + return + except RuntimeError: + time.sleep(1) + raise TimeoutError(f"timed out waiting for S3 API readiness: {client.endpoint}") + + +def create_bucket(client: S3Client, case: FixtureCase) -> None: + client.create_bucket(case.bucket) + + +def build_complete_multipart_xml(parts: list[dict[str, Any]]) -> bytes: + root = ET.Element("CompleteMultipartUpload") + for part in parts: + part_element = ET.SubElement(root, "Part") + ET.SubElement(part_element, "PartNumber").text = str(part["PartNumber"]) + ET.SubElement(part_element, "ETag").text = part["ETag"] + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +def upload_case(client: S3Client, case: FixtureCase, payload_file: Path) -> dict[str, Any]: + request_record = build_request_record(case) + headers = request_record["headers"] + headers["content-type"] = "binary/octet-stream" + + if not case.multipart: + client.put_object(case.bucket, case.object_name, payload_file.read_bytes(), headers) + return request_record + + upload_id = client.create_multipart_upload(case.bucket, case.object_name, headers) + + parts = [] + with payload_file.open("rb") as handle: + part_number = 1 + while True: + chunk = handle.read(MULTIPART_CHUNK_SIZE) + if not chunk: + break + part_file = payload_file.parent / f"part-{part_number}.bin" + part_file.write_bytes(chunk) + part_headers = headers if case.encryption == "SSE-C" else {} + etag = client.upload_part( + case.bucket, + case.object_name, + upload_id, + part_number, + chunk, + part_headers, + ) + parts.append({"ETag": etag, "PartNumber": part_number}) + part_number += 1 + + client.complete_multipart_upload(case.bucket, case.object_name, upload_id, parts) + return request_record + + +def head_case(client: S3Client, case: FixtureCase) -> dict[str, Any]: + headers = build_request_record(case)["headers"] if case.encryption == "SSE-C" else {} + return client.head_object(case.bucket, case.object_name, headers) + + +def stop_process(process: subprocess.Popen[str]) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=15) + + +def capture_case( + *, + paths: LabPaths, + launcher: MinioLauncher, + endpoint: str, + case: FixtureCase, + work_root: Path, + disk_count: int, + timeout_seconds: int, + preserve_workdir: bool, + kms_secret_key: KmsSecretKeyConfig, +) -> Path: + case_workdir = work_root / case.case_id + if case_workdir.exists(): + shutil.rmtree(case_workdir) + ensure_dir(case_workdir) + + payload_file = case_workdir / "payload.bin" + plaintext_sha256 = build_payload_file(payload_file, case.size_bytes) + disk_paths = create_disk_paths(case_workdir, disk_count) + certs_dir = None + if is_https_endpoint(endpoint): + host, _ = parse_endpoint(endpoint) + certs_dir = case_workdir / "certs" + ensure_local_tls_certificates(certs_dir, host) + server_command = build_server_command(launcher, endpoint, disk_paths, certs_dir) + server_log_path = case_workdir / "server.log" + + with server_log_path.open("w", encoding="utf-8") as server_log: + process = subprocess.Popen( + server_command, + cwd=str(launcher.workdir), + env=minio_env(kms_secret_key), + stdout=server_log, + stderr=subprocess.STDOUT, + text=True, + ) + try: + client = S3Client(endpoint) + wait_for_minio(endpoint, process, timeout_seconds) + wait_for_s3_ready(client, timeout_seconds) + create_bucket(client, case) + request_payload = upload_case(client, case, payload_file) + head_payload = head_case(client, case) + finally: + stop_process(process) + + case_dir = store_case_artifacts( + paths=paths, + case_id=case.case_id, + bucket=case.bucket, + object_name=case.object_name, + source_tree=case_workdir / "backend", + version_id=head_payload.get("VersionId") if "head_payload" in locals() else None, + request_payload=request_payload if "request_payload" in locals() else None, + head_payload=head_payload if "head_payload" in locals() else None, + plaintext_sha256=plaintext_sha256, + notes="Captured automatically from a local disposable MinIO run.", + capture_payload={ + "endpoint": endpoint, + "launcher_kind": launcher.kind, + "disk_count": disk_count, + "kms_key_id": kms_secret_key.key_id, + }, + ) + + if not preserve_workdir: + shutil.rmtree(case_workdir) + return case_dir + + +def select_cases(case_ids: list[str] | None) -> list[FixtureCase]: + cases = build_default_cases() + if not case_ids: + return cases + selected = [case for case in cases if case.case_id in set(case_ids)] + if len(selected) != len(set(case_ids)): + known = ", ".join(case.case_id for case in cases) + missing = sorted(set(case_ids) - {case.case_id for case in selected}) + raise ValueError(f"unknown case id(s): {', '.join(missing)}; known: {known}") + return selected + + +def cmd_capture_matrix(args: argparse.Namespace) -> int: + paths = resolve_lab_paths(args.root) + ensure_layout(paths) + ensure_dir(args.work_root) + + launcher = discover_minio_launcher(args.minio_binary, args.minio_root) + kms_secret_key = resolve_kms_secret_key(args.kms_secret_key) + cases = build_default_cases(kms_key_id=kms_secret_key.key_id) + if args.case_id: + selected_ids = set(args.case_id) + cases = [case for case in cases if case.case_id in selected_ids] + if len(cases) != len(selected_ids): + known = ", ".join(case.case_id for case in build_default_cases()) + missing = sorted(selected_ids - {case.case_id for case in cases}) + raise ValueError(f"unknown case id(s): {', '.join(missing)}; known: {known}") + for case in cases: + case_dir = capture_case( + paths=paths, + launcher=launcher, + endpoint=args.endpoint, + case=case, + work_root=args.work_root, + disk_count=args.disk_count, + timeout_seconds=args.timeout_seconds, + preserve_workdir=args.preserve_workdir, + kms_secret_key=kms_secret_key, + ) + print(f"{case.case_id}: {case_dir}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Capture real MinIO backend fixtures into a stable local layout." + ) + parser.set_defaults(func=None) + subparsers = parser.add_subparsers(dest="command") + + init_parser = subparsers.add_parser("init", help="initialize the fixture lab root") + init_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory") + init_parser.set_defaults(func=cmd_init) + + add_case_parser = subparsers.add_parser("add-case", help="capture one fixture case") + add_case_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory") + add_case_parser.add_argument("--case-id", required=True, help="stable fixture case identifier") + add_case_parser.add_argument("--bucket", required=True, help="bucket name") + add_case_parser.add_argument("--object", required=True, help="object key") + add_case_parser.add_argument("--version-id", help="object version id") + add_case_parser.add_argument("--source-tree", type=Path, required=True, help="backend object tree to capture") + add_case_parser.add_argument("--request-json", type=Path, help="request metadata JSON") + add_case_parser.add_argument("--head-json", type=Path, help="HEAD object metadata JSON") + add_case_parser.add_argument("--plaintext-sha256", type=Path, help="plaintext sha256 file") + add_case_parser.add_argument("--notes", help="free-form notes") + add_case_parser.set_defaults(func=cmd_add_case) + + capture_parser = subparsers.add_parser( + "capture-matrix", + help="run a disposable local MinIO instance and capture the default fixture matrix", + ) + capture_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory") + capture_parser.add_argument( + "--work-root", + type=Path, + default=DEFAULT_WORK_ROOT, + help="temporary runner workspace", + ) + capture_parser.add_argument( + "--minio-binary", + type=Path, + help="explicit path to a minio binary", + ) + capture_parser.add_argument( + "--minio-root", + type=Path, + help="optional directory containing minio.exe", + ) + capture_parser.add_argument( + "--endpoint", + default=DEFAULT_ENDPOINT, + help="local MinIO endpoint used for the disposable run", + ) + capture_parser.add_argument( + "--kms-secret-key", + help=( + "static MinIO KMS secret key in the form " + "':'; overrides " + f"{LAB_KMS_SECRET_KEY_ENV}, MINIO_KMS_SECRET_KEY, and the built-in default" + ), + ) + capture_parser.add_argument( + "--disk-count", + type=int, + default=DEFAULT_DISK_COUNT, + help="number of backend disks to provision per case", + ) + capture_parser.add_argument( + "--timeout-seconds", + type=int, + default=60, + help="startup timeout for MinIO health checks", + ) + capture_parser.add_argument( + "--preserve-workdir", + action="store_true", + help="keep per-case runner work directories after capture", + ) + capture_parser.add_argument( + "--case-id", + action="append", + help="capture only specific default case ids", + ) + capture_parser.set_defaults(func=cmd_capture_matrix) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + if args.func is None: + parser.print_help() + return 2 + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/crates/rio-v2/tests/minio_fixture_lab/test_lab.py b/crates/rio-v2/tests/minio_fixture_lab/test_lab.py new file mode 100644 index 000000000..611713b2b --- /dev/null +++ b/crates/rio-v2/tests/minio_fixture_lab/test_lab.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import importlib.util +import base64 +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +def load_lab_module(): + module_path = Path(__file__).with_name("lab.py") + spec = importlib.util.spec_from_file_location("minio_fixture_lab_lab", module_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lab = load_lab_module() + + +class DiscoverMinioLauncherTests(unittest.TestCase): + def test_prefers_explicit_binary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + binary = Path(temp_dir) / "minio.exe" + binary.write_text("", encoding="utf-8") + + launcher = lab.discover_minio_launcher(explicit_binary=binary, minio_root=None) + + self.assertEqual(launcher.command, [str(binary.resolve())]) + self.assertEqual(launcher.workdir, binary.resolve().parent) + self.assertEqual(launcher.kind, "binary") + + def test_defaults_to_bundled_binary_when_available(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + binary = Path(temp_dir) / "minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe" + binary.write_text("", encoding="utf-8") + + original_default_binary = lab.DEFAULT_MINIO_BINARY + try: + lab.DEFAULT_MINIO_BINARY = binary + launcher = lab.discover_minio_launcher(explicit_binary=None, minio_root=None) + finally: + lab.DEFAULT_MINIO_BINARY = original_default_binary + + self.assertEqual(launcher.command, [str(binary.resolve())]) + self.assertEqual(launcher.workdir, binary.resolve().parent) + self.assertEqual(launcher.kind, "binary") + + def test_rejects_source_checkout_without_binary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source_root = Path(temp_dir) / "minio-master" + source_root.mkdir() + (source_root / "main.go").write_text("package main\n", encoding="utf-8") + + original_default_binary = lab.DEFAULT_MINIO_BINARY + try: + lab.DEFAULT_MINIO_BINARY = Path(temp_dir) / "missing-bundled-binary.exe" + with mock.patch.object(lab.shutil, "which", return_value=None): + with self.assertRaises(FileNotFoundError): + lab.discover_minio_launcher(explicit_binary=None, minio_root=source_root) + finally: + lab.DEFAULT_MINIO_BINARY = original_default_binary + + def test_uses_binary_from_minio_root_directory(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + binary_root = Path(temp_dir) / "minio-dist" + binary_root.mkdir() + binary = binary_root / "minio.exe" + binary.write_text("", encoding="utf-8") + + launcher = lab.discover_minio_launcher(explicit_binary=None, minio_root=binary_root) + + self.assertEqual(launcher.command, [str(binary.resolve())]) + self.assertEqual(launcher.workdir, binary.resolve().parent) + self.assertEqual(launcher.kind, "binary") + + +class FixtureMatrixTests(unittest.TestCase): + def test_default_matrix_covers_sse_modes_and_shapes(self) -> None: + cases = lab.build_default_cases() + case_ids = {case.case_id for case in cases} + + self.assertEqual(len(cases), 6) + self.assertIn("sse-s3-singlepart-64k", case_ids) + self.assertIn("sse-kms-singlepart-64k", case_ids) + self.assertIn("sse-c-singlepart-64k", case_ids) + self.assertIn("sse-s3-multipart-8m", case_ids) + self.assertIn("sse-kms-multipart-8m", case_ids) + self.assertIn("sse-c-multipart-8m", case_ids) + + def test_default_matrix_uses_configured_kms_key_id(self) -> None: + cases = lab.build_default_cases(kms_key_id="local-fixture-key") + kms_cases = [case for case in cases if case.encryption == "SSE-KMS"] + + self.assertEqual(len(kms_cases), 2) + self.assertTrue(all(case.kms_key_id == "local-fixture-key" for case in kms_cases)) + + +class RequestRecordTests(unittest.TestCase): + def test_sse_kms_request_record_includes_context_header(self) -> None: + case = lab.FixtureCase( + case_id="sse-kms-singlepart-64k", + bucket="demo", + object_name="dir/object.bin", + encryption="SSE-KMS", + size_bytes=64 * 1024, + multipart=False, + kms_key_id="minio-default-key", + kms_context={"project": "rio-v2", "stage": "fixture"}, + ) + + record = lab.build_request_record(case) + + self.assertEqual(record["headers"]["x-amz-server-side-encryption"], "aws:kms") + self.assertEqual( + record["headers"]["x-amz-server-side-encryption-aws-kms-key-id"], + "minio-default-key", + ) + expected_context = base64.b64encode( + b'{"project":"rio-v2","stage":"fixture"}' + ).decode("ascii") + self.assertEqual( + record["headers"]["x-amz-server-side-encryption-context"], + expected_context, + ) + + def test_sse_s3_request_record_uses_aes256(self) -> None: + case = lab.FixtureCase( + case_id="sse-s3-singlepart-64k", + bucket="demo", + object_name="dir/object.bin", + encryption="SSE-S3", + size_bytes=64 * 1024, + multipart=False, + ) + + record = lab.build_request_record(case) + + self.assertEqual(record["headers"], {"x-amz-server-side-encryption": "AES256"}) + + def test_sse_c_request_record_includes_customer_key_headers(self) -> None: + case = lab.FixtureCase( + case_id="sse-c-singlepart-64k", + bucket="demo", + object_name="dir/object.bin", + encryption="SSE-C", + size_bytes=64 * 1024, + multipart=False, + ) + + record = lab.build_request_record(case) + + self.assertEqual( + record["headers"]["x-amz-server-side-encryption-customer-algorithm"], + "AES256", + ) + self.assertIn("x-amz-server-side-encryption-customer-key", record["headers"]) + self.assertIn("x-amz-server-side-encryption-customer-key-md5", record["headers"]) + + +class MultipartManifestTests(unittest.TestCase): + def test_complete_multipart_payload_is_xml(self) -> None: + parts = [ + {"ETag": '"etag-1"', "PartNumber": 1}, + {"ETag": '"etag-2"', "PartNumber": 2}, + ] + + payload = lab.build_complete_multipart_xml(parts).decode("utf-8") + + self.assertIn("", payload) + self.assertIn("1", payload) + self.assertIn("\"etag-1\"", payload) + self.assertIn("2", payload) + self.assertIn("\"etag-2\"", payload) + + +class KmsSecretKeyTests(unittest.TestCase): + def test_parse_kms_secret_key_extracts_key_id(self) -> None: + setting = lab.parse_kms_secret_key("local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + + self.assertEqual(setting.secret_key, "local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + self.assertEqual(setting.key_id, "local-fixture-key") + + def test_parse_kms_secret_key_rejects_missing_separator(self) -> None: + with self.assertRaises(ValueError): + lab.parse_kms_secret_key("invalid-secret-key") + + def test_minio_env_prefers_explicit_secret_key(self) -> None: + env = lab.minio_env( + kms_secret_key=lab.parse_kms_secret_key( + "local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + ) + ) + + self.assertEqual( + env["MINIO_KMS_SECRET_KEY"], + "local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/rio-v2/tests/minio_generated_fixtures.rs b/crates/rio-v2/tests/minio_generated_fixtures.rs new file mode 100644 index 000000000..d1181e923 --- /dev/null +++ b/crates/rio-v2/tests/minio_generated_fixtures.rs @@ -0,0 +1,288 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info}; +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +struct RequestRecord { + encryption: String, + headers: HashMap, + multipart: bool, + size_bytes: u64, + object: String, +} + +#[derive(Debug, Deserialize)] +struct HeadRecord { + #[serde(rename = "ContentLength")] + content_length: u64, + #[serde(rename = "ServerSideEncryption")] + server_side_encryption: Option, + #[serde(rename = "SSECustomerAlgorithm")] + sse_customer_algorithm: Option, + #[serde(rename = "SSECustomerKeyMD5")] + sse_customer_key_md5: Option, + #[serde(rename = "SSEKMSKeyId")] + sse_kms_key_id: Option, +} + +#[derive(Debug, Deserialize)] +struct ManifestRecord { + capture: Option, +} + +#[derive(Debug, Deserialize)] +struct CaptureRecord { + kms_key_id: Option, +} + +fn fixture_root() -> PathBuf { + std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/minio-generated")) +} + +fn case_dir(case_id: &str) -> PathBuf { + fixture_root().join("cases").join(case_id) +} + +fn read_json Deserialize<'de>>(path: &Path) -> T { + let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display())) +} + +fn find_object_xl_meta(case_dir: &Path) -> PathBuf { + let backend = case_dir.join("backend"); + for entry in walkdir::WalkDir::new(&backend) { + let entry = entry.unwrap_or_else(|err| panic!("walk {}: {err}", backend.display())); + if !entry.file_type().is_file() { + continue; + } + if entry.file_name() != "xl.meta" { + continue; + } + let path = entry.path(); + let path_text = path.to_string_lossy(); + if path_text.contains(".minio.sys") { + continue; + } + return path.to_path_buf(); + } + panic!("object xl.meta not found under {}", backend.display()); +} + +fn load_file_info(case_id: &str) -> FileInfo { + let case = case_dir(case_id); + let xl_meta = find_object_xl_meta(&case); + let data = fs::read(&xl_meta).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta.display())); + get_file_info( + &data, + "fixture-bucket", + "fixture-object", + "", + FileInfoOpts { + data: false, + include_free_versions: true, + }, + ) + .unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta.display())) +} + +fn read_manifest(case_id: &str) -> ManifestRecord { + read_json(&case_dir(case_id).join("manifest.json")) +} + +fn require_generated_fixture_root() { + let root = fixture_root(); + assert!( + root.join("cases").is_dir(), + "generated fixture root missing: {}. Run test/minio_fixture_lab/lab.py capture-matrix first.", + root.display() + ); +} + +fn metadata_keys(fi: &FileInfo) -> HashMap<&str, &str> { + fi.metadata.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect() +} + +fn expected_fixture_kms_key_id(case_id: &str, request: &RequestRecord) -> Option { + request + .headers + .get("x-amz-server-side-encryption-aws-kms-key-id") + .cloned() + .or_else(|| read_manifest(case_id).capture.and_then(|capture| capture.kms_key_id)) +} + +#[test] +#[ignore = "requires locally generated MinIO fixtures"] +fn parses_singlepart_sse_s3_fixture() { + require_generated_fixture_root(); + + let request: RequestRecord = read_json(&case_dir("sse-s3-singlepart-64k").join("request.json")); + let head: HeadRecord = read_json(&case_dir("sse-s3-singlepart-64k").join("head.json")); + let fi = load_file_info("sse-s3-singlepart-64k"); + let metadata = metadata_keys(&fi); + + assert_eq!(request.encryption, "SSE-S3"); + assert!(!request.multipart); + assert_eq!(request.object, "dir/object.bin"); + assert_eq!(head.content_length, request.size_bytes); + assert_eq!(head.server_side_encryption.as_deref(), Some("AES256")); + assert_eq!(fi.parts.len(), 1); + assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes); + assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key")); + assert_eq!(metadata.get("content-type"), Some(&"binary/octet-stream")); +} + +#[test] +#[ignore = "requires locally generated MinIO fixtures"] +fn parses_singlepart_sse_kms_fixture() { + require_generated_fixture_root(); + + let case_id = "sse-kms-singlepart-64k"; + let request: RequestRecord = read_json(&case_dir(case_id).join("request.json")); + let head: HeadRecord = read_json(&case_dir(case_id).join("head.json")); + let fi = load_file_info(case_id); + let metadata = metadata_keys(&fi); + let expected_key_id = expected_fixture_kms_key_id(case_id, &request).expect("fixture kms key id"); + + assert_eq!(request.encryption, "SSE-KMS"); + assert!(!request.multipart); + assert_eq!(head.content_length, request.size_bytes); + assert_eq!(head.server_side_encryption.as_deref(), Some("aws:kms")); + assert_eq!(request.headers.get("x-amz-server-side-encryption"), Some(&"aws:kms".to_string())); + assert_eq!(request.headers.get("x-amz-server-side-encryption-aws-kms-key-id"), Some(&expected_key_id)); + assert_eq!(head.sse_kms_key_id.as_deref(), Some(format!("arn:aws:kms:{expected_key_id}").as_str())); + assert_eq!(fi.parts.len(), 1); + assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes); + assert_eq!( + metadata.get("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"), + Some(&expected_key_id.as_str()) + ); + assert_eq!( + metadata.get("X-Minio-Internal-Server-Side-Encryption-Context").copied(), + request + .headers + .get("x-amz-server-side-encryption-context") + .map(String::as_str) + ); + assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key")); +} + +#[test] +#[ignore = "requires locally generated MinIO fixtures"] +fn parses_multipart_sse_s3_fixture() { + require_generated_fixture_root(); + + let case_id = "sse-s3-multipart-8m"; + let request: RequestRecord = read_json(&case_dir(case_id).join("request.json")); + let head: HeadRecord = read_json(&case_dir(case_id).join("head.json")); + let fi = load_file_info(case_id); + let metadata = metadata_keys(&fi); + + assert_eq!(request.encryption, "SSE-S3"); + assert!(request.multipart); + assert_eq!(head.content_length, request.size_bytes); + assert_eq!(head.server_side_encryption.as_deref(), Some("AES256")); + assert_eq!(fi.parts.len(), 2); + assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::(), request.size_bytes); + assert!( + metadata + .get("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id") + .is_some_and(|value| !value.is_empty()) + ); + assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key")); + assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart")); + assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608")); +} + +#[test] +#[ignore = "requires locally generated MinIO fixtures"] +fn parses_multipart_sse_kms_fixture() { + require_generated_fixture_root(); + + let case_id = "sse-kms-multipart-8m"; + let request: RequestRecord = read_json(&case_dir(case_id).join("request.json")); + let head: HeadRecord = read_json(&case_dir(case_id).join("head.json")); + let fi = load_file_info(case_id); + let metadata = metadata_keys(&fi); + let expected_key_id = expected_fixture_kms_key_id(case_id, &request).expect("fixture kms key id"); + + assert_eq!(request.encryption, "SSE-KMS"); + assert!(request.multipart); + assert_eq!(head.content_length, request.size_bytes); + assert_eq!(head.server_side_encryption.as_deref(), Some("aws:kms")); + assert_eq!(request.headers.get("x-amz-server-side-encryption-aws-kms-key-id"), Some(&expected_key_id)); + assert_eq!(head.sse_kms_key_id.as_deref(), Some(format!("arn:aws:kms:{expected_key_id}").as_str())); + assert_eq!(fi.parts.len(), 2); + assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::(), request.size_bytes); + assert_eq!( + metadata.get("X-Minio-Internal-Server-Side-Encryption-Context").copied(), + request + .headers + .get("x-amz-server-side-encryption-context") + .map(String::as_str) + ); + assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart")); + assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608")); +} + +#[test] +#[ignore = "requires locally generated MinIO fixtures"] +fn parses_singlepart_sse_c_fixture() { + require_generated_fixture_root(); + + let case_id = "sse-c-singlepart-64k"; + let request: RequestRecord = read_json(&case_dir(case_id).join("request.json")); + let head: HeadRecord = read_json(&case_dir(case_id).join("head.json")); + let fi = load_file_info(case_id); + let metadata = metadata_keys(&fi); + + assert_eq!(request.encryption, "SSE-C"); + assert!(!request.multipart); + assert_eq!(head.content_length, request.size_bytes); + assert_eq!(head.sse_customer_algorithm.as_deref(), Some("AES256")); + assert_eq!( + head.sse_customer_key_md5.as_deref(), + request + .headers + .get("x-amz-server-side-encryption-customer-key-md5") + .map(String::as_str) + ); + assert_eq!(fi.parts.len(), 1); + assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes); + assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key")); + assert!(!metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id")); +} + +#[test] +#[ignore = "requires locally generated MinIO fixtures"] +fn parses_multipart_sse_c_fixture() { + require_generated_fixture_root(); + + let case_id = "sse-c-multipart-8m"; + let request: RequestRecord = read_json(&case_dir(case_id).join("request.json")); + let head: HeadRecord = read_json(&case_dir(case_id).join("head.json")); + let fi = load_file_info(case_id); + let metadata = metadata_keys(&fi); + + assert_eq!(request.encryption, "SSE-C"); + assert!(request.multipart); + assert_eq!(head.content_length, request.size_bytes); + assert_eq!(head.sse_customer_algorithm.as_deref(), Some("AES256")); + assert_eq!( + head.sse_customer_key_md5.as_deref(), + request + .headers + .get("x-amz-server-side-encryption-customer-key-md5") + .map(String::as_str) + ); + assert_eq!(fi.parts.len(), 2); + assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::(), request.size_bytes); + assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key")); + assert!(!metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id")); + assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart")); + assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608")); +} diff --git a/crates/rio/src/compress_index.rs b/crates/rio/src/compress_index.rs index 2c8e5139b..75e415e26 100644 --- a/crates/rio/src/compress_index.rs +++ b/crates/rio/src/compress_index.rs @@ -394,12 +394,13 @@ impl Index { b = &b[n..]; if idx > 0 { - c_predict += c_off / 2; + let next_c_predict = c_predict + c_off / 2; let prev = self.info[idx - 1].compressed_offset; c_off += prev + c_predict; if c_off <= prev { return Err(io::Error::other("invalid offset")); } + c_predict = next_c_predict; } if c_off < 0 { return Err(io::Error::other("negative offset")); @@ -701,6 +702,7 @@ mod tests { let mut source = Index::new(); source.add(100, 1_000)?; source.add(300, 1_000 + MIN_INDEX_DIST)?; + source.add(650, 1_000 + MIN_INDEX_DIST * 2)?; let encoded = source.clone().into_vec(); @@ -713,8 +715,10 @@ mod tests { assert_eq!(decoded.info.len(), source.info.len()); assert_eq!(decoded.info[0].compressed_offset, source.info[0].compressed_offset); assert_eq!(decoded.info[0].uncompressed_offset, source.info[0].uncompressed_offset); + assert_eq!(decoded.info[1].compressed_offset, source.info[1].compressed_offset); assert_eq!(decoded.info[1].uncompressed_offset, source.info[1].uncompressed_offset); - assert!(decoded.info[1].compressed_offset > decoded.info[0].compressed_offset); + assert_eq!(decoded.info[2].compressed_offset, source.info[2].compressed_offset); + assert_eq!(decoded.info[2].uncompressed_offset, source.info[2].uncompressed_offset); Ok(()) } diff --git a/crates/s3select-api/src/object_store.rs b/crates/s3select-api/src/object_store.rs index 76f5d881b..9ce4fef00 100644 --- a/crates/s3select-api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -1046,7 +1046,10 @@ where AsyncTryStream::::new(|mut y| async move { pin_mut!(stream); let mut remaining: usize = content_length; - while let Some(result) = stream.next().await { + while remaining > 0 { + let Some(result) = stream.next().await else { + break; + }; let mut bytes = result.map_err(|e| o_Error::Generic { store: "", source: Box::new(e), @@ -1064,12 +1067,12 @@ where #[cfg(test)] mod test { use super::{ - ConvertStream, SelectScanRange, convert_field_delimiter_stream, extract_json_sub_path_from_expression, find_delimiter, - flatten_json_document_to_ndjson, http_range_spec_from_get_range, replace_symbol, scan_range_from_bounds, + ConvertStream, SelectScanRange, bytes_stream, convert_field_delimiter_stream, extract_json_sub_path_from_expression, + find_delimiter, flatten_json_document_to_ndjson, http_range_spec_from_get_range, replace_symbol, scan_range_from_bounds, scan_range_read_start, scan_range_stream, select_read_headers, }; use bytes::Bytes; - use futures::{StreamExt, stream}; + use futures::{StreamExt, TryStreamExt, stream}; use object_store::GetRange; use s3s::dto::{ CSVInput, CSVOutput, ExpressionType, InputSerialization, OutputSerialization, SelectObjectContentInput, @@ -1079,6 +1082,10 @@ mod test { X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, }; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use tokio::io::AsyncReadExt; use tokio_util::io::StreamReader; @@ -1285,6 +1292,29 @@ mod test { assert_eq!(output, b"a,"); } + #[tokio::test] + async fn test_bytes_stream_stops_at_content_length() { + let poll_count = Arc::new(AtomicUsize::new(0)); + let stream_poll_count = Arc::clone(&poll_count); + let source = stream::unfold(0, move |index| { + let stream_poll_count = Arc::clone(&stream_poll_count); + async move { + stream_poll_count.fetch_add(1, Ordering::SeqCst); + let bytes = match index { + 0 => Bytes::from_static(b"abcd"), + 1 => Bytes::from_static(b"efgh"), + _ => return None, + }; + Some((Ok::<_, std::io::Error>(bytes), index + 1)) + } + }); + + let chunks: Vec = bytes_stream(source, 4).try_collect().await.unwrap(); + + assert_eq!(chunks, vec![Bytes::from_static(b"abcd")]); + assert_eq!(poll_count.load(Ordering::SeqCst), 1); + } + /// A JSON array is split into one NDJSON line per element. #[test] fn test_flatten_array_produces_one_line_per_element() { diff --git a/crates/utils/src/http/metadata_compat.rs b/crates/utils/src/http/metadata_compat.rs index cdb511911..1846cf087 100644 --- a/crates/utils/src/http/metadata_compat.rs +++ b/crates/utils/src/http/metadata_compat.rs @@ -112,18 +112,27 @@ pub fn insert_str(map: &mut HashMap, suffix: &str, value: String pub fn get_str(map: &HashMap, suffix: &str) -> Option { let (k1, k2) = both_keys(suffix); - map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()) + map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()).or_else(|| { + map.iter() + .find(|(key, _)| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2)) + .map(|(_, value)| value.clone()) + }) } pub fn contains_key_str(map: &HashMap, suffix: &str) -> bool { let (k1, k2) = both_keys(suffix); - map.contains_key(&k1) || map.contains_key(&k2) + map.contains_key(&k1) + || map.contains_key(&k2) + || map + .keys() + .any(|key| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2)) } pub fn remove_str(map: &mut HashMap, suffix: &str) { let (k1, k2) = both_keys(suffix); map.remove(&k1); map.remove(&k2); + map.retain(|key, _| !key.eq_ignore_ascii_case(&k1) && !key.eq_ignore_ascii_case(&k2)); } // === Vec type (meta_sys) === @@ -178,4 +187,21 @@ mod tests { assert!(!has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_HEALING)); assert!(!has_internal_suffix("x-amz-meta-custom", SUFFIX_PURGESTATUS)); } + + #[test] + fn test_str_lookup_accepts_minio_metadata_case() { + let mut metadata = HashMap::from([ + ("X-Minio-Internal-compression".to_string(), "klauspost/compress/s2".to_string()), + ("X-Minio-Internal-actual-size".to_string(), "268435456".to_string()), + ]); + + assert!(contains_key_str(&metadata, SUFFIX_COMPRESSION)); + assert_eq!(get_str(&metadata, SUFFIX_COMPRESSION).as_deref(), Some("klauspost/compress/s2")); + assert_eq!(get_str(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("268435456")); + + remove_str(&mut metadata, SUFFIX_COMPRESSION); + assert!(!contains_key_str(&metadata, SUFFIX_COMPRESSION)); + assert!(!metadata.contains_key("X-Minio-Internal-compression")); + assert!(contains_key_str(&metadata, SUFFIX_ACTUAL_SIZE)); + } } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 9b53de4cc..5e636e963 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -54,6 +54,7 @@ io-scheduler-debug = [] # Enable debug information in I/O scheduler tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only) full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp"] manual-test-runners = [] +rio-v2 = ["rustfs-ecstore/rio-v2"] [lints] workspace = true @@ -138,6 +139,7 @@ astral-tokio-tar = { workspace = true } atoi = { workspace = true } atomic_enum = { workspace = true } base64 = { workspace = true } +hmac = { workspace = true } sha2 = { workspace = true } base64-simd.workspace = true clap = { workspace = true } diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 6075f4aa8..b27259581 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -28,7 +28,10 @@ use crate::storage::s3_api::multipart::{ ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, }; -use crate::storage::sse::{build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error}; +use crate::storage::sse::{ + build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, + extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata, +}; use crate::storage::*; use crate::table_catalog; use bytes::Bytes; @@ -46,13 +49,13 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag; use rustfs_ecstore::compress::is_compressible; use rustfs_ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found}; use rustfs_ecstore::new_object_layer_fn; +#[cfg(test)] +use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader}; +use rustfs_ecstore::rio::{HashReader, WritePlan}; use rustfs_ecstore::set_disk::is_valid_storage_class; use rustfs_ecstore::store_api::{CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions, PutObjReader}; use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; -use rustfs_rio::{CompressReader, EncryptReader, HashReader}; -#[cfg(test)] -use rustfs_rio::{DecryptReader, HardLimitReader, boxed_reader, wrap_reader}; use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; @@ -574,13 +577,16 @@ impl DefaultMultipartUsecase { metadata.extend(object_lock_metadata); } apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?; + let (_, sse_customer_key, _) = extract_ssec_params_from_headers(&req.headers)?; let encryption_request = PrepareEncryptionRequest { bucket: &bucket, key: &key, server_side_encryption, ssekms_key_id, + ssekms_context: extract_ssekms_context_from_headers(&req.headers)?, sse_customer_algorithm: sse_customer_algorithm.clone(), + sse_customer_key, sse_customer_key_md5: sse_customer_key_md5.clone(), }; @@ -589,7 +595,11 @@ impl DefaultMultipartUsecase { let server_side_encryption = Some(material.server_side_encryption.clone()); let ssekms_key_id = material.kms_key_id.clone(); - metadata.extend(encryption_material_to_metadata(&material)); + let mut encryption_metadata = encryption_material_to_metadata(&material); + if material.key_kind == crate::storage::sse::EncryptionKeyKind::Object { + mark_encrypted_multipart_metadata(&mut encryption_metadata); + } + metadata.extend(encryption_metadata); (server_side_encryption, ssekms_key_id) } @@ -600,7 +610,7 @@ impl DefaultMultipartUsecase { rustfs_utils::http::insert_str( &mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, - CompressionAlgorithm::default().to_string(), + rustfs_ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()), ); } @@ -757,7 +767,9 @@ impl DefaultMultipartUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); + let mut write_plan = WritePlan::new(); let mut reader = if is_compressible { + let algorithm = CompressionAlgorithm::default(); let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false) .map_err(ApiError::from)?; @@ -766,15 +778,8 @@ impl DefaultMultipartUsecase { } size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? + write_plan = write_plan.with_compression(algorithm); + hrd } else { HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? }; @@ -812,6 +817,7 @@ impl DefaultMultipartUsecase { key: &key, server_side_encryption: server_side_encryption.clone(), ssekms_key_id: ssekms_key_id.clone(), + ssekms_context: None, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key: sse_customer_key.clone(), sse_customer_key_md5: sse_customer_key_md5.clone(), @@ -819,35 +825,25 @@ impl DefaultMultipartUsecase { } .check_upload_part_customer_key_md5(&fi.user_defined, sse_customer_key_md5.clone())?; let (requested_sse, requested_kms_key_id) = if has_ssec { - let encryption_request = EncryptionRequest { + let ssec_material = sse_decryption(DecryptionRequest { bucket: &bucket, key: &key, - server_side_encryption, - ssekms_key_id, - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key, - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: actual_size, - }; - - match sse_encryption(encryption_request).await? { - Some(material) => { - let requested_sse = Some(material.server_side_encryption.clone()); - let requested_kms_key_id = material.kms_key_id.clone(); - let encrypted_reader = EncryptReader::new_multipart(reader, material.key_bytes, material.base_nonce, part_id); - reader = HashReader::from_reader( - encrypted_reader, - HashReader::SIZE_PRESERVE_LAYER, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)?; - (requested_sse, requested_kms_key_id) + metadata: &fi.user_defined, + sse_customer_key: sse_customer_key.as_ref(), + sse_customer_key_md5: sse_customer_key_md5.as_ref(), + }) + .await? + .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; + let ssec_write = match ssec_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => { + rustfs_ecstore::rio::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) } - None => (None, None), - } + crate::storage::sse::EncryptionKeyKind::Direct => { + rustfs_ecstore::rio::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id) + } + }; + write_plan = write_plan.with_encryption(ssec_write); + (Some(ssec_material.server_side_encryption), ssec_material.kms_key_id) } else if let Some(server_side_encryption) = server_side_encryption { let managed_material = sse_decryption(DecryptionRequest { bucket: &bucket, @@ -858,15 +854,24 @@ impl DefaultMultipartUsecase { }) .await? .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; - let encrypted_reader = - EncryptReader::new_multipart(reader, managed_material.key_bytes, managed_material.base_nonce, part_id); - reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + let managed_write = match managed_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => { + rustfs_ecstore::rio::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + } + crate::storage::sse::EncryptionKeyKind::Direct => rustfs_ecstore::rio::WriteEncryption::multipart( + managed_material.key_bytes, + managed_material.base_nonce, + part_id, + ), + }; + write_plan = write_plan.with_encryption(managed_write); (Some(server_side_encryption), ssekms_key_id) } else { (None, None) }; + reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; + let mut reader = PutObjReader::new(reader); let info = store @@ -1119,22 +1124,15 @@ impl DefaultMultipartUsecase { let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION); let actual_size = length; - let mut size = length; + let mut write_plan = WritePlan::new(); let mut reader = if is_compressible { - let hrd = HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?; - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? + let algorithm = CompressionAlgorithm::default(); + let hrd = HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?; + write_plan = write_plan.with_compression(algorithm); + hrd } else { - HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)? + HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)? }; let server_side_encryption = mp_info @@ -1158,6 +1156,7 @@ impl DefaultMultipartUsecase { key: &key, server_side_encryption: server_side_encryption.clone(), ssekms_key_id: ssekms_key_id.clone(), + ssekms_context: None, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key: sse_customer_key.clone(), sse_customer_key_md5: sse_customer_key_md5.clone(), @@ -1166,35 +1165,29 @@ impl DefaultMultipartUsecase { .check_upload_part_customer_key_md5(&mp_info.user_defined, sse_customer_key_md5.clone())?; let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec { - let encryption_request = EncryptionRequest { + let ssec_material = sse_decryption(DecryptionRequest { bucket: &bucket, key: &key, - server_side_encryption, - ssekms_key_id, - sse_customer_algorithm: sse_customer_algorithm.clone(), - sse_customer_key, - sse_customer_key_md5: sse_customer_key_md5.clone(), - content_size: actual_size, - }; - - match sse_encryption(encryption_request).await? { - Some(material) => { - let requested_sse = Some(material.server_side_encryption.clone()); - let requested_kms_key_id = material.kms_key_id.clone(); - let encrypted_reader = EncryptReader::new_multipart(reader, material.key_bytes, material.base_nonce, part_id); - reader = HashReader::from_reader( - encrypted_reader, - HashReader::SIZE_PRESERVE_LAYER, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)?; - (requested_sse, requested_kms_key_id, mp_info.user_defined.clone()) + metadata: &mp_info.user_defined, + sse_customer_key: sse_customer_key.as_ref(), + sse_customer_key_md5: sse_customer_key_md5.as_ref(), + }) + .await? + .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; + let ssec_write = match ssec_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => { + rustfs_ecstore::rio::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32) } - None => (None, None, mp_info.user_defined.clone()), - } + crate::storage::sse::EncryptionKeyKind::Direct => { + rustfs_ecstore::rio::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id) + } + }; + write_plan = write_plan.with_encryption(ssec_write); + ( + Some(ssec_material.server_side_encryption), + ssec_material.kms_key_id, + mp_info.user_defined.clone(), + ) } else if let Some(server_side_encryption) = server_side_encryption { let managed_material = sse_decryption(DecryptionRequest { bucket: &bucket, @@ -1205,15 +1198,24 @@ impl DefaultMultipartUsecase { }) .await? .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; - let encrypted_reader = - EncryptReader::new_multipart(reader, managed_material.key_bytes, managed_material.base_nonce, part_id); - reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + let managed_write = match managed_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => { + rustfs_ecstore::rio::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32) + } + crate::storage::sse::EncryptionKeyKind::Direct => rustfs_ecstore::rio::WriteEncryption::multipart( + managed_material.key_bytes, + managed_material.base_nonce, + part_id, + ), + }; + write_plan = write_plan.with_encryption(managed_write); (Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone()) } else { (None, None, mp_info.user_defined.clone()) }; + reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; + if let Some(checksum_algorithm) = mp_info .user_defined .get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) @@ -1384,18 +1386,17 @@ mod tests { key: "object", server_side_encryption: Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, + sse_customer_key: None, sse_customer_key_md5: None, }; let session_material = sse_prepare_encryption(prepare_request) .await .expect("prepare multipart encryption") .expect("managed multipart session material"); - let session_metadata = encryption_material_to_metadata(&session_material); - let session_nonce = session_metadata - .get("x-rustfs-encryption-iv") - .cloned() - .expect("session nonce metadata"); + let mut session_metadata = encryption_material_to_metadata(&session_material); + mark_encrypted_multipart_metadata(&mut session_metadata); let part_one_plaintext = vec![0x31; rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE + 23]; let part_two_plaintext = vec![0x32; rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE * 2 + 7]; @@ -1411,15 +1412,31 @@ mod tests { .expect("decrypt session one") .expect("part one material"); let mut encrypted_one = Vec::new(); - EncryptReader::new_multipart( + #[cfg(feature = "rio-v2")] + let mut part_one_reader = match part_one_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key( + Cursor::new(part_one_plaintext.clone()), + part_one_material.key_bytes, + 1, + ), + crate::storage::sse::EncryptionKeyKind::Direct => EncryptReader::new_multipart( + Cursor::new(part_one_plaintext.clone()), + part_one_material.key_bytes, + part_one_material.base_nonce, + 1, + ), + }; + #[cfg(not(feature = "rio-v2"))] + let mut part_one_reader = EncryptReader::new_multipart( Cursor::new(part_one_plaintext.clone()), part_one_material.key_bytes, part_one_material.base_nonce, 1, - ) - .read_to_end(&mut encrypted_one) - .await - .expect("read encrypted part one"); + ); + part_one_reader + .read_to_end(&mut encrypted_one) + .await + .expect("read encrypted part one"); let part_two_material = sse_decryption(DecryptionRequest { bucket: "bucket", @@ -1432,20 +1449,38 @@ mod tests { .expect("decrypt session two") .expect("part two material"); let mut encrypted_two = Vec::new(); - EncryptReader::new_multipart( + #[cfg(feature = "rio-v2")] + let mut part_two_reader = match part_two_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key( + Cursor::new(part_two_plaintext.clone()), + part_two_material.key_bytes, + 2, + ), + crate::storage::sse::EncryptionKeyKind::Direct => EncryptReader::new_multipart( + Cursor::new(part_two_plaintext.clone()), + part_two_material.key_bytes, + part_two_material.base_nonce, + 2, + ), + }; + #[cfg(not(feature = "rio-v2"))] + let mut part_two_reader = EncryptReader::new_multipart( Cursor::new(part_two_plaintext.clone()), part_two_material.key_bytes, part_two_material.base_nonce, 2, - ) - .read_to_end(&mut encrypted_two) - .await - .expect("read encrypted part two"); - - assert_eq!( - session_metadata.get("x-rustfs-encryption-iv").map(String::as_str), - Some(session_nonce.as_str()) ); + part_two_reader + .read_to_end(&mut encrypted_two) + .await + .expect("read encrypted part two"); + + if session_material.key_kind == crate::storage::sse::EncryptionKeyKind::Object { + assert!(session_metadata.contains_key("X-Minio-Internal-Encrypted-Multipart")); + assert!(session_metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key")); + } else { + assert!(session_metadata.contains_key("x-rustfs-encryption-iv")); + } let parts = vec![ ObjectPartInfo { @@ -1478,15 +1513,28 @@ mod tests { .expect("managed decryption material"); let plaintext_size = multipart_plaintext_size(&parts, -1); - let mut decrypted_reader = HardLimitReader::new( - boxed_reader(DecryptReader::new_multipart( + #[cfg(feature = "rio-v2")] + let decrypted_stream = match decryption_material.key_kind { + crate::storage::sse::EncryptionKeyKind::Object => boxed_reader(DecryptReader::new_multipart_with_object_key( + wrap_reader(Cursor::new(encrypted_stream)), + decryption_material.key_bytes, + multipart_part_numbers(&parts), + )), + crate::storage::sse::EncryptionKeyKind::Direct => boxed_reader(DecryptReader::new_multipart( wrap_reader(Cursor::new(encrypted_stream)), decryption_material.key_bytes, decryption_material.base_nonce, multipart_part_numbers(&parts), )), - plaintext_size, - ); + }; + #[cfg(not(feature = "rio-v2"))] + let decrypted_stream = boxed_reader(DecryptReader::new_multipart( + wrap_reader(Cursor::new(encrypted_stream)), + decryption_material.key_bytes, + decryption_material.base_nonce, + multipart_part_numbers(&parts), + )); + let mut decrypted_reader = HardLimitReader::new(decrypted_stream, plaintext_size); let mut decrypted = Vec::new(); decrypted_reader diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index e17c6f346..fecc7e90d 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -31,7 +31,10 @@ use crate::storage::options::{ }; use crate::storage::request_context::spawn_traced; use crate::storage::s3_api::multipart::parse_list_parts_params; -use crate::storage::sse::{SSEType, build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error}; +use crate::storage::sse::{ + SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssekms_context_from_headers, + map_get_object_reader_error, +}; use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; use crate::storage::*; use crate::table_catalog; @@ -71,6 +74,7 @@ use rustfs_ecstore::config::storageclass; use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found}; use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}; use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader}; use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class}; use rustfs_ecstore::store_api::{ HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, StorageAPI, @@ -84,7 +88,6 @@ use rustfs_io_metrics; use rustfs_lock::NamespaceLockGuard; use rustfs_notify::EventArgsBuilder; use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_rio::{CompressReader, DynReader, EncryptReader, HashReader, wrap_reader}; use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object}; use rustfs_s3select_api::object_store::bytes_stream; use rustfs_targets::{ @@ -1193,10 +1196,28 @@ impl DefaultObjectUsecase { where R: AsyncRead + Send + Sync + 'static, { - Some(StreamingBlob::wrap(bytes_stream( - ReaderStream::with_capacity(reader, optimal_buffer_size), - response_content_length as usize, - ))) + let expected = response_content_length.max(0) as usize; + #[cfg(feature = "tracing-chunk-debug")] + let stream = { + let mut emitted = 0usize; + ReaderStream::with_capacity(reader, optimal_buffer_size).inspect(move |item| match item { + Ok(bytes) => { + emitted += bytes.len(); + tracing::debug!(emitted, expected, chunk_len = bytes.len(), "GetObject ReaderStream emitted bytes"); + } + Err(err) => { + tracing::error!( + emitted, + expected, + error = %err, + "GetObject ReaderStream returned error" + ); + } + }) + }; + #[cfg(not(feature = "tracing-chunk-debug"))] + let stream = ReaderStream::with_capacity(reader, optimal_buffer_size); + Some(StreamingBlob::wrap(bytes_stream(stream, expected))) } fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result { @@ -1839,6 +1860,7 @@ impl DefaultObjectUsecase { validate_sse_headers_for_write( effective_sse.as_ref(), effective_kms_key_id.as_ref(), + extract_ssekms_context_from_headers(&req.headers)?.as_ref(), sse_customer_algorithm.as_ref(), sse_customer_key.as_ref(), sse_customer_key_md5.as_ref(), @@ -1906,9 +1928,15 @@ impl DefaultObjectUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); - let mut reader = if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 { + let should_compress = is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64; + let mut write_plan = WritePlan::new(); + let mut reader = if should_compress { let algorithm = CompressionAlgorithm::default(); - insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str( + &mut metadata, + SUFFIX_COMPRESSION, + rustfs_ecstore::rio::compression_metadata_value(algorithm), + ); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let mut hrd = @@ -1919,12 +1947,16 @@ impl DefaultObjectUsecase { } opts.want_checksum = hrd.checksum(); - insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string()); + insert_str( + &mut opts.user_defined, + SUFFIX_COMPRESSION, + rustfs_ecstore::rio::compression_metadata_value(algorithm), + ); insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader(CompressReader::new(hrd, algorithm), size, actual_size, None, None, false) - .map_err(ApiError::from)? + write_plan = write_plan.with_compression(algorithm); + hrd } else { HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)? }; @@ -1938,6 +1970,7 @@ impl DefaultObjectUsecase { } let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject); + let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?; // Apply encryption using unified SSE API. let encryption_request = EncryptionRequest { @@ -1945,6 +1978,7 @@ impl DefaultObjectUsecase { key: &key, server_side_encryption: effective_sse.clone(), ssekms_key_id: effective_kms_key_id.clone(), + ssekms_context, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key, sse_customer_key_md5: sse_customer_key_md5.clone(), @@ -1964,15 +1998,15 @@ impl DefaultObjectUsecase { effective_sse = Some(material.server_side_encryption.clone()); effective_kms_key_id = material.kms_key_id.clone(); - let encrypted_reader = EncryptReader::new(reader, material.key_bytes, material.base_nonce); - reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + write_plan = write_plan.with_encryption(material.write_encryption(None)); let encryption_metadata = encryption_material_to_metadata(&material); metadata.extend(encryption_metadata.clone()); opts.user_defined.extend(encryption_metadata); } + reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; + let mut reader = PutObjReader::new(reader); let mt2 = metadata.clone(); @@ -2864,14 +2898,18 @@ impl DefaultObjectUsecase { let actual_size = src_info.get_actual_size().map_err(ApiError::from)?; - let mut length = actual_size; + let length = actual_size; let mut compress_metadata = HashMap::new(); let should_compress = is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64; if should_compress { - insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + insert_str( + &mut compress_metadata, + SUFFIX_COMPRESSION, + rustfs_ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()), + ); insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); } else { remove_str(&mut user_defined, SUFFIX_COMPRESSION); @@ -2907,18 +2945,12 @@ impl DefaultObjectUsecase { } apply_bucket_default_lock_retention(&bucket, &mut user_defined, has_explicit_object_lock_retention).await?; + let mut write_plan = WritePlan::new(); let mut reader = if should_compress { + let algorithm = CompressionAlgorithm::default(); let hrd = HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?; - length = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - length, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? + write_plan = write_plan.with_compression(algorithm); + hrd } else { HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)? }; @@ -2928,6 +2960,7 @@ impl DefaultObjectUsecase { key: &key, server_side_encryption: effective_sse.clone(), ssekms_key_id: effective_kms_key_id.clone(), + ssekms_context: extract_ssekms_context_from_headers(&req.headers)?, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key, sse_customer_key_md5: sse_customer_key_md5.clone(), @@ -2938,13 +2971,13 @@ impl DefaultObjectUsecase { effective_sse = Some(material.server_side_encryption.clone()); effective_kms_key_id = material.kms_key_id.clone(); - let encrypted_reader = EncryptReader::new(reader, material.key_bytes, material.base_nonce); - reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + write_plan = write_plan.with_encryption(material.write_encryption(None)); user_defined.extend(encryption_material_to_metadata(&material)); } + reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?; + src_info.put_object_reader = Some(PutObjReader::new(reader)); // check quota @@ -4182,6 +4215,7 @@ impl DefaultObjectUsecase { validate_sse_headers_for_write( effective_sse.as_ref(), effective_kms_key_id.as_ref(), + extract_ssekms_context_from_headers(&req.headers)?.as_ref(), sse_customer_algorithm.as_ref(), sse_customer_key.as_ref(), sse_customer_key_md5.as_ref(), @@ -4379,24 +4413,22 @@ impl DefaultObjectUsecase { let should_compress = !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64; + let mut write_plan = WritePlan::new(); let mut hrd = if is_dir { HashReader::from_stream(std::io::Cursor::new(Vec::new()), size, actual_size, None, None, false) .map_err(ApiError::from)? } else if should_compress { - insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string()); + let algorithm = CompressionAlgorithm::default(); + insert_str( + &mut metadata, + SUFFIX_COMPRESSION, + rustfs_ecstore::rio::compression_metadata_value(algorithm), + ); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?; - size = HashReader::SIZE_PRESERVE_LAYER; - HashReader::from_reader( - CompressReader::new(hrd, CompressionAlgorithm::default()), - size, - actual_size, - None, - None, - false, - ) - .map_err(ApiError::from)? + write_plan = write_plan.with_compression(algorithm); + hrd } else { HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)? }; @@ -4413,6 +4445,7 @@ impl DefaultObjectUsecase { key: &fpath, server_side_encryption: effective_sse.clone(), ssekms_key_id: effective_kms_key_id.clone(), + ssekms_context: extract_ssekms_context_from_headers(&req.headers)?, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key: sse_customer_key.clone(), sse_customer_key_md5: sse_customer_key_md5.clone(), @@ -4423,14 +4456,13 @@ impl DefaultObjectUsecase { effective_sse = Some(material.server_side_encryption.clone()); effective_kms_key_id = material.kms_key_id.clone(); - let encrypted_reader = EncryptReader::new(hrd, material.key_bytes, material.base_nonce); - hrd = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false) - .map_err(ApiError::from)?; + write_plan = write_plan.with_encryption(material.write_encryption(None)); let encryption_metadata = encryption_material_to_metadata(&material); metadata.extend(encryption_metadata.clone()); opts.user_defined.extend(encryption_metadata); } + hrd = write_plan.apply(hrd, actual_size).map_err(ApiError::from)?; opts.user_defined.extend(metadata); let mut reader = PutObjReader::new(hrd); diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 6584a59bf..bf304166f 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -75,27 +75,63 @@ use aes_gcm::{ }; use async_trait::async_trait; use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; +#[cfg(feature = "rio-v2")] +use hmac::{Hmac, Mac}; use http::{HeaderMap, HeaderValue}; use rand::Rng; +#[cfg(feature = "rio-v2")] +use rand::RngExt; use rustfs_ecstore::error::StorageError; use rustfs_kms::{DataKey, service_manager::get_global_encryption_service, types::ObjectEncryptionContext}; use rustfs_utils::get_env_opt_str; use s3s::S3ErrorCode; use s3s::dto::ServerSideEncryption; +#[cfg(feature = "rio-v2")] +use sha2::Sha256; use std::collections::HashMap; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, LazyLock, RwLock}; use tracing::{debug, error}; const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; +const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key"; +const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv"; +const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm"; +const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size"; const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size"; +const MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER: &str = "X-Minio-Internal-Encrypted-Multipart"; +const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv"; +const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"; +const MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"; +const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"; +const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"; +const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"; +const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"; +const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context"; +#[cfg(feature = "rio-v2")] +const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256"; +#[cfg(feature = "rio-v2")] +const DARE_VERSION_20: u8 = 0x20; +#[cfg(feature = "rio-v2")] +const DARE_CIPHER_AES_256_GCM: u8 = 0x00; +#[cfg(feature = "rio-v2")] +const DARE_HEADER_SIZE: usize = 16; +#[cfg(feature = "rio-v2")] +const DARE_TAG_SIZE: usize = 16; +#[cfg(feature = "rio-v2")] +const SEALED_KEY_IV_SIZE: usize = 32; +#[cfg(feature = "rio-v2")] +const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE; +#[cfg(feature = "rio-v2")] +const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation"; use crate::error::ApiError; use rustfs_ecstore::bucket::metadata_sys; use rustfs_ecstore::error::Error; use rustfs_utils::http::headers::{ AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, - AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, + AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, }; +use rustfs_utils::path::path_join_buf; use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId}; // ============================================================================ @@ -333,6 +369,8 @@ pub struct EncryptionRequest<'a> { pub server_side_encryption: Option, /// KMS key ID (for SSE-KMS) pub ssekms_key_id: Option, + /// Optional client-provided KMS context for SSE-KMS. + pub ssekms_context: Option>, /// SSE-C algorithm (customer-provided key) pub sse_customer_algorithm: Option, /// SSE-C key (Base64-encoded) @@ -423,10 +461,28 @@ pub(crate) fn extract_server_side_encryption_from_headers(headers: &HeaderMap) - } } +pub(crate) fn extract_ssekms_context_from_headers(headers: &HeaderMap) -> Result>, ApiError> { + let Some(v) = headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT) else { + return Ok(None); + }; + + let value = v + .to_str() + .map_err(|_| sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid UTF-8."))?; + let decoded = BASE64_STANDARD.decode(value).map_err(|_| { + sse_invalid_argument("The x-amz-server-side-encryption-context header must be valid base64-encoded JSON.") + })?; + + serde_json::from_slice(&decoded).map(Some).map_err(|_| { + sse_invalid_argument("The x-amz-server-side-encryption-context header must be a base64-encoded JSON object.") + }) +} + #[inline] pub(crate) fn validate_sse_headers_for_write( server_side_encryption: Option<&ServerSideEncryption>, ssekms_key_id: Option<&SSEKMSKeyId>, + ssekms_context: Option<&HashMap>, sse_customer_algorithm: Option<&SSECustomerAlgorithm>, sse_customer_key: Option<&SSECustomerKey>, sse_customer_key_md5: Option<&SSECustomerKeyMD5>, @@ -442,7 +498,7 @@ pub(crate) fn validate_sse_headers_for_write( } let has_ssec_headers = sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some(); - let has_managed_headers = server_side_encryption.is_some() || ssekms_key_id.is_some(); + let has_managed_headers = server_side_encryption.is_some() || ssekms_key_id.is_some() || ssekms_context.is_some(); if has_ssec_headers { if has_managed_headers { @@ -477,6 +533,12 @@ pub(crate) fn validate_sse_headers_for_write( )); } + if ssekms_context.is_some() && server_side_encryption.is_none_or(|sse| sse.as_str() != ServerSideEncryption::AWS_KMS) { + return Err(sse_invalid_argument( + "The SSE-KMS context header can only be used when x-amz-server-side-encryption is set to aws:kms.", + )); + } + Ok(()) } @@ -487,7 +549,8 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap, || headers.contains_key("x-amz-server-side-encryption-customer-key-md5"); let has_req_sse = headers.contains_key("x-amz-server-side-encryption") - || headers.contains_key("x-amz-server-side-encryption-aws-kms-key-id"); + || headers.contains_key("x-amz-server-side-encryption-aws-kms-key-id") + || headers.contains_key("x-amz-server-side-encryption-context"); let is_object_ssec = metadata.contains_key("x-amz-server-side-encryption-customer-algorithm"); let is_object_sse = metadata.contains_key("x-amz-server-side-encryption"); @@ -604,6 +667,13 @@ pub struct EncryptionMaterial { pub customer_key_md5: Option, /// Original plaintext size when it should be persisted alongside metadata. pub original_size: Option, + /// Indicates whether `key_bytes` is a direct stream key or a MinIO-style object key. + pub key_kind: EncryptionKeyKind, + /// Original client-provided SSE-KMS context, stored in MinIO-compatible metadata. + pub managed_kms_context: Option>, + /// MinIO-compatible sealed object-key metadata for managed SSE object-key mode. + #[cfg_attr(not(feature = "rio-v2"), allow(dead_code))] + pub managed_sealed_key: Option, } /// Decryption material returned by `sse_decryption()`. @@ -620,6 +690,8 @@ pub struct DecryptionMaterial { pub key_bytes: [u8; 32], /// Base nonce/IV used by rio to derive block/part nonces. pub base_nonce: [u8; 12], + /// Indicates whether `key_bytes` is a direct stream key or a MinIO-style object key. + pub key_kind: EncryptionKeyKind, } /// Type of encryption used @@ -633,6 +705,243 @@ pub enum SSEType { SseC, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncryptionKeyKind { + Direct, + Object, +} + +#[derive(Debug, Clone)] +pub struct ManagedSealedKey { + #[cfg(feature = "rio-v2")] + pub iv: [u8; SEALED_KEY_IV_SIZE], + #[cfg(feature = "rio-v2")] + pub sealed_key: [u8; SEALED_KEY_SIZE], +} + +impl EncryptionMaterial { + pub fn write_encryption(&self, multipart_part_number: Option) -> rustfs_ecstore::rio::WriteEncryption { + match (self.key_kind, multipart_part_number) { + (EncryptionKeyKind::Object, Some(part_number)) => { + rustfs_ecstore::rio::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32) + } + (EncryptionKeyKind::Object, None) => rustfs_ecstore::rio::WriteEncryption::singlepart_object_key(self.key_bytes), + (EncryptionKeyKind::Direct, Some(part_number)) => { + rustfs_ecstore::rio::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number) + } + (EncryptionKeyKind::Direct, None) => { + rustfs_ecstore::rio::WriteEncryption::singlepart(self.key_bytes, self.base_nonce) + } + } + } +} + +#[cfg(feature = "rio-v2")] +type HmacSha256 = Hmac; + +#[cfg(feature = "rio-v2")] +fn managed_sse_domain(sse_type: SSEType) -> &'static str { + match sse_type { + SSEType::SseS3 => "SSE-S3", + SSEType::SseKms => "SSE-KMS", + SSEType::SseC => "SSE-C", + } +} + +fn canonical_kms_bucket_path(bucket: &str, key: &str) -> String { + path_join_buf(&[bucket, key]) +} + +fn build_kms_request_context( + bucket: &str, + key: &str, + provided_context: Option<&HashMap>, +) -> HashMap { + let mut context = provided_context.cloned().unwrap_or_default(); + context + .entry(bucket.to_string()) + .or_insert_with(|| canonical_kms_bucket_path(bucket, key)); + context +} + +fn build_object_encryption_context( + bucket: &str, + key: &str, + provided_context: Option<&HashMap>, +) -> ObjectEncryptionContext { + let mut context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); + for (ctx_key, ctx_value) in build_kms_request_context(bucket, key, provided_context) { + context = context.with_encryption_context(ctx_key, ctx_value); + } + context +} + +fn encode_minio_kms_context(context: &HashMap) -> Result { + let encoded = serde_json::to_vec(context) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to serialize KMS context: {e}"))))?; + Ok(BASE64_STANDARD.encode(encoded)) +} + +fn decode_minio_kms_context(metadata: &HashMap) -> Result>, ApiError> { + let Some(encoded) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) else { + return Ok(None); + }; + let decoded = BASE64_STANDARD + .decode(encoded) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode MinIO KMS context: {e}"))))?; + serde_json::from_slice(&decoded) + .map(Some) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse MinIO KMS context: {e}")))) +} + +#[cfg(feature = "rio-v2")] +fn canonical_sse_path(bucket: &str, object: &str) -> String { + let bucket = bucket.trim_matches('/'); + let object = object.trim_matches('/'); + if object.is_empty() { + bucket.to_string() + } else if bucket.is_empty() { + object.to_string() + } else { + format!("{bucket}/{object}") + } +} + +#[cfg(feature = "rio-v2")] +fn derive_object_key(external_key: [u8; 32]) -> [u8; 32] { + let mut random = [0u8; 32]; + rand::rng().fill(&mut random); + + let mut mac = HmacSha256::new_from_slice(&external_key).expect("32-byte HMAC key"); + mac.update(OBJECT_KEY_DERIVATION_CONTEXT); + mac.update(&random); + + let mut object_key = [0u8; 32]; + object_key.copy_from_slice(mac.finalize().into_bytes().as_slice()); + object_key +} + +#[cfg(feature = "rio-v2")] +fn derive_sealing_key( + external_key: [u8; 32], + iv: [u8; SEALED_KEY_IV_SIZE], + domain: &str, + bucket: &str, + object: &str, +) -> [u8; 32] { + let mut mac = HmacSha256::new_from_slice(&external_key).expect("32-byte HMAC key"); + mac.update(&iv); + mac.update(domain.as_bytes()); + mac.update(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.as_bytes()); + mac.update(canonical_sse_path(bucket, object).as_bytes()); + + let mut sealing_key = [0u8; 32]; + sealing_key.copy_from_slice(mac.finalize().into_bytes().as_slice()); + sealing_key +} + +#[cfg(feature = "rio-v2")] +fn seal_object_key( + object_key: [u8; 32], + external_key: [u8; 32], + sse_type: SSEType, + bucket: &str, + object: &str, +) -> Result { + let mut iv = [0u8; SEALED_KEY_IV_SIZE]; + rand::rng().fill(&mut iv); + let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(sse_type), bucket, object); + + let mut header = [0u8; DARE_HEADER_SIZE]; + header[0] = DARE_VERSION_20; + header[1] = DARE_CIPHER_AES_256_GCM; + header[2..4].copy_from_slice(&(31u16).to_le_bytes()); + let mut stream_nonce = [0u8; 12]; + rand::rng().fill(&mut stream_nonce); + stream_nonce[0] |= 0x80; + header[4..16].copy_from_slice(&stream_nonce); + + let cipher = Aes256Gcm::new_from_slice(&sealing_key) + .map_err(|err| ApiError::from(StorageError::other(format!("Invalid sealing key: {err}"))))?; + let nonce = Nonce::try_from(stream_nonce.as_slice()) + .map_err(|_| ApiError::from(StorageError::other("Invalid sealed object-key stream nonce")))?; + let ciphertext = cipher + .encrypt( + &nonce, + aes_gcm::aead::Payload { + msg: &object_key, + aad: &header[..4], + }, + ) + .map_err(|err| ApiError::from(StorageError::other(format!("Failed to seal object key: {err}"))))?; + + let mut sealed_key = [0u8; SEALED_KEY_SIZE]; + sealed_key[..DARE_HEADER_SIZE].copy_from_slice(&header); + sealed_key[DARE_HEADER_SIZE..].copy_from_slice(&ciphertext); + + Ok(ManagedSealedKey { iv, sealed_key }) +} + +#[cfg(feature = "rio-v2")] +fn unseal_object_key( + sealed: &ManagedSealedKey, + external_key: [u8; 32], + sse_type: SSEType, + bucket: &str, + object: &str, +) -> Result<[u8; 32], ApiError> { + let header = &sealed.sealed_key[..DARE_HEADER_SIZE]; + if header[0] != DARE_VERSION_20 || header[1] != DARE_CIPHER_AES_256_GCM { + return Err(ApiError::from(StorageError::other("Unsupported sealed object-key DARE header"))); + } + if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 { + return Err(ApiError::from(StorageError::other("Invalid sealed object-key payload header"))); + } + + let sealing_key = derive_sealing_key(external_key, sealed.iv, managed_sse_domain(sse_type), bucket, object); + let cipher = Aes256Gcm::new_from_slice(&sealing_key) + .map_err(|err| ApiError::from(StorageError::other(format!("Invalid sealing key: {err}"))))?; + let nonce = Nonce::try_from(&header[4..16]) + .map_err(|_| ApiError::from(StorageError::other("Invalid sealed object-key package nonce")))?; + let plaintext = cipher + .decrypt( + &nonce, + aes_gcm::aead::Payload { + msg: &sealed.sealed_key[DARE_HEADER_SIZE..], + aad: &header[..4], + }, + ) + .map_err(|err| ApiError::from(StorageError::other(format!("Failed to unseal object key: {err}"))))?; + + let object_key: [u8; 32] = plaintext + .as_slice() + .try_into() + .map_err(|_| ApiError::from(StorageError::other("Sealed object key must decrypt to 32 bytes")))?; + Ok(object_key) +} + +#[cfg(feature = "rio-v2")] +fn try_decode_minio_sealed_key(bytes: &str) -> Result, ApiError> { + let decoded = BASE64_STANDARD + .decode(bytes) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode sealed object key: {e}"))))?; + match decoded.as_slice().try_into() { + Ok(sealed_key) => Ok(Some(sealed_key)), + Err(_) => Ok(None), + } +} + +#[cfg(feature = "rio-v2")] +fn try_decode_minio_sealing_iv(bytes: &str) -> Result, ApiError> { + let decoded = BASE64_STANDARD + .decode(bytes) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode sealing IV: {e}"))))?; + match decoded.as_slice().try_into() { + Ok(iv) => Ok(Some(iv)), + Err(_) => Ok(None), + } +} + pub(crate) fn build_ssec_read_headers( algorithm: Option<&SSECustomerAlgorithm>, key: Option<&SSECustomerKey>, @@ -680,15 +989,25 @@ pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> HashMap if let Some(original_size) = material.original_size { metadata.insert(SSEC_ORIGINAL_SIZE_HEADER.to_string(), original_size.to_string()); } + + #[cfg(feature = "rio-v2")] + if let Some(sealed) = &material.managed_sealed_key { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealed.iv)); + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ); + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed.sealed_key), + ); + } } SSEType::SseS3 | SSEType::SseKms => { let encrypted_data_key = material .encrypted_data_key .as_deref() .expect("managed SSE materials must carry an encrypted data key"); - metadata.insert("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_data_key)); - metadata.insert("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(material.base_nonce)); - metadata.insert("x-rustfs-encryption-algorithm".to_string(), material.algorithm.as_str().to_string()); metadata.insert( "x-amz-server-side-encryption".to_string(), material.server_side_encryption.as_str().to_string(), @@ -705,7 +1024,80 @@ pub fn encryption_material_to_metadata(material: &EncryptionMaterial) -> HashMap } if let Some(original_size) = material.original_size { - metadata.insert("x-rustfs-encryption-original-size".to_string(), original_size.to_string()); + metadata.insert(INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), original_size.to_string()); + } + + if let Some(kms_context) = &material.managed_kms_context + && !kms_context.is_empty() + { + if let Ok(serialized) = serde_json::to_string(kms_context) { + metadata.insert("x-rustfs-encryption-context".to_string(), serialized); + } + if matches!(material.sse_type, SSEType::SseKms) + && let Ok(encoded) = encode_minio_kms_context(kms_context) + { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), encoded); + } + } + + if material.key_kind == EncryptionKeyKind::Direct { + metadata.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(encrypted_data_key)); + metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(material.base_nonce)); + metadata.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), material.algorithm.as_str().to_string()); + } + + #[cfg(feature = "rio-v2")] + if let Some(sealed) = &material.managed_sealed_key { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealed.iv)); + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ); + if let Some(kms_key_id) = &material.kms_key_id { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), kms_key_id.to_string()); + } + match material.sse_type { + SSEType::SseS3 => { + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed.sealed_key), + ); + } + SSEType::SseKms => { + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(sealed.sealed_key), + ); + } + SSEType::SseC => {} + } + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(encrypted_data_key), + ); + } else if cfg!(feature = "rio-v2") { + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), + BASE64_STANDARD.encode(material.base_nonce), + ); + metadata.insert( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ); + if let Some(kms_key_id) = &material.kms_key_id { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), kms_key_id.to_string()); + } + let encoded_key = BASE64_STANDARD.encode(encrypted_data_key); + match material.sse_type { + SSEType::SseS3 => { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), encoded_key); + } + SSEType::SseKms => { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), encoded_key.clone()); + metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), encoded_key); + } + SSEType::SseC => {} + } } } } @@ -755,6 +1147,7 @@ pub async fn sse_encryption(request: EncryptionRequest<'_>) -> Result) -> Result { pub server_side_encryption: Option, /// KMS key ID (for SSE-KMS) pub ssekms_key_id: Option, + /// Optional client-provided KMS context for SSE-KMS. + pub ssekms_context: Option>, /// SSE-C algorithm (customer-provided key) pub sse_customer_algorithm: Option, + /// SSE-C key (Base64-encoded) + pub sse_customer_key: Option, /// SSE-C key MD5 (Base64-encoded) pub sse_customer_key_md5: Option, } @@ -813,8 +1211,9 @@ pub async fn sse_prepare_encryption(request: PrepareEncryptionRequest<'_>) -> Re validate_sse_headers_for_write( request.server_side_encryption.as_ref(), request.ssekms_key_id.as_ref(), + request.ssekms_context.as_ref(), request.sse_customer_algorithm.as_ref(), - None, + request.sse_customer_key.as_ref(), request.sse_customer_key_md5.as_ref(), false, )?; @@ -823,7 +1222,7 @@ pub async fn sse_prepare_encryption(request: PrepareEncryptionRequest<'_>) -> Re request.bucket, request.server_side_encryption, request.sse_customer_algorithm, - None, + request.sse_customer_key.clone(), request.sse_customer_key_md5, request.ssekms_key_id, ) @@ -831,11 +1230,16 @@ pub async fn sse_prepare_encryption(request: PrepareEncryptionRequest<'_>) -> Re // apply encryption material let material = match sse_type { - Some(SseTypeV2::SseS3(sse)) => apply_managed_encryption_material(request.bucket, request.key, sse, None, 0).await?, - Some(SseTypeV2::SseKms(sse, kms_key_id)) => { - apply_managed_encryption_material(request.bucket, request.key, sse, kms_key_id, 0).await? + Some(SseTypeV2::SseS3(sse)) => { + apply_managed_encryption_material(request.bucket, request.key, sse, None, request.ssekms_context, 0).await? + } + Some(SseTypeV2::SseKms(sse, kms_key_id)) => { + apply_managed_encryption_material(request.bucket, request.key, sse, kms_key_id, request.ssekms_context, 0).await? + } + Some(SseTypeV2::SseC(algorithm, _, key_md5)) => { + apply_ssec_prepare_encryption_material(request.bucket, request.key, algorithm, request.sse_customer_key, key_md5) + .await? } - Some(SseTypeV2::SseC(algorithm, _, key_md5)) => apply_ssec_prepare_encryption_material(algorithm, key_md5).await?, None => return Ok(None), }; @@ -897,7 +1301,7 @@ pub async fn sse_decryption(request: DecryptionRequest<'_>) -> Result) -> Result, sse_key_md5: SSECustomerKeyMD5, ) -> Result { + #[cfg(feature = "rio-v2")] + let (key_bytes, base_nonce, key_kind, managed_sealed_key) = if let Some(sse_key) = sse_key { + let validated = validate_ssec_params(SsecParams { + algorithm: algorithm.clone(), + key: sse_key, + key_md5: sse_key_md5.clone(), + })?; + let object_key = derive_object_key(validated.key_bytes); + let sealed_key = seal_object_key(object_key, validated.key_bytes, SSEType::SseC, bucket, key)?; + (object_key, [0; 12], EncryptionKeyKind::Object, Some(sealed_key)) + } else { + ([0; 32], [0; 12], EncryptionKeyKind::Direct, None) + }; + + #[cfg(not(feature = "rio-v2"))] + let (key_bytes, base_nonce, key_kind, managed_sealed_key) = { + let _ = (bucket, key, sse_key); + ([0; 32], [0; 12], EncryptionKeyKind::Direct, None) + }; + Ok(EncryptionMaterial { sse_type: SSEType::SseC, server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AES256), kms_key_id: None, algorithm, - key_bytes: [0; 32], - base_nonce: [0; 12], + key_bytes, + base_nonce, encrypted_data_key: None, customer_key_md5: Some(sse_key_md5), original_size: None, + key_kind, + managed_kms_context: None, + managed_sealed_key, }) } @@ -942,8 +1372,18 @@ async fn apply_ssec_encryption_material( let validated = validate_ssec_params(params)?; - // Generate nonce (deterministic for SSE-C) - let base_nonce = generate_ssec_nonce(bucket, key); + #[cfg(feature = "rio-v2")] + let (key_bytes, base_nonce, key_kind, managed_sealed_key) = { + let object_key = derive_object_key(validated.key_bytes); + let sealed_key = seal_object_key(object_key, validated.key_bytes, SSEType::SseC, bucket, key)?; + (object_key, [0; 12], EncryptionKeyKind::Object, Some(sealed_key)) + }; + + #[cfg(not(feature = "rio-v2"))] + let (key_bytes, base_nonce, key_kind, managed_sealed_key) = { + let base_nonce = generate_ssec_nonce(bucket, key); + (validated.key_bytes, base_nonce, EncryptionKeyKind::Direct, None) + }; // Build metadata Ok(EncryptionMaterial { @@ -951,11 +1391,14 @@ async fn apply_ssec_encryption_material( server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AES256), kms_key_id: None, algorithm: validated.algorithm, - key_bytes: validated.key_bytes, + key_bytes, base_nonce, encrypted_data_key: None, customer_key_md5: Some(validated.key_md5), original_size: Some(content_size), + key_kind, + managed_kms_context: None, + managed_sealed_key, }) } @@ -980,8 +1423,23 @@ async fn apply_ssec_decryption_material( let validated = validate_ssec_params(params)?; - // Generate nonce (same as encryption) - let base_nonce = generate_ssec_nonce(bucket, key); + #[cfg(feature = "rio-v2")] + let (key_bytes, base_nonce, key_kind) = if let Some(sealed_key) = parse_minio_managed_sealed_key(metadata, SSEType::SseC)? { + ( + unseal_object_key(&sealed_key, validated.key_bytes, SSEType::SseC, bucket, key)?, + [0; 12], + EncryptionKeyKind::Object, + ) + } else { + let base_nonce = generate_ssec_nonce(bucket, key); + (validated.key_bytes, base_nonce, EncryptionKeyKind::Direct) + }; + + #[cfg(not(feature = "rio-v2"))] + let (key_bytes, base_nonce, key_kind) = { + let base_nonce = generate_ssec_nonce(bucket, key); + (validated.key_bytes, base_nonce, EncryptionKeyKind::Direct) + }; Ok(DecryptionMaterial { sse_type: SSEType::SseC, @@ -990,8 +1448,9 @@ async fn apply_ssec_decryption_material( algorithm: SSECustomerAlgorithm::from(algorithm), customer_key_md5: None, - key_bytes: validated.key_bytes, + key_bytes, base_nonce, + key_kind, }) } @@ -1004,6 +1463,7 @@ async fn apply_managed_encryption_material( key: &str, server_side_encryption: ServerSideEncryption, kms_key_id: Option, + ssekms_context: Option>, content_size: i64, ) -> Result { if !is_managed_sse(&server_side_encryption) { @@ -1041,43 +1501,85 @@ async fn apply_managed_encryption_material( }; let provider = get_sse_dek_provider().await?; + let object_context = build_object_encryption_context(bucket, key, ssekms_context.as_ref()); let (data_key, encrypted_data_key) = provider - .generate_sse_dek(bucket, key, &kms_key_to_use) + .generate_sse_dek(&object_context, &kms_key_to_use) .await .map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {e}"))))?; let algorithm = server_side_encryption.as_str().to_string(); + #[cfg(feature = "rio-v2")] + let (key_bytes, base_nonce, key_kind, managed_sealed_key) = { + let object_key = derive_object_key(data_key.plaintext_key); + let sealed_key = seal_object_key(object_key, data_key.plaintext_key, encryption_type, bucket, key)?; + (object_key, [0u8; 12], EncryptionKeyKind::Object, Some(sealed_key)) + }; + + #[cfg(not(feature = "rio-v2"))] + let (key_bytes, base_nonce, key_kind, managed_sealed_key) = + (data_key.plaintext_key, data_key.nonce, EncryptionKeyKind::Direct, None); Ok(EncryptionMaterial { sse_type: encryption_type, server_side_encryption, kms_key_id: matches!(encryption_type, SSEType::SseKms).then_some(kms_key_to_use), algorithm, - key_bytes: data_key.plaintext_key, - base_nonce: data_key.nonce, + key_bytes, + base_nonce, encrypted_data_key: Some(encrypted_data_key), customer_key_md5: None, original_size: Some(content_size), + key_kind, + managed_kms_context: matches!(encryption_type, SSEType::SseKms).then_some(ssekms_context.unwrap_or_default()), + managed_sealed_key, }) } async fn apply_managed_decryption_material( - _bucket: &str, - _key: &str, + bucket: &str, + key: &str, metadata: &HashMap, ) -> Result, ApiError> { - if !metadata.contains_key("x-rustfs-encryption-key") || !metadata.contains_key("x-amz-server-side-encryption") { + #[cfg(not(feature = "rio-v2"))] + let _ = (bucket, key); + if !contains_managed_encryption_metadata(metadata) || !metadata.contains_key("x-amz-server-side-encryption") { return Ok(None); } // Safe: presence is guaranteed by the contains_key check above. let server_side_encryption = metadata.get("x-amz-server-side-encryption").cloned().unwrap_or_default(); + let normalized_metadata = normalize_managed_metadata(metadata); - // Parse metadata - try using service if available, otherwise parse manually - let (encrypted_data_key, iv, algorithm) = if let Some(service) = get_global_encryption_service().await { + let encryption_type = match server_side_encryption.as_str() { + ServerSideEncryption::AES256 => SSEType::SseS3, + ServerSideEncryption::AWS_KMS => SSEType::SseKms, + _ => SSEType::SseS3, + }; + #[cfg(feature = "rio-v2")] + let minio_sealed_key = parse_minio_managed_sealed_key(metadata, encryption_type)?; + #[cfg(not(feature = "rio-v2"))] + let minio_sealed_key: Option = None; + + let (encrypted_data_key, iv, algorithm) = if minio_sealed_key.is_some() { + let encrypted_key_b64 = normalized_metadata + .get(INTERNAL_ENCRYPTION_KEY_HEADER) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)) + .ok_or_else(|| ApiError::from(StorageError::other("Missing encrypted key in metadata")))?; + let encrypted_data_key = BASE64_STANDARD + .decode(encrypted_key_b64) + .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode encrypted key: {e}"))))?; + ( + encrypted_data_key, + Vec::new(), + normalized_metadata + .get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER) + .cloned() + .unwrap_or_else(|| "AES256".to_string()), + ) + } else if let Some(service) = get_global_encryption_service().await { // Production mode: use service for metadata parsing let parsed = service - .headers_to_metadata(metadata) + .headers_to_metadata(&normalized_metadata) .map_err(|e| ApiError::from(StorageError::other(format!("Failed to parse encryption metadata: {e}"))))?; if parsed.iv.len() != 12 { @@ -1087,15 +1589,15 @@ async fn apply_managed_decryption_material( (parsed.encrypted_data_key, parsed.iv, parsed.algorithm) } else { // Test mode: parse metadata manually - let encrypted_key_b64 = metadata - .get("x-rustfs-encryption-key") + let encrypted_key_b64 = normalized_metadata + .get(INTERNAL_ENCRYPTION_KEY_HEADER) .ok_or_else(|| ApiError::from(StorageError::other("Missing encrypted key in metadata")))?; let encrypted_data_key = BASE64_STANDARD .decode(encrypted_key_b64) .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decode encrypted key: {e}"))))?; - let iv_b64 = metadata - .get("x-rustfs-encryption-iv") + let iv_b64 = normalized_metadata + .get(INTERNAL_ENCRYPTION_IV_HEADER) .ok_or_else(|| ApiError::from(StorageError::other("Missing IV in metadata")))?; let iv = BASE64_STANDARD .decode(iv_b64) @@ -1105,8 +1607,8 @@ async fn apply_managed_decryption_material( return Err(ApiError::from(StorageError::other("Invalid encryption nonce length; expected 12 bytes"))); } - let algorithm = metadata - .get("x-rustfs-encryption-algorithm") + let algorithm = normalized_metadata + .get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER) .cloned() .unwrap_or_else(|| "AES256".to_string()); @@ -1114,26 +1616,41 @@ async fn apply_managed_decryption_material( }; // Extract KMS key ID from metadata (optional, used for provider context) - let kms_key_id = metadata + let kms_key_id = normalized_metadata .get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) .or_else(|| metadata.get("x-amz-server-side-encryption-aws-kms-key-id")) .cloned() .unwrap_or_else(|| "default".to_string()); + let kms_context = if matches!(encryption_type, SSEType::SseKms) { + decode_minio_kms_context(metadata)? + } else { + None + }; + let object_context = build_object_encryption_context(bucket, key, kms_context.as_ref()); // Use factory pattern to get provider (test or production mode) let provider = get_sse_dek_provider().await?; - let key_bytes = provider - .decrypt_sse_dek(&encrypted_data_key, &kms_key_id) + let decrypted_data_key = provider + .decrypt_sse_dek(&encrypted_data_key, &kms_key_id, &object_context) .await .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {e}"))))?; - - let mut base_nonce = [0u8; 12]; - base_nonce.copy_from_slice(&iv[..12]); - - let encryption_type = match server_side_encryption.as_str() { - ServerSideEncryption::AES256 => SSEType::SseS3, - ServerSideEncryption::AWS_KMS => SSEType::SseKms, - _ => SSEType::SseS3, + #[cfg(feature = "rio-v2")] + let (key_bytes, base_nonce, key_kind) = if let Some(sealed_key) = minio_sealed_key { + ( + unseal_object_key(&sealed_key, decrypted_data_key, encryption_type, bucket, key)?, + [0u8; 12], + EncryptionKeyKind::Object, + ) + } else { + let mut base_nonce = [0u8; 12]; + base_nonce.copy_from_slice(&iv[..12]); + (decrypted_data_key, base_nonce, EncryptionKeyKind::Direct) + }; + #[cfg(not(feature = "rio-v2"))] + let (key_bytes, base_nonce, key_kind) = { + let mut base_nonce = [0u8; 12]; + base_nonce.copy_from_slice(&iv[..12]); + (decrypted_data_key, base_nonce, EncryptionKeyKind::Direct) }; Ok(Some(DecryptionMaterial { @@ -1145,6 +1662,7 @@ async fn apply_managed_decryption_material( key_bytes, base_nonce, + key_kind, })) } @@ -1183,10 +1701,16 @@ pub struct SsecParams { #[async_trait] pub trait SseDekProvider: Send + Sync { /// Generate an SSE data encryption key - async fn generate_sse_dek(&self, bucket: &str, key: &str, kms_key_id: &str) -> Result<(DataKey, Vec), ApiError>; + async fn generate_sse_dek(&self, context: &ObjectEncryptionContext, kms_key_id: &str) + -> Result<(DataKey, Vec), ApiError>; /// Decrypt an SSE data encryption key (returns only plaintext key, nonce should be read from metadata) - async fn decrypt_sse_dek(&self, encrypted_dek: &[u8], kms_key_id: &str) -> Result<[u8; 32], ApiError>; + async fn decrypt_sse_dek( + &self, + encrypted_dek: &[u8], + kms_key_id: &str, + context: &ObjectEncryptionContext, + ) -> Result<[u8; 32], ApiError>; } // ============================================================================ @@ -1213,29 +1737,34 @@ impl KmsSseDekProvider { #[async_trait] impl SseDekProvider for KmsSseDekProvider { - async fn generate_sse_dek(&self, bucket: &str, key: &str, kms_key_id: &str) -> Result<(DataKey, Vec), ApiError> { - let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); - + async fn generate_sse_dek( + &self, + context: &ObjectEncryptionContext, + kms_key_id: &str, + ) -> Result<(DataKey, Vec), ApiError> { let kms_key_option = Some(kms_key_id.to_string()); let service = Self::current_service() .await .ok_or_else(|| ApiError::from(StorageError::other("KMS encryption service is not initialized")))?; let (data_key, encrypted_data_key) = service - .create_data_key(&kms_key_option, &context) + .create_data_key(&kms_key_option, context) .await .map_err(|e| ApiError::from(StorageError::other(format!("Failed to create data key: {}", e))))?; Ok((data_key, encrypted_data_key)) } - async fn decrypt_sse_dek(&self, encrypted_dek: &[u8], _kms_key_id: &str) -> Result<[u8; 32], ApiError> { - // Create a minimal context for decryption - let context = ObjectEncryptionContext::new("".to_string(), "".to_string()); + async fn decrypt_sse_dek( + &self, + encrypted_dek: &[u8], + _kms_key_id: &str, + context: &ObjectEncryptionContext, + ) -> Result<[u8; 32], ApiError> { let service = Self::current_service() .await .ok_or_else(|| ApiError::from(StorageError::other("KMS encryption service is not initialized")))?; let data_key = service - .decrypt_data_key(encrypted_dek, &context) + .decrypt_data_key(encrypted_dek, context) .await .map_err(|e| ApiError::from(StorageError::other(format!("Failed to decrypt data key: {}", e))))?; @@ -1399,7 +1928,11 @@ impl TestSseDekProvider { #[async_trait] impl SseDekProvider for TestSseDekProvider { - async fn generate_sse_dek(&self, _bucket: &str, _key: &str, _kms_key_id: &str) -> Result<(DataKey, Vec), ApiError> { + async fn generate_sse_dek( + &self, + _context: &ObjectEncryptionContext, + _kms_key_id: &str, + ) -> Result<(DataKey, Vec), ApiError> { // Generate a 32-byte array as data key let mut dek = [0u8; 32]; rand::rng().fill_bytes(&mut dek); @@ -1421,7 +1954,12 @@ impl SseDekProvider for TestSseDekProvider { )) } - async fn decrypt_sse_dek(&self, encrypted_dek: &[u8], _kms_key_id: &str) -> Result<[u8; 32], ApiError> { + async fn decrypt_sse_dek( + &self, + encrypted_dek: &[u8], + _kms_key_id: &str, + _context: &ObjectEncryptionContext, + ) -> Result<[u8; 32], ApiError> { // Decrypt data key with master key let encrypted_dek_str = std::str::from_utf8(encrypted_dek) .map_err(|_| ApiError::from(StorageError::other("Invalid UTF-8 in encrypted DEK")))?; @@ -1435,7 +1973,7 @@ impl SseDekProvider for TestSseDekProvider { // ============================================================================ /// Global SSE DEK provider cache -static GLOBAL_SSE_DEK_PROVIDER: OnceLock> = OnceLock::new(); +static GLOBAL_SSE_DEK_PROVIDER: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); /// Get or initialize the global SSE DEK provider /// @@ -1454,16 +1992,23 @@ static GLOBAL_SSE_DEK_PROVIDER: OnceLock> = OnceLock::ne /// .await?; /// ``` pub async fn get_sse_dek_provider() -> Result, ApiError> { + if get_global_encryption_service().await.is_some() { + debug!("Using KmsSseDekProvider (KMS configured)"); + return Ok(Arc::new(KmsSseDekProvider::new().await?)); + } + // Check if already initialized - if let Some(provider) = GLOBAL_SSE_DEK_PROVIDER.get() { - return Ok(provider.clone()); + if let Some(provider) = GLOBAL_SSE_DEK_PROVIDER + .read() + .map_err(|_| ApiError::from(StorageError::other("Failed to read global SSE DEK provider cache")))? + .as_ref() + .cloned() + { + return Ok(provider); } // Determine provider: KMS when available, else test env, else local SSE-S3 fallback (no KMS) - let provider: Arc = if get_global_encryption_service().await.is_some() { - debug!("Using KmsSseDekProvider (KMS configured)"); - Arc::new(KmsSseDekProvider::new().await?) - } else if std::env::var("__RUSTFS_SSE_SIMPLE_CMK").is_ok() { + let provider: Arc = if std::env::var("__RUSTFS_SSE_SIMPLE_CMK").is_ok() { debug!("Using SimpleSseDekProvider (test mode) based on __RUSTFS_SSE_SIMPLE_CMK"); Arc::new(TestSseDekProvider::new()) } else { @@ -1471,10 +2016,13 @@ pub async fn get_sse_dek_provider() -> Result, ApiError> Arc::new(TestSseDekProvider::new_for_local_sse()) }; - // Store in global cache - GLOBAL_SSE_DEK_PROVIDER - .set(provider.clone()) - .map_err(|_| ApiError::from(StorageError::other("Failed to initialize global SSE DEK provider (already set)")))?; + let mut slot = GLOBAL_SSE_DEK_PROVIDER + .write() + .map_err(|_| ApiError::from(StorageError::other("Failed to update global SSE DEK provider cache")))?; + if let Some(existing) = slot.as_ref() { + return Ok(existing.clone()); + } + *slot = Some(provider.clone()); Ok(provider) } @@ -1486,8 +2034,9 @@ pub async fn get_sse_dek_provider() -> Result, ApiError> #[cfg(test)] #[allow(dead_code)] pub fn reset_sse_dek_provider() { - // OnceLock doesn't support reset - this is a documentation placeholder - // Consider using arc_swap::ArcSwap if runtime reset is needed + if let Ok(mut slot) = GLOBAL_SSE_DEK_PROVIDER.write() { + *slot = None; + } } // ============================================================================ @@ -1505,14 +2054,23 @@ pub fn is_managed_sse(server_side_encryption: &ServerSideEncryption) -> bool { /// Removes all managed SSE-related headers before returning object metadata to client. /// This is necessary because encryption is transparent to S3 clients. pub fn strip_managed_encryption_metadata(metadata: &mut HashMap) { - const KEYS: [&str; 7] = [ + const KEYS: [&str; 16] = [ "x-amz-server-side-encryption", "x-amz-server-side-encryption-aws-kms-key-id", - "x-rustfs-encryption-iv", + INTERNAL_ENCRYPTION_IV_HEADER, "x-rustfs-encryption-tag", - "x-rustfs-encryption-key", + INTERNAL_ENCRYPTION_KEY_HEADER, "x-rustfs-encryption-context", - "x-rustfs-encryption-original-size", + INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, + MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, + MINIO_INTERNAL_ENCRYPTION_IV_HEADER, + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, ]; for key in KEYS.iter() { @@ -1520,6 +2078,98 @@ pub fn strip_managed_encryption_metadata(metadata: &mut HashMap) } } +pub fn mark_encrypted_multipart_metadata(metadata: &mut HashMap) { + metadata.insert(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER.to_string(), String::new()); +} + +fn contains_managed_encryption_metadata(metadata: &HashMap) -> bool { + metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) + || metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) +} + +#[cfg(feature = "rio-v2")] +fn parse_minio_managed_sealed_key( + metadata: &HashMap, + sse_type: SSEType, +) -> Result, ApiError> { + let algorithm = match metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) { + Some(algorithm) => algorithm, + None => return Ok(None), + }; + if algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM { + return Ok(None); + } + + let iv = match metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) { + Some(iv) => match try_decode_minio_sealing_iv(iv)? { + Some(iv) => iv, + None => return Ok(None), + }, + None => return Ok(None), + }; + + let header_key = match sse_type { + SSEType::SseS3 => MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, + SSEType::SseKms => MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, + SSEType::SseC => MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + }; + let sealed_key = match metadata.get(header_key) { + Some(value) => match try_decode_minio_sealed_key(value)? { + Some(sealed_key) => sealed_key, + None => return Ok(None), + }, + None => return Ok(None), + }; + + Ok(Some(ManagedSealedKey { iv, sealed_key })) +} + +fn normalize_managed_metadata(metadata: &HashMap) -> HashMap { + let mut normalized = metadata.clone(); + + if !normalized.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER) + && let Some(value) = metadata + .get(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) + .or_else(|| metadata.get(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)) + { + normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_IV_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_ALGORITHM_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key(INTERNAL_ENCRYPTION_KEY_ID_HEADER) + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER) + { + normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.clone()); + } + + if !normalized.contains_key("x-rustfs-encryption-context") + && let Some(value) = metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) + && let Ok(decoded) = BASE64_STANDARD.decode(value) + && let Ok(context) = serde_json::from_slice::>(&decoded) + && let Ok(encoded) = serde_json::to_string(&context) + { + normalized.insert("x-rustfs-encryption-context".to_string(), encoded); + } + + normalized +} + // ============================================================================ // SSE-C Functions // ============================================================================ @@ -1658,6 +2308,14 @@ mod tests { use super::*; use http::HeaderValue; use rustfs_rio::{DecryptReader, EncryptReader}; + use std::sync::OnceLock; + use tokio::sync::Mutex; + + static SSE_TEST_LOCK: OnceLock> = OnceLock::new(); + + async fn lock_sse_test_state() -> tokio::sync::MutexGuard<'static, ()> { + SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await + } #[test] fn test_extract_ssec_params_from_headers() { @@ -1710,10 +2368,24 @@ mod tests { assert_eq!(err.code, S3ErrorCode::InvalidArgument); } + #[test] + fn test_extract_ssekms_context_from_headers_decodes_base64_json() { + let mut headers = http::HeaderMap::new(); + let encoded = + BASE64_STANDARD.encode(serde_json::to_vec(&HashMap::from([("tenant".to_string(), "alpha".to_string())])).unwrap()); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, HeaderValue::from_str(&encoded).unwrap()); + + let context = extract_ssekms_context_from_headers(&headers) + .expect("kms context header should parse") + .expect("kms context should exist"); + assert_eq!(context.get("tenant").map(String::as_str), Some("alpha")); + } + #[test] fn test_validate_sse_headers_for_write_rejects_algorithm_without_key() { let algorithm = SSECustomerAlgorithm::from("AES256".to_string()); let result = validate_sse_headers_for_write( + None, None, None, Some(&algorithm), @@ -1731,6 +2403,7 @@ mod tests { let algorithm = SSECustomerAlgorithm::from("AES256".to_string()); let key = SSECustomerKey::from("pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=".to_string()); let result = validate_sse_headers_for_write( + None, None, None, Some(&algorithm), @@ -1746,7 +2419,7 @@ mod tests { #[test] fn test_validate_sse_headers_for_write_rejects_invalid_sse_algorithm() { let bad_sse = ServerSideEncryption::from_static("aes:kms"); - let result = validate_sse_headers_for_write(Some(&bad_sse), None, None, None, None, true); + let result = validate_sse_headers_for_write(Some(&bad_sse), None, None, None, None, None, true); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.code, S3ErrorCode::InvalidArgument); @@ -1761,6 +2434,7 @@ mod tests { let result = validate_sse_headers_for_write( Some(&server_side_encryption), None, + None, Some(&algorithm), Some(&key), Some(&key_md5), @@ -1868,6 +2542,7 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: Some("AES256".to_string()), sse_customer_key: Some(sse_key.clone()), sse_customer_key_md5: None, @@ -1882,6 +2557,7 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: Some("AES256".to_string()), sse_customer_key: None, sse_customer_key_md5: Some(sse_key_md5.clone()), @@ -1896,6 +2572,7 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: Some(sse_key), sse_customer_key_md5: Some(sse_key_md5), @@ -1917,7 +2594,9 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, + sse_customer_key: None, sse_customer_key_md5: Some(sse_key_md5), }; @@ -1929,7 +2608,7 @@ mod tests { fn test_validate_sse_headers_for_write_allows_aws_kms_without_key_id() { let server_side_encryption: ServerSideEncryption = "aws:kms".to_string().into(); - let result = validate_sse_headers_for_write(Some(&server_side_encryption), None, None, None, None, true); + let result = validate_sse_headers_for_write(Some(&server_side_encryption), None, None, None, None, None, true); assert!(result.is_ok()); } @@ -1945,7 +2624,9 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: Some("AES256".to_string()), + sse_customer_key: None, sse_customer_key_md5: Some(sse_key_md5), }; @@ -1958,6 +2639,37 @@ mod tests { assert_eq!(metadata.get("x-amz-server-side-encryption-customer-algorithm").unwrap(), "AES256"); } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_sse_prepare_encryption_ssec_with_customer_key_stores_sealed_key_metadata() { + let bucket = "test-bucket"; + let key = "test-key"; + let customer_key_bytes = [0x24u8; 32]; + let customer_key = BASE64_STANDARD.encode(customer_key_bytes); + let sse_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0); + + let request = PrepareEncryptionRequest { + bucket, + key, + server_side_encryption: None, + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm: Some("AES256".to_string()), + sse_customer_key: Some(customer_key), + sse_customer_key_md5: Some(sse_key_md5), + }; + + let material = sse_prepare_encryption(request) + .await + .expect("prepare should accept full ssec headers") + .expect("ssec metadata should be generated"); + assert_eq!(material.key_kind, EncryptionKeyKind::Object); + + let metadata = encryption_material_to_metadata(&material); + assert!(metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_IV_HEADER)); + assert!(metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)); + } + #[test] fn test_encryption_material_to_metadata_persists_ssec_original_size() { let metadata = encryption_material_to_metadata(&EncryptionMaterial { @@ -1970,6 +2682,9 @@ mod tests { encrypted_data_key: None, customer_key_md5: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()), original_size: Some(1024), + key_kind: EncryptionKeyKind::Direct, + managed_kms_context: None, + managed_sealed_key: None, }); assert_eq!(metadata.get(SSEC_ORIGINAL_SIZE_HEADER).map(String::as_str), Some("1024")); @@ -1986,6 +2701,7 @@ mod tests { key, server_side_encryption: Some("AES256".to_string().into()), ssekms_key_id: Some("test-key".to_string()), + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, @@ -2007,6 +2723,7 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: Some("test-key".to_string()), + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, @@ -2030,6 +2747,7 @@ mod tests { key, server_side_encryption: Some("aws:kms".to_string().into()), ssekms_key_id: Some("test-key".to_string()), + ssekms_context: None, sse_customer_algorithm: Some("AES256".to_string()), sse_customer_key: Some(sse_key), sse_customer_key_md5: Some(sse_key_md5), @@ -2070,6 +2788,9 @@ mod tests { encrypted_data_key: Some(vec![1, 2, 3, 4]), customer_key_md5: None, original_size: Some(1024), + key_kind: EncryptionKeyKind::Direct, + managed_kms_context: None, + managed_sealed_key: None, }); assert_eq!(metadata.get("x-amz-server-side-encryption").map(String::as_str), Some("aws:kms")); @@ -2081,6 +2802,138 @@ mod tests { ); } + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_sse_kms_roundtrip_persists_and_uses_minio_context() { + use rustfs_kms::config::KmsConfig; + use rustfs_kms::types::{CreateKeyRequest, KeyUsage}; + use tempfile::TempDir; + let _guard = lock_sse_test_state().await; + + reset_sse_dek_provider(); + let manager = rustfs_kms::init_global_kms_service_manager(); + let temp_dir = TempDir::new().expect("temp dir"); + manager + .reconfigure(KmsConfig::local(temp_dir.path().to_path_buf())) + .await + .expect("kms reconfigure should succeed"); + manager + .get_encryption_service() + .await + .expect("encryption service should exist") + .create_key(CreateKeyRequest { + key_name: Some("kms-test".to_string()), + key_usage: KeyUsage::EncryptDecrypt, + description: None, + policy: None, + tags: HashMap::new(), + origin: None, + }) + .await + .expect("kms test key should be created"); + + let client_context = HashMap::from([("tenant".to_string(), "alpha".to_string())]); + let request = EncryptionRequest { + bucket: "bucket", + key: "dir/object", + server_side_encryption: Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS)), + ssekms_key_id: Some("kms-test".to_string()), + ssekms_context: Some(client_context.clone()), + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + content_size: 4096, + }; + + let material = sse_encryption(request) + .await + .expect("sse-kms encryption") + .expect("managed sse-kms material"); + let metadata = encryption_material_to_metadata(&material); + let encoded_context = metadata + .get(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) + .expect("minio kms context header should exist"); + let decoded_context: HashMap = + serde_json::from_slice(&BASE64_STANDARD.decode(encoded_context).expect("decode base64 context")) + .expect("decode json context"); + assert_eq!(decoded_context, client_context); + + let decrypted = sse_decryption(DecryptionRequest { + bucket: "bucket", + key: "dir/object", + metadata: &metadata, + sse_customer_key: None, + sse_customer_key_md5: None, + }) + .await + .expect("sse-kms decryption should succeed") + .expect("sse-kms material should exist"); + assert_eq!(decrypted.key_bytes, material.key_bytes); + + let mut wrong_metadata = metadata.clone(); + wrong_metadata.insert( + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), + BASE64_STANDARD.encode(serde_json::to_vec(&HashMap::from([("tenant".to_string(), "beta".to_string())])).unwrap()), + ); + let err = sse_decryption(DecryptionRequest { + bucket: "bucket", + key: "dir/object", + metadata: &wrong_metadata, + sse_customer_key: None, + sse_customer_key_md5: None, + }) + .await + .expect_err("mismatched kms context should fail"); + assert!( + err.message.contains("context") || err.message.contains("Context"), + "unexpected error for mismatched kms context: {}", + err.message + ); + + manager.stop().await.expect("kms service should stop cleanly"); + reset_sse_dek_provider(); + } + + #[cfg(feature = "rio-v2")] + #[test] + fn test_encryption_material_to_metadata_persists_minio_managed_headers() { + let encoded_nonce = BASE64_STANDARD.encode([9u8; 12]); + let encoded_key = BASE64_STANDARD.encode([1u8, 2, 3, 4]); + let metadata = encryption_material_to_metadata(&EncryptionMaterial { + sse_type: SSEType::SseKms, + server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + kms_key_id: Some("test-key".to_string()), + algorithm: SSECustomerAlgorithm::from(ServerSideEncryption::AWS_KMS.to_string()), + key_bytes: [7u8; 32], + base_nonce: [9u8; 12], + encrypted_data_key: Some(vec![1, 2, 3, 4]), + customer_key_md5: None, + original_size: Some(1024), + key_kind: EncryptionKeyKind::Direct, + managed_kms_context: None, + managed_sealed_key: None, + }); + + assert_eq!( + metadata.get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER).map(String::as_str), + Some(encoded_nonce.as_str()) + ); + assert_eq!( + metadata + .get(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) + .map(String::as_str), + Some(encoded_key.as_str()) + ); + assert_eq!( + metadata.get(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER).map(String::as_str), + Some("test-key") + ); + assert_eq!( + metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER).map(String::as_str), + Some(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM) + ); + } + #[tokio::test] async fn test_sse_encryption_omits_kms_header_for_sse_s3_objects() { let request = EncryptionRequest { @@ -2088,6 +2941,7 @@ mod tests { key: "test-key", server_side_encryption: Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, @@ -2109,15 +2963,157 @@ mod tests { let mut metadata = HashMap::new(); metadata.insert("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()); metadata.insert("x-rustfs-encryption-key".to_string(), "encrypted_key".to_string()); + metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(), "sealed".to_string()); metadata.insert("content-type".to_string(), "text/plain".to_string()); strip_managed_encryption_metadata(&mut metadata); assert!(!metadata.contains_key("x-amz-server-side-encryption")); assert!(!metadata.contains_key("x-rustfs-encryption-key")); + assert!(!metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)); assert!(metadata.contains_key("content-type")); } + #[cfg(feature = "rio-v2")] + #[test] + fn test_normalize_managed_metadata_accepts_minio_only_headers() { + let metadata = HashMap::from([ + ( + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), + BASE64_STANDARD.encode(b"encrypted-key"), + ), + (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x11u8; 12])), + ( + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), + MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), + ), + (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), + ]); + + let normalized = normalize_managed_metadata(&metadata); + + assert_eq!( + normalized.get(INTERNAL_ENCRYPTION_KEY_HEADER), + Some(&BASE64_STANDARD.encode(b"encrypted-key")) + ); + assert_eq!(normalized.get(INTERNAL_ENCRYPTION_IV_HEADER), Some(&BASE64_STANDARD.encode([0x11u8; 12]))); + assert_eq!( + normalized.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER), + Some(&MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string()) + ); + assert_eq!(normalized.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER), Some(&"default".to_string())); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_managed_sse_rio_v2_uses_object_key_metadata_roundtrip() { + let _guard = lock_sse_test_state().await; + reset_sse_dek_provider(); + if let Some(manager) = rustfs_kms::get_global_kms_service_manager() { + let _ = manager.stop().await; + } + + let request = EncryptionRequest { + bucket: "bucket", + key: "object", + server_side_encryption: Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + content_size: 4096, + }; + + let material = sse_encryption(request) + .await + .expect("managed sse encryption") + .expect("managed sse material"); + assert_eq!(material.key_kind, EncryptionKeyKind::Object); + + let metadata = encryption_material_to_metadata(&material); + assert!(!metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER)); + assert!(!metadata.contains_key(INTERNAL_ENCRYPTION_IV_HEADER)); + + let sealing_iv = metadata + .get(MINIO_INTERNAL_ENCRYPTION_IV_HEADER) + .expect("minio sealing iv should be stored"); + let sealed_key = metadata + .get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER) + .expect("minio sealed key should be stored"); + assert_eq!(BASE64_STANDARD.decode(sealing_iv).expect("decode iv").len(), SEALED_KEY_IV_SIZE); + assert_eq!(BASE64_STANDARD.decode(sealed_key).expect("decode sealed key").len(), SEALED_KEY_SIZE); + + let decrypted = sse_decryption(DecryptionRequest { + bucket: "bucket", + key: "object", + metadata: &metadata, + sse_customer_key: None, + sse_customer_key_md5: None, + }) + .await + .expect("managed sse decryption") + .expect("managed decryption material"); + + assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object); + assert_eq!(decrypted.key_bytes, material.key_bytes); + } + + #[cfg(feature = "rio-v2")] + #[tokio::test] + async fn test_ssec_rio_v2_uses_sealed_object_key_metadata_roundtrip() { + let customer_key_bytes = [0x42u8; 32]; + let customer_key = BASE64_STANDARD.encode(customer_key_bytes); + let customer_key_md5 = BASE64_STANDARD.encode(md5::compute(customer_key_bytes).0); + + let material = sse_encryption(EncryptionRequest { + bucket: "bucket", + key: "object", + server_side_encryption: None, + ssekms_key_id: None, + ssekms_context: None, + sse_customer_algorithm: Some("AES256".to_string()), + sse_customer_key: Some(customer_key.clone()), + sse_customer_key_md5: Some(customer_key_md5.clone()), + content_size: 4096, + }) + .await + .expect("sse-c encryption") + .expect("sse-c material"); + + assert_eq!(material.key_kind, EncryptionKeyKind::Object); + + let metadata = encryption_material_to_metadata(&material); + assert_eq!( + metadata.get(MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER).map(String::as_str), + Some(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM) + ); + assert!(metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_IV_HEADER)); + assert!(metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)); + + let decrypted = sse_decryption(DecryptionRequest { + bucket: "bucket", + key: "object", + metadata: &metadata, + sse_customer_key: Some(&customer_key), + sse_customer_key_md5: Some(&customer_key_md5), + }) + .await + .expect("sse-c decryption") + .expect("sse-c decryption material"); + + assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object); + assert_eq!(decrypted.key_bytes, material.key_bytes); + } + + #[cfg(feature = "rio-v2")] + #[test] + fn test_mark_encrypted_multipart_metadata_sets_minio_marker() { + let mut metadata = HashMap::new(); + mark_encrypted_multipart_metadata(&mut metadata); + assert!(metadata.contains_key(MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER)); + } + #[test] fn test_verify_ssec_key_match_success() { let md5 = "test_md5".to_string(); @@ -2151,6 +3147,7 @@ mod tests { key: "object", server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, @@ -2173,6 +3170,7 @@ mod tests { key: "object", server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, @@ -2199,9 +3197,10 @@ mod tests { let bucket = "test-bucket"; let key = "test-key"; let kms_key_id = "default"; // Key ID is ignored in simple provider + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); let (data_key, _encrypted_dek) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK"); @@ -2265,9 +3264,10 @@ mod tests { let bucket = "test-bucket"; let key = "test-key-large"; let kms_key_id = "default"; + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); let (data_key, _encrypted_dek) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK"); @@ -2316,15 +3316,16 @@ mod tests { let bucket = "test-bucket"; let key = "test-key"; let kms_key_id = "default"; + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); // Generate two different keys (with different nonces) let (data_key1, _) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK 1"); let (data_key2, _) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK 2"); @@ -2366,10 +3367,11 @@ mod tests { let bucket = "test-bucket"; let key = "test-key"; let kms_key_id = "default"; + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); // 1. Generate DEK and get encrypted DEK let (data_key, encrypted_dek) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK"); @@ -2381,7 +3383,7 @@ mod tests { // 3. Later, decrypt the DEK let decrypted_plaintext_key = provider - .decrypt_sse_dek(&encrypted_dek, kms_key_id) + .decrypt_sse_dek(&encrypted_dek, kms_key_id, &context) .await .expect("Failed to decrypt DEK"); @@ -2420,12 +3422,8 @@ mod tests { async fn test_kms_sse_dek_provider_uses_latest_reconfigured_service() { use rustfs_kms::config::KmsConfig; use rustfs_kms::types::{CreateKeyRequest, KeyUsage}; - use std::sync::OnceLock; use tempfile::TempDir; - use tokio::sync::Mutex; - - static KMS_TEST_LOCK: OnceLock> = OnceLock::new(); - let _guard = KMS_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await; + let _guard = lock_sse_test_state().await; let manager = rustfs_kms::init_global_kms_service_manager(); @@ -2450,8 +3448,9 @@ mod tests { .expect("first key should be created"); let provider = KmsSseDekProvider::new().await.expect("provider should initialize"); + let context = ObjectEncryptionContext::new("bucket".to_string(), "object".to_string()); provider - .generate_sse_dek("bucket", "object", "first-key") + .generate_sse_dek(&context, "first-key") .await .expect("provider should use the initial service"); @@ -2476,7 +3475,7 @@ mod tests { .expect("second key should be created"); provider - .generate_sse_dek("bucket", "object", "second-key") + .generate_sse_dek(&context, "second-key") .await .expect("provider should resolve the latest reconfigured service"); @@ -2716,6 +3715,7 @@ mod tests { key: "test-key", server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, sse_customer_key: None, sse_customer_key_md5: None, @@ -2748,6 +3748,7 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: Some("AES256".to_string()), sse_customer_key: Some(sse_key.clone()), sse_customer_key_md5: Some(wrong_md5), @@ -2761,6 +3762,7 @@ mod tests { key, server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: Some("unsupported-algo".to_string()), sse_customer_key: Some(sse_key), sse_customer_key_md5: Some(BASE64_STANDARD.encode(md5::compute([42u8; 32]).0)), @@ -2779,7 +3781,9 @@ mod tests { key: "test-key", server_side_encryption: None, ssekms_key_id: None, + ssekms_context: None, sse_customer_algorithm: None, + sse_customer_key: None, sse_customer_key_md5: None, }; let result = sse_prepare_encryption(request).await; diff --git a/rustfs/src/storage/sse_test.rs b/rustfs/src/storage/sse_test.rs index 06b414dd4..8ca6ff5db 100644 --- a/rustfs/src/storage/sse_test.rs +++ b/rustfs/src/storage/sse_test.rs @@ -16,6 +16,7 @@ mod tests { use crate::storage::sse::SseDekProvider; use crate::storage::sse::TestSseDekProvider; + use rustfs_kms::types::ObjectEncryptionContext; use rustfs_rio::{DecryptReader, EncryptReader}; use std::io::Cursor; use tokio::io::AsyncReadExt; @@ -36,9 +37,10 @@ mod tests { let bucket = "test-bucket"; let key = "test-key"; let kms_key_id = "default"; // Key ID is ignored in test provider + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); let (data_key, _encrypted_dek) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK"); @@ -104,9 +106,10 @@ mod tests { let bucket = "test-bucket"; let key = "test-key-large"; let kms_key_id = "default"; + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); let (data_key, _encrypted_dek) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK"); @@ -153,15 +156,16 @@ mod tests { let bucket = "test-bucket"; let key = "test-key"; let kms_key_id = "default"; + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); // Generate two different keys (with different nonces) let (data_key1, _) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK 1"); let (data_key2, _) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK 2"); @@ -201,10 +205,11 @@ mod tests { let bucket = "test-bucket"; let key = "test-key"; let kms_key_id = "default"; + let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string()); // Step 1: Generate DEK and get encrypted DEK let (data_key, encrypted_dek) = provider - .generate_sse_dek(bucket, key, kms_key_id) + .generate_sse_dek(&context, kms_key_id) .await .expect("Failed to generate DEK"); @@ -213,7 +218,7 @@ mod tests { // Step 2: Later, decrypt the DEK (simulating GET operation) let decrypted_plaintext_key = provider - .decrypt_sse_dek(&encrypted_dek, kms_key_id) + .decrypt_sse_dek(&encrypted_dek, kms_key_id, &context) .await .expect("Failed to decrypt DEK"); diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index 4ff7cca59..10920cfc8 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -23,6 +23,12 @@ export no_proxy="${NO_PROXY}" # Ensure user-level Python scripts are discoverable (awscurl/tox installed via --user) PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || echo "3.14") export PATH="$HOME/Library/Python/${PYTHON_VERSION}/bin:$HOME/.local/bin:$PATH" +if command -v uv >/dev/null 2>&1; then + UV_TOOL_BIN="$(uv tool dir --bin 2>/dev/null || true)" + if [ -n "${UV_TOOL_BIN}" ]; then + export PATH="${UV_TOOL_BIN}:$PATH" + fi +fi # Configuration S3_ACCESS_KEY="${S3_ACCESS_KEY:-rustfsadmin}" @@ -712,6 +718,17 @@ install_python_package() { local package=$1 local error_output + if command -v uv >/dev/null 2>&1; then + if uv tool install --upgrade "${package}"; then + UV_TOOL_BIN="$(uv tool dir --bin 2>/dev/null || true)" + if [ -n "${UV_TOOL_BIN}" ]; then + export PATH="${UV_TOOL_BIN}:$PATH" + fi + return 0 + fi + log_warn "Failed to install ${package} with uv, falling back to python3 -m pip" + fi + # Try --user first (works on most Linux systems) error_output=$(python3 -m pip install --user --upgrade pip "${package}" 2>&1) if [ $? -eq 0 ]; then