gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(rio): rio_v2 is compatible with minio for storing data. (#3115)

Browse Source 
* Set up a compatibility layer for replacing old Rio components with new ones.

* fix(rio). compress range

* feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level.

* feat(rio_v2): add sse-c test

* test compression component

* simple fix

* fix minlz encode

* fix metadata

* fix kms key cache error

* Update launch.json

* ci: set nix crate download user agent

* fix: gate obs pyroscope backend

* ignore minio test

* fix encrypt check

* fix

* fix

* fix

* Update object_usecase.rs

* Update ci.yml

* fix

* ci add rio-v2 test

* fix

* ci fix

* fix

* Reconstructed into a more reasonable compatibility mode

* fix

* fix

---------

Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: 唐小鸭 <tangtang1251@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
唐小鸭
2026-06-08 19:59:14 +08:00
committed by GitHub
parent 9504dff595
commit f7724d223b
47 changed files with 8742 additions and 682 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/AGENTS.md
+4 -1
View File
@@ -32,6 +32,9 @@ Current `test-and-lint` gate includes:
- `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes`
- `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path`
- `cargo fmt --all --check`
- `cargo clippy --all-targets --all-features -- -D warnings`
- `cargo clippy --all-targets -- -D warnings`
- `cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2`
- `cargo test -p rustfs --doc --features rio-v2`
- `cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings`
- `./scripts/check_layer_dependencies.sh`
- `./scripts/check_architecture_migration_rules.sh`
.github/workflows/ci.yml
+110 -2
View File
@@ -134,7 +134,7 @@ jobs:
run: ./scripts/check_unsafe_code_allowances.sh
- name: Run clippy lints
run: cargo clippy --all-targets --all-features -- -D warnings
run: cargo clippy --all-targets -- -D warnings
- name: Check layered dependencies
run: ./scripts/check_layer_dependencies.sh
@@ -142,6 +142,34 @@ jobs:
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-test-rio-v2-${{ hashFiles('**/Cargo.lock') }}
github-token: ${{ secrets.GITHUB_TOKEN }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Run rio-v2 feature tests
run: |
cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2
cargo test -p rustfs --doc --features rio-v2
- name: Run rio-v2 clippy lints
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
build-rustfs-debug-binary:
name: Build RustFS Debug Binary
needs: skip-check
@@ -175,6 +203,39 @@ jobs:
if-no-files-found: error
retention-days: 1
build-rustfs-debug-binary-rio-v2:
name: Build RustFS Debug Binary (rio-v2)
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
timeout-minutes: 30
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-rustfs-debug-binary-rio-v2-${{ hashFiles('**/Cargo.lock') }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build debug binary with rio-v2
run: |
touch rustfs/build.rs
cargo build -p rustfs --bins --features rio-v2 --jobs 2
- name: Upload debug binary
uses: actions/upload-artifact@v6
with:
name: rustfs-debug-binary-rio-v2
path: target/debug/rustfs
if-no-files-found: error
retention-days: 1
e2e-tests:
name: End-to-End Tests
needs: [ skip-check, build-rustfs-debug-binary ]
@@ -222,9 +283,56 @@ jobs:
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
retention-days: 3
e2e-tests-rio-v2:
name: End-to-End Tests (rio-v2)
needs: [ skip-check, build-rustfs-debug-binary-rio-v2 ]
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-2
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Clean up previous test run
run: |
rm -rf /tmp/rustfs
rm -f /tmp/rustfs.log
- name: Download debug binary
uses: actions/download-artifact@v7
with:
name: rustfs-debug-binary-rio-v2
path: target/debug
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
- name: Setup Rust toolchain for s3s-e2e installation
uses: dtolnay/rust-toolchain@stable
- name: Install s3s-e2e test tool
uses: taiki-e/cache-cargo-install-action@v2
with:
tool: s3s-e2e
git: https://github.com/s3s-project/s3s.git
rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52
- name: Run end-to-end tests
run: |
s3s-e2e --version
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v6
with:
name: e2e-test-logs-rio-v2-${{ github.run_number }}
path: /tmp/rustfs.log
retention-days: 3
s3-implemented-tests:
name: S3 Implemented Tests
needs: [ skip-check, build-rustfs-debug-binary ]
needs: [ skip-check, build-rustfs-debug-binary, e2e-tests ]
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
timeout-minutes: 60
.github/workflows/nix.yml
+4 -1
View File
@@ -48,6 +48,8 @@ jobs:
permissions:
contents: read
id-token: write
env:
NIX_CURL_FLAGS: -A cargo/1.95.0
steps:
- name: Checkout repository
uses: actions/checkout@v6
@@ -57,7 +59,8 @@ jobs:
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
extra-conf: |
experimental-features = nix-command flakes
experimental-features = nix-command flakes configurable-impure-env
impure-env = NIX_CURL_FLAGS
max-jobs = 1
- name: Cache Nix
.gitignore
+1
View File
@@ -42,6 +42,7 @@ artifacts/
PR_DESCRIPTION.md
scripts/s3-tests/selected_tests.txt
docs
__pycache__/
!docs/
docs/*
!docs/architecture/
.vscode/launch.json
Vendored
+42 -1
View File
@@ -224,6 +224,47 @@
"rust"
],
},
{
"type": "lldb",
"request": "launch",
"name": "Debug executable 'rustfs' with rio-v2 full matrix",
"cargo": {
"args": [
"build",
"--bin=rustfs",
"--package=rustfs",
"--features=full,rio-v2"
],
"filter": {
"name": "rustfs",
"kind": "bin"
}
},
"env": {
"RUST_LOG": "rustfs=error,ecstore=error,s3s=error,iam=error",
"RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS": "true",
"RUST_BACKTRACE": "full",
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
"RUSTFS_VOLUMES": "./target/rio-v2/test{1...4}",
"RUSTFS_SCANNER_ENABLED": "false",
"RUSTFS_HEAL_ENABLED": "false",
"RUSTFS_REGION": "us-east-1",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs/rio-v2",
"RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
"RUSTFS_COMPRESSION_ENABLED": "true",
"RUSTFS_KMS_ENABLE": "true",
"RUSTFS_KMS_BACKEND": "local",
"RUSTFS_KMS_KEY_DIR": "./target/rio-v2/kms-key-dir",
"RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
"RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"name": "E2E test executable target/debug/rustfs",
"type": "lldb",
@@ -256,4 +297,4 @@
],
},
]
}
}
AGENTS.md
+8
View File
@@ -103,6 +103,14 @@ Do not open a PR with code changes when the required checks fail.
- Use environment variables or vault tooling for sensitive configuration.
- For localhost-sensitive tests, verify proxy settings to avoid traffic leakage.
## Tools
### xl.meta decode tool Quick Use
```
cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta"
```
## Serde Safety
- Add `#[serde(deny_unknown_fields)]` to structs deserialized from untrusted input (S3 API XML/JSON, lifecycle rules, bucket policies, replication configs).
Cargo.lock
Generated
+32
View File
@@ -6255,6 +6255,15 @@ dependencies = [
"simd-adler32",
]
[[package]]
name = "minlz"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20c19c645132db7e3664aabe291a16896f601053c6b5f567f4c5a196e6817827"
dependencies = [
"crc",
]
[[package]]
name = "mio"
version = "1.2.1"
@@ -9047,6 +9056,7 @@ dependencies = [
"futures-util",
"hashbrown 0.17.1",
"hex-simd",
"hmac 0.13.0",
"http 1.4.1",
"http-body 1.0.1",
"http-body-util",
@@ -9328,6 +9338,7 @@ dependencies = [
"rustfs-policy",
"rustfs-protos",
"rustfs-rio",
"rustfs-rio-v2",
"rustfs-s3-types",
"rustfs-signer",
"rustfs-tls-runtime",
@@ -9846,6 +9857,27 @@ dependencies = [
"tracing",
]
[[package]]
name = "rustfs-rio-v2"
version = "1.0.0-beta.7"
dependencies = [
"aes-gcm",
"bytes",
"hex",
"hmac 0.13.0",
"minlz",
"pin-project-lite",
"rand 0.10.1",
"rustfs-filemeta",
"rustfs-rio",
"rustfs-utils",
"serde",
"serde_json",
"sha2 0.11.0",
"tokio",
"walkdir",
]
[[package]]
name = "rustfs-s3-ops"
version = "1.0.0-beta.7"
Cargo.toml
+2
View File
@@ -38,6 +38,7 @@ members = [
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
"crates/protos", # Protocol buffer definitions
"crates/rio", # Rust I/O utilities and abstractions
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
"crates/s3-types", # S3 event type definitions
"crates/s3-ops", # S3 operation definitions and mapping
@@ -104,6 +105,7 @@ rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.7" }
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.7" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.7" }
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.7" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.7" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.7" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.7" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.7" }
crates/ecstore/Cargo.toml
+2
View File
@@ -32,11 +32,13 @@ workspace = true
[features]
default = []
rio-v2 = ["dep:rustfs-rio-v2"]
[dependencies]
rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-rio.workspace = true
rustfs-rio-v2 = { workspace = true, optional = true }
rustfs-signer.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-checksums.workspace = true
crates/ecstore/src/erasure_coding/decode.rs
+57
View File
@@ -433,6 +433,63 @@ mod tests {
}
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn test_erasure_decode_preserves_compressed_stream_near_block_boundary() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 1024 * 1024;
use crate::rio::CompressReader;
use rustfs_utils::CompressionAlgorithm;
use tokio::io::AsyncReadExt;
let plaintext_size = 8 * BLOCK_SIZE + 123;
let plaintext = (0..plaintext_size)
.scan(0x9e37_79b9_7f4a_7c15u64, |state, _| {
*state ^= *state << 7;
*state ^= *state >> 9;
*state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9);
Some((*state >> 32) as u8)
})
.collect::<Vec<_>>();
let mut compressor = CompressReader::new(Cursor::new(plaintext), CompressionAlgorithm::default());
let mut compressed = Vec::new();
compressor.read_to_end(&mut compressed).await.unwrap();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
for block in compressed.chunks(BLOCK_SIZE) {
let shards = erasure.encode_data(block).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let mut decoded = Vec::new();
let (written, err) = erasure
.decode(&mut decoded, readers, 0, compressed.len(), compressed.len())
.await;
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, compressed.len());
assert_eq!(decoded, compressed);
}
#[tokio::test]
async fn test_parallel_reader_normal() {
const BLOCK_SIZE: usize = 64;
crates/ecstore/src/lib.rs
+10
View File
@@ -34,6 +34,7 @@ pub mod metrics_realtime;
pub mod notification_sys;
pub mod pools;
pub mod rebalance;
pub mod rio;
pub mod rpc;
pub mod set_disk;
mod sets;
@@ -60,3 +61,12 @@ pub use global::{get_global_lock_client, get_global_lock_clients, set_global_loc
pub use global::GLOBAL_Endpoints;
pub use store_api::StorageAPI;
#[cfg(test)]
mod rio_tests {
#[test]
fn uses_expected_rio_backend() {
let expected = if cfg!(feature = "rio-v2") { "rio-v2" } else { "legacy-rio" };
assert_eq!(crate::rio::backend_name(), expected);
}
}
crates/ecstore/src/rio.rs
+767
View File
@@ -0,0 +1,767 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(feature = "rio-v2")]
pub use rustfs_rio_v2::*;
#[cfg(not(feature = "rio-v2"))]
pub use rustfs_rio::*;
use bytes::Bytes;
use rustfs_utils::CompressionAlgorithm;
use std::str::FromStr;
use tokio::io::AsyncRead;
#[cfg(feature = "rio-v2")]
const MINIO_S2_COMPRESSION_SCHEME: &str = "klauspost/compress/s2";
#[cfg(feature = "rio-v2")]
const ENCRYPTED_S2_PADDING_MULTIPLE: usize = 256;
pub const fn backend_name() -> &'static str {
#[cfg(feature = "rio-v2")]
{
"rio-v2"
}
#[cfg(not(feature = "rio-v2"))]
{
"legacy-rio"
}
}
pub fn compression_metadata_value(algorithm: CompressionAlgorithm) -> String {
#[cfg(feature = "rio-v2")]
{
let _ = algorithm;
MINIO_S2_COMPRESSION_SCHEME.to_string()
}
#[cfg(not(feature = "rio-v2"))]
{
algorithm.to_string()
}
}
pub fn compression_scheme_to_algorithm(scheme: &str) -> std::io::Result<CompressionAlgorithm> {
#[cfg(feature = "rio-v2")]
if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
// rio_v2 currently routes all compressed-object handling through the S2
// reader implementation, so the enum is only a placeholder token here.
return Ok(CompressionAlgorithm::default());
}
CompressionAlgorithm::from_str(scheme)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadCompressionBackend {
Legacy,
V2,
}
pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(CompressionAlgorithm, ReadCompressionBackend)> {
#[cfg(feature = "rio-v2")]
if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
return Ok((CompressionAlgorithm::default(), ReadCompressionBackend::V2));
}
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionBackend {
Legacy,
V2,
}
pub fn compression_index_storage_bytes(index: &Index) -> Bytes {
#[cfg(feature = "rio-v2")]
{
minio_index_storage_bytes(index)
}
#[cfg(not(feature = "rio-v2"))]
{
index.clone().into_vec()
}
}
pub fn decode_compression_index_bytes(bytes: &Bytes) -> Option<Index> {
#[cfg(feature = "rio-v2")]
{
if let Some(decoded) = decode_minio_index_bytes(bytes) {
return Some(decoded);
}
}
let mut decoded = Index::new();
if decoded.load(bytes.as_ref()).is_ok() {
return Some(decoded);
}
#[cfg(feature = "rio-v2")]
{
let restored = restore_legacy_index_headers(bytes.as_ref());
let mut decoded = Index::new();
if decoded.load(&restored).is_ok() {
return Some(decoded);
}
}
None
}
pub fn compression_reader<R>(reader: R, algorithm: CompressionAlgorithm, encrypted: bool) -> CompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
#[cfg(feature = "rio-v2")]
{
if encrypted {
return CompressReader::with_encrypted_padding(reader, algorithm);
}
}
#[cfg(not(feature = "rio-v2"))]
let _ = encrypted;
CompressReader::new(reader, algorithm)
}
pub fn decompression_reader<R>(
reader: R,
algorithm: CompressionAlgorithm,
backend: ReadCompressionBackend,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
match backend {
ReadCompressionBackend::Legacy => Box::new(rustfs_rio::DecompressReader::new(reader, algorithm)),
ReadCompressionBackend::V2 => Box::new(rustfs_rio_v2::DecompressReader::new(reader, algorithm)),
}
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = backend;
Box::new(rustfs_rio::DecompressReader::new(reader, algorithm))
}
}
pub fn decrypt_reader<R>(
reader: R,
key: [u8; 32],
base_nonce: [u8; 12],
backend: ReadEncryptionBackend,
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
match backend {
ReadEncryptionBackend::Legacy => Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce)),
ReadEncryptionBackend::V2 => {
Box::new(rustfs_rio_v2::DecryptReader::new_with_sequence(reader, key, base_nonce, sequence_number))
}
}
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = (backend, sequence_number);
Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce))
}
}
pub fn decrypt_reader_with_object_key<R>(
reader: R,
object_key: [u8; 32],
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
Box::new(rustfs_rio_v2::DecryptReader::new_with_object_key_and_sequence(
reader,
object_key,
sequence_number,
))
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = sequence_number;
Box::new(rustfs_rio::DecryptReader::new(reader, object_key, [0u8; 12]))
}
}
pub fn decrypt_multipart_reader<R>(
reader: R,
key: [u8; 32],
base_nonce: [u8; 12],
multipart_parts: Vec<usize>,
backend: ReadEncryptionBackend,
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
match backend {
ReadEncryptionBackend::Legacy => {
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
}
ReadEncryptionBackend::V2 => Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_sequence(
reader,
key,
base_nonce,
multipart_parts,
sequence_number,
)),
}
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = (backend, sequence_number);
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
}
}
pub fn decrypt_multipart_reader_with_object_key<R>(
reader: R,
object_key: [u8; 32],
multipart_parts: Vec<usize>,
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_object_key_and_sequence(
reader,
object_key,
multipart_parts,
sequence_number,
))
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = sequence_number;
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, object_key, [0u8; 12], multipart_parts))
}
}
#[cfg(feature = "rio-v2")]
fn restore_legacy_index_headers(bytes: &[u8]) -> Vec<u8> {
if bytes.is_empty() {
return Vec::new();
}
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
let mut restored = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + bytes.len() + 4 + S2_INDEX_TRAILER.len());
restored.extend_from_slice(&[0x99, 0x2A, 0x4D, 0x18]);
restored.extend_from_slice(S2_INDEX_HEADER);
restored.extend_from_slice(bytes);
let total_size = (restored.len() + 4 + S2_INDEX_TRAILER.len()) as u32;
restored.extend_from_slice(&total_size.to_le_bytes());
restored.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = restored.len() - 4;
restored[1] = chunk_len as u8;
restored[2] = (chunk_len >> 8) as u8;
restored[3] = (chunk_len >> 16) as u8;
restored
}
#[derive(Debug, Clone, Copy)]
pub struct WriteEncryption {
key_bytes: [u8; 32],
mode: WriteEncryptionMode,
}
#[derive(Debug, Clone, Copy)]
enum WriteEncryptionMode {
SinglepartObjectKey,
Singlepart {
base_nonce: [u8; 12],
},
MultipartLegacy {
base_nonce: [u8; 12],
multipart_part_number: usize,
},
MultipartObjectKey {
multipart_part_number: u32,
},
}
impl WriteEncryption {
pub const fn singlepart_object_key(object_key: [u8; 32]) -> Self {
Self {
key_bytes: object_key,
mode: WriteEncryptionMode::SinglepartObjectKey,
}
}
pub const fn singlepart(key_bytes: [u8; 32], base_nonce: [u8; 12]) -> Self {
Self {
key_bytes,
mode: WriteEncryptionMode::Singlepart { base_nonce },
}
}
pub const fn multipart(key_bytes: [u8; 32], base_nonce: [u8; 12], multipart_part_number: usize) -> Self {
Self {
key_bytes,
mode: WriteEncryptionMode::MultipartLegacy {
base_nonce,
multipart_part_number,
},
}
}
pub const fn multipart_object_key(object_key: [u8; 32], multipart_part_number: u32) -> Self {
Self {
key_bytes: object_key,
mode: WriteEncryptionMode::MultipartObjectKey { multipart_part_number },
}
}
}
#[derive(Debug, Clone, Default)]
pub struct WritePlan {
compression: Option<CompressionAlgorithm>,
encryption: Option<WriteEncryption>,
}
impl WritePlan {
pub const fn new() -> Self {
Self {
compression: None,
encryption: None,
}
}
pub const fn with_compression(mut self, algorithm: CompressionAlgorithm) -> Self {
self.compression = Some(algorithm);
self
}
pub const fn with_encryption(mut self, encryption: WriteEncryption) -> Self {
self.encryption = Some(encryption);
self
}
pub const fn is_passthrough(&self) -> bool {
self.compression.is_none() && self.encryption.is_none()
}
pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result<HashReader> {
let encrypted = self.encryption.is_some();
if let Some(algorithm) = self.compression {
reader = HashReader::from_reader(
compression_reader(reader, algorithm, encrypted),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?;
}
if let Some(encryption) = self.encryption {
reader = match encryption.mode {
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
#[cfg(feature = "rio-v2")]
EncryptReader::new_with_object_key(reader, encryption.key_bytes),
#[cfg(not(feature = "rio-v2"))]
EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader(
EncryptReader::new(reader, encryption.key_bytes, base_nonce),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
WriteEncryptionMode::MultipartLegacy {
base_nonce,
multipart_part_number,
} => HashReader::from_reader(
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
#[cfg(feature = "rio-v2")]
EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
#[cfg(not(feature = "rio-v2"))]
EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
};
}
Ok(reader)
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_utils::CompressionAlgorithm;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
#[cfg(feature = "rio-v2")]
fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
let mut chunk_types = Vec::new();
let mut offset = 0usize;
while offset + 4 <= stream.len() {
let chunk_type = stream[offset];
let chunk_len =
(stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16);
chunk_types.push(chunk_type);
offset += 4 + chunk_len;
}
chunk_types
}
#[tokio::test]
async fn write_plan_passthrough_keeps_plaintext() {
let plaintext = b"write-plan-plain".to_vec();
let reader = HashReader::from_stream(
Cursor::new(plaintext.clone()),
plaintext.len() as i64,
plaintext.len() as i64,
None,
None,
false,
)
.expect("create hash reader");
let mut reader = WritePlan::new()
.apply(reader, plaintext.len() as i64)
.expect("apply passthrough plan");
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("read passthrough stream");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn write_plan_compress_then_encrypt_multipart_roundtrip() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".repeat(128);
let actual_size = plaintext.len() as i64;
let key_bytes = [0x5Au8; 32];
let base_nonce = [0xA5u8; 12];
let part_number = 7;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.with_encryption(WriteEncryption::multipart(key_bytes, base_nonce, part_number))
.apply(reader, actual_size)
.expect("apply transform plan");
let mut ciphertext = Vec::new();
transformed
.read_to_end(&mut ciphertext)
.await
.expect("read transformed ciphertext");
let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed
.read_to_end(&mut actual)
.await
.expect("decrypt and decompress transformed stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_supports_singlepart_object_key_encryption_roundtrip() {
let plaintext = b"singlepart-object-key".repeat(512);
let actual_size = plaintext.len() as i64;
let object_key = [0x7Cu8; 32];
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_encryption(WriteEncryption::singlepart_object_key(object_key))
.apply(reader, actual_size)
.expect("apply singlepart object-key plan");
let mut encrypted = Vec::new();
transformed
.read_to_end(&mut encrypted)
.await
.expect("read encrypted object-key stream");
let mut decrypted = DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key);
let mut actual = Vec::new();
decrypted.read_to_end(&mut actual).await.expect("decrypt object-key stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_supports_multipart_object_key_encryption_roundtrip() {
let plaintext = b"multipart-object-key-".repeat(4096);
let actual_size = plaintext.len() as i64;
let object_key = [0x2Du8; 32];
let part_number = 3u32;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_encryption(WriteEncryption::multipart_object_key(object_key, part_number))
.apply(reader, actual_size)
.expect("apply multipart object-key encryption");
let mut ciphertext = Vec::new();
transformed
.read_to_end(&mut ciphertext)
.await
.expect("read multipart object-key ciphertext");
let mut actual = Vec::new();
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), object_key, vec![part_number as usize])
.read_to_end(&mut actual)
.await
.expect("decrypt multipart object-key ciphertext");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_rio_v2_compression_emits_s2_stream_and_seekable_index() {
let plaintext = b"rustfs-rio-v2-s2-".repeat(600_000);
let actual_size = plaintext.len() as i64;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.apply(reader, actual_size)
.expect("apply compression plan");
let mut compressed = Vec::new();
transformed
.read_to_end(&mut compressed)
.await
.expect("read compressed stream");
assert!(
compressed.starts_with(b"\xff\x06\x00\x00S2sTwO"),
"rio_v2 compressed stream must start with the S2 stream identifier"
);
let index = transformed
.try_get_index()
.cloned()
.expect("rio_v2 compressed stream should expose a compression index");
let (compressed_offset, uncompressed_offset) = index.find(2 * 1024 * 1024).expect("seek into compression index");
assert!(compressed_offset > 0, "expected a non-zero compressed offset for the second block");
assert!(uncompressed_offset > 0, "expected a non-zero uncompressed offset for the second block");
let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_rio_v2_small_compression_skips_index_below_minio_threshold() {
let plaintext = b"rustfs-rio-v2-s2-".repeat(32_768);
let actual_size = plaintext.len() as i64;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.apply(reader, actual_size)
.expect("apply compression plan");
let mut compressed = Vec::new();
transformed
.read_to_end(&mut compressed)
.await
.expect("read compressed stream");
assert!(
transformed.try_get_index().is_none(),
"rio_v2 should match MinIO and skip compression indexes for small objects"
);
let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_singlepart_encrypt_decrypt_roundtrip_preserves_small_compressed_stream() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let key_bytes = [0x33u8; 32];
let base_nonce = [0x55u8; 12];
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut encrypted = Vec::new();
EncryptReader::new(Cursor::new(compressed), key_bytes, base_nonce)
.read_to_end(&mut encrypted)
.await
.expect("encrypt compressed stream");
let decrypt_reader = DecryptReader::new(Cursor::new(encrypted), key_bytes, base_nonce);
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed
.read_to_end(&mut actual)
.await
.expect("decrypt and decompress small stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_compress_then_encrypt_adds_s2_padding_frames() {
let plaintext = b"padding-check-".repeat(4097);
let actual_size = plaintext.len() as i64;
let key_bytes = [0x1Bu8; 32];
let base_nonce = [0xC4u8; 12];
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.with_encryption(WriteEncryption::singlepart(key_bytes, base_nonce))
.apply(reader, actual_size)
.expect("apply transform plan");
let mut ciphertext = Vec::new();
transformed
.read_to_end(&mut ciphertext)
.await
.expect("read transformed ciphertext");
let mut decrypted_compressed = Vec::new();
DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
.read_to_end(&mut decrypted_compressed)
.await
.expect("decrypt compressed stream");
assert_eq!(decrypted_compressed.len() % ENCRYPTED_S2_PADDING_MULTIPLE, 0);
let chunk_types = s2_chunk_types(&decrypted_compressed);
assert!(
chunk_types.contains(&0xfe),
"rio_v2 compressed+encrypted streams must include S2 padding frames before encryption"
);
let mut actual = Vec::new();
DecompressReader::new(Cursor::new(decrypted_compressed), CompressionAlgorithm::default())
.read_to_end(&mut actual)
.await
.expect("decompress padded stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_decompress_reader_returns_bytes_on_first_read() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut buf = [0u8; 64];
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
assert!(n > 0);
assert_eq!(&buf[..n], plaintext.as_slice());
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_decompress_reader_returns_bytes_on_first_large_read() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut buf = [0u8; 8192];
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
assert!(n > 0);
assert_eq!(&buf[..n], plaintext.as_slice());
}
}
crates/ecstore/src/set_disk.rs
+4 -3
View File
@@ -85,7 +85,6 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
use rustfs_s3_types::EventName;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
@@ -130,6 +129,8 @@ use tracing::error;
use tracing::{debug, info, warn};
use uuid::Uuid;
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
pub const MAX_PARTS_COUNT: usize = 10000;
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
@@ -1146,7 +1147,7 @@ impl ObjectIO for SetDisks {
insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string());
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes);
//TODO: userDefined
@@ -3035,7 +3036,7 @@ impl MultipartOperations for SetDisks {
)));
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes);
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
crates/ecstore/src/store_api.rs
+1 -2
View File
@@ -17,6 +17,7 @@ use crate::bucket::versioning::VersioningApi as _;
use crate::config::storageclass;
use crate::disk::DiskStore;
use crate::error::{Error, Result};
use crate::rio::{HashReader, LimitReader};
use crate::store_utils::clean_metadata;
use crate::{
bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
@@ -34,7 +35,6 @@ use rustfs_filemeta::{
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_rio::{DecompressReader, HashReader, LimitReader};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
@@ -44,7 +44,6 @@ use std::collections::HashMap;
use std::fmt::Debug;
use std::io::Cursor;
use std::pin::Pin;
use std::str::FromStr as _;
use std::sync::Arc;
use std::task::{Context, Poll};
use time::OffsetDateTime;
crates/ecstore/src/store_api/readers.rs
+1829 -252
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/store_api/types.rs
+25 -8
View File
@@ -369,13 +369,18 @@ impl ObjectInfo {
}
pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> {
let (algorithm, _, compressed) = self.compression_read_plan()?;
Ok((algorithm, compressed))
}
pub fn compression_read_plan(&self) -> Result<(CompressionAlgorithm, crate::rio::ReadCompressionBackend, bool)> {
let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
if let Some(scheme) = scheme {
let algorithm = CompressionAlgorithm::from_str(&scheme)?;
Ok((algorithm, true))
let (algorithm, backend) = crate::rio::compression_scheme_to_read_plan(&scheme)?;
Ok((algorithm, backend, true))
} else {
Ok((CompressionAlgorithm::None, false))
Ok((CompressionAlgorithm::None, crate::rio::ReadCompressionBackend::Legacy, false))
}
}
@@ -388,13 +393,18 @@ impl ObjectInfo {
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
self.user_defined.keys().any(|key| {
let key = key.to_lowercase();
key.starts_with("x-minio-encryption-")
let lower = key.to_ascii_lowercase();
lower.starts_with("x-minio-encryption-")
|| lower.starts_with("x-minio-internal-server-side-encryption-")
|| matches!(
key.as_str(),
"x-rustfs-encryption-key"
lower.as_str(),
"x-minio-internal-encrypted-multipart"
| "x-rustfs-encryption-key"
| "x-rustfs-encryption-algorithm"
| "x-rustfs-encryption-iv"
| "x-rustfs-encryption-key-id"
| "x-rustfs-encryption-context"
| "x-rustfs-encryption-tag"
| "x-amz-server-side-encryption-aws-kms-key-id"
| SSEC_ALGORITHM_HEADER
| SSEC_KEY_HEADER
@@ -405,10 +415,17 @@ impl ObjectInfo {
}
pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
if let Some(size_str) = self
.user_defined
.get("x-rustfs-encryption-original-size")
.or_else(|| self.user_defined.get("x-amz-server-side-encryption-customer-original-size"))
.map(String::as_str)
.or_else(|| {
self.user_defined
.get("x-amz-server-side-encryption-customer-original-size")
.map(String::as_str)
})
.or(actual_size.as_deref())
&& !size_str.is_empty()
{
let size = size_str
crates/ecstore/tests/README.md
+24
View File
@@ -0,0 +1,24 @@
# ecstore Integration Tests
## MinIO-generated encrypted fixtures
`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`.\rustfs\scripts\minio_fixture_lab\lab.py`.
It currently covers multipart fixtures for:
- `sse-s3-multipart-8m`
- `sse-kms-multipart-8m`
Required environment variables:
- `RUSTFS_MINIO_FIXTURE_ROOT`
- `RUSTFS_MINIO_STATIC_KMS_KEY_B64`
Example:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
cargo +1.95.0 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
```
crates/ecstore/tests/minio_generated_read_test.rs
+207
View File
@@ -0,0 +1,207 @@
#![cfg(feature = "rio-v2")]
use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use rustfs_ecstore::bitrot::create_bitrot_reader;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use temp_env::async_with_vars;
use tokio::io::AsyncReadExt;
#[derive(Debug, Deserialize)]
struct ManifestRecord {
bucket: String,
object: String,
backend_files: Vec<String>,
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn require_fixture_case(case_id: &str) -> PathBuf {
let path = case_dir(case_id);
assert!(
path.is_dir(),
"fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
path.display()
);
path
}
fn read_plaintext_sha256(case_dir: &Path) -> String {
fs::read_to_string(case_dir.join("plaintext.sha256"))
.unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
.trim()
.to_string()
}
fn minio_static_kms_key_b64() -> String {
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
}
fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object);
let relative = manifest
.backend_files
.iter()
.find(|entry| entry.as_str() == expected)
.unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}"));
case_dir.join("backend").join(relative)
}
fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo {
let xl_meta_path = object_xl_meta_path(case_dir, manifest);
let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display()));
get_file_info(
&xl_meta,
&manifest.bucket,
&manifest.object,
"",
FileInfoOpts {
data: true,
include_free_versions: true,
},
)
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display()))
}
fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo {
ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false)
}
fn sha256_hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
}
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
let disk_root = case_dir.join("backend").join("disk1");
let disk_root_str = disk_root
.to_str()
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open fixture disk");
let data_dir = file_info
.data_dir
.as_ref()
.unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object));
let mut encrypted = Vec::new();
for part in &file_info.parts {
let checksum_info = file_info.erasure.get_checksum_info(part.number);
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
let mut reader = create_bitrot_reader(
None,
Some(&disk),
&manifest.bucket,
&path,
0,
part.size,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
false,
false,
)
.await
.unwrap_or_else(|err| panic!("create bitrot reader for {path}: {err:?}"))
.unwrap_or_else(|| panic!("missing bitrot reader for {path}"));
let mut block = vec![0u8; file_info.erasure.shard_size()];
loop {
let n = reader
.read(&mut block)
.await
.unwrap_or_else(|err| panic!("read decoded encrypted bytes from {path}: {err}"));
if n == 0 {
break;
}
encrypted.extend_from_slice(&block[..n]);
if n < block.len() {
break;
}
}
}
disk.close().await.expect("close fixture disk");
encrypted
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_s3_multipart_fixture() {
assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_kms_multipart_fixture() {
assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
}
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
let case_dir = require_fixture_case(case_id);
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
let expected_sha256 = read_plaintext_sha256(&case_dir);
let file_info = load_file_info(&case_dir, &manifest);
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
let object_info = load_object_info(&file_info, &manifest);
let kms_key_b64 = minio_static_kms_key_b64();
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
],
async {
let (mut reader, offset, length) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&http::HeaderMap::new(),
)
.await
.expect("construct GetObjectReader from MinIO raw fixture");
let mut plaintext = Vec::new();
reader
.read_to_end(&mut plaintext)
.await
.expect("read plaintext from MinIO raw fixture");
assert_eq!(offset, 0);
assert_eq!(length, object_info.size);
assert_eq!(reader.object_info.size, expected_size);
assert_eq!(plaintext.len(), expected_size as usize);
assert_eq!(sha256_hex(&plaintext), expected_sha256);
},
)
.await;
}
crates/filemeta/examples/dump_fileinfo.rs
+247
View File
@@ -14,6 +14,30 @@
use rustfs_filemeta::{FileInfoOpts, get_file_info};
use std::{env, fs, path::PathBuf};
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
const CHUNK_TYPE_LEGACY_INDEX: u8 = 0x50;
const CHUNK_TYPE_MINIO_INDEX: u8 = 0x99;
const SKIPPABLE_FRAME_HEADER: usize = 4;
const MAX_INDEX_ENTRIES: i64 = 1 << 16;
#[derive(Debug)]
struct CompressionIndex {
format: &'static str,
storage: &'static str,
total_uncompressed: i64,
total_compressed: i64,
est_block_uncompressed: i64,
offsets: Vec<CompressionIndexOffset>,
}
#[derive(Debug)]
struct CompressionIndexOffset {
compressed: i64,
uncompressed: i64,
}
fn main() {
let path = env::args()
.nth(1)
@@ -40,6 +64,10 @@ fn main() {
"part#{idx}: number={} size={} actual_size={} etag={}",
part.number, part.size, part.actual_size, part.etag
);
match part.index.as_deref() {
Some(index_bytes) => print_compression_index(idx, index_bytes),
None => println!("part#{idx}.index: none"),
}
}
// Tier / transition fields
if !fi.transition_status.is_empty() {
@@ -61,3 +89,222 @@ fn main() {
println!("meta[{key}]={}", fi.metadata.get(&key).unwrap());
}
}
fn print_compression_index(part_idx: usize, bytes: &[u8]) {
match decode_compression_index(bytes) {
Ok(index) => {
println!(
"part#{part_idx}.index: bytes={} format={} storage={} entries={} total_uncompressed={} total_compressed={} est_block_uncompressed={}",
bytes.len(),
index.format,
index.storage,
index.offsets.len(),
index.total_uncompressed,
index.total_compressed,
index.est_block_uncompressed
);
for (idx, offset) in index.offsets.iter().take(5).enumerate() {
println!(
"part#{part_idx}.index.offset#{idx}: compressed={} uncompressed={}",
offset.compressed, offset.uncompressed
);
}
if index.offsets.len() > 5
&& let Some(offset) = index.offsets.last()
{
println!(
"part#{part_idx}.index.offset#last: compressed={} uncompressed={}",
offset.compressed, offset.uncompressed
);
}
}
Err(err) => {
println!("part#{part_idx}.index: bytes={} decode_error={err}", bytes.len());
}
}
}
fn decode_compression_index(bytes: &[u8]) -> Result<CompressionIndex, String> {
if bytes.is_empty() {
return Err("empty index".to_string());
}
let mut errors = Vec::new();
for candidate in index_candidates(bytes) {
match parse_index(&candidate.bytes, candidate.format, candidate.storage, candidate.signed_varint) {
Ok(index) => return Ok(index),
Err(err) => errors.push(format!("{} {}: {err}", candidate.format, candidate.storage)),
}
}
Err(errors.join("; "))
}
struct IndexCandidate {
bytes: Vec<u8>,
format: &'static str,
storage: &'static str,
signed_varint: bool,
}
fn index_candidates(bytes: &[u8]) -> Vec<IndexCandidate> {
let mut candidates = Vec::new();
match bytes[0] {
CHUNK_TYPE_MINIO_INDEX => candidates.push(IndexCandidate {
bytes: bytes.to_vec(),
format: "minio-s2",
storage: "full",
signed_varint: true,
}),
CHUNK_TYPE_LEGACY_INDEX => candidates.push(IndexCandidate {
bytes: bytes.to_vec(),
format: "legacy-rustfs",
storage: "full",
signed_varint: false,
}),
_ => {
candidates.push(IndexCandidate {
bytes: restore_index_headers(bytes, CHUNK_TYPE_MINIO_INDEX),
format: "minio-s2",
storage: "headerless",
signed_varint: true,
});
candidates.push(IndexCandidate {
bytes: restore_index_headers(bytes, CHUNK_TYPE_LEGACY_INDEX),
format: "legacy-rustfs",
storage: "headerless",
signed_varint: false,
});
}
}
candidates
}
fn parse_index(
bytes: &[u8],
format: &'static str,
storage: &'static str,
signed_varint: bool,
) -> Result<CompressionIndex, String> {
if bytes.len() <= SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() {
return Err("buffer too small".to_string());
}
if bytes[0] != CHUNK_TYPE_MINIO_INDEX && bytes[0] != CHUNK_TYPE_LEGACY_INDEX {
return Err(format!("invalid chunk type 0x{:02x}", bytes[0]));
}
let chunk_len = bytes[1] as usize | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16);
let chunk = bytes
.get(SKIPPABLE_FRAME_HEADER..SKIPPABLE_FRAME_HEADER + chunk_len)
.ok_or_else(|| "chunk length exceeds buffer".to_string())?;
let mut body = chunk
.strip_prefix(S2_INDEX_HEADER)
.ok_or_else(|| "invalid index header".to_string())?;
let (total_uncompressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
let (total_compressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
let (est_block_uncompressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
let (entries, used) = read_varint(body, signed_varint)?;
body = &body[used..];
if total_uncompressed < 0 || est_block_uncompressed < 0 || !(0..=MAX_INDEX_ENTRIES).contains(&entries) {
return Err("invalid index totals".to_string());
}
let has_uncompressed = *body.first().ok_or_else(|| "missing uncompressed-offset flag".to_string())?;
if has_uncompressed & 1 != has_uncompressed {
return Err("invalid uncompressed-offset flag".to_string());
}
body = &body[1..];
let mut offsets = (0..entries)
.map(|_| CompressionIndexOffset {
compressed: 0,
uncompressed: 0,
})
.collect::<Vec<_>>();
for idx in 0..offsets.len() {
let mut uncompressed = 0;
if has_uncompressed != 0 {
let (value, used) = read_varint(body, signed_varint)?;
uncompressed = value;
body = &body[used..];
}
if idx > 0 {
let prev = offsets[idx - 1].uncompressed;
uncompressed += prev + est_block_uncompressed;
}
offsets[idx].uncompressed = uncompressed;
}
let mut compressed_predict = est_block_uncompressed / 2;
for idx in 0..offsets.len() {
let (mut compressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
if idx > 0 {
let next_compressed_predict = compressed_predict + compressed / 2;
compressed += offsets[idx - 1].compressed + compressed_predict;
compressed_predict = next_compressed_predict;
}
offsets[idx].compressed = compressed;
}
if body.len() < 4 + S2_INDEX_TRAILER.len() {
return Err("missing index trailer".to_string());
}
body = &body[4..];
if !body.starts_with(S2_INDEX_TRAILER) {
return Err("invalid index trailer".to_string());
}
Ok(CompressionIndex {
format,
storage,
total_uncompressed,
total_compressed,
est_block_uncompressed,
offsets,
})
}
fn restore_index_headers(input: &[u8], chunk_type: u8) -> Vec<u8> {
let mut bytes = Vec::with_capacity(SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + input.len() + 4 + S2_INDEX_TRAILER.len());
bytes.extend_from_slice(&[chunk_type, 0, 0, 0]);
bytes.extend_from_slice(S2_INDEX_HEADER);
bytes.extend_from_slice(input);
bytes.extend_from_slice(&((bytes.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes());
bytes.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = bytes.len() - SKIPPABLE_FRAME_HEADER;
bytes[1] = chunk_len as u8;
bytes[2] = (chunk_len >> 8) as u8;
bytes[3] = (chunk_len >> 16) as u8;
bytes
}
fn read_varint(bytes: &[u8], signed: bool) -> Result<(i64, usize), String> {
let mut value = 0_u64;
let mut shift = 0_u32;
for (idx, byte) in bytes.iter().copied().enumerate() {
value |= u64::from(byte & 0x7f) << shift;
if byte < 0x80 {
let value = if signed {
((value >> 1) as i64) ^ (-((value & 1) as i64))
} else {
value as i64
};
return Ok((value, idx + 1));
}
shift += 7;
if shift >= 64 {
return Err("varint overflow".to_string());
}
}
Err("unexpected EOF while reading varint".to_string())
}
crates/kms/src/cache.rs
+8 -88
View File
@@ -14,22 +14,13 @@
//! Caching layer for KMS operations to improve performance
use crate::types::{KeyMetadata, KeySpec};
use crate::types::KeyMetadata;
use moka::future::Cache;
use std::time::Duration;
/// Cached data key entry
#[derive(Clone, Debug)]
pub struct CachedDataKey {
pub plaintext: Vec<u8>,
pub ciphertext: Vec<u8>,
pub key_spec: KeySpec,
}
/// KMS cache for storing frequently accessed keys and metadata
pub struct KmsCache {
key_metadata_cache: Cache<String, KeyMetadata>,
data_key_cache: Cache<String, CachedDataKey>,
}
impl KmsCache {
@@ -44,13 +35,9 @@ impl KmsCache {
pub fn new(capacity: u64) -> Self {
Self {
key_metadata_cache: Cache::builder()
.max_capacity(capacity / 2)
.max_capacity(capacity)
.time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
.build(),
data_key_cache: Cache::builder()
.max_capacity(capacity / 2)
.time_to_live(Duration::from_secs(60)) // 1 minute for data keys (shorter for security)
.build(),
}
}
@@ -77,35 +64,6 @@ impl KmsCache {
self.key_metadata_cache.run_pending_tasks().await;
}
/// Get data key from cache
///
/// # Arguments
/// * `key_id` - The ID of the key to retrieve the data key for
///
/// # Returns
/// An `Option` containing the `CachedDataKey` if found, or `None` if not found
///
pub async fn get_data_key(&self, key_id: &str) -> Option<CachedDataKey> {
self.data_key_cache.get(key_id).await
}
/// Put data key into cache
///
/// # Arguments
/// * `key_id` - The ID of the key to store the data key for
/// * `plaintext` - The plaintext data key bytes
/// * `ciphertext` - The ciphertext data key bytes
///
pub async fn put_data_key(&mut self, key_id: &str, plaintext: &[u8], ciphertext: &[u8]) {
let cached_key = CachedDataKey {
plaintext: plaintext.to_vec(),
ciphertext: ciphertext.to_vec(),
key_spec: KeySpec::Aes256, // Default to AES-256
};
self.data_key_cache.insert(key_id.to_string(), cached_key).await;
self.data_key_cache.run_pending_tasks().await;
}
/// Remove key metadata from cache
///
/// # Arguments
@@ -115,23 +73,12 @@ impl KmsCache {
self.key_metadata_cache.remove(key_id).await;
}
/// Remove data key from cache
///
/// # Arguments
/// * `key_id` - The ID of the key to remove the data key for
///
pub async fn remove_data_key(&mut self, key_id: &str) {
self.data_key_cache.remove(key_id).await;
}
/// Clear all cached entries
pub async fn clear(&mut self) {
self.key_metadata_cache.invalidate_all();
self.data_key_cache.invalidate_all();
// Wait for invalidation to complete
self.key_metadata_cache.run_pending_tasks().await;
self.data_key_cache.run_pending_tasks().await;
}
/// Get cache statistics (hit count, miss count)
@@ -140,13 +87,10 @@ impl KmsCache {
/// A tuple containing total entries and total misses
///
pub fn stats(&self) -> (u64, u64) {
let metadata_stats = (
(
self.key_metadata_cache.entry_count(),
0u64, // moka doesn't provide miss count directly
);
let data_key_stats = (self.data_key_cache.entry_count(), 0u64);
(metadata_stats.0 + data_key_stats.0, metadata_stats.1 + data_key_stats.1)
)
}
}
@@ -160,37 +104,30 @@ mod tests {
#[derive(Debug, Clone)]
struct CacheInfo {
key_metadata_count: u64,
data_key_count: u64,
}
impl CacheInfo {
fn total_entries(&self) -> u64 {
self.key_metadata_count + self.data_key_count
self.key_metadata_count
}
}
impl KmsCache {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration, data_key_ttl: Duration) -> Self {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
Self {
key_metadata_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(metadata_ttl).build(),
data_key_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(data_key_ttl).build(),
key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
}
}
fn info_for_tests(&self) -> CacheInfo {
CacheInfo {
key_metadata_count: self.key_metadata_cache.entry_count(),
data_key_count: self.data_key_cache.entry_count(),
}
}
fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
self.key_metadata_cache.contains_key(key_id)
}
fn contains_data_key_for_tests(&self, key_id: &str) -> bool {
self.data_key_cache.contains_key(key_id)
}
}
#[tokio::test]
@@ -216,23 +153,10 @@ mod tests {
assert!(retrieved.is_some());
assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
// Test data key caching
let plaintext = vec![1, 2, 3, 4];
let ciphertext = vec![5, 6, 7, 8];
cache.put_data_key("test-key-1", &plaintext, &ciphertext).await;
let cached_data_key = cache.get_data_key("test-key-1").await;
assert!(cached_data_key.is_some());
let cached_data_key = cached_data_key.expect("data key should be cached");
assert_eq!(cached_data_key.plaintext, plaintext);
assert_eq!(cached_data_key.ciphertext, ciphertext);
assert_eq!(cached_data_key.key_spec, KeySpec::Aes256);
// Test cache info
let info = cache.info_for_tests();
assert_eq!(info.key_metadata_count, 1);
assert_eq!(info.data_key_count, 1);
assert_eq!(info.total_entries(), 2);
assert_eq!(info.total_entries(), 1);
// Test cache clearing
cache.clear().await;
@@ -245,7 +169,6 @@ mod tests {
let mut cache = KmsCache::with_ttl_for_tests(
100,
Duration::from_millis(100), // Short TTL for testing
Duration::from_millis(50),
);
let metadata = KeyMetadata {
@@ -277,7 +200,6 @@ mod tests {
let mut cache = KmsCache::new(100);
assert!(!cache.contains_key_metadata_for_tests("nonexistent"));
assert!(!cache.contains_data_key_for_tests("nonexistent"));
let metadata = KeyMetadata {
key_id: "contains-test".to_string(),
@@ -292,9 +214,7 @@ mod tests {
};
cache.put_key_metadata("contains-test", &metadata).await;
cache.put_data_key("contains-test", &[1, 2, 3], &[4, 5, 6]).await;
assert!(cache.contains_key_metadata_for_tests("contains-test"));
assert!(cache.contains_data_key_for_tests("contains-test"));
}
}
crates/kms/src/lib.rs
+8
View File
@@ -33,6 +33,14 @@
//! - **Data Encryption Keys (DEK)**: Generated per object, encrypted by master keys
//! - **Object Data**: Encrypted using DEKs with AES-256-GCM or ChaCha20-Poly1305
//!
//! ## Caching Discipline
//!
//! KMS may cache stable master-key metadata, but it must not cache or reuse generated
//! data encryption keys by master key id alone. A generated DEK and its encrypted
//! ciphertext can be bound to the object encryption context, such as the bucket and
//! object path. Reusing it for another object can break context validation and would
//! also violate the expected per-object DEK model for SSE-S3 and SSE-KMS.
//!
//! ## Example
//!
//! ```rust,no_run
crates/kms/src/manager.rs
+60 -27
View File
@@ -71,32 +71,7 @@ impl KmsManager {
/// Generate a data encryption key
pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
// Check cache first if enabled
if self.config.enable_cache {
let cache = self.cache.read().await;
if let Some(cached_key) = cache.get_data_key(&request.key_id).await
&& cached_key.key_spec == request.key_spec
{
return Ok(GenerateDataKeyResponse {
key_id: request.key_id.clone(),
plaintext_key: cached_key.plaintext.clone(),
ciphertext_blob: cached_key.ciphertext,
});
}
}
// Generate new data key from backend
let response = self.backend.generate_data_key(request).await?;
// Cache the data key if enabled
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache
.put_data_key(&response.key_id, &response.plaintext_key, &response.ciphertext_blob)
.await;
}
Ok(response)
self.backend.generate_data_key(request).await
}
/// Describe a key
@@ -156,7 +131,6 @@ impl KmsManager {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.remove_key_metadata(&response.key_id).await;
cache.remove_data_key(&response.key_id).await;
}
Ok(response)
@@ -186,6 +160,7 @@ mod tests {
use super::*;
use crate::backends::local::LocalKmsBackend;
use crate::types::{KeySpec, KeyState, KeyUsage};
use std::collections::HashMap;
use tempfile::tempdir;
#[tokio::test]
@@ -237,4 +212,62 @@ mod tests {
let health = manager.health_check().await.expect("Health check failed");
assert!(health);
}
#[tokio::test]
async fn generate_data_key_does_not_reuse_context_bound_ciphertext() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf());
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
let manager = KmsManager::new(backend, config);
let create_response = manager
.create_key(CreateKeyRequest {
key_usage: KeyUsage::EncryptDecrypt,
description: Some("Context-bound data key test".to_string()),
..Default::default()
})
.await
.expect("Failed to create key");
let first_context = HashMap::from([
("bucket".to_string(), "sse-smoke".to_string()),
("object".to_string(), "first.bin".to_string()),
]);
let second_context = HashMap::from([
("bucket".to_string(), "sse-smoke".to_string()),
("object".to_string(), "second.bin".to_string()),
]);
let first = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: create_response.key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: first_context.clone(),
})
.await
.expect("Failed to generate first data key");
let second = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: create_response.key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: second_context.clone(),
})
.await
.expect("Failed to generate second data key");
assert_ne!(
first.ciphertext_blob, second.ciphertext_blob,
"data keys must not be cached only by KMS key id because ciphertext is bound to object context"
);
manager
.decrypt(DecryptRequest {
ciphertext: second.ciphertext_blob,
encryption_context: second_context,
grant_tokens: Vec::new(),
})
.await
.expect("second data key should decrypt with its own context");
}
}
crates/kms/src/service.rs
+59 -8
View File
@@ -49,6 +49,26 @@ pub struct ObjectEncryptionService {
kms_manager: KmsManager,
}
fn canonical_bucket_path(bucket: &str, object_key: &str) -> String {
let bucket = bucket.trim_matches('/');
let object_key = object_key.trim_matches('/');
if object_key.is_empty() {
bucket.to_string()
} else if bucket.is_empty() {
object_key.to_string()
} else {
format!("{bucket}/{object_key}")
}
}
fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap<String, String> {
let mut enc_context = context.encryption_context.clone();
enc_context
.entry(context.bucket.clone())
.or_insert_with(|| canonical_bucket_path(&context.bucket, &context.object_key));
enc_context
}
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
/// Result of object encryption
@@ -178,15 +198,10 @@ impl ObjectEncryptionService {
.or_else(|| self.kms_manager.get_default_key_id().map(|s| s.as_str()))
.ok_or_else(|| KmsError::configuration_error("No KMS key ID specified and no default configured"))?;
// Build encryption context
let mut enc_context = context.encryption_context.clone();
enc_context.insert("bucket".to_string(), context.bucket.clone());
enc_context.insert("object_key".to_string(), context.object_key.clone());
let request = GenerateDataKeyRequest {
key_id: actual_key_id.to_string(),
key_spec: KeySpec::Aes256,
encryption_context: enc_context,
encryption_context: request_encryption_context(context),
};
let data_key_response = self.kms_manager.generate_data_key(request).await?;
@@ -216,10 +231,10 @@ impl ObjectEncryptionService {
/// # Returns
/// DataKey with decrypted key
///
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], _context: &ObjectEncryptionContext) -> Result<DataKey> {
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], context: &ObjectEncryptionContext) -> Result<DataKey> {
let decrypt_request = DecryptRequest {
ciphertext: encrypted_key.to_vec(),
encryption_context: HashMap::new(),
encryption_context: request_encryption_context(context),
grant_tokens: Vec::new(),
};
@@ -864,4 +879,40 @@ mod tests {
.is_err()
);
}
#[tokio::test]
async fn test_decrypt_data_key_uses_object_encryption_context() {
let (service, _temp_dir) = create_test_service().await;
service
.create_key(CreateKeyRequest {
key_name: Some("test-key".to_string()),
key_usage: KeyUsage::EncryptDecrypt,
description: None,
policy: None,
tags: HashMap::new(),
origin: None,
})
.await
.expect("test key should be created");
let create_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string())
.with_encryption_context("tenant".to_string(), "alpha".to_string());
let kms_key = Some("test-key".to_string());
let (_data_key, encrypted_key) = service
.create_data_key(&kms_key, &create_context)
.await
.expect("create data key should succeed");
let wrong_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string())
.with_encryption_context("tenant".to_string(), "beta".to_string());
assert!(
service.decrypt_data_key(&encrypted_key, &wrong_context).await.is_err(),
"decrypt should reject mismatched KMS context"
);
let decrypted = service
.decrypt_data_key(&encrypted_key, &create_context)
.await
.expect("decrypt should accept matching KMS context");
assert_ne!(decrypted.plaintext_key, [0u8; 32]);
}
}
crates/obs/Cargo.toml
+3 -2
View File
@@ -28,6 +28,7 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
[features]
default = []
gpu = ["dep:nvml-wrapper"]
pyroscope = ["dep:jemalloc_pprof", "dep:pyroscope"]
[lints]
workspace = true
@@ -72,8 +73,8 @@ sysinfo = { workspace = true }
nvml-wrapper = { workspace = true, optional = true }
[target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
pyroscope = { workspace = true }
jemalloc_pprof = { workspace = true }
pyroscope = { workspace = true, features = ["backend-pprof-rs"], optional = true }
jemalloc_pprof = { workspace = true, optional = true }
[dev-dependencies]
crates/obs/src/telemetry/guard.rs
+26 -12
View File
@@ -28,6 +28,21 @@
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
pub(crate) type ProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
#[cfg(not(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
)))]
pub(crate) type ProfilingAgent = ();
#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
pub(crate) type MemoryProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
pub(crate) type MemoryProfilingAgent = ();
/// RAII guard that owns all active OpenTelemetry providers and the
/// `tracing_appender` worker guard.
///
@@ -41,10 +56,8 @@ pub struct OtelGuard {
pub(crate) meter_provider: Option<SdkMeterProvider>,
/// Optional logger provider for OTLP log export.
pub(crate) logger_provider: Option<SdkLoggerProvider>,
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
pub(crate) profiling_agent: Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>>,
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
pub(crate) memory_profiling_agent: Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>>,
pub(crate) profiling_agent: Option<ProfilingAgent>,
pub(crate) memory_profiling_agent: Option<MemoryProfilingAgent>,
/// Handle to the background log-cleanup task; aborted on drop.
pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
/// Worker guard that keeps the non-blocking `tracing_appender` thread
@@ -59,12 +72,10 @@ impl std::fmt::Debug for OtelGuard {
let mut s = f.debug_struct("OtelGuard");
s.field("tracer_provider", &self.tracer_provider.is_some())
.field("meter_provider", &self.meter_provider.is_some())
.field("logger_provider", &self.logger_provider.is_some());
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
s.field("profiling_agent", &self.profiling_agent.is_some());
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
s.field("memory_profiling_agent", &self.memory_profiling_agent.is_some());
s.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("logger_provider", &self.logger_provider.is_some())
.field("profiling_agent", &self.profiling_agent.is_some())
.field("memory_profiling_agent", &self.memory_profiling_agent.is_some())
.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("tracing_guard", &self.tracing_guard.is_some())
.field("stdout_guard", &self.stdout_guard.is_some())
.finish()
@@ -95,7 +106,10 @@ impl Drop for OtelGuard {
eprintln!("Logger shutdown error: {err:?}");
}
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
if let Some(agent) = self.profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
@@ -105,7 +119,7 @@ impl Drop for OtelGuard {
}
}
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
if let Some(agent) = self.memory_profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Memory profiling agent stop error: {err:?}"),
crates/obs/src/telemetry/local.rs
-4
View File
@@ -155,9 +155,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
tracer_provider: None,
meter_provider: None,
logger_provider: None,
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
profiling_agent: None,
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
memory_profiling_agent: None,
tracing_guard: Some(guard),
stdout_guard: None,
@@ -291,9 +289,7 @@ fn init_file_logging_internal(
tracer_provider: None,
meter_provider: None,
logger_provider: None,
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
profiling_agent: None,
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
memory_profiling_agent: None,
tracing_guard: Some(guard),
stdout_guard,
crates/obs/src/telemetry/otel.rs
+21 -10
View File
@@ -40,7 +40,7 @@ use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::set_observability_metric_enabled;
use crate::telemetry::filter::build_env_filter;
use crate::telemetry::guard::OtelGuard;
use crate::telemetry::guard::{MemoryProfilingAgent, OtelGuard, ProfilingAgent};
use crate::telemetry::local::spawn_cleanup_task;
use crate::telemetry::recorder::Recorder;
use crate::telemetry::resource::build_resource;
@@ -164,10 +164,7 @@ pub(super) fn init_observability_http(
// ── Meter provider (HTTP) ─────────────────────────────────────────────────
let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?;
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
let profiling_agent = init_profiler(config);
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
let memory_profiling_agent = init_memory_profiler(config);
// ── Logger Logic ──────────────────────────────────────────────────────────
@@ -317,9 +314,7 @@ pub(super) fn init_observability_http(
tracer_provider,
meter_provider,
logger_provider,
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
profiling_agent,
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
memory_profiling_agent,
tracing_guard,
stdout_guard,
@@ -496,8 +491,11 @@ fn build_logger_provider(
/// Returns `None` when profiling export is disabled, when no usable
/// profiling endpoint is configured, or when building or starting the agent
/// fails.
#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>> {
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend};
use pyroscope::pyroscope::PyroscopeAgentBuilder;
use rustfs_config::VERSION;
@@ -534,6 +532,14 @@ fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyrosc
}
}
#[cfg(not(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
)))]
fn init_profiler(_config: &OtelConfig) -> Option<ProfilingAgent> {
None
}
/// Initialise a Pyroscope agent for continuous **memory** profiling via jemalloc.
///
/// This is only available on `linux + gnu + x86_64` where tikv-jemallocator
@@ -541,8 +547,8 @@ fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyrosc
///
/// Returns `None` when profiling export is disabled, the endpoint is missing,
/// jemalloc profiling is not activated, or the agent fails to build/start.
#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
fn init_memory_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>> {
#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
fn init_memory_profiler(config: &OtelConfig) -> Option<MemoryProfilingAgent> {
use pyroscope::backend::jemalloc_backend;
use pyroscope::pyroscope::PyroscopeAgentBuilder;
use rustfs_config::VERSION;
@@ -588,6 +594,11 @@ fn init_memory_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent
}
}
#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
fn init_memory_profiler(_config: &OtelConfig) -> Option<MemoryProfilingAgent> {
None
}
/// Create a stdout periodic metrics reader for the given interval.
///
/// This helper is primarily used for local development and diagnostics when
crates/rio-v2/Cargo.toml
+48
View File
@@ -0,0 +1,48 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-rio-v2"
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
homepage.workspace = true
description = "Feature-gated rio_v2 compatibility facade for incremental RustFS migration."
keywords = ["asynchronous", "io", "compatibility", "rustfs", "minio"]
categories = ["web-programming", "development-tools", "asynchronous"]
documentation = "https://docs.rs/rustfs-rio-v2/latest/rustfs_rio_v2/"
[lints]
workspace = true
[dependencies]
aes-gcm.workspace = true
bytes.workspace = true
hex.workspace = true
hmac.workspace = true
minlz = "1.1.0"
pin-project-lite.workspace = true
rand.workspace = true
rustfs-rio.workspace = true
rustfs-utils.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tokio.workspace = true
[dev-dependencies]
rustfs-filemeta.workspace = true
walkdir.workspace = true
crates/rio-v2/src/compress_reader.rs
+755
View File
@@ -0,0 +1,755 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use minlz::{Encoder as MinlzEncoder, crc::crc, decode};
use pin_project_lite::pin_project;
use rand::RngExt;
use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex};
use rustfs_utils::CompressionAlgorithm;
use std::cmp::min;
use std::fmt;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
const MAGIC_CHUNK: &[u8] = b"\xff\x06\x00\x00S2sTwO";
const MAGIC_CHUNK_SNAPPY: &[u8] = b"\xff\x06\x00\x00sNaPpY";
const CHUNK_TYPE_COMPRESSED_DATA: u8 = 0x00;
const CHUNK_TYPE_UNCOMPRESSED_DATA: u8 = 0x01;
const CHUNK_TYPE_INDEX: u8 = 0x99;
const CHUNK_TYPE_PADDING: u8 = 0xfe;
const CHUNK_TYPE_STREAM_IDENTIFIER: u8 = 0xff;
const DEFAULT_BLOCK_SIZE: usize = 1 << 20;
const MAX_CHUNK_SIZE: usize = (1 << 24) - 1;
const CHECKSUM_SIZE: usize = 4;
const CHUNK_HEADER_LEN: usize = 4;
const ENCRYPTED_PADDING_MULTIPLE: usize = 256;
const MIN_INDEX_SIZE: usize = 8 << 20;
pin_project! {
#[derive(Debug)]
pub struct CompressReader<R> {
#[pin]
inner: R,
buffer: Vec<u8>,
pos: usize,
done: bool,
block_size: usize,
index: Index,
written: usize,
uncompressed_written: usize,
temp_buffer: Vec<u8>,
read_buffer: Vec<u8>,
wrote_stream_header: bool,
padding_multiple: Option<usize>,
block_encoder: S2BlockEncoder,
}
}
struct S2BlockEncoder {
inner: MinlzEncoder,
}
impl S2BlockEncoder {
fn new() -> Self {
Self {
inner: MinlzEncoder::new(),
}
}
fn encode(&mut self, uncompressed: &[u8]) -> Vec<u8> {
self.inner.encode(uncompressed)
}
}
impl fmt::Debug for S2BlockEncoder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("S2BlockEncoder").finish_non_exhaustive()
}
}
impl<R> CompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
Self::with_block_size(inner, DEFAULT_BLOCK_SIZE, CompressionAlgorithm::default())
}
pub fn with_block_size(inner: R, block_size: usize, _compression_algorithm: CompressionAlgorithm) -> Self {
Self {
inner,
buffer: Vec::new(),
pos: 0,
done: false,
block_size,
index: Index::new(),
written: 0,
uncompressed_written: 0,
temp_buffer: Vec::with_capacity(block_size),
read_buffer: vec![0u8; block_size],
wrote_stream_header: false,
padding_multiple: None,
block_encoder: S2BlockEncoder::new(),
}
}
pub fn with_encrypted_padding(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
let mut reader = Self::new(inner, CompressionAlgorithm::default());
reader.padding_multiple = Some(ENCRYPTED_PADDING_MULTIPLE);
reader
}
}
impl<R> TryGetIndex for CompressReader<R> {
fn try_get_index(&self) -> Option<&Index> {
(self.uncompressed_written > MIN_INDEX_SIZE).then_some(&self.index)
}
}
impl<R> AsyncRead for CompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.pos < this.buffer.len() {
let to_copy = min(buf.remaining(), this.buffer.len() - *this.pos);
buf.put_slice(&this.buffer[*this.pos..*this.pos + to_copy]);
*this.pos += to_copy;
if *this.pos == this.buffer.len() {
this.buffer.clear();
*this.pos = 0;
}
return Poll::Ready(Ok(()));
}
if *this.done {
return Poll::Ready(Ok(()));
}
while this.temp_buffer.len() < *this.block_size {
let remaining = *this.block_size - this.temp_buffer.len();
let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => {
return Poll::Pending;
}
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
break;
}
this.temp_buffer.extend_from_slice(read_buf.filled());
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
if this.temp_buffer.is_empty() {
if let Some(padding_multiple) = *this.padding_multiple
&& let Some(padding_chunk) = build_padding_chunk(*this.written, padding_multiple)?
{
*this.written += padding_chunk.len();
this.index.total_compressed = *this.written as i64;
*this.buffer = padding_chunk;
*this.pos = 0;
*this.done = true;
let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]);
*this.pos += to_copy;
if *this.pos == this.buffer.len() {
this.buffer.clear();
*this.pos = 0;
}
return Poll::Ready(Ok(()));
}
*this.done = true;
return Poll::Ready(Ok(()));
}
let mut out = Vec::new();
if !*this.wrote_stream_header {
out.extend_from_slice(MAGIC_CHUNK);
*this.written += MAGIC_CHUNK.len();
*this.wrote_stream_header = true;
}
if let Err(err) = this.index.add(*this.written as i64, *this.uncompressed_written as i64) {
return Poll::Ready(Err(err));
}
let block = build_s2_chunk(this.temp_buffer.as_slice(), this.block_encoder)?;
*this.uncompressed_written += this.temp_buffer.len();
*this.written += block.len();
this.index.total_uncompressed = *this.uncompressed_written as i64;
this.index.total_compressed = *this.written as i64;
out.extend_from_slice(&block);
this.temp_buffer.clear();
*this.buffer = out;
*this.pos = 0;
let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]);
*this.pos += to_copy;
if *this.pos == this.buffer.len() {
this.buffer.clear();
*this.pos = 0;
}
Poll::Ready(Ok(()))
}
}
impl<R> EtagResolvable for CompressReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for CompressReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
pin_project! {
#[derive(Debug)]
pub struct DecompressReader<R> {
#[pin]
inner: R,
buffer: Vec<u8>,
buffer_pos: usize,
finished: bool,
header_buf: [u8; CHUNK_HEADER_LEN],
header_read: usize,
chunk_type: u8,
chunk_buf: Vec<u8>,
chunk_len: usize,
chunk_read: usize,
reading_chunk: bool,
stream_initialized: bool,
}
}
impl<R> DecompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
Self {
inner,
buffer: Vec::new(),
buffer_pos: 0,
finished: false,
header_buf: [0u8; CHUNK_HEADER_LEN],
header_read: 0,
chunk_type: 0,
chunk_buf: Vec::new(),
chunk_len: 0,
chunk_read: 0,
reading_chunk: false,
stream_initialized: false,
}
}
}
impl<R> AsyncRead for DecompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.buffer_pos < this.buffer.len() {
let to_copy = min(buf.remaining(), this.buffer.len() - *this.buffer_pos);
buf.put_slice(&this.buffer[*this.buffer_pos..*this.buffer_pos + to_copy]);
*this.buffer_pos += to_copy;
if *this.buffer_pos == this.buffer.len() {
this.buffer.clear();
*this.buffer_pos = 0;
}
return Poll::Ready(Ok(()));
}
loop {
if *this.finished {
return Poll::Ready(Ok(()));
}
if !*this.reading_chunk {
while *this.header_read < CHUNK_HEADER_LEN {
let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
if *this.header_read == 0 {
*this.finished = true;
return Poll::Ready(Ok(()));
}
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading S2 chunk header",
)));
}
*this.header_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
*this.chunk_type = this.header_buf[0];
*this.chunk_len =
(this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16);
*this.header_read = 0;
if this.chunk_buf.len() < *this.chunk_len {
this.chunk_buf.resize(*this.chunk_len, 0);
}
*this.chunk_read = 0;
*this.reading_chunk = true;
}
while *this.chunk_read < *this.chunk_len {
let mut read_buf = ReadBuf::new(&mut this.chunk_buf[*this.chunk_read..*this.chunk_len]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading S2 chunk body",
)));
}
*this.chunk_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
let chunk = &this.chunk_buf[..*this.chunk_len];
*this.reading_chunk = false;
match *this.chunk_type {
CHUNK_TYPE_STREAM_IDENTIFIER => {
if chunk != &MAGIC_CHUNK[CHUNK_HEADER_LEN..] && chunk != &MAGIC_CHUNK_SNAPPY[CHUNK_HEADER_LEN..] {
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid S2 stream identifier")));
}
*this.stream_initialized = true;
continue;
}
CHUNK_TYPE_COMPRESSED_DATA => {
*this.stream_initialized = true;
let decompressed = decode_chunk(chunk, true)?;
*this.buffer = decompressed;
}
CHUNK_TYPE_UNCOMPRESSED_DATA => {
*this.stream_initialized = true;
let decompressed = decode_chunk(chunk, false)?;
*this.buffer = decompressed;
}
CHUNK_TYPE_INDEX | CHUNK_TYPE_PADDING | 0x80..=0xfd => {
*this.stream_initialized = true;
continue;
}
_ => {
if !*this.stream_initialized && *this.chunk_type != CHUNK_TYPE_COMPRESSED_DATA {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
)));
}
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
)));
}
}
*this.buffer_pos = 0;
let to_copy = min(buf.remaining(), this.buffer.len());
buf.put_slice(&this.buffer[..to_copy]);
*this.buffer_pos += to_copy;
if *this.buffer_pos == this.buffer.len() {
this.buffer.clear();
*this.buffer_pos = 0;
}
return Poll::Ready(Ok(()));
}
}
}
impl<R> EtagResolvable for DecompressReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for DecompressReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
fn build_s2_chunk(uncompressed: &[u8], encoder: &mut S2BlockEncoder) -> io::Result<Vec<u8>> {
let compressed = encode_block(uncompressed, encoder);
let checksum = crc(uncompressed);
let dst_limit = uncompressed.len().saturating_sub(uncompressed.len() / 32).saturating_sub(5);
let (chunk_type, payload) = if compressed.len() <= dst_limit {
(CHUNK_TYPE_COMPRESSED_DATA, compressed)
} else {
(CHUNK_TYPE_UNCOMPRESSED_DATA, uncompressed.to_vec())
};
let chunk_len = payload.len() + CHECKSUM_SIZE;
if chunk_len > MAX_CHUNK_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk exceeds 24-bit framing limit"));
}
let mut out = Vec::with_capacity(CHUNK_HEADER_LEN + chunk_len);
out.push(chunk_type);
out.push((chunk_len & 0xff) as u8);
out.push(((chunk_len >> 8) & 0xff) as u8);
out.push(((chunk_len >> 16) & 0xff) as u8);
out.extend_from_slice(&checksum.to_le_bytes());
out.extend_from_slice(&payload);
Ok(out)
}
fn encode_block(uncompressed: &[u8], encoder: &mut S2BlockEncoder) -> Vec<u8> {
encoder.encode(uncompressed)
}
fn build_padding_chunk(current_size: usize, padding_multiple: usize) -> io::Result<Option<Vec<u8>>> {
if padding_multiple == 0 || current_size.is_multiple_of(padding_multiple) {
return Ok(None);
}
let padding_len = (padding_multiple - ((current_size + CHUNK_HEADER_LEN) % padding_multiple)) % padding_multiple;
if padding_len > MAX_CHUNK_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 padding exceeds 24-bit framing limit"));
}
let mut out = Vec::with_capacity(CHUNK_HEADER_LEN + padding_len);
out.push(CHUNK_TYPE_PADDING);
out.push((padding_len & 0xff) as u8);
out.push(((padding_len >> 8) & 0xff) as u8);
out.push(((padding_len >> 16) & 0xff) as u8);
if padding_len > 0 {
let mut padding = vec![0u8; padding_len];
rand::rng().fill(padding.as_mut_slice());
out.extend_from_slice(&padding);
}
Ok(Some(out))
}
fn decode_chunk(chunk: &[u8], compressed: bool) -> io::Result<Vec<u8>> {
if chunk.len() < CHECKSUM_SIZE {
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk smaller than checksum header"));
}
let expected_crc = u32::from_le_bytes(chunk[..CHECKSUM_SIZE].try_into().expect("checksum header"));
let payload = &chunk[CHECKSUM_SIZE..];
let decompressed = if compressed {
decode(payload).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 decode error: {err}")))?
} else {
payload.to_vec()
};
let actual_crc = crc(&decompressed);
if actual_crc != expected_crc {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"S2 CRC mismatch: expected={expected_crc:08x} actual={actual_crc:08x} compressed={compressed} payload_len={} decompressed_len={}",
payload.len(),
decompressed.len()
),
));
}
Ok(decompressed)
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::AsyncReadExt;
struct PendingAfterBytes<R> {
inner: R,
max_chunk: usize,
pending_next: bool,
}
impl<R> PendingAfterBytes<R> {
fn new(inner: R, max_chunk: usize) -> Self {
Self {
inner,
max_chunk,
pending_next: false,
}
}
}
impl<R: AsyncRead + Unpin> AsyncRead for PendingAfterBytes<R> {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
if self.pending_next {
self.pending_next = false;
cx.waker().wake_by_ref();
return Poll::Pending;
}
let allowed = self.max_chunk.min(buf.remaining());
if allowed == 0 {
return Poll::Ready(Ok(()));
}
let mut scratch = vec![0u8; allowed];
let mut limited = ReadBuf::new(&mut scratch);
match Pin::new(&mut self.inner).poll_read(cx, &mut limited) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Ready(Ok(())) => {
let filled = limited.filled();
if !filled.is_empty() {
buf.put_slice(filled);
self.pending_next = true;
}
Poll::Ready(Ok(()))
}
}
}
}
fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
let mut chunk_types = Vec::new();
let mut offset = 0usize;
while offset + CHUNK_HEADER_LEN <= stream.len() {
let chunk_type = stream[offset];
let chunk_len =
(stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16);
chunk_types.push(chunk_type);
offset += CHUNK_HEADER_LEN + chunk_len;
}
chunk_types
}
#[test]
fn minlz_encoded_payload_decodes_with_minlz() {
let plaintext = b"compressible-rio-v2-block-".repeat(4096);
let mut encoder = S2BlockEncoder::new();
let compressed = encode_block(&plaintext, &mut encoder);
let decoded = decode(&compressed).expect("decode payload");
assert_eq!(decoded, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_roundtrip() {
let plaintext = b"hello-rio-v2-s2-".repeat(32_768);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
assert!(compressed.starts_with(MAGIC_CHUNK));
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_roundtrip_near_erasure_boundary() {
let size = 4 * 1024 * 1024 - 97;
let plaintext = pseudo_random_bytes(size);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
fn pseudo_random_bytes(size: usize) -> Vec<u8> {
(0..size)
.scan(0x9e37_79b9_7f4a_7c15u64, |state, _| {
*state ^= *state << 7;
*state ^= *state >> 9;
*state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9);
Some((*state >> 32) as u8)
})
.collect()
}
#[tokio::test]
async fn s2_compress_reader_roundtrip_large_random() {
let plaintext = pseudo_random_bytes(8 * 1024 * 1024 + 123);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_roundtrip_with_pending_source() {
let plaintext = pseudo_random_bytes(2 * 1024 * 1024 + 17);
let pending_reader = PendingAfterBytes::new(Cursor::new(plaintext.clone()), 257);
let mut reader = CompressReader::new(pending_reader, CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_decompress_reader_returns_bytes_on_first_read() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut buf = [0u8; 64];
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
assert!(n > 0);
assert_eq!(&buf[..n], plaintext.as_slice());
}
#[tokio::test]
async fn s2_decompress_reader_resumes_chunk_body_after_pending() {
let plaintext = pseudo_random_bytes(1024 * 1024 + 123);
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let pending_reader = PendingAfterBytes::new(Cursor::new(compressed), 257);
let mut decompressor = DecompressReader::new(pending_reader, CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_decompress_reader_handles_concatenated_streams_after_pending() {
let first = pseudo_random_bytes(16 * 1024 * 1024);
let second = pseudo_random_bytes(12 * 1024 * 1024 + 123);
let mut first_compressed = Vec::new();
CompressReader::new(Cursor::new(first.clone()), CompressionAlgorithm::default())
.read_to_end(&mut first_compressed)
.await
.expect("compress first stream");
let mut second_compressed = Vec::new();
CompressReader::new(Cursor::new(second.clone()), CompressionAlgorithm::default())
.read_to_end(&mut second_compressed)
.await
.expect("compress second stream");
first_compressed.extend_from_slice(&second_compressed);
let pending_reader = PendingAfterBytes::new(Cursor::new(first_compressed), 4096);
let mut decompressor = DecompressReader::new(pending_reader, CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
let mut expected = first;
expected.extend_from_slice(&second);
assert_eq!(actual, expected);
}
#[tokio::test]
async fn s2_compress_reader_with_encrypted_padding_emits_padding_frame() {
let plaintext = b"encrypted-padding-check-".repeat(8192);
let mut reader = CompressReader::with_encrypted_padding(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
assert_eq!(compressed.len() % ENCRYPTED_PADDING_MULTIPLE, 0);
assert!(s2_chunk_types(&compressed).contains(&CHUNK_TYPE_PADDING));
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn s2_compress_reader_skips_index_for_small_streams() {
let plaintext = b"index-threshold-check-".repeat(16_384);
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("read compressed data");
assert!(reader.try_get_index().is_none());
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressor.read_to_end(&mut actual).await.expect("read decompressed data");
assert_eq!(actual, plaintext);
}
}
crates/rio-v2/src/encrypt_reader.rs
+759
View File
@@ -0,0 +1,759 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use aes_gcm::aead::Aead;
use aes_gcm::{Aes256Gcm, KeyInit, Nonce};
use hmac::{Hmac, Mac};
use pin_project_lite::pin_project;
use rand::RngExt;
use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex, multipart_part_nonce};
use sha2::Sha256;
use std::cmp::min;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncRead, ReadBuf};
const DARE_VERSION_20: u8 = 0x20;
const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
const DARE_HEADER_SIZE: usize = 16;
const DARE_TAG_SIZE: usize = 16;
const DARE_PAYLOAD_SIZE: usize = 64 * 1024;
type HmacSha256 = Hmac<Sha256>;
#[derive(Clone, Copy)]
enum MultipartKeySource {
LegacyNonce { base_nonce: [u8; 12] },
ObjectKey { object_key: [u8; 32] },
}
pin_project! {
pub struct EncryptReader<R> {
#[pin]
inner: R,
cipher: Aes256Gcm,
base_nonce: [u8; 12],
sequence_number: u32,
temp_buffer: Vec<u8>,
read_buffer: Vec<u8>,
output_buffer: Vec<u8>,
output_pos: usize,
finished: bool,
}
}
impl<R> EncryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new_with_object_key(inner: R, object_key: [u8; 32]) -> Self {
Self::new(inner, object_key, random_stream_nonce())
}
pub fn new(inner: R, key: [u8; 32], nonce: [u8; 12]) -> Self {
Self::new_with_sequence(inner, key, nonce, 0)
}
pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
base_nonce: nonce,
sequence_number,
temp_buffer: Vec::with_capacity(DARE_PAYLOAD_SIZE + 1),
read_buffer: vec![0u8; DARE_PAYLOAD_SIZE + 1],
output_buffer: Vec::new(),
output_pos: 0,
finished: false,
}
}
pub fn new_multipart(inner: R, key: [u8; 32], base_nonce: [u8; 12], part_number: usize) -> Self {
Self::new(inner, key, multipart_part_nonce(base_nonce, part_number))
}
pub fn new_multipart_with_object_key(inner: R, object_key: [u8; 32], part_number: u32) -> Self {
Self::new(inner, derive_part_key(object_key, part_number), random_stream_nonce())
}
pub fn new_multipart_with_sequence(
inner: R,
key: [u8; 32],
base_nonce: [u8; 12],
part_number: usize,
sequence_number: u32,
) -> Self {
Self::new_with_sequence(inner, key, multipart_part_nonce(base_nonce, part_number), sequence_number)
}
}
impl<R> AsyncRead for EncryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.output_pos < this.output_buffer.len() {
let to_copy = min(buf.remaining(), this.output_buffer.len() - *this.output_pos);
buf.put_slice(&this.output_buffer[*this.output_pos..*this.output_pos + to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
return Poll::Ready(Ok(()));
}
if *this.finished {
return Poll::Ready(Ok(()));
}
loop {
if this.temp_buffer.len() > DARE_PAYLOAD_SIZE {
let package = build_dare_package(
this.cipher,
*this.sequence_number,
*this.base_nonce,
&this.temp_buffer[..DARE_PAYLOAD_SIZE],
false,
)?;
let carry = this.temp_buffer.split_off(DARE_PAYLOAD_SIZE);
*this.temp_buffer = carry;
*this.sequence_number = this.sequence_number.wrapping_add(1);
*this.output_buffer = package;
break;
}
let remaining = DARE_PAYLOAD_SIZE + 1 - this.temp_buffer.len();
let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => {
if this.temp_buffer.len() > DARE_PAYLOAD_SIZE {
continue;
}
return Poll::Pending;
}
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
if this.temp_buffer.is_empty() {
*this.finished = true;
return Poll::Ready(Ok(()));
}
let package =
build_dare_package(this.cipher, *this.sequence_number, *this.base_nonce, this.temp_buffer, true)?;
this.temp_buffer.clear();
*this.sequence_number = this.sequence_number.wrapping_add(1);
*this.output_buffer = package;
*this.finished = true;
break;
}
this.temp_buffer.extend_from_slice(read_buf.filled());
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
let to_copy = min(buf.remaining(), this.output_buffer.len());
buf.put_slice(&this.output_buffer[..to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
Poll::Ready(Ok(()))
}
}
impl<R> EtagResolvable for EncryptReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for EncryptReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
impl<R> TryGetIndex for EncryptReader<R>
where
R: TryGetIndex,
{
fn try_get_index(&self) -> Option<&Index> {
self.inner.try_get_index()
}
}
pin_project! {
pub struct DecryptReader<R> {
#[pin]
inner: R,
cipher: Aes256Gcm,
expected_base_nonce: Option<[u8; 12]>,
sequence_number: u32,
multipart_parts: Vec<usize>,
current_part_index: usize,
multipart_mode: bool,
multipart_key_source: Option<MultipartKeySource>,
header_buf: [u8; DARE_HEADER_SIZE],
header_read: usize,
ciphertext_buf: Vec<u8>,
ciphertext_len: usize,
ciphertext_read: usize,
ref_nonce: Option<[u8; 12]>,
finalized: bool,
finished: bool,
output_buffer: Vec<u8>,
output_pos: usize,
}
}
impl<R> DecryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
pub fn new_with_object_key(inner: R, object_key: [u8; 32]) -> Self {
Self::new_with_object_key_and_sequence(inner, object_key, 0)
}
pub fn new(inner: R, key: [u8; 32], nonce: [u8; 12]) -> Self {
Self::new_with_sequence(inner, key, nonce, 0)
}
pub fn new_with_object_key_and_sequence(inner: R, object_key: [u8; 32], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&object_key).expect("valid AES-256-GCM key"),
expected_base_nonce: None,
sequence_number,
multipart_parts: Vec::new(),
current_part_index: 0,
multipart_mode: false,
multipart_key_source: None,
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
pub fn new_with_sequence(inner: R, key: [u8; 32], nonce: [u8; 12], sequence_number: u32) -> Self {
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
expected_base_nonce: Some(nonce),
sequence_number,
multipart_parts: Vec::new(),
current_part_index: 0,
multipart_mode: false,
multipart_key_source: None,
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
pub fn new_multipart(inner: R, key: [u8; 32], base_nonce: [u8; 12], multipart_parts: Vec<usize>) -> Self {
Self::new_multipart_with_sequence(inner, key, base_nonce, multipart_parts, 0)
}
pub fn new_multipart_with_object_key(inner: R, object_key: [u8; 32], multipart_parts: Vec<usize>) -> Self {
Self::new_multipart_with_object_key_and_sequence(inner, object_key, multipart_parts, 0)
}
pub fn new_multipart_with_sequence(
inner: R,
key: [u8; 32],
base_nonce: [u8; 12],
multipart_parts: Vec<usize>,
sequence_number: u32,
) -> Self {
let first_part = multipart_parts.first().copied().unwrap_or(1);
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&key).expect("valid AES-256-GCM key"),
expected_base_nonce: Some(multipart_part_nonce(base_nonce, first_part)),
sequence_number,
multipart_parts,
current_part_index: 0,
multipart_mode: true,
multipart_key_source: Some(MultipartKeySource::LegacyNonce { base_nonce }),
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
pub fn new_multipart_with_object_key_and_sequence(
inner: R,
object_key: [u8; 32],
multipart_parts: Vec<usize>,
sequence_number: u32,
) -> Self {
let first_part = multipart_parts.first().copied().unwrap_or(1);
let first_key = derive_part_key(object_key, first_part as u32);
Self {
inner,
cipher: Aes256Gcm::new_from_slice(&first_key).expect("valid AES-256-GCM key"),
expected_base_nonce: None,
sequence_number,
multipart_parts,
current_part_index: 0,
multipart_mode: true,
multipart_key_source: Some(MultipartKeySource::ObjectKey { object_key }),
header_buf: [0u8; DARE_HEADER_SIZE],
header_read: 0,
ciphertext_buf: Vec::new(),
ciphertext_len: 0,
ciphertext_read: 0,
ref_nonce: None,
finalized: false,
finished: false,
output_buffer: Vec::new(),
output_pos: 0,
}
}
}
impl<R> AsyncRead for DecryptReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
let mut this = self.project();
if *this.output_pos < this.output_buffer.len() {
let to_copy = min(buf.remaining(), this.output_buffer.len() - *this.output_pos);
buf.put_slice(&this.output_buffer[*this.output_pos..*this.output_pos + to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
return Poll::Ready(Ok(()));
}
loop {
if *this.finished {
return Poll::Ready(Ok(()));
}
if *this.finalized {
if !this.multipart_parts.is_empty() && *this.current_part_index + 1 < this.multipart_parts.len() {
let next_part = this.multipart_parts[*this.current_part_index + 1];
*this.current_part_index += 1;
match this.multipart_key_source.as_ref().copied() {
Some(MultipartKeySource::LegacyNonce { base_nonce }) => {
*this.expected_base_nonce = Some(multipart_part_nonce(base_nonce, next_part));
}
Some(MultipartKeySource::ObjectKey { object_key }) => {
let part_key = derive_part_key(object_key, next_part as u32);
*this.cipher = Aes256Gcm::new_from_slice(&part_key).expect("valid AES-256-GCM key");
*this.expected_base_nonce = None;
}
None => {}
}
*this.sequence_number = 0;
*this.ref_nonce = None;
*this.finalized = false;
continue;
}
*this.finished = true;
return Poll::Ready(Ok(()));
}
while *this.header_read < DARE_HEADER_SIZE {
let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
if *this.header_read == 0 {
*this.finished = true;
return Poll::Ready(Ok(()));
}
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading DARE header",
)));
}
*this.header_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
let header = this.header_buf;
let payload_len = usize::from(u16::from_le_bytes([header[2], header[3]])) + 1;
let package_len = payload_len + DARE_TAG_SIZE;
if payload_len == 0 || payload_len > DARE_PAYLOAD_SIZE {
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid DARE payload size")));
}
if !is_final_header(*header) && payload_len != DARE_PAYLOAD_SIZE {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::InvalidData,
"non-final DARE package must carry a full 64KiB payload",
)));
}
if this.ciphertext_buf.len() < package_len {
this.ciphertext_buf.resize(package_len, 0);
}
*this.ciphertext_len = package_len;
*this.ciphertext_read = 0;
while *this.ciphertext_read < *this.ciphertext_len {
let mut read_buf = ReadBuf::new(&mut this.ciphertext_buf[*this.ciphertext_read..*this.ciphertext_len]);
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF while reading DARE ciphertext",
)));
}
*this.ciphertext_read += n;
}
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
}
}
match open_dare_package(
this.cipher,
*this.sequence_number,
*this.expected_base_nonce,
*header,
&this.ciphertext_buf[..*this.ciphertext_len],
this.ref_nonce,
) {
Ok((plaintext, ref_nonce, finalized)) => {
*this.ref_nonce = Some(ref_nonce);
*this.finalized = finalized;
*this.header_read = 0;
*this.ciphertext_len = 0;
*this.ciphertext_read = 0;
*this.sequence_number = this.sequence_number.wrapping_add(1);
*this.output_buffer = plaintext;
let to_copy = min(buf.remaining(), this.output_buffer.len());
buf.put_slice(&this.output_buffer[..to_copy]);
*this.output_pos += to_copy;
if *this.output_pos == this.output_buffer.len() {
this.output_buffer.clear();
*this.output_pos = 0;
}
return Poll::Ready(Ok(()));
}
Err(err) => return Poll::Ready(Err(err)),
}
}
}
}
impl<R> EtagResolvable for DecryptReader<R>
where
R: EtagResolvable,
{
fn try_resolve_etag(&mut self) -> Option<String> {
self.inner.try_resolve_etag()
}
}
impl<R> HashReaderDetector for DecryptReader<R>
where
R: HashReaderDetector,
{
fn is_hash_reader(&self) -> bool {
self.inner.is_hash_reader()
}
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
self.inner.as_hash_reader_mut()
}
}
impl<R> TryGetIndex for DecryptReader<R>
where
R: TryGetIndex,
{
fn try_get_index(&self) -> Option<&Index> {
self.inner.try_get_index()
}
}
fn build_dare_package(
cipher: &Aes256Gcm,
sequence_number: u32,
base_nonce: [u8; 12],
plaintext: &[u8],
final_package: bool,
) -> io::Result<Vec<u8>> {
let mut header = [0u8; DARE_HEADER_SIZE];
header[0] = DARE_VERSION_20;
header[1] = DARE_CIPHER_AES_256_GCM;
header[2..4].copy_from_slice(
&u16::try_from(plaintext.len() - 1)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "DARE payload too large for Version 2.0 framing"))?
.to_le_bytes(),
);
header[4..16].copy_from_slice(&base_nonce);
if final_package {
header[4] |= 0x80;
} else {
header[4] &= 0x7F;
}
let mut package_nonce = header[4..16].try_into().expect("nonce slice");
xor_sequence_into_nonce(&mut package_nonce, sequence_number);
let nonce = Nonce::try_from(package_nonce.as_slice())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid DARE nonce length"))?;
let ciphertext = cipher
.encrypt(
&nonce,
aes_gcm::aead::Payload {
msg: plaintext,
aad: &header[..4],
},
)
.map_err(|err| io::Error::other(format!("failed to encrypt DARE package: {err}")))?;
let mut package = Vec::with_capacity(DARE_HEADER_SIZE + ciphertext.len());
package.extend_from_slice(&header);
package.extend_from_slice(&ciphertext);
Ok(package)
}
fn open_dare_package(
cipher: &Aes256Gcm,
sequence_number: u32,
expected_base_nonce: Option<[u8; 12]>,
header: [u8; DARE_HEADER_SIZE],
ciphertext: &[u8],
ref_nonce: &mut Option<[u8; 12]>,
) -> io::Result<(Vec<u8>, [u8; 12], bool)> {
if header[0] != DARE_VERSION_20 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE version"));
}
if header[1] != DARE_CIPHER_AES_256_GCM {
return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported DARE cipher suite"));
}
let header_nonce: [u8; 12] = header[4..16].try_into().expect("nonce slice");
if let Some(expected_base_nonce) = expected_base_nonce {
let masked_expected = apply_final_flag(expected_base_nonce, is_final_header(header));
if header_nonce != masked_expected {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"DARE package nonce does not match the configured stream nonce",
));
}
}
let current_ref = ref_nonce.get_or_insert(header_nonce);
let expected_ref = apply_final_flag(*current_ref, is_final_header(header));
if header_nonce != expected_ref {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"DARE package nonce does not match the stream reference nonce",
));
}
let mut package_nonce = header_nonce;
xor_sequence_into_nonce(&mut package_nonce, sequence_number);
let nonce = Nonce::try_from(package_nonce.as_slice())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid DARE nonce length"))?;
let plaintext = cipher
.decrypt(
&nonce,
aes_gcm::aead::Payload {
msg: ciphertext,
aad: &header[..4],
},
)
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("DARE authentication failed: {err}")))?;
Ok((plaintext, *current_ref, is_final_header(header)))
}
fn xor_sequence_into_nonce(nonce: &mut [u8; 12], sequence_number: u32) {
let last = u32::from_le_bytes([nonce[8], nonce[9], nonce[10], nonce[11]]) ^ sequence_number;
nonce[8..12].copy_from_slice(&last.to_le_bytes());
}
fn apply_final_flag(mut nonce: [u8; 12], final_package: bool) -> [u8; 12] {
if final_package {
nonce[0] |= 0x80;
} else {
nonce[0] &= 0x7F;
}
nonce
}
fn is_final_header(header: [u8; DARE_HEADER_SIZE]) -> bool {
header[4] & 0x80 != 0
}
pub fn derive_part_key(object_key: [u8; 32], part_number: u32) -> [u8; 32] {
let mut mac = HmacSha256::new_from_slice(&object_key).expect("HMAC-SHA256 accepts 32-byte object keys");
mac.update(&part_number.to_le_bytes());
let mut part_key = [0u8; 32];
part_key.copy_from_slice(mac.finalize().into_bytes().as_slice());
part_key
}
fn random_stream_nonce() -> [u8; 12] {
let mut nonce = [0u8; 12];
rand::rng().fill(&mut nonce);
nonce
}
#[cfg(test)]
mod tests {
use super::*;
use hex::encode as hex_encode;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
#[tokio::test]
async fn decrypt_reader_can_start_from_non_zero_sequence_number() {
let plaintext = vec![0xAB; DARE_PAYLOAD_SIZE * 2 + 19];
let key_bytes = [0x44u8; 32];
let base_nonce = [0x66u8; 12];
let mut encrypted = Vec::new();
EncryptReader::new(Cursor::new(plaintext.clone()), key_bytes, base_nonce)
.read_to_end(&mut encrypted)
.await
.expect("encrypt plaintext");
let tail = encrypted[DARE_PACKAGE_SIZE..].to_vec();
let mut decrypted = Vec::new();
DecryptReader::new_with_sequence(Cursor::new(tail), key_bytes, base_nonce, 1)
.read_to_end(&mut decrypted)
.await
.expect("decrypt tail packages with non-zero sequence");
assert_eq!(decrypted, plaintext[DARE_PAYLOAD_SIZE..]);
}
#[tokio::test]
async fn singlepart_object_key_roundtrip_uses_header_nonce() {
let object_key = [0x51u8; 32];
let plaintext = vec![0xAC; DARE_PAYLOAD_SIZE + 17];
let mut encrypted = Vec::new();
EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key)
.read_to_end(&mut encrypted)
.await
.expect("encrypt object-key singlepart stream");
let mut decrypted = Vec::new();
DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key)
.read_to_end(&mut decrypted)
.await
.expect("decrypt object-key singlepart stream");
assert_eq!(decrypted, plaintext);
}
#[test]
fn derive_part_key_matches_minio_test_vectors() {
assert_eq!(
hex_encode(derive_part_key([0u8; 32], 0)),
"aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"
);
assert_eq!(
hex_encode(derive_part_key([0u8; 32], 1)),
"a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"
);
assert_eq!(
hex_encode(derive_part_key([0u8; 32], 10_000)),
"f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"
);
}
#[tokio::test]
async fn multipart_object_key_roundtrip_resets_sequence_per_part() {
let object_key = [0x19u8; 32];
let part_one_plaintext = vec![0xA1; DARE_PAYLOAD_SIZE + 31];
let part_two_plaintext = vec![0xB2; DARE_PAYLOAD_SIZE * 2 + 7];
let mut encrypted_one = Vec::new();
EncryptReader::new_multipart_with_object_key(Cursor::new(part_one_plaintext.clone()), object_key, 1)
.read_to_end(&mut encrypted_one)
.await
.expect("encrypt multipart part one with object key");
let mut encrypted_two = Vec::new();
EncryptReader::new_multipart_with_object_key(Cursor::new(part_two_plaintext.clone()), object_key, 2)
.read_to_end(&mut encrypted_two)
.await
.expect("encrypt multipart part two with object key");
let mut encrypted = encrypted_one.clone();
encrypted.extend_from_slice(&encrypted_two);
let mut decrypted = Vec::new();
DecryptReader::new_multipart_with_object_key(Cursor::new(encrypted), object_key, vec![1, 2])
.read_to_end(&mut decrypted)
.await
.expect("decrypt multipart object-key stream");
let mut expected = part_one_plaintext;
expected.extend_from_slice(&part_two_plaintext);
assert_eq!(decrypted, expected);
assert_ne!(encrypted_one, encrypted_two[..encrypted_one.len()]);
}
}
crates/rio-v2/src/lib.rs
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! rio_v2 selectively replaces legacy rio components while keeping the
//! remaining API surface stable for feature-gated integration.
mod compress_reader;
mod encrypt_reader;
mod s2_index;
pub use compress_reader::{CompressReader, DecompressReader};
pub use encrypt_reader::{DecryptReader, EncryptReader, derive_part_key};
pub use rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE;
pub use rustfs_rio::DynReader;
pub use rustfs_rio::EtagReader;
pub use rustfs_rio::EtagResolvable;
pub use rustfs_rio::HardLimitReader;
pub use rustfs_rio::HashReader;
pub use rustfs_rio::HashReaderDetector;
pub use rustfs_rio::HashReaderMut;
pub use rustfs_rio::Index;
pub use rustfs_rio::LimitReader;
pub use rustfs_rio::ReadStream;
pub use rustfs_rio::Reader;
pub use rustfs_rio::ReaderCapabilities;
pub use rustfs_rio::TryGetIndex;
pub use rustfs_rio::WarpReader;
pub use rustfs_rio::boxed_reader;
pub use rustfs_rio::read_checksums;
pub use rustfs_rio::resolve_etag_generic;
pub use rustfs_rio::wrap_reader;
pub use s2_index::{decode_minio_index_bytes, minio_index_storage_bytes};
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const DARE_VERSION_20: u8 = 0x20;
const DARE_HEADER_SIZE: usize = 16;
const DARE_TAG_SIZE: usize = 16;
const DARE_PAYLOAD_SIZE: usize = 64 * 1024;
const DARE_PACKAGE_SIZE: usize = DARE_HEADER_SIZE + DARE_PAYLOAD_SIZE + DARE_TAG_SIZE;
#[tokio::test]
async fn encrypt_reader_emits_dare_v2_packages() {
let plaintext = vec![0x5Au8; DARE_PAYLOAD_SIZE + 17];
let key_bytes = [0x11u8; 32];
let base_nonce = [0x22u8; 12];
let mut encrypted = Vec::new();
EncryptReader::new(Cursor::new(plaintext), key_bytes, base_nonce)
.read_to_end(&mut encrypted)
.await
.expect("encrypt plaintext into rio_v2 stream");
assert!(
encrypted.len() > DARE_PACKAGE_SIZE,
"expected at least one full DARE package and one final package"
);
assert_eq!(encrypted[0], DARE_VERSION_20, "rio_v2 encrypted streams must start with a DARE V2 header");
assert_eq!(
&encrypted[4..16],
&base_nonce,
"rio_v2 should preserve the configured nonce in the first DARE header"
);
let second_header_offset = DARE_PACKAGE_SIZE;
assert_eq!(
encrypted[second_header_offset], DARE_VERSION_20,
"rio_v2 should emit subsequent DARE V2 package headers at 64KiB boundaries"
);
assert_ne!(
encrypted[second_header_offset + 4] & 0x80,
0,
"the final DARE package must set the final flag"
);
}
}
crates/rio-v2/src/s2_index.rs
+434
View File
@@ -0,0 +1,434 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use bytes::Bytes;
use rustfs_rio::Index;
use serde::Deserialize;
use std::io;
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
const CHUNK_TYPE_INDEX: u8 = 0x99;
const SKIPPABLE_FRAME_HEADER: usize = 4;
const MAX_INDEX_ENTRIES: usize = 1 << 16;
#[derive(Debug, Deserialize)]
struct LegacyIndexJson {
total_uncompressed: i64,
total_compressed: i64,
offsets: Vec<LegacyIndexOffset>,
est_block_uncompressed: i64,
}
#[derive(Debug, Deserialize)]
struct LegacyIndexOffset {
compressed: i64,
uncompressed: i64,
}
#[derive(Debug, Clone)]
struct S2IndexInfo {
compressed_offset: i64,
uncompressed_offset: i64,
}
#[derive(Debug, Clone)]
struct S2Index {
total_uncompressed: i64,
total_compressed: i64,
est_block_uncompressed: i64,
info: Vec<S2IndexInfo>,
}
pub fn minio_index_storage_bytes(index: &Index) -> Bytes {
let decoded = legacy_index_to_s2_index(index).unwrap_or_else(|_| S2Index {
total_uncompressed: index.total_uncompressed,
total_compressed: index.total_compressed,
est_block_uncompressed: 0,
info: Vec::new(),
});
let encoded = decoded.into_full_bytes();
remove_index_headers(encoded.as_ref())
.map(Bytes::copy_from_slice)
.unwrap_or(encoded)
}
pub fn decode_minio_index_bytes(bytes: &Bytes) -> Option<Index> {
let decoded = S2Index::load(bytes.as_ref())
.or_else(|_| S2Index::load(&restore_index_headers(bytes.as_ref())))
.ok()?;
let mut index = Index::new();
for info in decoded.info {
index.add(info.compressed_offset, info.uncompressed_offset).ok()?;
}
index.total_uncompressed = decoded.total_uncompressed;
index.total_compressed = decoded.total_compressed;
Some(index)
}
fn legacy_index_to_s2_index(index: &Index) -> io::Result<S2Index> {
let json = index
.to_json()
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
let decoded: LegacyIndexJson =
serde_json::from_slice(&json).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
Ok(S2Index {
total_uncompressed: decoded.total_uncompressed,
total_compressed: decoded.total_compressed,
est_block_uncompressed: decoded.est_block_uncompressed,
info: decoded
.offsets
.into_iter()
.map(|offset| S2IndexInfo {
compressed_offset: offset.compressed,
uncompressed_offset: offset.uncompressed,
})
.collect(),
})
}
impl S2Index {
fn into_full_bytes(self) -> Bytes {
let mut out = Vec::new();
out.extend_from_slice(&[CHUNK_TYPE_INDEX, 0, 0, 0]);
out.extend_from_slice(S2_INDEX_HEADER);
write_varint(&mut out, self.total_uncompressed);
write_varint(&mut out, self.total_compressed);
write_varint(&mut out, self.est_block_uncompressed);
write_varint(&mut out, self.info.len() as i64);
let has_uncompressed = self.has_explicit_uncompressed_offsets();
out.push(u8::from(has_uncompressed));
if has_uncompressed {
for (idx, info) in self.info.iter().enumerate() {
let mut offset = info.uncompressed_offset;
if idx > 0 {
let prev = &self.info[idx - 1];
offset -= prev.uncompressed_offset + self.est_block_uncompressed;
}
write_varint(&mut out, offset);
}
}
let mut compressed_predict = self.est_block_uncompressed / 2;
for (idx, info) in self.info.iter().enumerate() {
let mut offset = info.compressed_offset;
if idx > 0 {
let prev = &self.info[idx - 1];
offset -= prev.compressed_offset + compressed_predict;
compressed_predict += offset / 2;
}
write_varint(&mut out, offset);
}
let mut total_size = [0u8; 4];
total_size.copy_from_slice(&((out.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes());
out.extend_from_slice(&total_size);
out.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = out.len() - SKIPPABLE_FRAME_HEADER;
out[1] = chunk_len as u8;
out[2] = (chunk_len >> 8) as u8;
out[3] = (chunk_len >> 16) as u8;
Bytes::from(out)
}
fn has_explicit_uncompressed_offsets(&self) -> bool {
for (idx, info) in self.info.iter().enumerate() {
if idx == 0 {
if info.uncompressed_offset != 0 {
return true;
}
continue;
}
if info.uncompressed_offset != self.info[idx - 1].uncompressed_offset + self.est_block_uncompressed {
return true;
}
}
false
}
fn load(mut bytes: &[u8]) -> io::Result<Self> {
if bytes.len() <= SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
if bytes[0] != CHUNK_TYPE_INDEX {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index chunk type"));
}
let chunk_len = (bytes[1] as usize) | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16);
bytes = &bytes[SKIPPABLE_FRAME_HEADER..];
if bytes.len() < chunk_len {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
bytes = &bytes[..chunk_len];
if !bytes.starts_with(S2_INDEX_HEADER) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index header"));
}
bytes = &bytes[S2_INDEX_HEADER.len()..];
let (total_uncompressed, used) = read_varint(bytes)?;
if total_uncompressed < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed size"));
}
bytes = &bytes[used..];
let (total_compressed, used) = read_varint(bytes)?;
bytes = &bytes[used..];
let (est_block_uncompressed, used) = read_varint(bytes)?;
if est_block_uncompressed < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid block size"));
}
bytes = &bytes[used..];
let (entries, used) = read_varint(bytes)?;
if entries < 0 || entries > MAX_INDEX_ENTRIES as i64 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid number of entries"));
}
bytes = &bytes[used..];
if bytes.is_empty() {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
let has_uncompressed = bytes[0];
if has_uncompressed & 1 != has_uncompressed {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed flag"));
}
bytes = &bytes[1..];
let mut info = vec![
S2IndexInfo {
compressed_offset: 0,
uncompressed_offset: 0,
};
entries as usize
];
for idx in 0..info.len() {
let mut uncompressed_offset = 0_i64;
if has_uncompressed != 0 {
let (value, used) = read_varint(bytes)?;
uncompressed_offset = value;
bytes = &bytes[used..];
}
if idx > 0 {
let prev = info[idx - 1].uncompressed_offset;
uncompressed_offset += prev + est_block_uncompressed;
if uncompressed_offset <= prev {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid uncompressed offset"));
}
}
if uncompressed_offset < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "negative uncompressed offset"));
}
info[idx].uncompressed_offset = uncompressed_offset;
}
let mut compressed_predict = est_block_uncompressed / 2;
for idx in 0..info.len() {
let (mut compressed_offset, used) = read_varint(bytes)?;
bytes = &bytes[used..];
if idx > 0 {
let next_compressed_predict = compressed_predict + compressed_offset / 2;
let prev = info[idx - 1].compressed_offset;
compressed_offset += prev + compressed_predict;
if compressed_offset <= prev {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressed offset"));
}
compressed_predict = next_compressed_predict;
}
if compressed_offset < 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "negative compressed offset"));
}
info[idx].compressed_offset = compressed_offset;
}
if bytes.len() < 4 + S2_INDEX_TRAILER.len() {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "buffer too small"));
}
bytes = &bytes[4..];
if !bytes.starts_with(S2_INDEX_TRAILER) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid index trailer"));
}
Ok(Self {
total_uncompressed,
total_compressed,
est_block_uncompressed,
info,
})
}
}
fn remove_index_headers(bytes: &[u8]) -> Option<&[u8]> {
let save = SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() + 4;
if bytes.len() <= save || bytes[0] != CHUNK_TYPE_INDEX {
return None;
}
let chunk_len = (bytes[1] as usize) | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16);
let bytes = &bytes[SKIPPABLE_FRAME_HEADER..];
if bytes.len() < chunk_len {
return None;
}
let bytes = &bytes[..chunk_len];
let bytes = bytes.strip_prefix(S2_INDEX_HEADER)?;
let bytes = bytes.strip_suffix(S2_INDEX_TRAILER)?;
if bytes.len() < 4 {
return None;
}
Some(&bytes[..bytes.len() - 4])
}
fn restore_index_headers(input: &[u8]) -> Vec<u8> {
if input.is_empty() {
return Vec::new();
}
let mut bytes = Vec::with_capacity(SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + input.len() + 4 + S2_INDEX_TRAILER.len());
bytes.extend_from_slice(&[CHUNK_TYPE_INDEX, 0, 0, 0]);
bytes.extend_from_slice(S2_INDEX_HEADER);
bytes.extend_from_slice(input);
bytes.extend_from_slice(&((bytes.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes());
bytes.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = bytes.len() - SKIPPABLE_FRAME_HEADER;
bytes[1] = chunk_len as u8;
bytes[2] = (chunk_len >> 8) as u8;
bytes[3] = (chunk_len >> 16) as u8;
bytes
}
fn write_varint(out: &mut Vec<u8>, value: i64) {
let mut unsigned = ((value as u64) << 1) ^ ((value >> 63) as u64);
while unsigned >= 0x80 {
out.push((unsigned as u8) | 0x80);
unsigned >>= 7;
}
out.push(unsigned as u8);
}
fn read_varint(bytes: &[u8]) -> io::Result<(i64, usize)> {
let (unsigned, used) = read_uvarint(bytes)?;
let value = ((unsigned >> 1) as i64) ^ (-((unsigned & 1) as i64));
Ok((value, used))
}
fn read_uvarint(bytes: &[u8]) -> io::Result<(u64, usize)> {
let mut value = 0_u64;
let mut shift = 0_u32;
for (idx, byte) in bytes.iter().copied().enumerate() {
if byte < 0x80 {
if idx > 9 || (idx == 9 && byte > 1) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "varint overflow"));
}
return Ok((value | ((byte as u64) << shift), idx + 1));
}
value |= ((byte & 0x7f) as u64) << shift;
shift += 7;
}
Err(io::Error::new(io::ErrorKind::UnexpectedEof, "unexpected EOF while reading varint"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn signed_varint_matches_go_binary_varint_examples() {
let cases = [
(0, vec![0x00]),
(-1, vec![0x01]),
(1, vec![0x02]),
(-2, vec![0x03]),
(64, vec![0x80, 0x01]),
(-64, vec![0x7f]),
];
for (value, expected) in cases {
let mut encoded = Vec::new();
write_varint(&mut encoded, value);
assert_eq!(encoded, expected);
assert_eq!(read_varint(&encoded).unwrap(), (value, encoded.len()));
}
}
#[test]
fn minio_storage_bytes_round_trip_through_headerless_form() {
let mut source = Index::new();
source.add(0, 0).unwrap();
source.add(1_048_576, 2_097_152).unwrap();
source.total_uncompressed = 4_194_304;
source.total_compressed = 3_000_000;
let stored = minio_index_storage_bytes(&source);
assert!(!stored.starts_with(&[CHUNK_TYPE_INDEX, 0x2a, 0x4d, 0x18]));
assert_eq!(
S2Index::load(&restore_index_headers(&stored))
.expect("restore full index")
.info
.len(),
2
);
let decoded = decode_minio_index_bytes(&stored).expect("decode headerless MinIO index");
assert_eq!(decoded.total_uncompressed, source.total_uncompressed);
assert_eq!(decoded.total_compressed, source.total_compressed);
assert_eq!(decoded.find(2_097_152).unwrap(), (1_048_576, 2_097_152));
}
#[test]
fn minio_index_allows_unknown_total_compressed_size() {
let index = S2Index {
total_uncompressed: 4_194_304,
total_compressed: -1,
est_block_uncompressed: 0,
info: vec![
S2IndexInfo {
compressed_offset: 0,
uncompressed_offset: 0,
},
S2IndexInfo {
compressed_offset: 1_048_576,
uncompressed_offset: 2_097_152,
},
],
};
let full = index.into_full_bytes();
assert_eq!(full[0], CHUNK_TYPE_INDEX);
let headerless = Bytes::copy_from_slice(remove_index_headers(full.as_ref()).expect("strip index headers"));
let restored = restore_index_headers(&headerless);
assert_eq!(restored[0], CHUNK_TYPE_INDEX);
let decoded = decode_minio_index_bytes(&headerless).expect("decode index with unknown compressed total");
assert_eq!(decoded.total_uncompressed, 4_194_304);
assert_eq!(decoded.total_compressed, -1);
assert_eq!(decoded.find(2_097_152).unwrap(), (1_048_576, 2_097_152));
}
}
crates/rio-v2/tests/README.md
+55
View File
@@ -0,0 +1,55 @@
# rio-v2 Fixture Tests
This directory holds interoperability tests for `rustfs-rio-v2`.
## Generated MinIO fixtures
The integration test `minio_generated_fixtures.rs` reads raw MinIO backend data generated by
`crates/rio-v2/tests/minio_fixture_lab/lab.py`.
Default fixture root:
```text
crates/rio-v2/tests/fixtures/minio-generated
```
The generated MinIO fixture data is intentionally not checked in and these tests are
ignored by default, so they are not part of automated CI runs.
To generate or expand fixtures locally, use:
```powershell
uv run python .\minio_fixture_lab\lab.py capture-matrix `
--root .\rustfs\crates\rio-v2\tests\fixtures\minio-generated `
--minio-binary .\rustfs\tmp\minio.darwin-arm64.RELEASE.2025-09-07T16-13-09Z `
--endpoint https://127.0.0.1:19000 `
--disk-count 1
```
Or point the tests at another generated root:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-smoke'
cargo +1.95.0 test -p rustfs-rio-v2 --test minio_generated_fixtures -- --ignored
```
## Scope
The current fixture test covers the full default capture matrix:
- `sse-s3-singlepart-64k`
- `sse-s3-multipart-8m`
- `sse-kms-singlepart-64k`
- `sse-kms-multipart-8m`
- `sse-c-singlepart-64k`
- `sse-c-multipart-8m`
It checks that:
- raw `xl.meta` can be parsed through `rustfs-filemeta`
- singlepart and multipart fixture structures are recognized
- SSE-S3, SSE-KMS, and SSE-C metadata markers survive capture
- KMS key ids are derived from each fixture's own `manifest.json`, so local static-KMS runs are not tied to one hard-coded key name
- SSE-C `HEAD` responses round-trip the expected customer algorithm and customer-key MD5
These tests do not yet validate full plaintext reconstruction from MinIO-written encrypted data.
crates/rio-v2/tests/fixtures/.gitignore
+2
View File
@@ -0,0 +1,2 @@
*
!.gitignore
crates/rio-v2/tests/minio_fixture_lab/README.md
+150
View File
@@ -0,0 +1,150 @@
# MinIO Fixture Lab
This lab captures real MinIO backend artifacts into a repeatable local layout for RustFS compatibility tests.
It now supports two workflows:
- manual capture from an already exported backend tree
- automated capture from a disposable local MinIO run
## Scope
Use the manual path after you already have:
- a running MinIO instance
- one or more uploaded objects you want to preserve as fixtures
- the backend object tree or object-version directory you want to export
Use the automated path when you want the lab to:
- locate a local `minio` binary
- start a disposable MinIO instance
- upload a predefined SSE fixture case
- export the generated backend tree into the lab layout
## Layout
The default root is `artifacts/minio-fixture-lab`, which is already ignored by the repository.
Each case is stored under:
```text
artifacts/minio-fixture-lab/
cases/
<case-id>/
backend/
request.json
head.json
plaintext.sha256
manifest.json
```
`manifest.json` is the source of truth for the captured case.
## Commands
Initialize the lab root:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py init
```
Capture one case from an existing MinIO backend tree:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py add-case `
--case-id sse-kms-singlepart-64k `
--bucket demo `
--object dir/object.bin `
--source-tree D:\minio-data-export\case-tree `
--head-json D:\minio-data-export\head.json `
--request-json D:\minio-data-export\request.json `
--plaintext-sha256 D:\minio-data-export\plaintext.sha256
```
Capture the default automated matrix:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix `
--root D:\Github\rustfs\artifacts\minio-fixture-lab `
--minio-binary D:\go\bin\minio.exe `
--endpoint https://127.0.0.1:19000
```
Capture only one automated case:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix `
--root D:\Github\rustfs\artifacts\minio-fixture-lab `
--minio-binary D:\Github\rustfs\tmp\minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe `
--endpoint https://127.0.0.1:19000 `
--case-id sse-s3-singlepart-64k
```
The automated default matrix is intentionally small:
- `sse-s3-singlepart-64k`
- `sse-kms-singlepart-64k`
- `sse-c-singlepart-64k`
- `sse-s3-multipart-8m`
- `sse-kms-multipart-8m`
- `sse-c-multipart-8m`
`64 KiB multipart` is intentionally excluded because S3 multipart semantics require a larger non-final part size.
## Automated Runner Prerequisites
The automated runner expects:
- either `minio` in `PATH`, the bundled `D:\Github\rustfs\tmp\minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe`, `--minio-binary`, or `--minio-root` pointing at a directory containing `minio.exe`
- a free local endpoint port
When the selected matrix includes SSE-C cases, use an `https://` endpoint. The lab will mint a short-lived local self-signed certificate and use its built-in SigV4 S3 client with certificate verification disabled for the disposable local MinIO run.
The runner provisions backend directories under `--work-root` and exports the resulting backend tree into the lab.
For local static-KMS runs, pass `--kms-secret-key` or set
`MINIO_FIXTURE_LAB_KMS_SECRET_KEY` using MinIO's
`<key-id>:<base64-32byte-key>` format. The runner derives the SSE-KMS request
key id from that configured key name automatically.
On some Windows MinIO builds, a multi-disk backend may not come online when all disk directories live on the same volume. If you only want a local smoke run of the upload/export pipeline, try:
```powershell
uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capture-matrix `
--root D:\Github\rustfs\artifacts\minio-fixture-lab `
--minio-binary D:\go\bin\minio.exe `
--endpoint https://127.0.0.1:19000 `
--disk-count 1 `
--case-id sse-s3-singlepart-64k
```
That is a runner smoke path only. Real compatibility fixtures should still prefer the intended multi-disk backend layout when the local environment can support it.
## Capture Guidance
For each case, preserve these inputs when possible:
- the exact backend tree that contains `xl.meta` and part files
- the request shape used to create the object
- the API metadata returned by `HEAD Object`
- the plaintext digest used for byte-for-byte read verification
Recommended early matrix:
- singlepart SSE-S3
- singlepart SSE-KMS
- multipart SSE-S3
- multipart SSE-KMS
- compressed + encrypted
- range-sensitive sizes around `64 KiB` and `8 MiB`
## Next Stage
The current runner covers launch, upload, HEAD capture, and backend export.
The next iteration should focus on:
- proving the multi-disk backend path in the target local environment
- adding compressed fixtures to the automated matrix
- tightening exported tree selection if a narrower object-level slice becomes practical
crates/rio-v2/tests/minio_fixture_lab/lab.py
+963
View File
@@ -0,0 +1,963 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import base64
import hashlib
import hmac
import ipaddress
import json
import os
import shutil
import ssl
import subprocess
import sys
import time
import urllib.parse
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
DEFAULT_ROOT = Path("artifacts/minio-fixture-lab")
DEFAULT_WORK_ROOT = DEFAULT_ROOT / "_runner"
DEFAULT_MINIO_BINARY = (
Path(__file__).resolve().parents[4]
/ "tmp"
/ "minio.darwin-arm64.RELEASE.2025-09-07T16-13-09Z"
)
DEFAULT_KMS_KEY_ID = "minio-default-key"
DEFAULT_KMS_SECRET_KEY = (
"minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g="
)
LAB_KMS_SECRET_KEY_ENV = "MINIO_FIXTURE_LAB_KMS_SECRET_KEY"
DEFAULT_ENDPOINT = "http://127.0.0.1:9000"
DEFAULT_DISK_COUNT = 4
DEFAULT_ACCESS_KEY = "minioadmin"
DEFAULT_SECRET_KEY = "minioadmin"
DEFAULT_REGION = "us-east-1"
MULTIPART_CHUNK_SIZE = 5 * 1024 * 1024
DEFAULT_BUCKET = "demo"
DEFAULT_OBJECT = "dir/object.bin"
DEFAULT_SSE_C_KEY_BYTES = bytes(range(32))
DEFAULT_SSE_C_KEY_B64 = base64.b64encode(DEFAULT_SSE_C_KEY_BYTES).decode("ascii")
DEFAULT_SSE_C_KEY_MD5_B64 = base64.b64encode(hashlib.md5(DEFAULT_SSE_C_KEY_BYTES).digest()).decode("ascii")
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def ensure_dir(path: Path) -> Path:
path.mkdir(parents=True, exist_ok=True)
return path
def copy_optional_file(source: Path | None, destination: Path) -> str | None:
if source is None:
return None
if not source.is_file():
raise FileNotFoundError(f"expected file: {source}")
shutil.copy2(source, destination)
return destination.name
def copy_tree(source: Path, destination: Path) -> list[str]:
if not source.exists():
raise FileNotFoundError(f"source tree does not exist: {source}")
if source.is_file():
raise ValueError(f"source tree must be a directory: {source}")
if destination.exists():
shutil.rmtree(destination)
shutil.copytree(source, destination)
return sorted(
str(path.relative_to(destination)).replace("\\", "/")
for path in destination.rglob("*")
if path.is_file()
)
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def read_optional_json(path: Path | None) -> dict[str, Any] | None:
if path is None:
return None
return json.loads(path.read_text(encoding="utf-8"))
def format_context_header(context: dict[str, str] | None) -> str | None:
if not context:
return None
json_bytes = json.dumps(context, sort_keys=True, separators=(",", ":")).encode("utf-8")
return base64.b64encode(json_bytes).decode("ascii")
def format_size_label(size_bytes: int) -> str:
if size_bytes == 64 * 1024:
return "64k"
if size_bytes == 8 * 1024 * 1024:
return "8m"
return f"{size_bytes}b"
@dataclass
class LabPaths:
root: Path
cases: Path
@dataclass
class MinioLauncher:
kind: str
command: list[str]
workdir: Path
@dataclass
class FixtureCase:
case_id: str
bucket: str
object_name: str
encryption: str
size_bytes: int
multipart: bool
kms_key_id: str | None = None
kms_context: dict[str, str] | None = None
@dataclass(frozen=True)
class KmsSecretKeyConfig:
secret_key: str
key_id: str
def resolve_lab_paths(root: Path) -> LabPaths:
root = root.resolve()
return LabPaths(root=root, cases=root / "cases")
def resolve_existing_command(name: str) -> Path:
resolved = shutil.which(name)
if resolved is None:
raise FileNotFoundError(f"required command not found in PATH: {name}")
return Path(resolved).resolve()
def discover_minio_launcher(
explicit_binary: Path | None, minio_root: Path | None
) -> MinioLauncher:
if explicit_binary is not None:
binary = explicit_binary.resolve()
if not binary.is_file():
raise FileNotFoundError(f"minio binary not found: {binary}")
return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent)
resolved_root = minio_root.resolve() if minio_root is not None else None
if resolved_root is not None:
candidate_binary = resolved_root / "minio.exe"
if candidate_binary.is_file():
return MinioLauncher(
kind="binary",
command=[str(candidate_binary.resolve())],
workdir=candidate_binary.resolve().parent,
)
if DEFAULT_MINIO_BINARY.is_file():
binary = DEFAULT_MINIO_BINARY.resolve()
return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent)
path_binary = shutil.which("minio")
if path_binary is not None:
binary = Path(path_binary).resolve()
return MinioLauncher(kind="binary", command=[str(binary)], workdir=binary.parent)
raise FileNotFoundError(
"unable to locate MinIO. Provide --minio-binary, point --minio-root at a directory "
"containing minio.exe, or ensure minio is in PATH."
)
def parse_kms_secret_key(secret_key: str) -> KmsSecretKeyConfig:
key_id, separator, raw_key = secret_key.partition(":")
if not separator or not key_id or not raw_key:
raise ValueError(
"kms secret key must be in the form '<key-id>:<base64-32byte-key>'"
)
return KmsSecretKeyConfig(secret_key=secret_key, key_id=key_id)
def resolve_kms_secret_key(explicit_secret_key: str | None) -> KmsSecretKeyConfig:
secret_key = (
explicit_secret_key
or os.environ.get(LAB_KMS_SECRET_KEY_ENV)
or os.environ.get("MINIO_KMS_SECRET_KEY")
or DEFAULT_KMS_SECRET_KEY
)
return parse_kms_secret_key(secret_key)
def build_default_cases(kms_key_id: str = DEFAULT_KMS_KEY_ID) -> list[FixtureCase]:
kms_context = {"project": "rio-v2", "stage": "fixture"}
cases = []
for encryption in ("SSE-S3", "SSE-KMS", "SSE-C"):
for multipart, size_bytes in ((False, 64 * 1024), (True, 8 * 1024 * 1024)):
shape = "multipart" if multipart else "singlepart"
case = FixtureCase(
case_id=f"{encryption.lower()}-{shape}-{format_size_label(size_bytes)}",
bucket=DEFAULT_BUCKET,
object_name=DEFAULT_OBJECT,
encryption=encryption,
size_bytes=size_bytes,
multipart=multipart,
kms_key_id=kms_key_id if encryption == "SSE-KMS" else None,
kms_context=kms_context if encryption == "SSE-KMS" else None,
)
cases.append(case)
return cases
def build_request_record(case: FixtureCase) -> dict[str, Any]:
headers: dict[str, str] = {}
if case.encryption == "SSE-S3":
headers["x-amz-server-side-encryption"] = "AES256"
elif case.encryption == "SSE-KMS":
headers["x-amz-server-side-encryption"] = "aws:kms"
if case.kms_key_id is not None:
headers["x-amz-server-side-encryption-aws-kms-key-id"] = case.kms_key_id
context_header = format_context_header(case.kms_context)
if context_header is not None:
headers["x-amz-server-side-encryption-context"] = context_header
elif case.encryption == "SSE-C":
headers["x-amz-server-side-encryption-customer-algorithm"] = "AES256"
headers["x-amz-server-side-encryption-customer-key"] = DEFAULT_SSE_C_KEY_B64
headers["x-amz-server-side-encryption-customer-key-md5"] = DEFAULT_SSE_C_KEY_MD5_B64
else:
raise ValueError(f"unsupported encryption mode: {case.encryption}")
record: dict[str, Any] = {
"case_id": case.case_id,
"bucket": case.bucket,
"object": case.object_name,
"encryption": case.encryption,
"multipart": case.multipart,
"size_bytes": case.size_bytes,
"headers": headers,
}
if case.multipart:
record["multipart_chunk_size_bytes"] = MULTIPART_CHUNK_SIZE
return record
def cmd_init(args: argparse.Namespace) -> int:
paths = resolve_lab_paths(args.root)
ensure_layout(paths)
print(str(paths.root))
return 0
def ensure_layout(paths: LabPaths) -> None:
ensure_dir(paths.root)
ensure_dir(paths.cases)
write_json(
paths.root / "layout.json",
{
"schema_version": 1,
"created_at_utc": utc_now(),
"root": ".",
"description": "Captured MinIO fixture cases for RustFS compatibility validation.",
},
)
def store_case_artifacts(
*,
paths: LabPaths,
case_id: str,
bucket: str,
object_name: str,
source_tree: Path,
version_id: str | None,
request_payload: dict[str, Any] | None,
head_payload: dict[str, Any] | None,
plaintext_sha256: str | None,
notes: str | None,
capture_payload: dict[str, Any] | None = None,
) -> Path:
ensure_dir(paths.cases)
case_dir = paths.cases / case_id
backend_dir = case_dir / "backend"
if case_dir.exists():
shutil.rmtree(case_dir)
ensure_dir(case_dir)
exported_files = copy_tree(source_tree.resolve(), backend_dir)
request_file = None
if request_payload is not None:
write_json(case_dir / "request.json", request_payload)
request_file = "request.json"
head_file = None
if head_payload is not None:
write_json(case_dir / "head.json", head_payload)
head_file = "head.json"
plaintext_file = None
if plaintext_sha256 is not None:
(case_dir / "plaintext.sha256").write_text(plaintext_sha256 + "\n", encoding="utf-8")
plaintext_file = "plaintext.sha256"
manifest = {
"schema_version": 1,
"case_id": case_id,
"created_at_utc": utc_now(),
"bucket": bucket,
"object": object_name,
"version_id": version_id,
"source_tree": str(source_tree).replace("\\", "/"),
"notes": notes,
"artifacts": {
"backend_dir": "backend",
"request_json": request_file,
"head_json": head_file,
"plaintext_sha256": plaintext_file,
},
"backend_files": exported_files,
}
if capture_payload is not None:
manifest["capture"] = capture_payload
write_json(case_dir / "manifest.json", manifest)
return case_dir
def cmd_add_case(args: argparse.Namespace) -> int:
paths = resolve_lab_paths(args.root)
case_dir = store_case_artifacts(
paths=paths,
case_id=args.case_id,
bucket=args.bucket,
object_name=args.object,
source_tree=args.source_tree,
version_id=args.version_id,
request_payload=read_optional_json(args.request_json.resolve() if args.request_json else None),
head_payload=read_optional_json(args.head_json.resolve() if args.head_json else None),
plaintext_sha256=(
args.plaintext_sha256.resolve().read_text(encoding="utf-8").strip()
if args.plaintext_sha256
else None
),
notes=args.notes,
)
print(str(case_dir))
return 0
def parse_endpoint(endpoint: str) -> tuple[str, int]:
parsed = urlparse(endpoint)
if parsed.scheme not in {"http", "https"}:
raise ValueError(f"unsupported endpoint scheme: {endpoint}")
if parsed.hostname is None or parsed.port is None:
raise ValueError(f"endpoint must include host and port: {endpoint}")
return parsed.hostname, parsed.port
def is_https_endpoint(endpoint: str) -> bool:
return urlparse(endpoint).scheme == "https"
def ensure_local_tls_certificates(certs_dir: Path, hostname: str) -> None:
public_crt = certs_dir / "public.crt"
private_key = certs_dir / "private.key"
if public_crt.is_file() and private_key.is_file():
return
ensure_dir(certs_dir)
openssl = resolve_existing_command("openssl")
san_entries = ["DNS:localhost", "IP:127.0.0.1"]
try:
ipaddress.ip_address(hostname)
san_entries.append(f"IP:{hostname}")
except ValueError:
san_entries.append(f"DNS:{hostname}")
config_path = certs_dir / "openssl.cnf"
config_path.write_text(
"\n".join(
[
"[req]",
"distinguished_name = req_distinguished_name",
"x509_extensions = v3_req",
"prompt = no",
"[req_distinguished_name]",
f"CN = {hostname}",
"[v3_req]",
f"subjectAltName = {','.join(san_entries)}",
"",
]
),
encoding="utf-8",
)
subprocess.run(
[
str(openssl),
"req",
"-x509",
"-newkey",
"rsa:2048",
"-nodes",
"-days",
"30",
"-keyout",
str(private_key),
"-out",
str(public_crt),
"-config",
str(config_path),
"-extensions",
"v3_req",
],
check=True,
capture_output=True,
text=True,
)
def build_server_command(
launcher: MinioLauncher, endpoint: str, disk_paths: list[Path], certs_dir: Path | None = None
) -> list[str]:
host, port = parse_endpoint(endpoint)
normalized_disk_paths = [str(path.resolve()).replace("\\", "/") for path in disk_paths]
command = launcher.command + [
"server",
"--address",
f"{host}:{port}",
"--console-address",
f"{host}:{port + 1}",
]
if certs_dir is not None:
command += ["--certs-dir", str(certs_dir.resolve())]
command += normalized_disk_paths
return command
def create_disk_paths(workdir: Path, disk_count: int) -> list[Path]:
disks = []
for index in range(1, disk_count + 1):
disk = ensure_dir(workdir / "backend" / f"disk{index}")
disks.append(disk)
return disks
def build_payload_file(path: Path, size_bytes: int) -> str:
hasher = hashlib.sha256()
remaining = size_bytes
pattern = bytes(range(251))
with path.open("wb") as handle:
while remaining > 0:
chunk = pattern[: min(len(pattern), remaining)]
handle.write(chunk)
hasher.update(chunk)
remaining -= len(chunk)
return hasher.hexdigest()
def minio_env(kms_secret_key: KmsSecretKeyConfig) -> dict[str, str]:
env = dict(os.environ)
env["CI"] = "on"
env["MINIO_CI_CD"] = "1"
env["MINIO_ROOT_USER"] = "minioadmin"
env["MINIO_ROOT_PASSWORD"] = "minioadmin"
env["MINIO_KMS_SECRET_KEY"] = kms_secret_key.secret_key
env["MINIO_UPDATE"] = "off"
return env
class S3Client:
def __init__(
self,
endpoint: str,
access_key: str = DEFAULT_ACCESS_KEY,
secret_key: str = DEFAULT_SECRET_KEY,
region: str = DEFAULT_REGION,
) -> None:
self.endpoint = endpoint.rstrip("/")
self.access_key = access_key
self.secret_key = secret_key
self.region = region
self.parsed_endpoint = urlparse(self.endpoint)
self.ssl_context = ssl._create_unverified_context() if is_https_endpoint(endpoint) else None
def request(
self,
method: str,
bucket: str | None = None,
key: str | None = None,
*,
query: dict[str, str] | None = None,
headers: dict[str, str] | None = None,
body: bytes = b"",
) -> tuple[int, dict[str, str], bytes]:
path = "/"
if bucket is not None:
path += urllib.parse.quote(bucket, safe="")
if key is not None:
path += "/" + urllib.parse.quote(key, safe="/")
query = query or {}
headers = dict(headers or {})
signed_headers = self.sign_headers(method, path, query, headers, body)
url = self.endpoint + path
if query:
url += "?" + urllib.parse.urlencode(sorted(query.items()), quote_via=urllib.parse.quote)
request = urllib.request.Request(url, data=body if method != "HEAD" else None, method=method)
for name, value in signed_headers.items():
request.add_header(name, value)
try:
with urllib.request.urlopen(request, timeout=30, context=self.ssl_context) as response:
return response.status, {k.lower(): v for k, v in response.headers.items()}, response.read()
except urllib.error.HTTPError as err:
payload = err.read()
raise RuntimeError(
f"S3 request failed: {method} {url}\nstatus: {err.code}\nbody:\n{payload.decode('utf-8', errors='replace')}"
) from err
def sign_headers(
self,
method: str,
canonical_uri: str,
query: dict[str, str],
headers: dict[str, str],
body: bytes,
) -> dict[str, str]:
now = datetime.now(timezone.utc)
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
date_stamp = now.strftime("%Y%m%d")
payload_hash = hashlib.sha256(body).hexdigest()
host = self.parsed_endpoint.netloc
signed_headers = {k.lower(): v.strip() for k, v in headers.items()}
signed_headers["host"] = host
signed_headers["x-amz-content-sha256"] = payload_hash
signed_headers["x-amz-date"] = amz_date
canonical_query = "&".join(
f"{urllib.parse.quote(k, safe='-_.~')}={urllib.parse.quote(v, safe='-_.~')}"
for k, v in sorted(query.items())
)
canonical_headers = "".join(f"{k}:{signed_headers[k]}\n" for k in sorted(signed_headers))
signed_header_names = ";".join(sorted(signed_headers))
canonical_request = "\n".join(
[
method,
canonical_uri,
canonical_query,
canonical_headers,
signed_header_names,
payload_hash,
]
)
scope = f"{date_stamp}/{self.region}/s3/aws4_request"
string_to_sign = "\n".join(
[
"AWS4-HMAC-SHA256",
amz_date,
scope,
hashlib.sha256(canonical_request.encode("utf-8")).hexdigest(),
]
)
signature = hmac.new(
self.signing_key(date_stamp),
string_to_sign.encode("utf-8"),
hashlib.sha256,
).hexdigest()
signed_headers["authorization"] = (
f"AWS4-HMAC-SHA256 Credential={self.access_key}/{scope}, "
f"SignedHeaders={signed_header_names}, Signature={signature}"
)
return signed_headers
def signing_key(self, date_stamp: str) -> bytes:
key = ("AWS4" + self.secret_key).encode("utf-8")
for value in (date_stamp, self.region, "s3", "aws4_request"):
key = hmac.new(key, value.encode("utf-8"), hashlib.sha256).digest()
return key
def list_buckets(self) -> None:
self.request("GET")
def create_bucket(self, bucket: str) -> None:
self.request("PUT", bucket=bucket)
def put_object(
self,
bucket: str,
key: str,
body: bytes,
headers: dict[str, str],
) -> None:
self.request("PUT", bucket=bucket, key=key, headers=headers, body=body)
def create_multipart_upload(self, bucket: str, key: str, headers: dict[str, str]) -> str:
_, _, body = self.request("POST", bucket=bucket, key=key, query={"uploads": ""}, headers=headers)
root = ET.fromstring(body)
upload_id = root.findtext("{*}UploadId")
if upload_id is None:
raise RuntimeError(f"missing UploadId in create multipart response: {body!r}")
return upload_id
def upload_part(
self,
bucket: str,
key: str,
upload_id: str,
part_number: int,
body: bytes,
headers: dict[str, str],
) -> str:
_, response_headers, _ = self.request(
"PUT",
bucket=bucket,
key=key,
query={"partNumber": str(part_number), "uploadId": upload_id},
headers=headers,
body=body,
)
etag = response_headers.get("etag")
if etag is None:
raise RuntimeError("missing ETag in upload part response")
return etag
def complete_multipart_upload(
self,
bucket: str,
key: str,
upload_id: str,
parts: list[dict[str, Any]],
) -> None:
body = build_complete_multipart_xml(parts)
self.request(
"POST",
bucket=bucket,
key=key,
query={"uploadId": upload_id},
headers={"content-type": "application/xml"},
body=body,
)
def head_object(self, bucket: str, key: str, headers: dict[str, str]) -> dict[str, Any]:
_, response_headers, _ = self.request("HEAD", bucket=bucket, key=key, headers=headers)
return {
"ContentLength": int(response_headers.get("content-length", "0")),
"ServerSideEncryption": response_headers.get("x-amz-server-side-encryption"),
"SSECustomerAlgorithm": response_headers.get("x-amz-server-side-encryption-customer-algorithm"),
"SSECustomerKeyMD5": response_headers.get("x-amz-server-side-encryption-customer-key-md5"),
"SSEKMSKeyId": response_headers.get("x-amz-server-side-encryption-aws-kms-key-id"),
"VersionId": response_headers.get("x-amz-version-id"),
}
def wait_for_minio(endpoint: str, process: subprocess.Popen[str], timeout_seconds: int) -> None:
deadline = time.time() + timeout_seconds
health_url = endpoint.rstrip("/") + "/minio/health/live"
ssl_context = ssl._create_unverified_context() if is_https_endpoint(endpoint) else None
while time.time() < deadline:
if process.poll() is not None:
raise RuntimeError("MinIO exited before becoming healthy")
try:
with urllib.request.urlopen(health_url, timeout=2, context=ssl_context) as response:
if response.status == 200:
return
except (urllib.error.URLError, TimeoutError):
time.sleep(1)
raise TimeoutError(f"timed out waiting for MinIO health endpoint: {health_url}")
def wait_for_s3_ready(client: S3Client, timeout_seconds: int) -> None:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
try:
client.list_buckets()
return
except RuntimeError:
time.sleep(1)
raise TimeoutError(f"timed out waiting for S3 API readiness: {client.endpoint}")
def create_bucket(client: S3Client, case: FixtureCase) -> None:
client.create_bucket(case.bucket)
def build_complete_multipart_xml(parts: list[dict[str, Any]]) -> bytes:
root = ET.Element("CompleteMultipartUpload")
for part in parts:
part_element = ET.SubElement(root, "Part")
ET.SubElement(part_element, "PartNumber").text = str(part["PartNumber"])
ET.SubElement(part_element, "ETag").text = part["ETag"]
return ET.tostring(root, encoding="utf-8", xml_declaration=True)
def upload_case(client: S3Client, case: FixtureCase, payload_file: Path) -> dict[str, Any]:
request_record = build_request_record(case)
headers = request_record["headers"]
headers["content-type"] = "binary/octet-stream"
if not case.multipart:
client.put_object(case.bucket, case.object_name, payload_file.read_bytes(), headers)
return request_record
upload_id = client.create_multipart_upload(case.bucket, case.object_name, headers)
parts = []
with payload_file.open("rb") as handle:
part_number = 1
while True:
chunk = handle.read(MULTIPART_CHUNK_SIZE)
if not chunk:
break
part_file = payload_file.parent / f"part-{part_number}.bin"
part_file.write_bytes(chunk)
part_headers = headers if case.encryption == "SSE-C" else {}
etag = client.upload_part(
case.bucket,
case.object_name,
upload_id,
part_number,
chunk,
part_headers,
)
parts.append({"ETag": etag, "PartNumber": part_number})
part_number += 1
client.complete_multipart_upload(case.bucket, case.object_name, upload_id, parts)
return request_record
def head_case(client: S3Client, case: FixtureCase) -> dict[str, Any]:
headers = build_request_record(case)["headers"] if case.encryption == "SSE-C" else {}
return client.head_object(case.bucket, case.object_name, headers)
def stop_process(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
process.terminate()
try:
process.wait(timeout=15)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=15)
def capture_case(
*,
paths: LabPaths,
launcher: MinioLauncher,
endpoint: str,
case: FixtureCase,
work_root: Path,
disk_count: int,
timeout_seconds: int,
preserve_workdir: bool,
kms_secret_key: KmsSecretKeyConfig,
) -> Path:
case_workdir = work_root / case.case_id
if case_workdir.exists():
shutil.rmtree(case_workdir)
ensure_dir(case_workdir)
payload_file = case_workdir / "payload.bin"
plaintext_sha256 = build_payload_file(payload_file, case.size_bytes)
disk_paths = create_disk_paths(case_workdir, disk_count)
certs_dir = None
if is_https_endpoint(endpoint):
host, _ = parse_endpoint(endpoint)
certs_dir = case_workdir / "certs"
ensure_local_tls_certificates(certs_dir, host)
server_command = build_server_command(launcher, endpoint, disk_paths, certs_dir)
server_log_path = case_workdir / "server.log"
with server_log_path.open("w", encoding="utf-8") as server_log:
process = subprocess.Popen(
server_command,
cwd=str(launcher.workdir),
env=minio_env(kms_secret_key),
stdout=server_log,
stderr=subprocess.STDOUT,
text=True,
)
try:
client = S3Client(endpoint)
wait_for_minio(endpoint, process, timeout_seconds)
wait_for_s3_ready(client, timeout_seconds)
create_bucket(client, case)
request_payload = upload_case(client, case, payload_file)
head_payload = head_case(client, case)
finally:
stop_process(process)
case_dir = store_case_artifacts(
paths=paths,
case_id=case.case_id,
bucket=case.bucket,
object_name=case.object_name,
source_tree=case_workdir / "backend",
version_id=head_payload.get("VersionId") if "head_payload" in locals() else None,
request_payload=request_payload if "request_payload" in locals() else None,
head_payload=head_payload if "head_payload" in locals() else None,
plaintext_sha256=plaintext_sha256,
notes="Captured automatically from a local disposable MinIO run.",
capture_payload={
"endpoint": endpoint,
"launcher_kind": launcher.kind,
"disk_count": disk_count,
"kms_key_id": kms_secret_key.key_id,
},
)
if not preserve_workdir:
shutil.rmtree(case_workdir)
return case_dir
def select_cases(case_ids: list[str] | None) -> list[FixtureCase]:
cases = build_default_cases()
if not case_ids:
return cases
selected = [case for case in cases if case.case_id in set(case_ids)]
if len(selected) != len(set(case_ids)):
known = ", ".join(case.case_id for case in cases)
missing = sorted(set(case_ids) - {case.case_id for case in selected})
raise ValueError(f"unknown case id(s): {', '.join(missing)}; known: {known}")
return selected
def cmd_capture_matrix(args: argparse.Namespace) -> int:
paths = resolve_lab_paths(args.root)
ensure_layout(paths)
ensure_dir(args.work_root)
launcher = discover_minio_launcher(args.minio_binary, args.minio_root)
kms_secret_key = resolve_kms_secret_key(args.kms_secret_key)
cases = build_default_cases(kms_key_id=kms_secret_key.key_id)
if args.case_id:
selected_ids = set(args.case_id)
cases = [case for case in cases if case.case_id in selected_ids]
if len(cases) != len(selected_ids):
known = ", ".join(case.case_id for case in build_default_cases())
missing = sorted(selected_ids - {case.case_id for case in cases})
raise ValueError(f"unknown case id(s): {', '.join(missing)}; known: {known}")
for case in cases:
case_dir = capture_case(
paths=paths,
launcher=launcher,
endpoint=args.endpoint,
case=case,
work_root=args.work_root,
disk_count=args.disk_count,
timeout_seconds=args.timeout_seconds,
preserve_workdir=args.preserve_workdir,
kms_secret_key=kms_secret_key,
)
print(f"{case.case_id}: {case_dir}")
return 0
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Capture real MinIO backend fixtures into a stable local layout."
)
parser.set_defaults(func=None)
subparsers = parser.add_subparsers(dest="command")
init_parser = subparsers.add_parser("init", help="initialize the fixture lab root")
init_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory")
init_parser.set_defaults(func=cmd_init)
add_case_parser = subparsers.add_parser("add-case", help="capture one fixture case")
add_case_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory")
add_case_parser.add_argument("--case-id", required=True, help="stable fixture case identifier")
add_case_parser.add_argument("--bucket", required=True, help="bucket name")
add_case_parser.add_argument("--object", required=True, help="object key")
add_case_parser.add_argument("--version-id", help="object version id")
add_case_parser.add_argument("--source-tree", type=Path, required=True, help="backend object tree to capture")
add_case_parser.add_argument("--request-json", type=Path, help="request metadata JSON")
add_case_parser.add_argument("--head-json", type=Path, help="HEAD object metadata JSON")
add_case_parser.add_argument("--plaintext-sha256", type=Path, help="plaintext sha256 file")
add_case_parser.add_argument("--notes", help="free-form notes")
add_case_parser.set_defaults(func=cmd_add_case)
capture_parser = subparsers.add_parser(
"capture-matrix",
help="run a disposable local MinIO instance and capture the default fixture matrix",
)
capture_parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="fixture lab root directory")
capture_parser.add_argument(
"--work-root",
type=Path,
default=DEFAULT_WORK_ROOT,
help="temporary runner workspace",
)
capture_parser.add_argument(
"--minio-binary",
type=Path,
help="explicit path to a minio binary",
)
capture_parser.add_argument(
"--minio-root",
type=Path,
help="optional directory containing minio.exe",
)
capture_parser.add_argument(
"--endpoint",
default=DEFAULT_ENDPOINT,
help="local MinIO endpoint used for the disposable run",
)
capture_parser.add_argument(
"--kms-secret-key",
help=(
"static MinIO KMS secret key in the form "
"'<key-id>:<base64-32byte-key>'; overrides "
f"{LAB_KMS_SECRET_KEY_ENV}, MINIO_KMS_SECRET_KEY, and the built-in default"
),
)
capture_parser.add_argument(
"--disk-count",
type=int,
default=DEFAULT_DISK_COUNT,
help="number of backend disks to provision per case",
)
capture_parser.add_argument(
"--timeout-seconds",
type=int,
default=60,
help="startup timeout for MinIO health checks",
)
capture_parser.add_argument(
"--preserve-workdir",
action="store_true",
help="keep per-case runner work directories after capture",
)
capture_parser.add_argument(
"--case-id",
action="append",
help="capture only specific default case ids",
)
capture_parser.set_defaults(func=cmd_capture_matrix)
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
if args.func is None:
parser.print_help()
return 2
return args.func(args)
if __name__ == "__main__":
raise SystemExit(main())
crates/rio-v2/tests/minio_fixture_lab/test_lab.py
+207
View File
@@ -0,0 +1,207 @@
from __future__ import annotations
import importlib.util
import base64
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
def load_lab_module():
module_path = Path(__file__).with_name("lab.py")
spec = importlib.util.spec_from_file_location("minio_fixture_lab_lab", module_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
lab = load_lab_module()
class DiscoverMinioLauncherTests(unittest.TestCase):
def test_prefers_explicit_binary(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary = Path(temp_dir) / "minio.exe"
binary.write_text("", encoding="utf-8")
launcher = lab.discover_minio_launcher(explicit_binary=binary, minio_root=None)
self.assertEqual(launcher.command, [str(binary.resolve())])
self.assertEqual(launcher.workdir, binary.resolve().parent)
self.assertEqual(launcher.kind, "binary")
def test_defaults_to_bundled_binary_when_available(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary = Path(temp_dir) / "minio.windows-amd64.RELEASE.2025-09-07T16-13-09Z.exe"
binary.write_text("", encoding="utf-8")
original_default_binary = lab.DEFAULT_MINIO_BINARY
try:
lab.DEFAULT_MINIO_BINARY = binary
launcher = lab.discover_minio_launcher(explicit_binary=None, minio_root=None)
finally:
lab.DEFAULT_MINIO_BINARY = original_default_binary
self.assertEqual(launcher.command, [str(binary.resolve())])
self.assertEqual(launcher.workdir, binary.resolve().parent)
self.assertEqual(launcher.kind, "binary")
def test_rejects_source_checkout_without_binary(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
source_root = Path(temp_dir) / "minio-master"
source_root.mkdir()
(source_root / "main.go").write_text("package main\n", encoding="utf-8")
original_default_binary = lab.DEFAULT_MINIO_BINARY
try:
lab.DEFAULT_MINIO_BINARY = Path(temp_dir) / "missing-bundled-binary.exe"
with mock.patch.object(lab.shutil, "which", return_value=None):
with self.assertRaises(FileNotFoundError):
lab.discover_minio_launcher(explicit_binary=None, minio_root=source_root)
finally:
lab.DEFAULT_MINIO_BINARY = original_default_binary
def test_uses_binary_from_minio_root_directory(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary_root = Path(temp_dir) / "minio-dist"
binary_root.mkdir()
binary = binary_root / "minio.exe"
binary.write_text("", encoding="utf-8")
launcher = lab.discover_minio_launcher(explicit_binary=None, minio_root=binary_root)
self.assertEqual(launcher.command, [str(binary.resolve())])
self.assertEqual(launcher.workdir, binary.resolve().parent)
self.assertEqual(launcher.kind, "binary")
class FixtureMatrixTests(unittest.TestCase):
def test_default_matrix_covers_sse_modes_and_shapes(self) -> None:
cases = lab.build_default_cases()
case_ids = {case.case_id for case in cases}
self.assertEqual(len(cases), 6)
self.assertIn("sse-s3-singlepart-64k", case_ids)
self.assertIn("sse-kms-singlepart-64k", case_ids)
self.assertIn("sse-c-singlepart-64k", case_ids)
self.assertIn("sse-s3-multipart-8m", case_ids)
self.assertIn("sse-kms-multipart-8m", case_ids)
self.assertIn("sse-c-multipart-8m", case_ids)
def test_default_matrix_uses_configured_kms_key_id(self) -> None:
cases = lab.build_default_cases(kms_key_id="local-fixture-key")
kms_cases = [case for case in cases if case.encryption == "SSE-KMS"]
self.assertEqual(len(kms_cases), 2)
self.assertTrue(all(case.kms_key_id == "local-fixture-key" for case in kms_cases))
class RequestRecordTests(unittest.TestCase):
def test_sse_kms_request_record_includes_context_header(self) -> None:
case = lab.FixtureCase(
case_id="sse-kms-singlepart-64k",
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-KMS",
size_bytes=64 * 1024,
multipart=False,
kms_key_id="minio-default-key",
kms_context={"project": "rio-v2", "stage": "fixture"},
)
record = lab.build_request_record(case)
self.assertEqual(record["headers"]["x-amz-server-side-encryption"], "aws:kms")
self.assertEqual(
record["headers"]["x-amz-server-side-encryption-aws-kms-key-id"],
"minio-default-key",
)
expected_context = base64.b64encode(
b'{"project":"rio-v2","stage":"fixture"}'
).decode("ascii")
self.assertEqual(
record["headers"]["x-amz-server-side-encryption-context"],
expected_context,
)
def test_sse_s3_request_record_uses_aes256(self) -> None:
case = lab.FixtureCase(
case_id="sse-s3-singlepart-64k",
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-S3",
size_bytes=64 * 1024,
multipart=False,
)
record = lab.build_request_record(case)
self.assertEqual(record["headers"], {"x-amz-server-side-encryption": "AES256"})
def test_sse_c_request_record_includes_customer_key_headers(self) -> None:
case = lab.FixtureCase(
case_id="sse-c-singlepart-64k",
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-C",
size_bytes=64 * 1024,
multipart=False,
)
record = lab.build_request_record(case)
self.assertEqual(
record["headers"]["x-amz-server-side-encryption-customer-algorithm"],
"AES256",
)
self.assertIn("x-amz-server-side-encryption-customer-key", record["headers"])
self.assertIn("x-amz-server-side-encryption-customer-key-md5", record["headers"])
class MultipartManifestTests(unittest.TestCase):
def test_complete_multipart_payload_is_xml(self) -> None:
parts = [
{"ETag": '"etag-1"', "PartNumber": 1},
{"ETag": '"etag-2"', "PartNumber": 2},
]
payload = lab.build_complete_multipart_xml(parts).decode("utf-8")
self.assertIn("<CompleteMultipartUpload>", payload)
self.assertIn("<PartNumber>1</PartNumber>", payload)
self.assertIn("<ETag>\"etag-1\"</ETag>", payload)
self.assertIn("<PartNumber>2</PartNumber>", payload)
self.assertIn("<ETag>\"etag-2\"</ETag>", payload)
class KmsSecretKeyTests(unittest.TestCase):
def test_parse_kms_secret_key_extracts_key_id(self) -> None:
setting = lab.parse_kms_secret_key("local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
self.assertEqual(setting.secret_key, "local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")
self.assertEqual(setting.key_id, "local-fixture-key")
def test_parse_kms_secret_key_rejects_missing_separator(self) -> None:
with self.assertRaises(ValueError):
lab.parse_kms_secret_key("invalid-secret-key")
def test_minio_env_prefers_explicit_secret_key(self) -> None:
env = lab.minio_env(
kms_secret_key=lab.parse_kms_secret_key(
"local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
)
)
self.assertEqual(
env["MINIO_KMS_SECRET_KEY"],
"local-fixture-key:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
)
if __name__ == "__main__":
unittest.main()
crates/rio-v2/tests/minio_generated_fixtures.rs
+288
View File
@@ -0,0 +1,288 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
#[derive(Debug, Deserialize)]
struct RequestRecord {
encryption: String,
headers: HashMap<String, String>,
multipart: bool,
size_bytes: u64,
object: String,
}
#[derive(Debug, Deserialize)]
struct HeadRecord {
#[serde(rename = "ContentLength")]
content_length: u64,
#[serde(rename = "ServerSideEncryption")]
server_side_encryption: Option<String>,
#[serde(rename = "SSECustomerAlgorithm")]
sse_customer_algorithm: Option<String>,
#[serde(rename = "SSECustomerKeyMD5")]
sse_customer_key_md5: Option<String>,
#[serde(rename = "SSEKMSKeyId")]
sse_kms_key_id: Option<String>,
}
#[derive(Debug, Deserialize)]
struct ManifestRecord {
capture: Option<CaptureRecord>,
}
#[derive(Debug, Deserialize)]
struct CaptureRecord {
kms_key_id: Option<String>,
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn find_object_xl_meta(case_dir: &Path) -> PathBuf {
let backend = case_dir.join("backend");
for entry in walkdir::WalkDir::new(&backend) {
let entry = entry.unwrap_or_else(|err| panic!("walk {}: {err}", backend.display()));
if !entry.file_type().is_file() {
continue;
}
if entry.file_name() != "xl.meta" {
continue;
}
let path = entry.path();
let path_text = path.to_string_lossy();
if path_text.contains(".minio.sys") {
continue;
}
return path.to_path_buf();
}
panic!("object xl.meta not found under {}", backend.display());
}
fn load_file_info(case_id: &str) -> FileInfo {
let case = case_dir(case_id);
let xl_meta = find_object_xl_meta(&case);
let data = fs::read(&xl_meta).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta.display()));
get_file_info(
&data,
"fixture-bucket",
"fixture-object",
"",
FileInfoOpts {
data: false,
include_free_versions: true,
},
)
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta.display()))
}
fn read_manifest(case_id: &str) -> ManifestRecord {
read_json(&case_dir(case_id).join("manifest.json"))
}
fn require_generated_fixture_root() {
let root = fixture_root();
assert!(
root.join("cases").is_dir(),
"generated fixture root missing: {}. Run test/minio_fixture_lab/lab.py capture-matrix first.",
root.display()
);
}
fn metadata_keys(fi: &FileInfo) -> HashMap<&str, &str> {
fi.metadata.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect()
}
fn expected_fixture_kms_key_id(case_id: &str, request: &RequestRecord) -> Option<String> {
request
.headers
.get("x-amz-server-side-encryption-aws-kms-key-id")
.cloned()
.or_else(|| read_manifest(case_id).capture.and_then(|capture| capture.kms_key_id))
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_singlepart_sse_s3_fixture() {
require_generated_fixture_root();
let request: RequestRecord = read_json(&case_dir("sse-s3-singlepart-64k").join("request.json"));
let head: HeadRecord = read_json(&case_dir("sse-s3-singlepart-64k").join("head.json"));
let fi = load_file_info("sse-s3-singlepart-64k");
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-S3");
assert!(!request.multipart);
assert_eq!(request.object, "dir/object.bin");
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("AES256"));
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"));
assert_eq!(metadata.get("content-type"), Some(&"binary/octet-stream"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_singlepart_sse_kms_fixture() {
require_generated_fixture_root();
let case_id = "sse-kms-singlepart-64k";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
let expected_key_id = expected_fixture_kms_key_id(case_id, &request).expect("fixture kms key id");
assert_eq!(request.encryption, "SSE-KMS");
assert!(!request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("aws:kms"));
assert_eq!(request.headers.get("x-amz-server-side-encryption"), Some(&"aws:kms".to_string()));
assert_eq!(request.headers.get("x-amz-server-side-encryption-aws-kms-key-id"), Some(&expected_key_id));
assert_eq!(head.sse_kms_key_id.as_deref(), Some(format!("arn:aws:kms:{expected_key_id}").as_str()));
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes);
assert_eq!(
metadata.get("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"),
Some(&expected_key_id.as_str())
);
assert_eq!(
metadata.get("X-Minio-Internal-Server-Side-Encryption-Context").copied(),
request
.headers
.get("x-amz-server-side-encryption-context")
.map(String::as_str)
);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_multipart_sse_s3_fixture() {
require_generated_fixture_root();
let case_id = "sse-s3-multipart-8m";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-S3");
assert!(request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("AES256"));
assert_eq!(fi.parts.len(), 2);
assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::<u64>(), request.size_bytes);
assert!(
metadata
.get("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id")
.is_some_and(|value| !value.is_empty())
);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"));
assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_multipart_sse_kms_fixture() {
require_generated_fixture_root();
let case_id = "sse-kms-multipart-8m";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
let expected_key_id = expected_fixture_kms_key_id(case_id, &request).expect("fixture kms key id");
assert_eq!(request.encryption, "SSE-KMS");
assert!(request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.server_side_encryption.as_deref(), Some("aws:kms"));
assert_eq!(request.headers.get("x-amz-server-side-encryption-aws-kms-key-id"), Some(&expected_key_id));
assert_eq!(head.sse_kms_key_id.as_deref(), Some(format!("arn:aws:kms:{expected_key_id}").as_str()));
assert_eq!(fi.parts.len(), 2);
assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::<u64>(), request.size_bytes);
assert_eq!(
metadata.get("X-Minio-Internal-Server-Side-Encryption-Context").copied(),
request
.headers
.get("x-amz-server-side-encryption-context")
.map(String::as_str)
);
assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_singlepart_sse_c_fixture() {
require_generated_fixture_root();
let case_id = "sse-c-singlepart-64k";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-C");
assert!(!request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(
head.sse_customer_key_md5.as_deref(),
request
.headers
.get("x-amz-server-side-encryption-customer-key-md5")
.map(String::as_str)
);
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].actual_size as u64, request.size_bytes);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
assert!(!metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"));
}
#[test]
#[ignore = "requires locally generated MinIO fixtures"]
fn parses_multipart_sse_c_fixture() {
require_generated_fixture_root();
let case_id = "sse-c-multipart-8m";
let request: RequestRecord = read_json(&case_dir(case_id).join("request.json"));
let head: HeadRecord = read_json(&case_dir(case_id).join("head.json"));
let fi = load_file_info(case_id);
let metadata = metadata_keys(&fi);
assert_eq!(request.encryption, "SSE-C");
assert!(request.multipart);
assert_eq!(head.content_length, request.size_bytes);
assert_eq!(head.sse_customer_algorithm.as_deref(), Some("AES256"));
assert_eq!(
head.sse_customer_key_md5.as_deref(),
request
.headers
.get("x-amz-server-side-encryption-customer-key-md5")
.map(String::as_str)
);
assert_eq!(fi.parts.len(), 2);
assert_eq!(fi.parts.iter().map(|part| part.actual_size as u64).sum::<u64>(), request.size_bytes);
assert!(metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
assert!(!metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"));
assert!(metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert_eq!(metadata.get("X-Minio-Internal-actual-size"), Some(&"8388608"));
}
crates/rio/src/compress_index.rs
+6 -2
View File
@@ -394,12 +394,13 @@ impl Index {
b = &b[n..];
if idx > 0 {
c_predict += c_off / 2;
let next_c_predict = c_predict + c_off / 2;
let prev = self.info[idx - 1].compressed_offset;
c_off += prev + c_predict;
if c_off <= prev {
return Err(io::Error::other("invalid offset"));
}
c_predict = next_c_predict;
}
if c_off < 0 {
return Err(io::Error::other("negative offset"));
@@ -701,6 +702,7 @@ mod tests {
let mut source = Index::new();
source.add(100, 1_000)?;
source.add(300, 1_000 + MIN_INDEX_DIST)?;
source.add(650, 1_000 + MIN_INDEX_DIST * 2)?;
let encoded = source.clone().into_vec();
@@ -713,8 +715,10 @@ mod tests {
assert_eq!(decoded.info.len(), source.info.len());
assert_eq!(decoded.info[0].compressed_offset, source.info[0].compressed_offset);
assert_eq!(decoded.info[0].uncompressed_offset, source.info[0].uncompressed_offset);
assert_eq!(decoded.info[1].compressed_offset, source.info[1].compressed_offset);
assert_eq!(decoded.info[1].uncompressed_offset, source.info[1].uncompressed_offset);
assert!(decoded.info[1].compressed_offset > decoded.info[0].compressed_offset);
assert_eq!(decoded.info[2].compressed_offset, source.info[2].compressed_offset);
assert_eq!(decoded.info[2].uncompressed_offset, source.info[2].uncompressed_offset);
Ok(())
}
crates/s3select-api/src/object_store.rs
+34 -4
View File
@@ -1046,7 +1046,10 @@ where
AsyncTryStream::<Bytes, o_Error, _>::new(|mut y| async move {
pin_mut!(stream);
let mut remaining: usize = content_length;
while let Some(result) = stream.next().await {
while remaining > 0 {
let Some(result) = stream.next().await else {
break;
};
let mut bytes = result.map_err(|e| o_Error::Generic {
store: "",
source: Box::new(e),
@@ -1064,12 +1067,12 @@ where
#[cfg(test)]
mod test {
use super::{
ConvertStream, SelectScanRange, convert_field_delimiter_stream, extract_json_sub_path_from_expression, find_delimiter,
flatten_json_document_to_ndjson, http_range_spec_from_get_range, replace_symbol, scan_range_from_bounds,
ConvertStream, SelectScanRange, bytes_stream, convert_field_delimiter_stream, extract_json_sub_path_from_expression,
find_delimiter, flatten_json_document_to_ndjson, http_range_spec_from_get_range, replace_symbol, scan_range_from_bounds,
scan_range_read_start, scan_range_stream, select_read_headers,
};
use bytes::Bytes;
use futures::{StreamExt, stream};
use futures::{StreamExt, TryStreamExt, stream};
use object_store::GetRange;
use s3s::dto::{
CSVInput, CSVOutput, ExpressionType, InputSerialization, OutputSerialization, SelectObjectContentInput,
@@ -1079,6 +1082,10 @@ mod test {
X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5,
};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use tokio::io::AsyncReadExt;
use tokio_util::io::StreamReader;
@@ -1285,6 +1292,29 @@ mod test {
assert_eq!(output, b"a,");
}
#[tokio::test]
async fn test_bytes_stream_stops_at_content_length() {
let poll_count = Arc::new(AtomicUsize::new(0));
let stream_poll_count = Arc::clone(&poll_count);
let source = stream::unfold(0, move |index| {
let stream_poll_count = Arc::clone(&stream_poll_count);
async move {
stream_poll_count.fetch_add(1, Ordering::SeqCst);
let bytes = match index {
0 => Bytes::from_static(b"abcd"),
1 => Bytes::from_static(b"efgh"),
_ => return None,
};
Some((Ok::<_, std::io::Error>(bytes), index + 1))
}
});
let chunks: Vec<Bytes> = bytes_stream(source, 4).try_collect().await.unwrap();
assert_eq!(chunks, vec![Bytes::from_static(b"abcd")]);
assert_eq!(poll_count.load(Ordering::SeqCst), 1);
}
/// A JSON array is split into one NDJSON line per element.
#[test]
fn test_flatten_array_produces_one_line_per_element() {
crates/utils/src/http/metadata_compat.rs
+28 -2
View File
@@ -112,18 +112,27 @@ pub fn insert_str(map: &mut HashMap<String, String>, suffix: &str, value: String
pub fn get_str(map: &HashMap<String, String>, suffix: &str) -> Option<String> {
let (k1, k2) = both_keys(suffix);
map.get(&k1).cloned().or_else(|| map.get(&k2).cloned())
map.get(&k1).cloned().or_else(|| map.get(&k2).cloned()).or_else(|| {
map.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2))
.map(|(_, value)| value.clone())
})
}
pub fn contains_key_str(map: &HashMap<String, String>, suffix: &str) -> bool {
let (k1, k2) = both_keys(suffix);
map.contains_key(&k1) || map.contains_key(&k2)
map.contains_key(&k1)
|| map.contains_key(&k2)
|| map
.keys()
.any(|key| key.eq_ignore_ascii_case(&k1) || key.eq_ignore_ascii_case(&k2))
}
pub fn remove_str(map: &mut HashMap<String, String>, suffix: &str) {
let (k1, k2) = both_keys(suffix);
map.remove(&k1);
map.remove(&k2);
map.retain(|key, _| !key.eq_ignore_ascii_case(&k1) && !key.eq_ignore_ascii_case(&k2));
}
// === Vec<u8> type (meta_sys) ===
@@ -178,4 +187,21 @@ mod tests {
assert!(!has_internal_suffix("x-rustfs-internal-purgestatus", SUFFIX_HEALING));
assert!(!has_internal_suffix("x-amz-meta-custom", SUFFIX_PURGESTATUS));
}
#[test]
fn test_str_lookup_accepts_minio_metadata_case() {
let mut metadata = HashMap::from([
("X-Minio-Internal-compression".to_string(), "klauspost/compress/s2".to_string()),
("X-Minio-Internal-actual-size".to_string(), "268435456".to_string()),
]);
assert!(contains_key_str(&metadata, SUFFIX_COMPRESSION));
assert_eq!(get_str(&metadata, SUFFIX_COMPRESSION).as_deref(), Some("klauspost/compress/s2"));
assert_eq!(get_str(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("268435456"));
remove_str(&mut metadata, SUFFIX_COMPRESSION);
assert!(!contains_key_str(&metadata, SUFFIX_COMPRESSION));
assert!(!metadata.contains_key("X-Minio-Internal-compression"));
assert!(contains_key_str(&metadata, SUFFIX_ACTUAL_SIZE));
}
}
rustfs/Cargo.toml
+2
View File
@@ -54,6 +54,7 @@ io-scheduler-debug = [] # Enable debug information in I/O scheduler
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp"]
manual-test-runners = []
rio-v2 = ["rustfs-ecstore/rio-v2"]
[lints]
workspace = true
@@ -138,6 +139,7 @@ astral-tokio-tar = { workspace = true }
atoi = { workspace = true }
atomic_enum = { workspace = true }
base64 = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
base64-simd.workspace = true
clap = { workspace = true }
rustfs/src/app/multipart_usecase.rs
+159 -111
View File
@@ -28,7 +28,10 @@ use crate::storage::s3_api::multipart::{
ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output,
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
};
use crate::storage::sse::{build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error};
use crate::storage::sse::{
build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers,
extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata,
};
use crate::storage::*;
use crate::table_catalog;
use bytes::Bytes;
@@ -46,13 +49,13 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
use rustfs_ecstore::compress::is_compressible;
use rustfs_ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
use rustfs_ecstore::new_object_layer_fn;
#[cfg(test)]
use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
use rustfs_ecstore::rio::{HashReader, WritePlan};
use rustfs_ecstore::set_disk::is_valid_storage_class;
use rustfs_ecstore::store_api::{CompletePart, HTTPRangeSpec, MultipartUploadResult, ObjectIO, ObjectOptions, PutObjReader};
use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations};
use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
use rustfs_rio::{CompressReader, EncryptReader, HashReader};
#[cfg(test)]
use rustfs_rio::{DecryptReader, HardLimitReader, boxed_reader, wrap_reader};
use rustfs_s3_ops::S3Operation;
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
@@ -574,13 +577,16 @@ impl DefaultMultipartUsecase {
metadata.extend(object_lock_metadata);
}
apply_bucket_default_lock_retention(&bucket, &mut metadata, has_explicit_object_lock_retention).await?;
let (_, sse_customer_key, _) = extract_ssec_params_from_headers(&req.headers)?;
let encryption_request = PrepareEncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
};
@@ -589,7 +595,11 @@ impl DefaultMultipartUsecase {
let server_side_encryption = Some(material.server_side_encryption.clone());
let ssekms_key_id = material.kms_key_id.clone();
metadata.extend(encryption_material_to_metadata(&material));
let mut encryption_metadata = encryption_material_to_metadata(&material);
if material.key_kind == crate::storage::sse::EncryptionKeyKind::Object {
mark_encrypted_multipart_metadata(&mut encryption_metadata);
}
metadata.extend(encryption_metadata);
(server_side_encryption, ssekms_key_id)
}
@@ -600,7 +610,7 @@ impl DefaultMultipartUsecase {
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
CompressionAlgorithm::default().to_string(),
rustfs_ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
);
}
@@ -757,7 +767,9 @@ impl DefaultMultipartUsecase {
let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
let mut write_plan = WritePlan::new();
let mut reader = if is_compressible {
let algorithm = CompressionAlgorithm::default();
let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false)
.map_err(ApiError::from)?;
@@ -766,15 +778,8 @@ impl DefaultMultipartUsecase {
}
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
};
@@ -812,6 +817,7 @@ impl DefaultMultipartUsecase {
key: &key,
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
ssekms_context: None,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
@@ -819,35 +825,25 @@ impl DefaultMultipartUsecase {
}
.check_upload_part_customer_key_md5(&fi.user_defined, sse_customer_key_md5.clone())?;
let (requested_sse, requested_kms_key_id) = if has_ssec {
let encryption_request = EncryptionRequest {
let ssec_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
};
match sse_encryption(encryption_request).await? {
Some(material) => {
let requested_sse = Some(material.server_side_encryption.clone());
let requested_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new_multipart(reader, material.key_bytes, material.base_nonce, part_id);
reader = HashReader::from_reader(
encrypted_reader,
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?;
(requested_sse, requested_kms_key_id)
metadata: &fi.user_defined,
sse_customer_key: sse_customer_key.as_ref(),
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
rustfs_ecstore::rio::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
}
None => (None, None),
}
crate::storage::sse::EncryptionKeyKind::Direct => {
rustfs_ecstore::rio::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
}
};
write_plan = write_plan.with_encryption(ssec_write);
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
} else if let Some(server_side_encryption) = server_side_encryption {
let managed_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
@@ -858,15 +854,24 @@ impl DefaultMultipartUsecase {
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let encrypted_reader =
EncryptReader::new_multipart(reader, managed_material.key_bytes, managed_material.base_nonce, part_id);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
let managed_write = match managed_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
rustfs_ecstore::rio::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
}
crate::storage::sse::EncryptionKeyKind::Direct => rustfs_ecstore::rio::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
),
};
write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id)
} else {
(None, None)
};
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(reader);
let info = store
@@ -1119,22 +1124,15 @@ impl DefaultMultipartUsecase {
let is_compressible = rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
let actual_size = length;
let mut size = length;
let mut write_plan = WritePlan::new();
let mut reader = if is_compressible {
let hrd = HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?;
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
let algorithm = CompressionAlgorithm::default();
let hrd = HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(src_stream, size, actual_size, None, None, false).map_err(ApiError::from)?
HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?
};
let server_side_encryption = mp_info
@@ -1158,6 +1156,7 @@ impl DefaultMultipartUsecase {
key: &key,
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
ssekms_context: None,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
@@ -1166,35 +1165,29 @@ impl DefaultMultipartUsecase {
.check_upload_part_customer_key_md5(&mp_info.user_defined, sse_customer_key_md5.clone())?;
let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec {
let encryption_request = EncryptionRequest {
let ssec_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
};
match sse_encryption(encryption_request).await? {
Some(material) => {
let requested_sse = Some(material.server_side_encryption.clone());
let requested_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new_multipart(reader, material.key_bytes, material.base_nonce, part_id);
reader = HashReader::from_reader(
encrypted_reader,
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?;
(requested_sse, requested_kms_key_id, mp_info.user_defined.clone())
metadata: &mp_info.user_defined,
sse_customer_key: sse_customer_key.as_ref(),
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
rustfs_ecstore::rio::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
}
None => (None, None, mp_info.user_defined.clone()),
}
crate::storage::sse::EncryptionKeyKind::Direct => {
rustfs_ecstore::rio::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
}
};
write_plan = write_plan.with_encryption(ssec_write);
(
Some(ssec_material.server_side_encryption),
ssec_material.kms_key_id,
mp_info.user_defined.clone(),
)
} else if let Some(server_side_encryption) = server_side_encryption {
let managed_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
@@ -1205,15 +1198,24 @@ impl DefaultMultipartUsecase {
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let encrypted_reader =
EncryptReader::new_multipart(reader, managed_material.key_bytes, managed_material.base_nonce, part_id);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
let managed_write = match managed_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
rustfs_ecstore::rio::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
}
crate::storage::sse::EncryptionKeyKind::Direct => rustfs_ecstore::rio::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
),
};
write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
} else {
(None, None, mp_info.user_defined.clone())
};
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
if let Some(checksum_algorithm) = mp_info
.user_defined
.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
@@ -1384,18 +1386,17 @@ mod tests {
key: "object",
server_side_encryption: Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)),
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm: None,
sse_customer_key: None,
sse_customer_key_md5: None,
};
let session_material = sse_prepare_encryption(prepare_request)
.await
.expect("prepare multipart encryption")
.expect("managed multipart session material");
let session_metadata = encryption_material_to_metadata(&session_material);
let session_nonce = session_metadata
.get("x-rustfs-encryption-iv")
.cloned()
.expect("session nonce metadata");
let mut session_metadata = encryption_material_to_metadata(&session_material);
mark_encrypted_multipart_metadata(&mut session_metadata);
let part_one_plaintext = vec![0x31; rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE + 23];
let part_two_plaintext = vec![0x32; rustfs_rio::DEFAULT_ENCRYPTION_BLOCK_SIZE * 2 + 7];
@@ -1411,15 +1412,31 @@ mod tests {
.expect("decrypt session one")
.expect("part one material");
let mut encrypted_one = Vec::new();
EncryptReader::new_multipart(
#[cfg(feature = "rio-v2")]
let mut part_one_reader = match part_one_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key(
Cursor::new(part_one_plaintext.clone()),
part_one_material.key_bytes,
1,
),
crate::storage::sse::EncryptionKeyKind::Direct => EncryptReader::new_multipart(
Cursor::new(part_one_plaintext.clone()),
part_one_material.key_bytes,
part_one_material.base_nonce,
1,
),
};
#[cfg(not(feature = "rio-v2"))]
let mut part_one_reader = EncryptReader::new_multipart(
Cursor::new(part_one_plaintext.clone()),
part_one_material.key_bytes,
part_one_material.base_nonce,
1,
)
.read_to_end(&mut encrypted_one)
.await
.expect("read encrypted part one");
);
part_one_reader
.read_to_end(&mut encrypted_one)
.await
.expect("read encrypted part one");
let part_two_material = sse_decryption(DecryptionRequest {
bucket: "bucket",
@@ -1432,20 +1449,38 @@ mod tests {
.expect("decrypt session two")
.expect("part two material");
let mut encrypted_two = Vec::new();
EncryptReader::new_multipart(
#[cfg(feature = "rio-v2")]
let mut part_two_reader = match part_two_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key(
Cursor::new(part_two_plaintext.clone()),
part_two_material.key_bytes,
2,
),
crate::storage::sse::EncryptionKeyKind::Direct => EncryptReader::new_multipart(
Cursor::new(part_two_plaintext.clone()),
part_two_material.key_bytes,
part_two_material.base_nonce,
2,
),
};
#[cfg(not(feature = "rio-v2"))]
let mut part_two_reader = EncryptReader::new_multipart(
Cursor::new(part_two_plaintext.clone()),
part_two_material.key_bytes,
part_two_material.base_nonce,
2,
)
.read_to_end(&mut encrypted_two)
.await
.expect("read encrypted part two");
assert_eq!(
session_metadata.get("x-rustfs-encryption-iv").map(String::as_str),
Some(session_nonce.as_str())
);
part_two_reader
.read_to_end(&mut encrypted_two)
.await
.expect("read encrypted part two");
if session_material.key_kind == crate::storage::sse::EncryptionKeyKind::Object {
assert!(session_metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
assert!(session_metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"));
} else {
assert!(session_metadata.contains_key("x-rustfs-encryption-iv"));
}
let parts = vec![
ObjectPartInfo {
@@ -1478,15 +1513,28 @@ mod tests {
.expect("managed decryption material");
let plaintext_size = multipart_plaintext_size(&parts, -1);
let mut decrypted_reader = HardLimitReader::new(
boxed_reader(DecryptReader::new_multipart(
#[cfg(feature = "rio-v2")]
let decrypted_stream = match decryption_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => boxed_reader(DecryptReader::new_multipart_with_object_key(
wrap_reader(Cursor::new(encrypted_stream)),
decryption_material.key_bytes,
multipart_part_numbers(&parts),
)),
crate::storage::sse::EncryptionKeyKind::Direct => boxed_reader(DecryptReader::new_multipart(
wrap_reader(Cursor::new(encrypted_stream)),
decryption_material.key_bytes,
decryption_material.base_nonce,
multipart_part_numbers(&parts),
)),
plaintext_size,
);
};
#[cfg(not(feature = "rio-v2"))]
let decrypted_stream = boxed_reader(DecryptReader::new_multipart(
wrap_reader(Cursor::new(encrypted_stream)),
decryption_material.key_bytes,
decryption_material.base_nonce,
multipart_part_numbers(&parts),
));
let mut decrypted_reader = HardLimitReader::new(decrypted_stream, plaintext_size);
let mut decrypted = Vec::new();
decrypted_reader
rustfs/src/app/object_usecase.rs
+75 -43
View File
@@ -31,7 +31,10 @@ use crate::storage::options::{
};
use crate::storage::request_context::spawn_traced;
use crate::storage::s3_api::multipart::parse_list_parts_params;
use crate::storage::sse::{SSEType, build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error};
use crate::storage::sse::{
SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssekms_context_from_headers,
map_get_object_reader_error,
};
use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
use crate::storage::*;
use crate::table_catalog;
@@ -71,6 +74,7 @@ use rustfs_ecstore::config::storageclass;
use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
use rustfs_ecstore::new_object_layer_fn;
use rustfs_ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
use rustfs_ecstore::store_api::{
HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, StorageAPI,
@@ -84,7 +88,6 @@ use rustfs_io_metrics;
use rustfs_lock::NamespaceLockGuard;
use rustfs_notify::EventArgsBuilder;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_rio::{CompressReader, DynReader, EncryptReader, HashReader, wrap_reader};
use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object};
use rustfs_s3select_api::object_store::bytes_stream;
use rustfs_targets::{
@@ -1193,10 +1196,28 @@ impl DefaultObjectUsecase {
where
R: AsyncRead + Send + Sync + 'static,
{
Some(StreamingBlob::wrap(bytes_stream(
ReaderStream::with_capacity(reader, optimal_buffer_size),
response_content_length as usize,
)))
let expected = response_content_length.max(0) as usize;
#[cfg(feature = "tracing-chunk-debug")]
let stream = {
let mut emitted = 0usize;
ReaderStream::with_capacity(reader, optimal_buffer_size).inspect(move |item| match item {
Ok(bytes) => {
emitted += bytes.len();
tracing::debug!(emitted, expected, chunk_len = bytes.len(), "GetObject ReaderStream emitted bytes");
}
Err(err) => {
tracing::error!(
emitted,
expected,
error = %err,
"GetObject ReaderStream returned error"
);
}
})
};
#[cfg(not(feature = "tracing-chunk-debug"))]
let stream = ReaderStream::with_capacity(reader, optimal_buffer_size);
Some(StreamingBlob::wrap(bytes_stream(stream, expected)))
}
fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
@@ -1839,6 +1860,7 @@ impl DefaultObjectUsecase {
validate_sse_headers_for_write(
effective_sse.as_ref(),
effective_kms_key_id.as_ref(),
extract_ssekms_context_from_headers(&req.headers)?.as_ref(),
sse_customer_algorithm.as_ref(),
sse_customer_key.as_ref(),
sse_customer_key_md5.as_ref(),
@@ -1906,9 +1928,15 @@ impl DefaultObjectUsecase {
let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
let mut reader = if is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64 {
let should_compress = is_compressible(&req.headers, &key) && size > MIN_COMPRESSIBLE_SIZE as i64;
let mut write_plan = WritePlan::new();
let mut reader = if should_compress {
let algorithm = CompressionAlgorithm::default();
insert_str(&mut metadata, SUFFIX_COMPRESSION, algorithm.to_string());
insert_str(
&mut metadata,
SUFFIX_COMPRESSION,
rustfs_ecstore::rio::compression_metadata_value(algorithm),
);
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
let mut hrd =
@@ -1919,12 +1947,16 @@ impl DefaultObjectUsecase {
}
opts.want_checksum = hrd.checksum();
insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, algorithm.to_string());
insert_str(
&mut opts.user_defined,
SUFFIX_COMPRESSION,
rustfs_ecstore::rio::compression_metadata_value(algorithm),
);
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(CompressReader::new(hrd, algorithm), size, actual_size, None, None, false)
.map_err(ApiError::from)?
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
};
@@ -1938,6 +1970,7 @@ impl DefaultObjectUsecase {
}
let mut helper = OperationHelper::new(&req, event_name, S3Operation::PutObject);
let ssekms_context = extract_ssekms_context_from_headers(&req.headers)?;
// Apply encryption using unified SSE API.
let encryption_request = EncryptionRequest {
@@ -1945,6 +1978,7 @@ impl DefaultObjectUsecase {
key: &key,
server_side_encryption: effective_sse.clone(),
ssekms_key_id: effective_kms_key_id.clone(),
ssekms_context,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
@@ -1964,15 +1998,15 @@ impl DefaultObjectUsecase {
effective_sse = Some(material.server_side_encryption.clone());
effective_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new(reader, material.key_bytes, material.base_nonce);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
write_plan = write_plan.with_encryption(material.write_encryption(None));
let encryption_metadata = encryption_material_to_metadata(&material);
metadata.extend(encryption_metadata.clone());
opts.user_defined.extend(encryption_metadata);
}
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
let mut reader = PutObjReader::new(reader);
let mt2 = metadata.clone();
@@ -2864,14 +2898,18 @@ impl DefaultObjectUsecase {
let actual_size = src_info.get_actual_size().map_err(ApiError::from)?;
let mut length = actual_size;
let length = actual_size;
let mut compress_metadata = HashMap::new();
let should_compress = is_compressible(&req.headers, &key) && actual_size > MIN_COMPRESSIBLE_SIZE as i64;
if should_compress {
insert_str(&mut compress_metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
insert_str(
&mut compress_metadata,
SUFFIX_COMPRESSION,
rustfs_ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
);
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
} else {
remove_str(&mut user_defined, SUFFIX_COMPRESSION);
@@ -2907,18 +2945,12 @@ impl DefaultObjectUsecase {
}
apply_bucket_default_lock_retention(&bucket, &mut user_defined, has_explicit_object_lock_retention).await?;
let mut write_plan = WritePlan::new();
let mut reader = if should_compress {
let algorithm = CompressionAlgorithm::default();
let hrd = HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
length = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
length,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(gr.stream, length, actual_size, None, None, false).map_err(ApiError::from)?
};
@@ -2928,6 +2960,7 @@ impl DefaultObjectUsecase {
key: &key,
server_side_encryption: effective_sse.clone(),
ssekms_key_id: effective_kms_key_id.clone(),
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
@@ -2938,13 +2971,13 @@ impl DefaultObjectUsecase {
effective_sse = Some(material.server_side_encryption.clone());
effective_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new(reader, material.key_bytes, material.base_nonce);
reader = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
write_plan = write_plan.with_encryption(material.write_encryption(None));
user_defined.extend(encryption_material_to_metadata(&material));
}
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
src_info.put_object_reader = Some(PutObjReader::new(reader));
// check quota
@@ -4182,6 +4215,7 @@ impl DefaultObjectUsecase {
validate_sse_headers_for_write(
effective_sse.as_ref(),
effective_kms_key_id.as_ref(),
extract_ssekms_context_from_headers(&req.headers)?.as_ref(),
sse_customer_algorithm.as_ref(),
sse_customer_key.as_ref(),
sse_customer_key_md5.as_ref(),
@@ -4379,24 +4413,22 @@ impl DefaultObjectUsecase {
let should_compress = !is_dir && is_compressible(&HeaderMap::new(), &fpath) && size > MIN_COMPRESSIBLE_SIZE as i64;
let mut write_plan = WritePlan::new();
let mut hrd = if is_dir {
HashReader::from_stream(std::io::Cursor::new(Vec::new()), size, actual_size, None, None, false)
.map_err(ApiError::from)?
} else if should_compress {
insert_str(&mut metadata, SUFFIX_COMPRESSION, CompressionAlgorithm::default().to_string());
let algorithm = CompressionAlgorithm::default();
insert_str(
&mut metadata,
SUFFIX_COMPRESSION,
rustfs_ecstore::rio::compression_metadata_value(algorithm),
);
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?;
size = HashReader::SIZE_PRESERVE_LAYER;
HashReader::from_reader(
CompressReader::new(hrd, CompressionAlgorithm::default()),
size,
actual_size,
None,
None,
false,
)
.map_err(ApiError::from)?
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?
};
@@ -4413,6 +4445,7 @@ impl DefaultObjectUsecase {
key: &fpath,
server_side_encryption: effective_sse.clone(),
ssekms_key_id: effective_kms_key_id.clone(),
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
@@ -4423,14 +4456,13 @@ impl DefaultObjectUsecase {
effective_sse = Some(material.server_side_encryption.clone());
effective_kms_key_id = material.kms_key_id.clone();
let encrypted_reader = EncryptReader::new(hrd, material.key_bytes, material.base_nonce);
hrd = HashReader::from_reader(encrypted_reader, HashReader::SIZE_PRESERVE_LAYER, actual_size, None, None, false)
.map_err(ApiError::from)?;
write_plan = write_plan.with_encryption(material.write_encryption(None));
let encryption_metadata = encryption_material_to_metadata(&material);
metadata.extend(encryption_metadata.clone());
opts.user_defined.extend(encryption_metadata);
}
hrd = write_plan.apply(hrd, actual_size).map_err(ApiError::from)?;
opts.user_defined.extend(metadata);
let mut reader = PutObjReader::new(hrd);
rustfs/src/storage/sse.rs
+1097 -93
View File
File diff suppressed because it is too large Load Diff
rustfs/src/storage/sse_test.rs
+11 -6
View File
@@ -16,6 +16,7 @@
mod tests {
use crate::storage::sse::SseDekProvider;
use crate::storage::sse::TestSseDekProvider;
use rustfs_kms::types::ObjectEncryptionContext;
use rustfs_rio::{DecryptReader, EncryptReader};
use std::io::Cursor;
use tokio::io::AsyncReadExt;
@@ -36,9 +37,10 @@ mod tests {
let bucket = "test-bucket";
let key = "test-key";
let kms_key_id = "default"; // Key ID is ignored in test provider
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
let (data_key, _encrypted_dek) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.generate_sse_dek(&context, kms_key_id)
.await
.expect("Failed to generate DEK");
@@ -104,9 +106,10 @@ mod tests {
let bucket = "test-bucket";
let key = "test-key-large";
let kms_key_id = "default";
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
let (data_key, _encrypted_dek) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.generate_sse_dek(&context, kms_key_id)
.await
.expect("Failed to generate DEK");
@@ -153,15 +156,16 @@ mod tests {
let bucket = "test-bucket";
let key = "test-key";
let kms_key_id = "default";
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
// Generate two different keys (with different nonces)
let (data_key1, _) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.generate_sse_dek(&context, kms_key_id)
.await
.expect("Failed to generate DEK 1");
let (data_key2, _) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.generate_sse_dek(&context, kms_key_id)
.await
.expect("Failed to generate DEK 2");
@@ -201,10 +205,11 @@ mod tests {
let bucket = "test-bucket";
let key = "test-key";
let kms_key_id = "default";
let context = ObjectEncryptionContext::new(bucket.to_string(), key.to_string());
// Step 1: Generate DEK and get encrypted DEK
let (data_key, encrypted_dek) = provider
.generate_sse_dek(bucket, key, kms_key_id)
.generate_sse_dek(&context, kms_key_id)
.await
.expect("Failed to generate DEK");
@@ -213,7 +218,7 @@ mod tests {
// Step 2: Later, decrypt the DEK (simulating GET operation)
let decrypted_plaintext_key = provider
.decrypt_sse_dek(&encrypted_dek, kms_key_id)
.decrypt_sse_dek(&encrypted_dek, kms_key_id, &context)
.await
.expect("Failed to decrypt DEK");
scripts/s3-tests/run.sh
+17
View File
@@ -23,6 +23,12 @@ export no_proxy="${NO_PROXY}"
# Ensure user-level Python scripts are discoverable (awscurl/tox installed via --user)
PYTHON_VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" 2>/dev/null || echo "3.14")
export PATH="$HOME/Library/Python/${PYTHON_VERSION}/bin:$HOME/.local/bin:$PATH"
if command -v uv >/dev/null 2>&1; then
UV_TOOL_BIN="$(uv tool dir --bin 2>/dev/null || true)"
if [ -n "${UV_TOOL_BIN}" ]; then
export PATH="${UV_TOOL_BIN}:$PATH"
fi
fi
# Configuration
S3_ACCESS_KEY="${S3_ACCESS_KEY:-rustfsadmin}"
@@ -712,6 +718,17 @@ install_python_package() {
local package=$1
local error_output
if command -v uv >/dev/null 2>&1; then
if uv tool install --upgrade "${package}"; then
UV_TOOL_BIN="$(uv tool dir --bin 2>/dev/null || true)"
if [ -n "${UV_TOOL_BIN}" ]; then
export PATH="${UV_TOOL_BIN}:$PATH"
fi
return 0
fi
log_warn "Failed to install ${package} with uv, falling back to python3 -m pip"
fi
# Try --user first (works on most Linux systems)
error_output=$(python3 -m pip install --user --upgrade pip "${package}" 2>&1)
if [ $? -eq 0 ]; then