From 035a6f431a9b9c1481f5ddf67c5bac4b9c2068d7 Mon Sep 17 00:00:00 2001 From: overtrue Date: Wed, 19 Aug 2026 08:36:18 +0800 Subject: [PATCH] docs(sse): state MinIO interop by measured shape, and pin the production key path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration warning said RustFS cannot read MinIO-encrypted objects at all. That is no longer true for the shapes this branch fixes, and a blanket 'no' costs a migrating operator a full decrypt-and-recopy they may not need. It is replaced by a table that states support per object shape, with the test that establishes each row. The table's honesty depends on one caveat worth stating in code rather than prose: every interop reader test injected its master key through __RUSTFS_SSE_SIMPLE_CMK, which is #[cfg(test)]-only and never reaches LocalSseDekProvider::new_from_env. Those tests going green therefore said nothing about a deployment. A new case reads a fixture through RUSTFS_SSE_S3_MASTER_KEY alone — the sole production entry point — so the claim now rests on the path operators actually run. Single-part and SSE-C objects are listed as unverified rather than unsupported, because the fixture harness cannot yet load them (small objects live inline in xl.meta, sharded across disks) and they have never been read in a test either way. Conflating 'untested' with 'broken' in the other direction would be the same failure the old warning made. The reverse direction gets an explicit ruling: MinIO cannot read RustFS-written objects, and the MinIO-branded headers RustFS writes are RustFS-internal rather than a compatibility promise — no coexistence plan should assume two-way reads. StaticConfig's doc comment is corrected on the same basis: it described MinIO's ciphertext as the legacy JSON encoding, and claimed MinIO-written objects cannot be read at all, which now belongs to the object read path rather than to this backend. Refs rustfs/backlog#1638. --- crates/kms/src/config.rs | 17 +++-- docs/operations/kms-backend-security.md | 29 ++++++--- .../src/storage/minio_generated_read_test.rs | 63 +++++++++++++++++++ 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 6e8cc1d06..6003efcec 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -324,11 +324,18 @@ impl Default for LocalConfig { /// wraps data encryption keys — there is no HMAC-SHA256 derivation step — and /// each wrapped DEK is serialized as a RustFS `DataKeyEnvelope` JSON blob. /// -/// This mirrors the *concept* of MinIO's builtin/static single-key KMS, but is -/// not wire-compatible with it: MinIO wraps DEKs in a different (`{"aead": ...}`) -/// blob that this backend neither produces nor accepts, so KMS ciphertext -/// written by MinIO cannot be opened here. Reading MinIO-written SSE objects is -/// tracked separately in rustfs/backlog#1638. +/// This mirrors the *concept* of MinIO's builtin/static single-key KMS but is +/// not wire-compatible with it. MinIO seals a DEK as `sealed || iv[16] || +/// nonce[12]` under a per-ciphertext key derived from the master secret, with +/// a legacy JSON encoding of the same layout; this backend neither produces +/// nor accepts either, so pointing it at MinIO's master key does **not** make +/// MinIO-written objects readable through it. +/// +/// Reading MinIO-written SSE objects is a property of the object read path, not +/// of this backend: that path decodes MinIO's format directly, keyed by +/// `RUSTFS_SSE_S3_MASTER_KEY`. See the migration section of +/// `docs/operations/kms-backend-security.md` for which object shapes are +/// covered, and rustfs/backlog#1638 for the remainder. #[derive(Clone, Default, Serialize, Deserialize)] pub struct StaticConfig { /// Key identifier (name) for the single configured key diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 3f1dfdc7e..11a73dfae 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -14,24 +14,39 @@ For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vaul | Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | | AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments already rooted in AWS IAM that want AWS as the cryptographic root — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first | -## Migrating from MinIO: encrypted objects do not carry over +## Migrating from MinIO: what carries over, and what does not -> **Warning: RustFS does not currently support reading objects that MinIO encrypted.** -> This applies to SSE-S3, SSE-KMS, and SSE-C, in every released binary and container image, and it holds regardless of which KMS backend you configure. Configuring the `Static` backend with the same key material MinIO used does **not** make those objects readable — MinIO wraps data keys in a different envelope format that no RustFS backend produces or accepts (`crates/kms/src/config.rs:304-308`). Plan for this **before** moving data. Tracked in rustfs/backlog#1638. +> **Read this before moving data.** Some MinIO-encrypted objects are readable by RustFS and some are not, and the boundary is not where you would guess. Verify against a sample of your own objects rather than assuming either answer. Tracked in rustfs/backlog#1638. -The read does fail closed — ciphertext is never served as plaintext. MinIO's internal encryption headers mark the object as encrypted (`crates/utils/src/http/header_compat.rs:50-67`), so the read path demands encryption material and refuses when none resolves (`crates/ecstore/src/object_api/readers.rs:559-568`). Two properties still make the problem easy to discover late: +Support is stated per shape below because that is how far it has been *measured* — against fixtures captured from a real MinIO server (`minio/minio:RELEASE.2025-09-07T16-13-09Z`), not inferred from the code: + +| MinIO object | RustFS read | Evidence | +| --- | --- | --- | +| SSE-S3, multipart | **Yes** | `reads_minio_generated_sse_s3_multipart_fixture` | +| SSE-KMS, multipart | **Yes** | `reads_minio_generated_sse_kms_multipart_fixture` | +| SSE-S3 / SSE-KMS, single-part | **Unverified** | No fixture coverage — see below | +| SSE-C | **Unverified** | No fixture coverage | +| Sealed by KES, a KMS plugin, or MinKMS | **No**, and not planned | Re-encrypt at the source before migrating | + +Reading a supported object requires RustFS to hold the same master key MinIO used, supplied through `RUSTFS_SSE_S3_MASTER_KEY` (the production entry point, exercised by `reads_minio_generated_sse_s3_fixture_through_production_master_key_env`). MinIO's builtin KMS derives a per-ciphertext sealing key from that master secret, so the *same* secret is required — not merely an equivalently configured backend. + +**"Unverified" means unknown, not broken.** Single-part objects below MinIO's small-file threshold carry their data inline in `xl.meta`, sharded across disks, and the interop fixture harness cannot yet load that shape — so those objects have never been read in a test either way. Do not read the table's "Yes" rows as covering them. + +Whatever the table says, verify before you commit: **read a sample of encrypted objects, not just their listings.** A read that is not supported fails closed — ciphertext is never served as plaintext — but two properties still make it easy to discover late: - **The error does not say what happened.** It surfaces as a 500 `InternalError`, which reads as a RustFS fault rather than "another implementation encrypted this object". - **Surrounding metadata migrates fine.** The object's `xl.meta` parses, so encrypted objects list and HEAD normally and report plausible sizes. The failure appears only when something reads the payload. -Read a sample of encrypted objects, not just their listings, before decommissioning the MinIO deployment. - -Current options for a migration whose source contains encrypted objects: +For any shape that does not read, the options are unchanged: - Decrypt on the MinIO side first, migrate plaintext, then let RustFS re-encrypt with its own KMS. - Copy through the S3 API rather than moving drives — MinIO decrypts on read, and RustFS encrypts on write. This re-encrypts rather than preserving ciphertext and costs a full data transfer. - Leave encrypted objects on MinIO and migrate only unencrypted data. +### The reverse direction does not work + +MinIO cannot read objects RustFS encrypted, and that is a deliberate, documented position rather than a gap awaiting a fix. RustFS fills MinIO's metadata slots — the sealed-key and IV headers are MinIO-shaped — but the data key in `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key` is a RustFS envelope, which MinIO's KMS cannot open. **Treat the MinIO-branded headers on a RustFS-written object as RustFS-internal.** Their presence is not a statement that MinIO can read the object, and no coexistence plan should assume two-way reads. + Inventory the source before choosing: bucket default-encryption settings mean objects can be encrypted without any client having sent SSE headers. The same limitation applies in reverse — objects RustFS encrypts are not readable by MinIO. For the code-level breakdown of which seams block each SSE mode, see [MinIO file-format interoperability, Part C](../architecture/minio-file-format-compat.md#part-c--server-side-encryption-sse). diff --git a/rustfs/src/storage/minio_generated_read_test.rs b/rustfs/src/storage/minio_generated_read_test.rs index 1937eff36..dc51e985e 100644 --- a/rustfs/src/storage/minio_generated_read_test.rs +++ b/rustfs/src/storage/minio_generated_read_test.rs @@ -260,6 +260,20 @@ async fn reads_minio_generated_sse_kms_multipart_fixture() { assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await; } +/// The interop claim must hold on the production key entry point, not only on +/// the test-only injection channel every other case here uses. +#[tokio::test] +#[ignore = "requires generated MinIO fixture data and a local static KMS key"] +async fn reads_minio_generated_sse_s3_fixture_through_production_master_key_env() { + let (object_info, encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await; + + let plaintext = read_fixture_plaintext_via_production_env(encrypted, object_info, minio_static_kms_key_b64()) + .await + .expect("fixture must restore through RUSTFS_SSE_S3_MASTER_KEY"); + + assert_eq!(sha256_hex(&plaintext), expected_sha256); +} + #[tokio::test] #[ignore = "requires generated MinIO fixture data and a local static KMS key"] async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() { @@ -288,6 +302,55 @@ async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() { } } +/// Read a fixture through the **production** provider selection. +/// +/// [`read_fixture_plaintext`] injects the master key through +/// `__RUSTFS_SSE_SIMPLE_CMK`, which is `#[cfg(test)]`-only, so on its own it +/// proves nothing about a deployment: it never reaches +/// `LocalSseDekProvider::new_from_env`. This variant sets only +/// `RUSTFS_SSE_S3_MASTER_KEY` — the sole production entry point — so the +/// interop claim rests on the path operators actually run (backlog#1638). +async fn read_fixture_plaintext_via_production_env( + encrypted: Vec, + object_info: ObjectInfo, + master_key_b64: String, +) -> Result, String> { + let object_size = object_info.size; + reset_sse_dek_provider(); + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", None::), + ("RUSTFS_SSE_S3_MASTER_KEY", Some(master_key_b64)), + ], + async move { + let resolver = SseObjectEncryptionResolver; + let (mut reader, offset, length) = GetObjectReader::new_with_resolver( + Box::new(Cursor::new(encrypted)), + None, + &object_info, + &ObjectOptions::default(), + &http::HeaderMap::new(), + Some(&resolver), + ) + .await + .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?; + + if offset != 0 || length != object_size { + return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}")); + } + + let mut plaintext = Vec::new(); + reader + .read_to_end(&mut plaintext) + .await + .map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?; + Ok(plaintext) + }, + ) + .await +} + async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) { let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await; // `ObjectInfo.size` is the on-disk size. For SSE objects that is the