diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 6e8cc1d06..6003efcec 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -324,11 +324,18 @@ impl Default for LocalConfig { /// wraps data encryption keys — there is no HMAC-SHA256 derivation step — and /// each wrapped DEK is serialized as a RustFS `DataKeyEnvelope` JSON blob. /// -/// This mirrors the *concept* of MinIO's builtin/static single-key KMS, but is -/// not wire-compatible with it: MinIO wraps DEKs in a different (`{"aead": ...}`) -/// blob that this backend neither produces nor accepts, so KMS ciphertext -/// written by MinIO cannot be opened here. Reading MinIO-written SSE objects is -/// tracked separately in rustfs/backlog#1638. +/// This mirrors the *concept* of MinIO's builtin/static single-key KMS but is +/// not wire-compatible with it. MinIO seals a DEK as `sealed || iv[16] || +/// nonce[12]` under a per-ciphertext key derived from the master secret, with +/// a legacy JSON encoding of the same layout; this backend neither produces +/// nor accepts either, so pointing it at MinIO's master key does **not** make +/// MinIO-written objects readable through it. +/// +/// Reading MinIO-written SSE objects is a property of the object read path, not +/// of this backend: that path decodes MinIO's format directly, keyed by +/// `RUSTFS_SSE_S3_MASTER_KEY`. See the migration section of +/// `docs/operations/kms-backend-security.md` for which object shapes are +/// covered, and rustfs/backlog#1638 for the remainder. #[derive(Clone, Default, Serialize, Deserialize)] pub struct StaticConfig { /// Key identifier (name) for the single configured key diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 3f1dfdc7e..11a73dfae 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -14,24 +14,39 @@ For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vaul | Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | | AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments already rooted in AWS IAM that want AWS as the cryptographic root — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first | -## Migrating from MinIO: encrypted objects do not carry over +## Migrating from MinIO: what carries over, and what does not -> **Warning: RustFS does not currently support reading objects that MinIO encrypted.** -> This applies to SSE-S3, SSE-KMS, and SSE-C, in every released binary and container image, and it holds regardless of which KMS backend you configure. Configuring the `Static` backend with the same key material MinIO used does **not** make those objects readable — MinIO wraps data keys in a different envelope format that no RustFS backend produces or accepts (`crates/kms/src/config.rs:304-308`). Plan for this **before** moving data. Tracked in rustfs/backlog#1638. +> **Read this before moving data.** Some MinIO-encrypted objects are readable by RustFS and some are not, and the boundary is not where you would guess. Verify against a sample of your own objects rather than assuming either answer. Tracked in rustfs/backlog#1638. -The read does fail closed — ciphertext is never served as plaintext. MinIO's internal encryption headers mark the object as encrypted (`crates/utils/src/http/header_compat.rs:50-67`), so the read path demands encryption material and refuses when none resolves (`crates/ecstore/src/object_api/readers.rs:559-568`). Two properties still make the problem easy to discover late: +Support is stated per shape below because that is how far it has been *measured* — against fixtures captured from a real MinIO server (`minio/minio:RELEASE.2025-09-07T16-13-09Z`), not inferred from the code: + +| MinIO object | RustFS read | Evidence | +| --- | --- | --- | +| SSE-S3, multipart | **Yes** | `reads_minio_generated_sse_s3_multipart_fixture` | +| SSE-KMS, multipart | **Yes** | `reads_minio_generated_sse_kms_multipart_fixture` | +| SSE-S3 / SSE-KMS, single-part | **Unverified** | No fixture coverage — see below | +| SSE-C | **Unverified** | No fixture coverage | +| Sealed by KES, a KMS plugin, or MinKMS | **No**, and not planned | Re-encrypt at the source before migrating | + +Reading a supported object requires RustFS to hold the same master key MinIO used, supplied through `RUSTFS_SSE_S3_MASTER_KEY` (the production entry point, exercised by `reads_minio_generated_sse_s3_fixture_through_production_master_key_env`). MinIO's builtin KMS derives a per-ciphertext sealing key from that master secret, so the *same* secret is required — not merely an equivalently configured backend. + +**"Unverified" means unknown, not broken.** Single-part objects below MinIO's small-file threshold carry their data inline in `xl.meta`, sharded across disks, and the interop fixture harness cannot yet load that shape — so those objects have never been read in a test either way. Do not read the table's "Yes" rows as covering them. + +Whatever the table says, verify before you commit: **read a sample of encrypted objects, not just their listings.** A read that is not supported fails closed — ciphertext is never served as plaintext — but two properties still make it easy to discover late: - **The error does not say what happened.** It surfaces as a 500 `InternalError`, which reads as a RustFS fault rather than "another implementation encrypted this object". - **Surrounding metadata migrates fine.** The object's `xl.meta` parses, so encrypted objects list and HEAD normally and report plausible sizes. The failure appears only when something reads the payload. -Read a sample of encrypted objects, not just their listings, before decommissioning the MinIO deployment. - -Current options for a migration whose source contains encrypted objects: +For any shape that does not read, the options are unchanged: - Decrypt on the MinIO side first, migrate plaintext, then let RustFS re-encrypt with its own KMS. - Copy through the S3 API rather than moving drives — MinIO decrypts on read, and RustFS encrypts on write. This re-encrypts rather than preserving ciphertext and costs a full data transfer. - Leave encrypted objects on MinIO and migrate only unencrypted data. +### The reverse direction does not work + +MinIO cannot read objects RustFS encrypted, and that is a deliberate, documented position rather than a gap awaiting a fix. RustFS fills MinIO's metadata slots — the sealed-key and IV headers are MinIO-shaped — but the data key in `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key` is a RustFS envelope, which MinIO's KMS cannot open. **Treat the MinIO-branded headers on a RustFS-written object as RustFS-internal.** Their presence is not a statement that MinIO can read the object, and no coexistence plan should assume two-way reads. + Inventory the source before choosing: bucket default-encryption settings mean objects can be encrypted without any client having sent SSE headers. The same limitation applies in reverse — objects RustFS encrypts are not readable by MinIO. For the code-level breakdown of which seams block each SSE mode, see [MinIO file-format interoperability, Part C](../architecture/minio-file-format-compat.md#part-c--server-side-encryption-sse). diff --git a/rustfs/src/storage/minio_generated_read_test.rs b/rustfs/src/storage/minio_generated_read_test.rs index 1937eff36..dc51e985e 100644 --- a/rustfs/src/storage/minio_generated_read_test.rs +++ b/rustfs/src/storage/minio_generated_read_test.rs @@ -260,6 +260,20 @@ async fn reads_minio_generated_sse_kms_multipart_fixture() { assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await; } +/// The interop claim must hold on the production key entry point, not only on +/// the test-only injection channel every other case here uses. +#[tokio::test] +#[ignore = "requires generated MinIO fixture data and a local static KMS key"] +async fn reads_minio_generated_sse_s3_fixture_through_production_master_key_env() { + let (object_info, encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await; + + let plaintext = read_fixture_plaintext_via_production_env(encrypted, object_info, minio_static_kms_key_b64()) + .await + .expect("fixture must restore through RUSTFS_SSE_S3_MASTER_KEY"); + + assert_eq!(sha256_hex(&plaintext), expected_sha256); +} + #[tokio::test] #[ignore = "requires generated MinIO fixture data and a local static KMS key"] async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() { @@ -288,6 +302,55 @@ async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() { } } +/// Read a fixture through the **production** provider selection. +/// +/// [`read_fixture_plaintext`] injects the master key through +/// `__RUSTFS_SSE_SIMPLE_CMK`, which is `#[cfg(test)]`-only, so on its own it +/// proves nothing about a deployment: it never reaches +/// `LocalSseDekProvider::new_from_env`. This variant sets only +/// `RUSTFS_SSE_S3_MASTER_KEY` — the sole production entry point — so the +/// interop claim rests on the path operators actually run (backlog#1638). +async fn read_fixture_plaintext_via_production_env( + encrypted: Vec, + object_info: ObjectInfo, + master_key_b64: String, +) -> Result, String> { + let object_size = object_info.size; + reset_sse_dek_provider(); + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", None::), + ("RUSTFS_SSE_S3_MASTER_KEY", Some(master_key_b64)), + ], + async move { + let resolver = SseObjectEncryptionResolver; + let (mut reader, offset, length) = GetObjectReader::new_with_resolver( + Box::new(Cursor::new(encrypted)), + None, + &object_info, + &ObjectOptions::default(), + &http::HeaderMap::new(), + Some(&resolver), + ) + .await + .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?; + + if offset != 0 || length != object_size { + return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}")); + } + + let mut plaintext = Vec::new(); + reader + .read_to_end(&mut plaintext) + .await + .map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?; + Ok(plaintext) + }, + ) + .await +} + async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) { let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await; // `ObjectInfo.size` is the on-disk size. For SSE objects that is the