diff --git a/.gitignore b/.gitignore index d66fee9fb..f5c8ecfe3 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,7 @@ worktrees/* # Local AI-agent review artifacts (omo evidence dumps) .omo/ + +# insta scratch files; the accepted .snap files ARE the assertions and are committed +*.snap.new +*.pending-snap diff --git a/crates/kms/AGENTS.md b/crates/kms/AGENTS.md index 5555c8071..41e66eb66 100644 --- a/crates/kms/AGENTS.md +++ b/crates/kms/AGENTS.md @@ -26,6 +26,41 @@ NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1 ``` +### Black-box behavior suite and the Vault lane + +`crates/kms/tests/behavior_*.rs` drive the crate through its public entry +points only. By default they run against the Local and Static backends. + +Setting `RUSTFS_KMS_VAULT_TOKEN` adds the Vault KV2 and Vault Transit backends +to every `for_each_backend` spec, against a live server +(`RUSTFS_KMS_VAULT_ADDR`, default `http://127.0.0.1:8200`): + +```bash +NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ +RUSTFS_KMS_VAULT_TOKEN= cargo test -p rustfs-kms +``` + +The server needs a KV v2 engine at `secret/` and a Transit engine at +`transit/`, matching the crate's config defaults. + +**Run the Vault lane whenever you touch rotation or versioning.** `rotate` and +`versioning` are advertised only by the Vault backends, so without it every +capability-gated branch for them takes the `UnsupportedCapability` side and +`behavior_rotation.rs` never asserts the working half — a rotation that dropped +prior key versions would go green. + +The lane creates real keys under unique names (`behavior-kv2-*`, +`behavior-transit-*`) and does not remove them, so a dev Vault accumulates them +across runs. Clear them out periodically — against a dev server only: + +```bash +vault list -format=json transit/keys | jq -r '.[] | select(startswith("behavior-transit-"))' | while read -r k; do vault write "transit/keys/$k/config" deletion_allowed=true >/dev/null && vault delete "transit/keys/$k"; done +``` + +```bash +vault list -format=json secret/metadata/rustfs/kms/keys | jq -r '.[] | select(startswith("behavior-kv2-"))' | xargs -I{} vault kv metadata delete secret/rustfs/kms/keys/{} +``` + ## Local Key Export for SSE-S3 Migration Tests Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index d7e550b5d..ee3be16c5 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -1450,7 +1450,23 @@ impl LocalKmsClient { let key_info = self.describe_key(&request.key_id, context).await?; ensure_key_status_permits(&request.key_id, &key_info.status, StateGatedOperation::Encrypt)?; - let (ciphertext, _nonce) = self.encrypt_with_master_key(&request.key_id, &request.plaintext).await?; + let (encrypted_key, nonce) = self.encrypt_with_master_key(&request.key_id, &request.plaintext).await?; + + // The ciphertext must be the same envelope `decrypt` parses: the nonce + // and the bound context live in it, so handing back the bare AES-GCM + // output would make every `encrypt` result permanently unopenable. + let envelope = DataKeyEnvelope { + key_id: uuid::Uuid::new_v4().to_string(), + master_key_id: request.key_id.clone(), + key_spec: key_info.algorithm.clone(), + encrypted_key, + nonce, + encryption_context: request.encryption_context.clone(), + created_at: Zoned::now(), + // Local rotation is rejected, so the key has a single material version. + master_key_version: None, + }; + let ciphertext = serde_json::to_vec(&envelope)?; Ok(EncryptResponse { ciphertext, @@ -1466,6 +1482,13 @@ impl LocalKmsClient { // Parse the data key envelope from ciphertext let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)?; + // NOTE: this comparison is an authorization check, not a cryptographic + // binding. `DekCrypto` seals only the plaintext, so `encryption_context` + // rides in the envelope unauthenticated: anyone able to rewrite the + // stored envelope can rewrite this field and present a matching context. + // The Static and Vault Transit backends do bind it (as AEAD AAD and as + // the Transit KDF context respectively); closing the gap here needs a + // versioned envelope, since existing ciphertext was sealed without AAD. // Verify encryption context matches // Check that all keys in envelope.encryption_context are present in request.encryption_context // and their values match. This ensures the context used for decryption matches what was used for encryption. @@ -1849,10 +1872,14 @@ impl KmsBackend for LocalKmsBackend { async fn decrypt(&self, request: DecryptRequest) -> Result { let plaintext = self.client.decrypt(&request, None).await?; - // For simplicity, return basic response - in real implementation would extract more info from ciphertext + // The envelope that was just opened names the master key that opened it. + // Reporting "unknown" left every caller unable to tell which key was + // actually used, which is what audit and key-rotation checks read. + let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)?; + Ok(DecryptResponse { plaintext, - key_id: "unknown".to_string(), // Would be extracted from ciphertext metadata + key_id: envelope.master_key_id, encryption_algorithm: Some("AES-256-GCM".to_string()), }) } diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index d28d1fe3f..f6dfa6672 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -26,7 +26,7 @@ use crate::backends::{BackendCapabilities, KmsBackend, empty_key_page, list_keys_page_size}; use crate::config::{BackendConfig, KmsConfig}; -use crate::encryption::DataKeyEnvelope; +use crate::encryption::{DataKeyEnvelope, context_aad}; use crate::error::{KmsError, Result}; use crate::types::*; use aes_gcm::{ @@ -36,7 +36,7 @@ use aes_gcm::{ use async_trait::async_trait; use jiff::Zoned; use rand::RngExt; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use tracing::debug; use zeroize::Zeroizing; @@ -45,11 +45,6 @@ const NONCE_SIZE: usize = 12; /// AES-256 key size in bytes. const KEY_SIZE: usize = 32; -fn context_aad(context: &HashMap) -> Result> { - let canonical: BTreeMap<&str, &str> = context.iter().map(|(key, value)| (key.as_str(), value.as_str())).collect(); - serde_json::to_vec(&canonical).map_err(Into::into) -} - /// Static single-key KMS backend. /// /// Uses a pre-configured AES-256 key to derive data encryption keys. This is a @@ -113,8 +108,18 @@ impl StaticKmsBackend { let mut nonce_bytes = [0u8; NONCE_SIZE]; rand::rng().fill(&mut nonce_bytes[..]); - // Generate 32 random bytes as plaintext DEK - let mut plaintext = [0u8; KEY_SIZE]; + // The requested spec decides the DEK length; a caller that asked for + // AES_128 and silently got 256 bits would build objects whose recorded + // spec does not match their key material. + // Lengths track `KeySpec::key_size`; the request carries the spec as a + // string, so the mapping is repeated here rather than shared. + let key_length = match request.key_spec.as_str() { + "AES_256" | "ChaCha20" => 32, + "AES_128" => 16, + _ => return Err(KmsError::unsupported_algorithm(&request.key_spec)), + }; + + let mut plaintext = vec![0u8; key_length]; rand::rng().fill(&mut plaintext[..]); // Encrypt DEK with AES-256-GCM using the static key directly @@ -127,7 +132,7 @@ impl StaticKmsBackend { .encrypt( &nonce, Payload { - msg: plaintext.as_ref(), + msg: plaintext.as_slice(), aad: &aad, }, ) diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index c928e0f9b..57eabe3d8 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -857,6 +857,13 @@ impl VaultKmsClient { let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext) .map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?; + // NOTE: this comparison is an authorization check, not a cryptographic + // binding. `DekCrypto` seals only the plaintext, so `encryption_context` + // rides in the envelope unauthenticated: anyone able to rewrite the + // stored envelope can rewrite this field and present a matching context. + // The Static and Vault Transit backends do bind it (as AEAD AAD and as + // the Transit KDF context respectively); closing the gap here needs a + // versioned envelope, since existing ciphertext was sealed without AAD. // Verify encryption context matches // Check that all keys in envelope.encryption_context are present in request.encryption_context // and their values match. This ensures the context used for decryption matches what was used for encryption. @@ -1580,9 +1587,14 @@ impl KmsBackend for VaultKmsBackend { async fn decrypt(&self, request: DecryptRequest) -> Result { let plaintext = self.client.decrypt(&request, None).await?; + // The envelope that was just opened names the master key that opened it. + // Reporting "unknown" left every caller unable to tell which key was + // actually used, which is what audit and key-rotation checks read. + let envelope: DataKeyEnvelope = serde_json::from_slice(&request.ciphertext)?; + Ok(DecryptResponse { plaintext, - key_id: "unknown".to_string(), // Would be extracted from ciphertext metadata + key_id: envelope.master_key_id, encryption_algorithm: Some("AES-256-GCM".to_string()), }) } diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 7d5d9b0e5..4e0797625 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -772,7 +772,7 @@ impl VaultTransitKmsClient { let metadata = self .ensure_key_state_allows(&request.key_id, StateGatedOperation::Encrypt) .await?; - let ciphertext = match self + let encrypted = match self .transit_encrypt(&request.key_id, &request.plaintext, &request.encryption_context) .await { @@ -783,8 +783,25 @@ impl VaultTransitKmsClient { } }; + // The ciphertext must be the same envelope `decrypt` parses — it is what + // carries the key id and the bound context. Returning the bare Transit + // string made every `encrypt` result permanently unopenable. + let envelope = DataKeyEnvelope { + key_id: uuid::Uuid::new_v4().to_string(), + master_key_id: request.key_id.clone(), + key_spec: "AES_256".to_string(), + encrypted_key: encrypted.into_bytes(), + nonce: Vec::new(), + encryption_context: request.encryption_context.clone(), + created_at: Zoned::now(), + // Transit ciphertext already self-describes its key version + // ("vault:vN:..."), so the envelope never carries one. + master_key_version: None, + }; + let ciphertext = serde_json::to_vec(&envelope)?; + Ok(EncryptResponse { - ciphertext: ciphertext.into_bytes(), + ciphertext, key_id: request.key_id.clone(), key_version: metadata.current_version, algorithm: "vault-transit".to_string(), @@ -1698,7 +1715,9 @@ mod tests { ) .await .expect("encrypt must retry past a transient 429"); - assert_eq!(response.ciphertext, b"vault:v1:scripted".to_vec()); + let envelope: DataKeyEnvelope = serde_json::from_slice(&response.ciphertext).expect("encrypt must return an envelope"); + assert_eq!(envelope.encrypted_key, b"vault:v1:scripted".to_vec()); + assert_eq!(envelope.master_key_id, "wired-key"); let requests = vault.requests(); assert_eq!(requests.len(), 3, "metadata read plus two encrypt attempts: {requests:?}"); diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 97fce0772..849f7c2e5 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -1833,6 +1833,10 @@ mod tests { ("RUSTFS_KMS_BACKEND", Some("vault")), ("RUSTFS_KMS_VAULT_ADDRESS", Some("https://vault.example.com")), (ENV_KMS_VAULT_TOKEN_FILE, Some("/run/vault-agent/token")), + // Cleared explicitly: a static token in the ambient environment + // outranks the token file, so leaving it up to the caller's shell + // would make this assertion depend on who runs the test. + ("RUSTFS_KMS_VAULT_TOKEN", None), ], || { let config = KmsConfig::from_env().expect("kms config should load from env"); diff --git a/crates/kms/src/encryption/dek.rs b/crates/kms/src/encryption/dek.rs index f1e98f755..ca3f2658d 100644 --- a/crates/kms/src/encryption/dek.rs +++ b/crates/kms/src/encryption/dek.rs @@ -27,7 +27,7 @@ use jiff::Zoned; use rand::Rng; use serde::de::{self, IgnoredAny, MapAccess, Visitor}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::sync::atomic::{AtomicU64, Ordering}; @@ -229,6 +229,23 @@ struct DataKeyEnvelopeMarker { _created_at: IgnoredAny, } +/// Serialize an encryption context into deterministic AAD bytes. +/// +/// The AAD has to be reproducible byte-for-byte at decrypt time. A `HashMap` +/// serializes in its own iteration order, which differs between instances — so +/// a context rebuilt from storage (or from headers) would produce different +/// bytes than the one used to seal, and the sealed data would never open +/// again. Ordering by key removes that dependency. +/// +/// Shared by every layer that binds a context as additional data. It lives +/// here rather than beside one caller because a second, subtly different copy +/// is exactly how the object layer ended up serializing a `HashMap` directly +/// while the Static backend was already canonicalizing. +pub fn context_aad(context: &HashMap) -> Result> { + let canonical: BTreeMap<&str, &str> = context.iter().map(|(key, value)| (key.as_str(), value.as_str())).collect(); + serde_json::to_vec(&canonical).map_err(Into::into) +} + /// Returns whether ciphertext is a RustFS KMS data-key envelope. pub fn is_data_key_envelope(ciphertext: &[u8]) -> bool { ciphertext.iter().copied().find(|byte| !byte.is_ascii_whitespace()) == Some(b'{') diff --git a/crates/kms/src/encryption/mod.rs b/crates/kms/src/encryption/mod.rs index d18a1df34..fff0073a2 100644 --- a/crates/kms/src/encryption/mod.rs +++ b/crates/kms/src/encryption/mod.rs @@ -17,4 +17,4 @@ pub mod ciphers; pub mod dek; -pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material, is_data_key_envelope}; +pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, context_aad, generate_key_material, is_data_key_envelope}; diff --git a/crates/kms/src/service.rs b/crates/kms/src/service.rs index 400ccbee8..a0b1bfb95 100644 --- a/crates/kms/src/service.rs +++ b/crates/kms/src/service.rs @@ -19,6 +19,7 @@ use crate::api_types::{ }; use crate::cache::KmsCacheStats; use crate::encryption::ciphers::{create_cipher, generate_iv}; +use crate::encryption::context_aad; use crate::error::{KmsError, Result}; use crate::manager::KmsManager; use crate::types::*; @@ -82,6 +83,15 @@ fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap stored.clone(), + None => context_aad(&metadata.encryption_context)?, + }; // Get tag from metadata let tag = metadata @@ -634,7 +653,7 @@ impl ObjectEncryptionService { ("sse_type".to_string(), "customer".to_string()), ]); - let aad = serde_json::to_vec(&context)?; + let aad = context_aad(&context)?; // Encrypt the data let (ciphertext, tag) = cipher.encrypt(&data, &iv, &aad)?; @@ -648,6 +667,9 @@ impl ObjectEncryptionService { tag: Some(tag), encryption_context: context, encrypted_at: Zoned::now(), + // Pinned to the bytes actually fed to the AEAD, so the projection + // below can store them verbatim instead of re-deriving them. + context_aad: Some(aad), original_size, encrypted_data_key: Vec::new(), // Empty for SSE-C }; @@ -702,7 +724,13 @@ impl ObjectEncryptionService { let cipher = create_cipher(&algorithm, customer_key)?; // Build AAD from encryption context - let aad = serde_json::to_vec(&metadata.encryption_context)?; + // Prefer the bytes the object was sealed under. Deriving them from the + // parsed map would re-order a pre-canonicalization context and fail the + // AEAD on an object that is otherwise perfectly readable. + let aad = match metadata.context_aad.as_ref() { + Some(stored) => stored.clone(), + None => context_aad(&metadata.encryption_context)?, + }; // Get tag from metadata let tag = metadata @@ -774,6 +802,9 @@ impl ObjectEncryptionService { headers.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), metadata.key_id.clone()); } + // Record the cipher separately from the SSE mode advertised above. + headers.insert(INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), metadata.algorithm.clone()); + // Internal headers for decryption headers.insert( "x-rustfs-encryption-iv".to_string(), @@ -792,9 +823,16 @@ impl ObjectEncryptionService { base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key), ); + // Whatever the object was sealed under is what gets stored: for a + // pre-canonicalization object that is its original ordering, which must + // survive a re-projection rather than being rewritten into sorted form. + let context_bytes = match metadata.context_aad.as_ref() { + Some(stored) => stored.clone(), + None => context_aad(&metadata.encryption_context).unwrap_or_default(), + }; headers.insert( "x-rustfs-encryption-context".to_string(), - serde_json::to_string(&metadata.encryption_context).unwrap_or_default(), + String::from_utf8_lossy(&context_bytes).into_owned(), ); headers @@ -809,18 +847,27 @@ impl ObjectEncryptionService { /// EncryptionMetadata parsed from headers /// pub fn headers_to_metadata(&self, headers: &HashMap) -> Result { - let algorithm = headers + let sse_mode = headers .get("x-amz-server-side-encryption") .ok_or_else(|| KmsError::validation_error("Missing encryption algorithm header"))? .clone(); - let key_id = if algorithm == "AES256" && headers.contains_key("x-amz-server-side-encryption-customer-algorithm") { + // Prefer the recorded cipher; fall back to the SSE mode for objects + // written before that header existed, where `AES256`/`aws:kms` was the + // only thing stored and AES-256-GCM was the only cipher in use. + let algorithm = match headers.get(INTERNAL_ENCRYPTION_ALGORITHM_HEADER) { + Some(algorithm) => algorithm.clone(), + None if sse_mode == "aws:kms" => EncryptionAlgorithm::Aes256.as_str().to_string(), + None => sse_mode.clone(), + }; + + let key_id = if sse_mode == "AES256" && headers.contains_key("x-amz-server-side-encryption-customer-algorithm") { "sse-c".to_string() } else if let Some(key_id) = headers.get(INTERNAL_ENCRYPTION_KEY_ID_HEADER) { key_id.clone() } else if let Some(kms_key_id) = headers.get("x-amz-server-side-encryption-aws-kms-key-id") { kms_key_id.clone() - } else if algorithm == "AES256" { + } else if sse_mode == "AES256" { self.get_default_key_id() .cloned() .ok_or_else(|| KmsError::validation_error("Missing key ID"))? @@ -853,11 +900,17 @@ impl ObjectEncryptionService { Vec::new() // Empty for SSE-C }; - let encryption_context = if let Some(context_str) = headers.get("x-rustfs-encryption-context") { - serde_json::from_str(context_str) - .map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {e}")))? - } else { - HashMap::new() + // The stored string is the AAD verbatim. It is parsed into a map for + // callers that inspect the context, but the bytes are carried through + // untouched: re-serializing the parsed map is exactly how the original + // ordering — and with it the ability to open the object — was lost. + let (encryption_context, context_aad) = match headers.get("x-rustfs-encryption-context") { + Some(context_str) => ( + serde_json::from_str(context_str) + .map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {e}")))?, + Some(context_str.as_bytes().to_vec()), + ), + None => (HashMap::new(), None), }; Ok(EncryptionMetadata { @@ -870,6 +923,7 @@ impl ObjectEncryptionService { encrypted_at: Zoned::now(), original_size: 0, // Not available from headers encrypted_data_key, + context_aad, }) } } @@ -986,6 +1040,8 @@ mod tests { encrypted_at: Zoned::now(), original_size: 100, encrypted_data_key: vec![1, 2, 3, 4], + // A hand-built record with no sealed bytes to defer to. + context_aad: None, }; // Convert to headers diff --git a/crates/kms/src/types.rs b/crates/kms/src/types.rs index a19eae996..484df55f6 100644 --- a/crates/kms/src/types.rs +++ b/crates/kms/src/types.rs @@ -636,6 +636,20 @@ pub struct EncryptionMetadata { pub original_size: u64, /// Encrypted data key pub encrypted_data_key: Vec, + /// The exact AAD bytes this object was sealed under. + /// + /// The AAD is the serialized encryption context, and the serialization is + /// what must be reproduced byte-for-byte — not the map. Objects written + /// before the context was canonicalized carry whichever `HashMap` order + /// happened to be in effect when they were sealed, and + /// `x-rustfs-encryption-context` preserves that exact byte sequence. It is + /// therefore recoverable, but only while it is never round-tripped through + /// a `HashMap` and re-serialized. + /// + /// `None` means "derive it from `encryption_context`", which is correct + /// only when no stored serialization exists to defer to. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context_aad: Option>, } /// Health status information diff --git a/crates/kms/tests/behavior_backup.rs b/crates/kms/tests/behavior_backup.rs new file mode 100644 index 000000000..598ec4eca --- /dev/null +++ b/crates/kms/tests/behavior_backup.rs @@ -0,0 +1,498 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: exporting the Local backend as a sealed backup bundle. +//! +//! A backup is only worth having if it is *provably* restorable, so the bundle +//! format is fail-closed at every step. The properties asserted here: +//! +//! * **Nothing leaves unwrapped.** Every artifact is AEAD-encrypted under a +//! backup KEK that is deliberately outside the KMS hierarchy — including +//! plaintext-dev-only key records, whose material would otherwise be readable +//! straight out of the bundle. +//! * **Sealing is all-or-nothing.** The manifest is written last and carries a +//! completeness marker plus a digest over its own canonical bytes, so an +//! interrupted export is permanently non-restorable rather than subtly short. +//! * **The KEK is checked before anything else.** Presenting the wrong KEK +//! identity fails on identity, not on a decryption error, so a mismatch is +//! diagnosable rather than looking like corruption. +//! * **Tampering is detected.** The manifest digest covers the manifest, and +//! each artifact carries a digest of its *encrypted* bytes, so verification +//! never needs the KEK. + +mod common; + +use std::path::{Path, PathBuf}; + +use common::{TestKms, without_probe_key}; +use rustfs_kms::backends::local::LocalKmsClient; +use rustfs_kms::backup::{ + ArtifactKind, BackupError, BackupKek, BackupManifest, CompletenessState, ContentDigest, LOCAL_BUNDLE_MANIFEST_FILE, + LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup, read_local_bundle_manifest, +}; +use rustfs_kms::{KmsError, LocalConfig}; +use tempfile::TempDir; + +const KEK_ID: &str = "backup-kek"; +const KEK_VERSION: u32 = 3; + +fn kek() -> BackupKek { + BackupKek::new(KEK_ID, KEK_VERSION, [0x7eu8; 32]).expect("KEK should build") +} + +fn export_request(destination: PathBuf) -> LocalBackupExportRequest { + LocalBackupExportRequest { + backup_id: "backup-behavior-001".to_string(), + deployment_identity: "deployment-under-test".to_string(), + rustfs_version: "0.0.0-behavior".to_string(), + snapshot_generation: 7, + destination, + // These specs cover the key-material path; the sanitized configuration + // is the admin layer's own artifact and is exercised separately. + sanitized_config: None, + } +} + +/// A KMS with a few keys, plus a client over the same directory — the shape an +/// admin-layer export takes. +async fn seeded_kms(key_names: &[&str]) -> (TestKms, LocalKmsClient) { + let kms = TestKms::local().await; + for name in key_names { + kms.create_key(name).await; + } + let key_dir = kms.key_dir().expect("local backend has a key dir"); + let client = LocalKmsClient::new(LocalConfig { + key_dir, + master_key: None, + file_permissions: Some(0o600), + }) + .await + .expect("client over the same key directory"); + (kms, client) +} + +async fn read_bytes(path: &Path) -> Vec { + tokio::fs::read(path).await.expect("bundle file should be readable") +} + +#[tokio::test] +async fn a_sealed_bundle_describes_and_yields_its_contents() { + let (_kms, client) = seeded_kms(&["backup-a", "backup-b"]).await; + let out = TempDir::new().expect("temp dir"); + let bundle_dir = out.path().join("bundle"); + + let manifest = export_local_backup(&client, &kek(), &export_request(bundle_dir.clone())) + .await + .expect("export should succeed"); + + // --- manifest identity ------------------------------------------------ + assert_eq!(manifest.format_version, BackupManifest::FORMAT_VERSION); + assert_eq!(manifest.backup_id, "backup-behavior-001"); + assert_eq!(manifest.deployment_identity, "deployment-under-test"); + assert_eq!(manifest.rustfs_version, "0.0.0-behavior"); + assert_eq!( + manifest.snapshot_generation, 7, + "the caller-supplied generation must be recorded verbatim" + ); + assert_eq!(manifest.completeness, CompletenessState::Complete, "a returned manifest must be sealed"); + assert_eq!(manifest.backup_kek.kek_id, KEK_ID, "the manifest records the KEK identity"); + assert_eq!(manifest.backup_kek.kek_version, KEK_VERSION); + assert!( + manifest.local_kdf.is_some(), + "a Local bundle must record its KDF parameters so a restore can detect drift" + ); + assert!( + manifest.key_versions.is_none() && manifest.capability_discovery.is_none(), + "reserved slots must stay empty in format version 1" + ); + manifest.validate().expect("the produced manifest must validate"); + manifest.verify_digest().expect("the produced manifest's digest must verify"); + + // --- what was read back off disk agrees ------------------------------- + let from_disk = read_local_bundle_manifest(&bundle_dir) + .await + .expect("the written manifest must read back"); + assert_eq!(from_disk, manifest, "the manifest on disk must equal the returned one"); + + // --- artifacts -------------------------------------------------------- + assert!(!manifest.artifacts.is_empty(), "a bundle must carry artifacts"); + let key_material = manifest + .require_artifact(ArtifactKind::KeyMaterial) + .expect("a Local bundle must carry key material"); + assert!(!key_material.path.starts_with('/'), "artifact paths must stay bundle-relative"); + assert!(!key_material.path.contains(".."), "artifact paths must not traverse"); + + for artifact in &manifest.artifacts { + assert!(!artifact.kind.is_reserved(), "reserved artifact kinds must not be produced"); + let payload = read_bytes(&bundle_dir.join(&artifact.path)).await; + assert_eq!( + payload.len() as u64, + artifact.len, + "artifact {} must be exactly the declared length", + artifact.path + ); + assert_eq!( + ContentDigest::sha256_of(&payload), + artifact.encrypted_digest, + "artifact {} must match its declared digest of the *encrypted* bytes", + artifact.path + ); + + // The KEK opens it, and the plaintext is real content. + let plaintext = decrypt_bundle_artifact(&bundle_dir, &manifest, artifact, &kek()) + .await + .unwrap_or_else(|error| panic!("artifact {} must decrypt under the right KEK: {error:?}", artifact.path)); + assert!(!plaintext.is_empty(), "artifact {} decrypted to nothing", artifact.path); + } + + // Key material is one artifact per key, so every key must be represented + // and every artifact must decrypt to the record it claims. + let exported = exported_key_ids(&bundle_dir, &manifest).await; + assert_eq!( + exported, + vec!["backup-a".to_string(), "backup-b".to_string()], + "every key must be exported exactly once" + ); + for key_id in ["backup-a", "backup-b"] { + assert!( + manifest + .artifacts + .iter() + .any(|artifact| artifact.kind == ArtifactKind::KeyMaterial && artifact.path.contains(key_id)), + "the bundle must carry a dedicated key-material artifact for {key_id}" + ); + } +} + +/// Decrypt every key-material artifact and return the key ids it contains, +/// sorted, so a bundle's coverage can be asserted as a set. +async fn exported_key_ids(bundle_dir: &Path, manifest: &BackupManifest) -> Vec { + let mut found = Vec::new(); + for artifact in manifest + .artifacts + .iter() + .filter(|artifact| artifact.kind == ArtifactKind::KeyMaterial) + { + let plaintext = decrypt_bundle_artifact(bundle_dir, manifest, artifact, &kek()) + .await + .unwrap_or_else(|error| panic!("artifact {} must decrypt: {error:?}", artifact.path)); + let record: serde_json::Value = serde_json::from_slice(&plaintext) + .unwrap_or_else(|error| panic!("artifact {} must be a key record: {error:?}", artifact.path)); + let key_id = record + .get("key_id") + .and_then(|value| value.as_str()) + .unwrap_or_else(|| panic!("artifact {} record has no key_id", artifact.path)); + found.push(key_id.to_string()); + } + found.sort(); + // The probe key is exported like any other — correctly so — but these specs + // assert over the keys they seeded. + without_probe_key(found) +} + +#[tokio::test] +async fn nothing_readable_leaves_the_bundle_unwrapped() { + // Dev-mode Local records store key material in the clear on disk; the whole + // point of the bundle KEK is that such material must not be readable from a + // backup. Compare the on-disk record against every bundle byte. + let (kms, client) = seeded_kms(&["leak-check"]).await; + let key_dir = kms.key_dir().expect("key dir"); + let on_disk = read_bytes(&key_dir.join("leak-check.key")).await; + + let out = TempDir::new().expect("temp dir"); + let bundle_dir = out.path().join("bundle"); + let manifest = export_local_backup(&client, &kek(), &export_request(bundle_dir.clone())) + .await + .expect("export should succeed"); + + for artifact in &manifest.artifacts { + let payload = read_bytes(&bundle_dir.join(&artifact.path)).await; + assert!( + !payload + .windows(on_disk.len().min(payload.len()).max(1)) + .any(|window| window == on_disk.as_slice()), + "artifact {} carries the raw on-disk record", + artifact.path + ); + // A cheap structural check too: an encrypted payload is not JSON. + assert_ne!(payload.first(), Some(&b'{'), "artifact {} looks like plaintext JSON", artifact.path); + } + + // The manifest itself is not encrypted, so assert directly that it carries + // no material — only identities and digests. + let manifest_bytes = read_bytes(&bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE)).await; + let manifest_text = String::from_utf8(manifest_bytes).expect("the manifest is JSON"); + assert!( + !manifest_text.contains("encrypted_key_material"), + "the manifest must not inline key material" + ); +} + +#[tokio::test] +async fn the_wrong_kek_is_refused_on_identity_not_on_decryption() { + let (_kms, client) = seeded_kms(&["kek-check"]).await; + let out = TempDir::new().expect("temp dir"); + let bundle_dir = out.path().join("bundle"); + let manifest = export_local_backup(&client, &kek(), &export_request(bundle_dir.clone())) + .await + .expect("export should succeed"); + let artifact = manifest + .require_artifact(ArtifactKind::KeyMaterial) + .expect("key material artifact"); + + // Wrong id, wrong version, and right identity with wrong material are all + // distinct failure modes and must be reported as such. + let wrong_id = BackupKek::new("some-other-kek", KEK_VERSION, [0x7eu8; 32]).expect("KEK"); + match decrypt_bundle_artifact(&bundle_dir, &manifest, artifact, &wrong_id).await { + Err(KmsError::Backup(BackupError::WrongKek { + required_kek_id, + supplied_kek_id, + .. + })) => { + assert_eq!(required_kek_id, KEK_ID); + assert_eq!(supplied_kek_id, "some-other-kek"); + } + other => panic!("expected WrongKek for a mismatched id, got {other:?}"), + } + + let wrong_version = BackupKek::new(KEK_ID, KEK_VERSION + 1, [0x7eu8; 32]).expect("KEK"); + match decrypt_bundle_artifact(&bundle_dir, &manifest, artifact, &wrong_version).await { + Err(KmsError::Backup(BackupError::WrongKek { + required_kek_version, + supplied_kek_version, + .. + })) => { + assert_eq!(required_kek_version, KEK_VERSION); + assert_eq!(supplied_kek_version, KEK_VERSION + 1); + } + other => panic!("expected WrongKek for a mismatched version, got {other:?}"), + } + + // Right identity, wrong material: identity passes, AEAD does not. + let impostor = BackupKek::new(KEK_ID, KEK_VERSION, [0x00u8; 32]).expect("KEK"); + let error = decrypt_bundle_artifact(&bundle_dir, &manifest, artifact, &impostor) + .await + .expect_err("a KEK with the right identity but wrong material must not open the bundle"); + assert!( + !matches!(error, KmsError::Backup(BackupError::WrongKek { .. })), + "an identity match followed by an AEAD failure must not be reported as WrongKek: {error:?}" + ); + + // The correct KEK still works, so the failures above are about the KEK. + decrypt_bundle_artifact(&bundle_dir, &manifest, artifact, &kek()) + .await + .expect("the correct KEK must still open the artifact"); +} + +#[tokio::test] +async fn a_tampered_bundle_is_detected() { + let (_kms, client) = seeded_kms(&["tamper-check"]).await; + let out = TempDir::new().expect("temp dir"); + let bundle_dir = out.path().join("bundle"); + let manifest = export_local_backup(&client, &kek(), &export_request(bundle_dir.clone())) + .await + .expect("export should succeed"); + let artifact = manifest + .require_artifact(ArtifactKind::KeyMaterial) + .expect("key material artifact") + .clone(); + let artifact_path = bundle_dir.join(&artifact.path); + let original = read_bytes(&artifact_path).await; + + // Flipping a byte breaks the declared digest, which is checked before the + // KEK is ever applied — so detection does not depend on holding the KEK. + let mut tampered = original.clone(); + let middle = tampered.len() / 2; + tampered[middle] ^= 0xff; + tokio::fs::write(&artifact_path, &tampered).await.expect("write tampered"); + assert_ne!( + ContentDigest::sha256_of(&tampered), + artifact.encrypted_digest, + "the tampered payload must no longer match its declared digest" + ); + assert!( + decrypt_bundle_artifact(&bundle_dir, &manifest, &artifact, &kek()) + .await + .is_err(), + "a tampered artifact must be refused" + ); + + // Truncation is caught by the declared length. + tokio::fs::write(&artifact_path, &original[..original.len() / 2]) + .await + .expect("write truncated"); + match decrypt_bundle_artifact(&bundle_dir, &manifest, &artifact, &kek()).await { + Err(KmsError::Backup(BackupError::Truncated { .. })) => {} + other => panic!("expected Truncated for a short artifact, got {other:?}"), + } + + // A missing artifact is its own failure mode. + tokio::fs::remove_file(&artifact_path).await.expect("remove artifact"); + match decrypt_bundle_artifact(&bundle_dir, &manifest, &artifact, &kek()).await { + Err(KmsError::Backup(BackupError::MissingArtifact { .. })) => {} + other => panic!("expected MissingArtifact, got {other:?}"), + } + + // Restoring the bytes restores the bundle: the failures were the tampering. + tokio::fs::write(&artifact_path, &original).await.expect("restore artifact"); + decrypt_bundle_artifact(&bundle_dir, &manifest, &artifact, &kek()) + .await + .expect("the restored artifact must open again"); +} + +#[tokio::test] +async fn a_tampered_manifest_fails_its_own_digest() { + let (_kms, client) = seeded_kms(&["manifest-tamper"]).await; + let out = TempDir::new().expect("temp dir"); + let bundle_dir = out.path().join("bundle"); + export_local_backup(&client, &kek(), &export_request(bundle_dir.clone())) + .await + .expect("export should succeed"); + + let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE); + let original = read_bytes(&manifest_path).await; + let text = String::from_utf8(original.clone()).expect("manifest is JSON"); + + // Rewrite a semantically meaningful field, leaving the digest untouched. + let tampered = text.replace("\"snapshot_generation\":7", "\"snapshot_generation\":99"); + assert_ne!(tampered, text, "the rewrite must actually change the manifest"); + tokio::fs::write(&manifest_path, tampered.as_bytes()) + .await + .expect("write tampered manifest"); + + assert!( + read_local_bundle_manifest(&bundle_dir).await.is_err(), + "a manifest whose contents no longer match its digest must be rejected" + ); + + // Truncating the manifest is rejected too, and so is deleting it — the + // latter is exactly what an interrupted export leaves behind. + tokio::fs::write(&manifest_path, &original[..original.len() / 2]) + .await + .expect("write truncated manifest"); + assert!( + read_local_bundle_manifest(&bundle_dir).await.is_err(), + "a truncated manifest must be rejected" + ); + + tokio::fs::remove_file(&manifest_path).await.expect("remove manifest"); + match read_local_bundle_manifest(&bundle_dir).await { + Err(KmsError::Backup(BackupError::IncompleteBundle { .. })) => {} + other => panic!("a bundle with no manifest is an unsealed export, got {other:?}"), + } + + tokio::fs::write(&manifest_path, &original).await.expect("restore manifest"); + read_local_bundle_manifest(&bundle_dir) + .await + .expect("the restored manifest must read back"); +} + +#[tokio::test] +async fn export_refuses_inputs_that_would_produce_an_unrestorable_bundle() { + let (_kms, client) = seeded_kms(&["input-validation"]).await; + let out = TempDir::new().expect("temp dir"); + + // Identity fields are the only way a restore can tell bundles apart. + for (field, mutate) in [ + ( + "backup_id", + (|r: &mut LocalBackupExportRequest| r.backup_id.clear()) as fn(&mut LocalBackupExportRequest), + ), + ("deployment_identity", |r: &mut LocalBackupExportRequest| r.deployment_identity.clear()), + ("rustfs_version", |r: &mut LocalBackupExportRequest| r.rustfs_version.clear()), + ] { + let destination = out.path().join(format!("bundle-{field}")); + let mut request = export_request(destination.clone()); + mutate(&mut request); + let error = export_local_backup(&client, &kek(), &request) + .await + .err() + .unwrap_or_else(|| panic!("an empty {field} must be refused, but the export produced a bundle")); + match error { + KmsError::ValidationError { message } => { + assert!(message.contains(field), "the refusal must name the offending field {field}: {message}") + } + other => panic!("an empty {field} must be a validation error, got {other:?}"), + } + assert!( + !destination.join(LOCAL_BUNDLE_MANIFEST_FILE).exists(), + "a refused export must not leave a sealed manifest behind for {field}" + ); + } + + // An empty KEK id cannot identify a trust root. + assert!(BackupKek::new("", KEK_VERSION, [0u8; 32]).is_err(), "an empty KEK id must be refused"); +} + +#[tokio::test] +async fn exporting_an_empty_key_directory_is_refused() { + // An empty bundle would restore to an empty KMS, silently destroying state. + let kms = TestKms::local().await; + let key_dir = kms.key_dir().expect("key dir"); + let client = LocalKmsClient::new(LocalConfig { + key_dir, + master_key: None, + file_permissions: Some(0o600), + }) + .await + .expect("client"); + + let out = TempDir::new().expect("temp dir"); + let error = export_local_backup(&client, &kek(), &export_request(out.path().join("empty"))) + .await + .expect_err("an export with no key records must be refused"); + assert!( + matches!(error, KmsError::InvalidOperation { .. }), + "refusing an empty bundle is an invalid-operation, got {error:?}" + ); +} + +#[tokio::test] +async fn a_bundle_taken_after_a_change_reflects_that_change() { + // Two generations of the same deployment: the second bundle must contain + // the key the first did not, proving the export reads live state rather + // than a cached snapshot. + let (kms, client) = seeded_kms(&["generation-one"]).await; + let out = TempDir::new().expect("temp dir"); + + let first_dir = out.path().join("gen-1"); + let first = export_local_backup(&client, &kek(), &export_request(first_dir.clone())) + .await + .expect("first export"); + assert_eq!( + exported_key_ids(&first_dir, &first).await, + vec!["generation-one".to_string()], + "the first bundle must contain exactly the key that existed when it was taken" + ); + + kms.create_key("generation-two").await; + + let second_dir = out.path().join("gen-2"); + let mut request = export_request(second_dir.clone()); + request.backup_id = "backup-behavior-002".to_string(); + request.snapshot_generation = 8; + let second = export_local_backup(&client, &kek(), &request).await.expect("second export"); + assert_eq!(second.snapshot_generation, 8, "the newer bundle carries the newer generation"); + + assert_eq!( + exported_key_ids(&second_dir, &second).await, + vec!["generation-one".to_string(), "generation-two".to_string()], + "the second bundle must contain both the old and the newly created key" + ); + + // The first bundle is untouched by the second export. + read_local_bundle_manifest(&first_dir) + .await + .expect("the earlier bundle must still verify"); +} diff --git a/crates/kms/tests/behavior_cache.rs b/crates/kms/tests/behavior_cache.rs new file mode 100644 index 000000000..26d0a06fc --- /dev/null +++ b/crates/kms/tests/behavior_cache.rs @@ -0,0 +1,323 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: the KMS metadata cache. +//! +//! The cache exists to keep `describe_key` off the backend on hot paths. Two +//! properties matter far more than its hit rate: +//! +//! * **It must never outlive the truth it caches.** Every state mutation has to +//! drop the entry, or the state gate in front of encryption would consult a +//! stale `Enabled` snapshot and let a disabled key keep minting data keys. +//! * **It must never extend to data keys.** `lib.rs` allows caching stable +//! master-key metadata and forbids caching generated DEKs, because a DEK is +//! bound to one object's encryption context. This file asserts the boundary +//! holds with the cache both enabled and disabled. + +mod common; + +use common::{TestKms, assert_invalid_operation, ctx}; +use rustfs_kms::{ + CancelKeyDeletionRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyState, KmsManager, +}; + +async fn describe_state(kms: &KmsManager, key_id: &str) -> KeyState { + kms.describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .expect("describe should succeed") + .key_metadata + .key_state +} + +async fn entry_count(kms: &KmsManager) -> u64 { + kms.cache_stats().await.expect("cache is enabled").entries +} + +#[tokio::test] +async fn cache_reporting_follows_the_enable_flag() { + let enabled = TestKms::local().await; + let manager = enabled.kms().await; + assert!(manager.cache_stats().await.is_some(), "a cache-enabled service must report statistics"); + + let disabled = TestKms::local_with(|config| config.enable_cache = false).await; + let disabled_manager = disabled.kms().await; + assert!( + disabled_manager.cache_stats().await.is_none(), + "a cache-disabled service must report no statistics at all" + ); + + // Clearing a cache that does not exist is a no-op, not an error. + disabled_manager + .clear_cache() + .await + .expect("clear_cache must succeed even when caching is off"); +} + +#[tokio::test] +async fn cache_population_and_clearing_are_observable() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + + assert_eq!(entry_count(&manager).await, 0, "a fresh service caches nothing"); + + // Creating a key populates the cache eagerly. + kms.create_key("cached-a").await; + assert_eq!(entry_count(&manager).await, 1, "create_key caches the new key's metadata"); + + kms.create_key("cached-b").await; + assert_eq!(entry_count(&manager).await, 2); + + // Repeated describes of a cached key add no entries. + for _ in 0..5 { + assert_eq!(describe_state(&manager, "cached-a").await, KeyState::Enabled); + } + assert_eq!(entry_count(&manager).await, 2, "repeat describes must not grow the cache"); + + // A failed describe must not create a negative-cache entry. + assert!( + manager + .describe_key(DescribeKeyRequest { + key_id: "never-existed".to_string(), + }) + .await + .is_err() + ); + assert_eq!(entry_count(&manager).await, 2, "a missing key must not be cached"); + + manager.clear_cache().await.expect("clear should succeed"); + assert_eq!(entry_count(&manager).await, 0, "clear_cache must empty the cache"); + + // Clearing does not lose data: the backend is still the source of truth. + assert_eq!(describe_state(&manager, "cached-a").await, KeyState::Enabled); + assert_eq!(entry_count(&manager).await, 1, "a describe after clearing repopulates"); +} + +/// A stale cache entry would silently defeat the state gate, so every mutation +/// path is checked: the gate must see the post-mutation state, and the cached +/// entry must be gone rather than merely overwritten later. +#[tokio::test] +async fn every_state_mutation_invalidates_the_cached_entry() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + let key_id = kms.create_key("invalidated").await; + let context = ctx(&[("bucket", "cache-behavior")]); + + let generate = || GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }; + + // Warm the cache so a missing invalidation would be observable. + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + manager + .generate_data_key(generate()) + .await + .expect("Enabled permits generation"); + + // Invalidation is asserted behaviourally, not by counting entries: moka's + // `entry_count` is eventually consistent, so a count is not a reliable + // observable. What must hold is that the next read sees backend truth and + // the state gate acts on it. + manager.disable_key(&key_id).await.expect("disable"); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::Disabled, + "the read after a disable must not be served from the pre-mutation snapshot" + ); + assert_invalid_operation(manager.generate_data_key(generate()).await, "is disabled"); + + manager.enable_key(&key_id).await.expect("enable"); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::Enabled, + "the read after an enable must not be served from the Disabled snapshot" + ); + manager + .generate_data_key(generate()) + .await + .expect("re-enabling must restore generation"); + + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await + .expect("schedule deletion"); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::PendingDeletion, + "the read after a scheduled deletion must not be served from the Enabled snapshot" + ); + assert_invalid_operation(manager.generate_data_key(generate()).await, "pending deletion"); + + manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await + .expect("cancel deletion"); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::Enabled, + "the cache must not resurrect the PendingDeletion snapshot after a cancel" + ); + manager + .generate_data_key(generate()) + .await + .expect("a cancelled key must generate again"); +} + +#[tokio::test] +async fn a_destroyed_key_cannot_be_served_from_cache() { + let kms = TestKms::local_with(|config| config.allow_immediate_deletion = true).await; + let manager = kms.kms().await; + let key_id = kms.create_key("destroyed").await; + + // Warm the cache, then destroy the key outright. + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: None, + force_immediate: Some(true), + confirm_key_id: Some(key_id.clone()), + }) + .await + .expect("forced deletion"); + + assert!( + manager + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .is_err(), + "a destroyed key must not be served from the cache" + ); + assert!( + manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: ctx(&[("bucket", "cache-behavior")]), + }) + .await + .is_err(), + "a destroyed key must not keep minting data keys through a cached snapshot" + ); +} + +/// The invariant `lib.rs` states outright: metadata may be cached, data keys +/// may not. Asserted with the cache both on and off so a caching change cannot +/// quietly extend to DEKs. +#[tokio::test] +async fn caching_never_extends_to_data_keys() { + for enable_cache in [true, false] { + let kms = TestKms::local_with(|config| config.enable_cache = enable_cache).await; + let manager = kms.kms().await; + let key_id = kms.create_key("dek-freshness").await; + let context = ctx(&[("bucket", "cache-behavior"), ("object", "same.bin")]); + + // Warm the metadata cache first: if DEK generation ever consulted it, + // this is where a reused key would come from. + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + + let mut seen_plaintext = Vec::new(); + let mut seen_ciphertext = Vec::new(); + for _ in 0..16 { + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("generate should succeed"); + assert!( + !seen_plaintext.contains(&dek.plaintext_key), + "cache={enable_cache}: a data key was reused across calls with identical inputs" + ); + assert!( + !seen_ciphertext.contains(&dek.ciphertext_blob), + "cache={enable_cache}: a wrapped data key was reused across calls with identical inputs" + ); + seen_plaintext.push(dek.plaintext_key); + seen_ciphertext.push(dek.ciphertext_blob); + } + } +} + +#[tokio::test] +async fn disabling_the_cache_changes_no_observable_behavior() { + // Same script under both settings: results must be identical apart from + // the statistics, so the cache is a pure performance concern. + for enable_cache in [true, false] { + let kms = TestKms::local_with(|config| config.enable_cache = enable_cache).await; + let manager = kms.kms().await; + let key_id = kms.create_key("parity").await; + + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled, "cache={enable_cache}"); + manager.disable_key(&key_id).await.expect("disable"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Disabled, "cache={enable_cache}"); + manager.enable_key(&key_id).await.expect("enable"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled, "cache={enable_cache}"); + + assert!( + manager + .describe_key(DescribeKeyRequest { + key_id: "absent".to_string(), + }) + .await + .is_err(), + "cache={enable_cache}: an unknown key is an error either way" + ); + } +} + +/// `cache_stats` reports live counters a caller can compute a hit rate from. +/// +/// Pinned because the numbers are operator-facing: a counter frozen at zero +/// reads as "the cache is never helping" and invites someone to tune away a +/// cache that is in fact working. +#[tokio::test] +async fn cache_stats_reports_hits_and_misses_separately() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + kms.create_key("stats-a").await; + kms.create_key("stats-b").await; + + // Traffic a real hit/miss counter has to move: the same key read over and + // over, plus a lookup that can never be served from cache. + for _ in 0..10 { + assert_eq!(describe_state(&manager, "stats-a").await, KeyState::Enabled); + assert!( + manager + .describe_key(DescribeKeyRequest { + key_id: "absent".to_string(), + }) + .await + .is_err() + ); + } + + let stats = manager.cache_stats().await.expect("cache is enabled"); + assert_eq!(stats.entries, 2, "both created keys are cached and the absent one is not"); + assert!(stats.hits > 0, "re-reading one key ten times must register hits, got {stats:?}"); + assert!( + stats.misses > 0, + "a lookup that cannot be served from cache must register a miss, got {stats:?}" + ); +} diff --git a/crates/kms/tests/behavior_concurrency.rs b/crates/kms/tests/behavior_concurrency.rs new file mode 100644 index 000000000..344ccbfd2 --- /dev/null +++ b/crates/kms/tests/behavior_concurrency.rs @@ -0,0 +1,560 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: concurrency and durability across a restart. +//! +//! Two things a single-threaded test can never show: +//! +//! * **Concurrency.** Lifecycle operations are serialized behind one lock, and +//! key operations run in parallel against a shared on-disk store. Under load +//! the guarantees that matter are that a race has exactly one winner, that no +//! interleaving produces a panic, and that parallel data-key generation never +//! collides. +//! * **Durability.** Everything the KMS promises is worthless if a restart +//! loses it. The restart cases below drop the whole service and bring a +//! brand-new manager up over the same directory, so anything that still holds +//! afterwards genuinely came off disk rather than out of a warm cache. + +mod common; + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use common::{TestKms, assert_invalid_operation, ctx}; +use rustfs_kms::{ + CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyState, KmsConfig, + KmsError, KmsServiceManager, KmsServiceStatus, +}; +use tempfile::TempDir; + +fn context() -> std::collections::HashMap { + ctx(&[("bucket", "concurrency-behavior")]) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_reconfiguration_is_serialized_and_versions_stay_monotonic() { + let dir = TempDir::new().expect("temp dir"); + let manager = Arc::new(KmsServiceManager::new()); + let config = KmsConfig::local(dir.path().to_path_buf()).with_insecure_development_defaults(); + manager.configure(config.clone()).await.expect("configure"); + manager.start().await.expect("start"); + assert_eq!(manager.get_service_version().await, Some(1)); + + const RECONFIGURATIONS: u64 = 8; + let mut handles = Vec::new(); + for index in 0..RECONFIGURATIONS { + let manager = manager.clone(); + let mut candidate = config.clone(); + // Vary a field that is allowed to change so each call does real work. + candidate.timeout = Duration::from_secs(30 + index); + handles.push(tokio::spawn(async move { manager.reconfigure(candidate).await })); + } + + for (index, handle) in handles.into_iter().enumerate() { + handle + .await + .expect("reconfigure task must not panic") + .unwrap_or_else(|error| panic!("reconfigure {index} should succeed: {error:?}")); + } + + assert_eq!( + manager.get_service_version().await, + Some(1 + RECONFIGURATIONS), + "each serialized reconfigure must consume exactly one version" + ); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert!(manager.health_check().await.expect("health check"), "the survivor must be healthy"); + + // Exactly one candidate won, and the published config is one of the ones + // that was actually submitted. + let published = manager.get_config().await.expect("config").timeout; + assert!( + (30..30 + RECONFIGURATIONS).contains(&published.as_secs()), + "the published timeout must be one of the submitted candidates, got {published:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_race_to_create_the_same_key_has_exactly_one_winner() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + + const RACERS: usize = 8; + let mut handles = Vec::new(); + for _ in 0..RACERS { + let manager = manager.clone(); + handles.push(tokio::spawn(async move { + manager + .create_key(CreateKeyRequest { + key_name: Some("contested".to_string()), + ..Default::default() + }) + .await + })); + } + + let mut winners = 0; + let mut conflicts = 0; + for handle in handles { + match handle.await.expect("create task must not panic") { + Ok(response) => { + assert_eq!(response.key_id, "contested"); + winners += 1; + } + Err(KmsError::KeyAlreadyExists { key_id }) => { + assert_eq!(key_id, "contested"); + conflicts += 1; + } + Err(other) => panic!("a create race must resolve to success or KeyAlreadyExists, got {other:?}"), + } + } + assert_eq!(winners, 1, "exactly one racer may create the key"); + assert_eq!(conflicts, RACERS - 1, "every other racer must see a conflict"); + + // The single surviving key is intact and usable. + let described = manager + .describe_key(DescribeKeyRequest { + key_id: "contested".to_string(), + }) + .await + .expect("the contested key must exist"); + assert_eq!(described.key_metadata.key_state, KeyState::Enabled); + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "contested".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("the contested key must work"); + manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob, + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .expect("the contested key's material must be coherent, not a torn write"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn parallel_data_key_generation_never_collides() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + let key_id = kms.create_key("parallel-dek").await; + + const TASKS: usize = 16; + const PER_TASK: usize = 8; + let mut handles = Vec::new(); + for _ in 0..TASKS { + let manager = manager.clone(); + let key_id = key_id.clone(); + handles.push(tokio::spawn(async move { + let mut produced = Vec::new(); + for _ in 0..PER_TASK { + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("generate should succeed under load"); + produced.push((dek.plaintext_key, dek.ciphertext_blob)); + } + produced + })); + } + + let mut plaintexts = HashSet::new(); + let mut ciphertexts = HashSet::new(); + let mut all = Vec::new(); + for handle in handles { + for (plaintext, ciphertext) in handle.await.expect("generation task must not panic") { + assert!(plaintexts.insert(plaintext.clone()), "a data key was handed out twice under load"); + assert!( + ciphertexts.insert(ciphertext.clone()), + "a wrapped data key was handed out twice under load" + ); + all.push((plaintext, ciphertext)); + } + } + assert_eq!(all.len(), TASKS * PER_TASK, "every request must be answered"); + + // Every blob still opens to its own key: concurrency must not have crossed + // wires between requests. + for (index, (expected, blob)) in all.into_iter().enumerate() { + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: blob, + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("blob {index} should decrypt: {error:?}")); + assert_eq!(decrypted.plaintext, expected, "blob {index} opened to another request's key"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lifecycle_churn_against_live_traffic_stays_coherent() { + // A disable/enable loop running against concurrent data-key generation. + // Each generation must either succeed outright or be refused by the state + // gate — never panic, never return a broken key, never see a torn record. + let kms = TestKms::local().await; + let manager = kms.kms().await; + let key_id = kms.create_key("churned").await; + + let churn = { + let manager = manager.clone(); + let key_id = key_id.clone(); + tokio::spawn(async move { + for _ in 0..20 { + manager.disable_key(&key_id).await.expect("disable should succeed"); + tokio::task::yield_now().await; + manager.enable_key(&key_id).await.expect("enable should succeed"); + tokio::task::yield_now().await; + } + }) + }; + + let mut workers = Vec::new(); + for _ in 0..4 { + let manager = manager.clone(); + let key_id = key_id.clone(); + workers.push(tokio::spawn(async move { + let mut succeeded = 0usize; + let mut refused = 0usize; + for _ in 0..40 { + match manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + { + Ok(dek) => { + assert_eq!(dek.plaintext_key.len(), 32, "a key handed out under churn must be well formed"); + // A key produced while the master key was enabled must + // remain decryptable regardless of later state changes. + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob, + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .expect("a key issued under churn must stay decryptable"); + assert_eq!(decrypted.plaintext, dek.plaintext_key); + succeeded += 1; + } + Err(KmsError::InvalidOperation { message }) => { + assert!( + message.contains("disabled"), + "the only acceptable refusal under this churn is the disabled gate, got {message:?}" + ); + refused += 1; + } + Err(other) => panic!("unexpected error under lifecycle churn: {other:?}"), + } + tokio::task::yield_now().await; + } + (succeeded, refused) + })); + } + + churn.await.expect("churn task must not panic"); + let mut total = 0usize; + for worker in workers { + let (succeeded, refused) = worker.await.expect("worker task must not panic"); + total += succeeded + refused; + } + assert_eq!(total, 4 * 40, "every request must be accounted for"); + + // The totals above say nothing about the state gate on their own: if the + // disable/enable loop happens to fall entirely between request windows, + // every request succeeds and the count still balances — and an + // implementation that refused everything would balance too. Asserting + // `refused > 0` on the concurrent phase would only trade that hole for a + // scheduling-dependent flake, so both branches are pinned deterministically + // here instead. Removing the state gate, or breaking progress in the + // enabled state, now fails this test. + let gated_request = || GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }; + + manager.disable_key(&key_id).await.expect("disable for the gated check"); + assert_invalid_operation(manager.generate_data_key(gated_request()).await, "is disabled"); + + manager.enable_key(&key_id).await.expect("enable for the gated check"); + let after_enable = manager + .generate_data_key(gated_request()) + .await + .expect("an enabled key must generate again after the churn"); + assert_eq!(after_enable.plaintext_key.len(), 32, "the post-churn key must be well formed"); + + // The key survives the churn in a well-defined state. + manager.enable_key(&key_id).await.expect("final enable"); + assert_eq!( + manager + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .expect("describe") + .key_metadata + .key_state, + KeyState::Enabled + ); +} + +#[tokio::test] +async fn a_reconfigure_mid_flight_does_not_orphan_in_progress_work() { + let dir = TempDir::new().expect("temp dir"); + let manager = Arc::new(KmsServiceManager::new()); + let config = KmsConfig::local(dir.path().to_path_buf()).with_insecure_development_defaults(); + manager.configure(config.clone()).await.expect("configure"); + manager.start().await.expect("start"); + + let old_kms = manager.get_manager().await.expect("manager v1"); + old_kms + .create_key(CreateKeyRequest { + key_name: Some("spans-reconfigure".to_string()), + ..Default::default() + }) + .await + .expect("create"); + + // A caller that grabbed the handle before the swap keeps working with it. + let dek = old_kms + .generate_data_key(GenerateDataKeyRequest { + key_id: "spans-reconfigure".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("generate on the old generation"); + + let mut next = config.clone(); + next.timeout = Duration::from_secs(42); + manager.reconfigure(next).await.expect("reconfigure"); + let new_kms = manager.get_manager().await.expect("manager v2"); + assert!(!Arc::ptr_eq(&old_kms, &new_kms), "the reconfigure must have swapped the handle"); + + // The old handle finishes its work... + let via_old = old_kms + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .expect("the pre-swap handle must complete its in-flight work"); + assert_eq!(via_old.plaintext, dek.plaintext_key); + + // ...and the new handle can read what the old one wrote, because both are + // backed by the same key directory. + let via_new = new_kms + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .expect("the post-swap handle must read the old generation's output"); + assert_eq!(via_new.plaintext, dek.plaintext_key); + new_kms + .describe_key(DescribeKeyRequest { + key_id: "spans-reconfigure".to_string(), + }) + .await + .expect("a key created before the swap must be visible after it"); +} + +/// The durability case: several keys in different states, a full restart, and +/// then every promise re-checked against the new process. +#[tokio::test] +async fn key_states_and_ciphertext_survive_a_restart() { + let mut kms = TestKms::local().await; + let manager = kms.kms().await; + + for key_id in ["survivor-enabled", "survivor-disabled", "survivor-pending"] { + kms.create_key(key_id).await; + } + + // Mint ciphertext under each key *before* the restart, so the assertions + // afterwards prove the material itself survived, not just the metadata. + let mut blobs = Vec::new(); + for key_id in ["survivor-enabled", "survivor-disabled", "survivor-pending"] { + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("generate before restart"); + blobs.push((key_id, dek.plaintext_key, dek.ciphertext_blob)); + } + + manager.disable_key("survivor-disabled").await.expect("disable"); + manager + .delete_key(DeleteKeyRequest { + key_id: "survivor-pending".to_string(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await + .expect("schedule deletion"); + + // --- restart --------------------------------------------------------- + kms.restart().await; + let manager = kms.kms().await; + + // Every state came back exactly as it was left. + for (key_id, expected) in [ + ("survivor-enabled", KeyState::Enabled), + ("survivor-disabled", KeyState::Disabled), + ("survivor-pending", KeyState::PendingDeletion), + ] { + let described = manager + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .unwrap_or_else(|error| panic!("{key_id} must survive the restart: {error:?}")) + .key_metadata; + assert_eq!(described.key_state, expected, "{key_id} must come back in its persisted state"); + if expected == KeyState::PendingDeletion { + assert!(described.deletion_date.is_some(), "{key_id} must come back with its deadline intact"); + } + } + + // Ciphertext written before the restart still opens under every state, + // including the disabled and pending-deletion keys. + for (key_id, expected_plaintext, blob) in &blobs { + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("{key_id}'s pre-restart ciphertext must still decrypt: {error:?}")); + assert_eq!(&decrypted.plaintext, expected_plaintext, "{key_id} decrypted to the wrong key"); + } + + // The state gate is re-applied from persisted state, not re-derived as + // Enabled: a disabled key must still refuse new work after a restart. + assert_invalid_operation( + manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "survivor-disabled".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await, + "is disabled", + ); + assert_invalid_operation( + manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "survivor-pending".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await, + "pending deletion", + ); + manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "survivor-enabled".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("the enabled key must accept new work after a restart"); + + // Recovery still works across the restart boundary. + manager + .enable_key("survivor-disabled") + .await + .expect("re-enable after restart"); + manager + .cancel_key_deletion(rustfs_kms::CancelKeyDeletionRequest { + key_id: "survivor-pending".to_string(), + }) + .await + .expect("cancel after restart"); + for key_id in ["survivor-disabled", "survivor-pending"] { + manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .unwrap_or_else(|error| panic!("{key_id} must be usable after recovery: {error:?}")); + } +} + +#[tokio::test] +async fn a_destroyed_key_stays_destroyed_across_a_restart() { + let mut kms = TestKms::local_with(|config| config.allow_immediate_deletion = true).await; + let manager = kms.kms().await; + kms.create_key("gone-for-good").await; + kms.create_key("kept").await; + + manager + .delete_key(DeleteKeyRequest { + key_id: "gone-for-good".to_string(), + pending_window_in_days: None, + force_immediate: Some(true), + confirm_key_id: Some("gone-for-good".to_string()), + }) + .await + .expect("forced deletion"); + + kms.restart().await; + let manager = kms.kms().await; + + assert!( + manager + .describe_key(DescribeKeyRequest { + key_id: "gone-for-good".to_string(), + }) + .await + .is_err(), + "a destroyed key must not reappear after a restart" + ); + assert!( + !manager + .list_keys(rustfs_kms::ListKeysRequest::default()) + .await + .expect("list") + .keys + .iter() + .any(|key| key.key_id == "gone-for-good"), + "a destroyed key must not reappear in listings after a restart" + ); + manager + .describe_key(DescribeKeyRequest { + key_id: "kept".to_string(), + }) + .await + .expect("the untouched key must survive the restart"); +} diff --git a/crates/kms/tests/behavior_crypto.rs b/crates/kms/tests/behavior_crypto.rs new file mode 100644 index 000000000..30dc6670c --- /dev/null +++ b/crates/kms/tests/behavior_crypto.rs @@ -0,0 +1,572 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: master-key crypto and data-encryption-key semantics. +//! +//! Three invariants carry most of the weight here: +//! +//! 1. **Every DEK is fresh.** `lib.rs` forbids caching a generated data key by +//! master key id: a DEK and its ciphertext are bound to one object's +//! encryption context, so reuse would both break context validation and +//! violate the per-object DEK model SSE-S3 and SSE-KMS assume. +//! 2. **The encryption context is authenticated.** It is the AEAD's additional +//! data, so a wrong value must fail decryption rather than silently return +//! the wrong plaintext. +//! 3. **Corrupt input fails cleanly.** Tampered, truncated, or foreign +//! ciphertext returns a typed error and never panics — this input is +//! attacker-reachable through object metadata. +//! +//! One deliberate carve-out is pinned below: an *empty* request context skips +//! the "missing context key" check so legacy objects, written before contexts +//! were bound, stay readable. A *wrong* value is still rejected. + +mod common; + +use common::{BackendCase, BackendKind, TestKms, assert_context_mismatch, ctx, flip_middle_bit, for_each_backend, payload}; +use rustfs_kms::{ + DecryptRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec, KmsError, ObjectEncryptionContext, is_data_key_envelope, +}; + +fn context() -> std::collections::HashMap { + ctx(&[("bucket", "crypto-behavior"), ("object", "alpha.bin")]) +} + +#[tokio::test] +async fn master_key_encrypt_decrypt_round_trips_under_the_same_context() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + let plaintext = payload(1024); + + let encrypted = manager + .encrypt(EncryptRequest { + key_id: case.key_id.clone(), + plaintext: plaintext.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] encrypt should succeed: {error:?}")); + + assert!(!encrypted.ciphertext.is_empty(), "[{label}] ciphertext must not be empty"); + assert_ne!(encrypted.ciphertext, plaintext, "[{label}] ciphertext must not equal the plaintext"); + assert_eq!(encrypted.key_id, case.key_id, "[{label}] the response names the key used"); + assert!(!encrypted.algorithm.is_empty(), "[{label}] the algorithm must be reported"); + + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: encrypted.ciphertext.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] decrypt should succeed: {error:?}")); + assert_eq!(decrypted.plaintext, plaintext, "[{label}] round-trip must return the input"); + assert_eq!(decrypted.key_id, case.key_id, "[{label}] decrypt reports the key it used"); + + // Encrypting the same plaintext twice must not produce the same + // ciphertext: a fresh nonce per call is what keeps AES-GCM safe. + let again = manager + .encrypt(EncryptRequest { + key_id: case.key_id.clone(), + plaintext: plaintext.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] second encrypt should succeed: {error:?}")); + assert_ne!( + again.ciphertext, encrypted.ciphertext, + "[{label}] repeated encryption of identical plaintext must not be deterministic" + ); + }) + .await; +} + +#[tokio::test] +async fn empty_plaintext_round_trips() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + + let encrypted = manager + .encrypt(EncryptRequest { + key_id: case.key_id.clone(), + plaintext: Vec::new(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] encrypting nothing should still succeed: {error:?}")); + + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: encrypted.ciphertext, + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] decrypt should succeed: {error:?}")); + assert!(decrypted.plaintext.is_empty(), "[{label}] empty in, empty out"); + }) + .await; +} + +#[tokio::test] +async fn encryption_context_is_authenticated() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: case.key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] generate should succeed: {error:?}")); + + // A changed value for a bound key is a mismatch. + assert_context_mismatch( + manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: ctx(&[("bucket", "crypto-behavior"), ("object", "other.bin")]), + grant_tokens: Vec::new(), + }) + .await, + ); + + // A non-empty context that omits a bound key is also a mismatch. + assert_context_mismatch( + manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: ctx(&[("bucket", "crypto-behavior")]), + grant_tokens: Vec::new(), + }) + .await, + ); + + // Extra keys beyond the bound set are tolerated: only the bound pairs + // are authenticated, so adding context cannot lock an object out. + let with_extra = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: ctx(&[("bucket", "crypto-behavior"), ("object", "alpha.bin"), ("unrelated", "value")]), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] a superset context should decrypt: {error:?}")); + assert_eq!(with_extra.plaintext, dek.plaintext_key); + + // The documented legacy carve-out: a fully empty request context skips + // the missing-key check so pre-context objects stay readable. + let legacy = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: Default::default(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] the empty-context legacy path must work: {error:?}")); + assert_eq!( + legacy.plaintext, dek.plaintext_key, + "[{label}] the legacy path must return the same data key" + ); + }) + .await; +} + +#[tokio::test] +async fn every_generated_data_key_is_fresh() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + + // Same key id, same context, repeated: nothing may be reused. + let mut plaintexts = Vec::new(); + let mut ciphertexts = Vec::new(); + for _ in 0..8 { + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: case.key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] generate should succeed: {error:?}")); + + assert_eq!(dek.plaintext_key.len(), 32, "[{label}] an AES-256 DEK is 32 bytes"); + assert!(!dek.ciphertext_blob.is_empty(), "[{label}] the wrapped DEK must not be empty"); + assert!( + !dek.ciphertext_blob + .windows(dek.plaintext_key.len()) + .any(|window| window == dek.plaintext_key), + "[{label}] the wrapped blob must never contain the plaintext data key" + ); + plaintexts.push(dek.plaintext_key); + ciphertexts.push(dek.ciphertext_blob); + } + + for i in 0..plaintexts.len() { + for j in (i + 1)..plaintexts.len() { + assert_ne!( + plaintexts[i], plaintexts[j], + "[{label}] data keys must not repeat across calls (indices {i} and {j})" + ); + assert_ne!( + ciphertexts[i], ciphertexts[j], + "[{label}] wrapped data keys must not repeat across calls (indices {i} and {j})" + ); + } + } + + // Each wrapped blob still opens to exactly its own plaintext. + for (index, (expected, blob)) in plaintexts.iter().zip(ciphertexts.iter()).enumerate() { + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] blob {index} should decrypt: {error:?}")); + assert_eq!(&decrypted.plaintext, expected, "[{label}] blob {index} opened to the wrong key"); + } + }) + .await; +} + +#[tokio::test] +async fn data_key_spec_controls_the_length_of_the_generated_key() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + + // A backend that accepts a `key_spec` must honour it. Silently + // returning a different size means the caller builds a cipher from + // material it did not ask for, and the envelope records a spec its + // payload does not match. + // ChaCha20 material is 32 random bytes, exactly like AES_256, so a + // backend that mints DEKs itself has no technical reason to refuse it. + // Static accepts it; Local and both Vault backends route through + // `generate_key_material`, which only knows the two AES specs. That + // split is pinned per backend rather than tolerated on both sides: a + // blanket "honoured or refused" contract would accept a backend + // regressing from working into refusing, which is exactly how a + // silently dropped spec would ship. + for spec in [KeySpec::Aes256, KeySpec::Aes128, KeySpec::ChaCha20] { + let must_be_honoured = spec != KeySpec::ChaCha20 || case.kind() == BackendKind::Static; + match manager + .generate_data_key(GenerateDataKeyRequest { + key_id: case.key_id.clone(), + key_spec: spec.clone(), + encryption_context: context(), + }) + .await + { + Ok(generated) => assert_eq!( + generated.plaintext_key.len(), + spec.key_size(), + "[{label}] {spec:?} must yield a {}-byte data key", + spec.key_size() + ), + Err(KmsError::UnsupportedAlgorithm { .. }) if !must_be_honoured => {} + Err(error) => panic!("[{label}] {spec:?} must yield a {}-byte data key: {error:?}", spec.key_size()), + } + } + + let aes128 = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: case.key_id.clone(), + key_spec: KeySpec::Aes128, + encryption_context: context(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] AES-128 generate should succeed: {error:?}")); + + // Whatever the length, the blob still round-trips. + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: aes128.ciphertext_blob, + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] AES-128 blob should decrypt: {error:?}")); + assert_eq!(decrypted.plaintext, aes128.plaintext_key); + }) + .await; +} + +#[tokio::test] +async fn corrupt_ciphertext_fails_cleanly() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: case.key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] generate should succeed: {error:?}")); + + let decrypt = |ciphertext: Vec| { + let manager = manager.clone(); + async move { + manager + .decrypt(DecryptRequest { + ciphertext, + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + } + }; + + // A single flipped bit anywhere in the envelope must not decrypt. + let tampered = flip_middle_bit(&dek.ciphertext_blob); + assert!(decrypt(tampered).await.is_err(), "[{label}] a bit-flipped envelope must not decrypt"); + + // Truncation, emptiness, and non-envelope bytes are all typed errors. + for (name, input) in [ + ("truncated", dek.ciphertext_blob[..dek.ciphertext_blob.len() / 2].to_vec()), + ("empty", Vec::new()), + ("not-json", b"absolutely not an envelope".to_vec()), + ("json-but-wrong-shape", br#"{"hello":"world"}"#.to_vec()), + ] { + let error = decrypt(input) + .await + .expect_err(&format!("[{label}] {name} input must be rejected")); + assert!( + !matches!(error, KmsError::InternalError { .. }), + "[{label}] {name} input must map to a specific error, not InternalError: {error:?}" + ); + } + + // Truncating only the AEAD tail (keeping the envelope parseable) must + // fail authentication rather than return partial plaintext. + let mut short_envelope = dek.ciphertext_blob.clone(); + short_envelope.pop(); + assert!(decrypt(short_envelope).await.is_err(), "[{label}] a truncated envelope must not decrypt"); + }) + .await; +} + +#[tokio::test] +async fn a_data_key_is_not_transferable_between_master_keys() { + // Two independent master keys on the same backend: a blob wrapped by one + // must not open under the other, even with an identical context. + let kms = TestKms::local_with(|config| config.allow_immediate_deletion = true).await; + let manager = kms.kms().await; + kms.create_key("wrapper-a").await; + kms.create_key("wrapper-b").await; + + let from_a = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "wrapper-a".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("generate under wrapper-a"); + let from_b = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "wrapper-b".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .expect("generate under wrapper-b"); + + assert_ne!( + from_a.plaintext_key, from_b.plaintext_key, + "different master keys must produce different data keys" + ); + + // The envelope names its own master key, so each opens under its own. + for (label, dek) in [("wrapper-a", &from_a), ("wrapper-b", &from_b)] { + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("{label} blob should decrypt under its own key: {error:?}")); + assert_eq!(&decrypted.plaintext, &dek.plaintext_key); + } + + // Deleting wrapper-a makes its blobs undecryptable while wrapper-b's keep + // working — the blobs are genuinely bound to distinct material. + manager + .delete_key(rustfs_kms::DeleteKeyRequest { + key_id: "wrapper-a".to_string(), + pending_window_in_days: None, + force_immediate: Some(true), + confirm_key_id: Some("wrapper-a".to_string()), + }) + .await + .expect("forced deletion should succeed"); + + assert!( + manager + .decrypt(DecryptRequest { + ciphertext: from_a.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .is_err(), + "a blob wrapped by a destroyed key must not decrypt" + ); + manager + .decrypt(DecryptRequest { + ciphertext: from_b.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .expect("an unrelated key's blobs must be unaffected by the deletion"); +} + +#[tokio::test] +async fn data_key_envelope_detection_matches_produced_blobs() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: case.key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] generate should succeed: {error:?}")); + + assert!( + is_data_key_envelope(&dek.ciphertext_blob), + "[{label}] a freshly wrapped data key must be recognised as an envelope" + ); + + for (name, input) in [ + ("empty", Vec::new()), + ("raw bytes", vec![0x00, 0x01, 0x02]), + ("plain text", b"not an envelope".to_vec()), + ("unrelated json", br#"{"unrelated":true}"#.to_vec()), + ("json array", b"[]".to_vec()), + ] { + assert!( + !is_data_key_envelope(&input), + "[{label}] {name} must not be mistaken for a data key envelope" + ); + } + }) + .await; +} + +#[tokio::test] +async fn object_data_keys_are_bound_to_their_object() { + // `ObjectEncryptionService::create_data_key` derives the encryption context + // from bucket + object key, so a DEK minted for one object must not open + // under another object's context. + let kms = TestKms::local().await; + let service = kms.service().await; + kms.create_key("object-binding").await; + let key_id = Some("object-binding".to_string()); + + let alpha = ObjectEncryptionContext::new("bucket-x".to_string(), "alpha.bin".to_string()); + let beta = ObjectEncryptionContext::new("bucket-x".to_string(), "beta.bin".to_string()); + + let (alpha_key, alpha_blob) = service + .create_data_key(&key_id, &alpha) + .await + .expect("create_data_key for alpha"); + let (beta_key, beta_blob) = service + .create_data_key(&key_id, &beta) + .await + .expect("create_data_key for beta"); + + assert_ne!(alpha_key.plaintext_key, beta_key.plaintext_key, "each object gets its own data key"); + assert_ne!(alpha_blob, beta_blob, "each object gets its own wrapped data key"); + assert_ne!(alpha_key.nonce, beta_key.nonce, "each object gets its own base nonce for streaming"); + + let recovered = service + .decrypt_data_key(&alpha_blob, &alpha) + .await + .expect("alpha's blob must open under alpha's context"); + assert_eq!( + recovered.plaintext_key, alpha_key.plaintext_key, + "the recovered data key must match the one handed out" + ); + + assert_context_mismatch(service.decrypt_data_key(&alpha_blob, &beta).await); + assert_context_mismatch(service.decrypt_data_key(&beta_blob, &alpha).await); + + // A different bucket with the same object name is also a different object. + let other_bucket = ObjectEncryptionContext::new("bucket-y".to_string(), "alpha.bin".to_string()); + assert_context_mismatch(service.decrypt_data_key(&alpha_blob, &other_bucket).await); + + // The legacy path intentionally drops the context; it is the only way to + // read objects written before per-object binding existed. + let legacy = service + .decrypt_legacy_data_key(&alpha_blob) + .await + .expect("the legacy path must still open the blob"); + assert_eq!(legacy.plaintext_key, alpha_key.plaintext_key); + assert_eq!( + legacy.nonce, [0u8; 12], + "the legacy path returns a zero nonce; callers substitute the stored one" + ); +} + +#[tokio::test] +async fn create_data_key_requires_a_resolvable_key_id() { + let kms = TestKms::local().await; + let service = kms.service().await; + let context = ObjectEncryptionContext::new("bucket".to_string(), "object".to_string()); + + // No explicit id and no configured default: a configuration error, not a + // silent fallback to some arbitrary key. + match service.create_data_key(&None, &context).await { + Err(KmsError::ConfigurationError { message }) => { + assert!(message.contains("No KMS key ID"), "should explain the missing key id: {message}") + } + other => panic!("expected ConfigurationError, got {other:?}"), + } + + // With a default configured, the same call resolves to it. + let kms = TestKms::local_with(|config| config.default_key_id = Some("default-key".to_string())).await; + let service = kms.service().await; + kms.create_key("default-key").await; + assert_eq!( + service.get_default_key_id().map(String::as_str), + Some("default-key"), + "the configured default must be visible to callers" + ); + let (_key, blob) = service + .create_data_key(&None, &context) + .await + .expect("the default key must be used when none is given"); + service + .decrypt_data_key(&blob, &context) + .await + .expect("the default key's blob must round-trip"); +} diff --git a/crates/kms/tests/behavior_deletion.rs b/crates/kms/tests/behavior_deletion.rs new file mode 100644 index 000000000..20533039e --- /dev/null +++ b/crates/kms/tests/behavior_deletion.rs @@ -0,0 +1,470 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: completing a scheduled key deletion. +//! +//! `KmsBackend::remove_expired_key` is the only operation in this crate that +//! destroys key material, and the background sweep may call it concurrently on +//! several nodes and again after a crash. Its documented contract is therefore +//! unusually strict, and each clause is asserted below: +//! +//! * the deadline is honoured — a key that is not yet due is never removed; +//! * a cancellation observed after the caller inspected the key **wins**, so a +//! racing sweep reports `StateChanged` rather than destroying a key the +//! operator just rescued; +//! * removal is idempotent — re-running it after a crash, or on a key that is +//! already gone, succeeds rather than erroring; +//! * the deadline is persisted, so a restart does not reset the clock. +//! +//! The clock is a parameter of the operation, so every case here is +//! deterministic: nothing sleeps and nothing waits on wall-clock time. +//! +//! Backends without deletion support must report the capability gap instead of +//! silently doing nothing. + +mod common; + +use std::sync::Arc; +use std::time::Duration; + +use common::{TestKms, assert_key_not_found, assert_unsupported_capability, ctx}; +use jiff::Zoned; +use rustfs_kms::backends::local::LocalKmsBackend; +use rustfs_kms::backends::static_kms::StaticKmsBackend; +use rustfs_kms::backends::{ExpiredKeyRemoval, KmsBackend}; +use rustfs_kms::{ + CancelKeyDeletionRequest, CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, + KeySpec, KeyState, KeyUsage, KmsConfig, +}; +use tempfile::TempDir; + +/// A backend built the way the service manager builds it, so the deletion +/// contract is exercised on the same object production uses. +async fn local_backend(dir: &TempDir) -> Arc { + let config = KmsConfig::local(dir.path().to_path_buf()).with_insecure_development_defaults(); + Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build")) +} + +async fn create(backend: &LocalKmsBackend, key_id: &str) { + backend + .create_key(CreateKeyRequest { + key_name: Some(key_id.to_string()), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + }) + .await + .expect("key should be created"); +} + +async fn schedule(backend: &LocalKmsBackend, key_id: &str, days: u32) { + backend + .delete_key(DeleteKeyRequest { + key_id: key_id.to_string(), + pending_window_in_days: Some(days), + force_immediate: None, + confirm_key_id: None, + }) + .await + .expect("deletion should be scheduled"); +} + +fn days_from_now(days: u64) -> Zoned { + Zoned::now() + Duration::from_secs(days * 86_400) +} + +async fn exists(backend: &LocalKmsBackend, key_id: &str) -> bool { + backend + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .is_ok() +} + +#[tokio::test] +async fn removal_waits_for_the_deadline() { + let dir = TempDir::new().expect("temp dir"); + let backend = local_backend(&dir).await; + create(&backend, "not-yet-due").await; + schedule(&backend, "not-yet-due", 30).await; + + // Right now, and at every point strictly inside the window, the key stays. + for elapsed_days in [0u64, 1, 15, 29] { + assert_eq!( + backend + .remove_expired_key("not-yet-due", &days_from_now(elapsed_days)) + .await + .expect("the check itself must succeed"), + ExpiredKeyRemoval::NotExpired, + "a key {elapsed_days} days into a 30-day window is not due" + ); + assert!(exists(&backend, "not-yet-due").await, "a key that is not due must survive"); + } + + // Past the deadline it goes. + assert_eq!( + backend + .remove_expired_key("not-yet-due", &days_from_now(31)) + .await + .expect("removal should succeed"), + ExpiredKeyRemoval::Removed + ); + assert!(!exists(&backend, "not-yet-due").await, "an expired key must be gone"); +} + +#[tokio::test] +async fn removal_is_idempotent_across_restarts_and_nodes() { + let dir = TempDir::new().expect("temp dir"); + let backend = local_backend(&dir).await; + create(&backend, "idempotent").await; + schedule(&backend, "idempotent", 7).await; + + let past_deadline = days_from_now(8); + + assert_eq!( + backend + .remove_expired_key("idempotent", &past_deadline) + .await + .expect("first removal"), + ExpiredKeyRemoval::Removed + ); + + // A second sweep — another node, or the same node after a restart — must + // report success rather than failing on the now-missing record. + for attempt in 0..3 { + assert_eq!( + backend + .remove_expired_key("idempotent", &past_deadline) + .await + .unwrap_or_else(|error| panic!("repeat removal {attempt} must succeed: {error:?}")), + ExpiredKeyRemoval::Removed, + "removing an already-removed key is a no-op success" + ); + } + + // A key that never existed is treated the same way, so a stale sweep entry + // cannot wedge the worker. + assert_eq!( + backend + .remove_expired_key("never-existed", &past_deadline) + .await + .expect("unknown key must not error"), + ExpiredKeyRemoval::Removed + ); +} + +#[tokio::test] +async fn a_cancellation_beats_a_racing_sweep() { + let dir = TempDir::new().expect("temp dir"); + let backend = local_backend(&dir).await; + create(&backend, "rescued").await; + create(&backend, "doomed").await; + schedule(&backend, "rescued", 7).await; + schedule(&backend, "doomed", 7).await; + + // The sweep has already decided both keys are due; the operator cancels one + // before the removal actually runs. + backend + .cancel_key_deletion(CancelKeyDeletionRequest { + key_id: "rescued".to_string(), + }) + .await + .expect("cancel should succeed"); + + let past_deadline = days_from_now(8); + assert_eq!( + backend + .remove_expired_key("rescued", &past_deadline) + .await + .expect("the check must succeed"), + ExpiredKeyRemoval::StateChanged, + "a key rescued after inspection must report StateChanged, not be destroyed" + ); + assert_eq!( + backend + .remove_expired_key("doomed", &past_deadline) + .await + .expect("removal should succeed"), + ExpiredKeyRemoval::Removed + ); + + // The rescued key is not merely present: it is fully usable again. + let described = backend + .describe_key(DescribeKeyRequest { + key_id: "rescued".to_string(), + }) + .await + .expect("the rescued key must survive"); + assert_eq!(described.key_metadata.key_state, KeyState::Enabled); + assert!(described.key_metadata.deletion_date.is_none(), "the rescued key must carry no deadline"); + backend + .generate_data_key(GenerateDataKeyRequest { + key_id: "rescued".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: ctx(&[("bucket", "deletion-behavior")]), + }) + .await + .expect("the rescued key must accept new work"); + + assert!(!exists(&backend, "doomed").await, "the un-rescued key must be gone"); +} + +#[tokio::test] +async fn keys_that_are_not_pending_deletion_are_never_removed() { + let dir = TempDir::new().expect("temp dir"); + let backend = local_backend(&dir).await; + create(&backend, "healthy").await; + create(&backend, "disabled").await; + backend.disable_key("disabled").await.expect("disable should succeed"); + + let far_future = days_from_now(3_650); + for key_id in ["healthy", "disabled"] { + assert_eq!( + backend + .remove_expired_key(key_id, &far_future) + .await + .expect("the check must succeed"), + ExpiredKeyRemoval::StateChanged, + "{key_id} is not pending deletion, so no deadline can apply to it" + ); + assert!(exists(&backend, key_id).await, "{key_id} must survive"); + } + + // Their states are untouched by the attempt. + assert_eq!( + backend + .describe_key(DescribeKeyRequest { + key_id: "healthy".to_string() + }) + .await + .expect("describe") + .key_metadata + .key_state, + KeyState::Enabled + ); + assert_eq!( + backend + .describe_key(DescribeKeyRequest { + key_id: "disabled".to_string() + }) + .await + .expect("describe") + .key_metadata + .key_state, + KeyState::Disabled + ); +} + +#[tokio::test] +async fn the_deletion_deadline_survives_a_restart() { + let dir = TempDir::new().expect("temp dir"); + + // Schedule the deletion, then drop the backend entirely — a process restart. + { + let backend = local_backend(&dir).await; + create(&backend, "scheduled-before-restart").await; + schedule(&backend, "scheduled-before-restart", 7).await; + } + + let backend = local_backend(&dir).await; + let described = backend + .describe_key(DescribeKeyRequest { + key_id: "scheduled-before-restart".to_string(), + }) + .await + .expect("the key must survive the restart"); + assert_eq!( + described.key_metadata.key_state, + KeyState::PendingDeletion, + "the pending state must be persisted, not held in memory" + ); + assert!( + described.key_metadata.deletion_date.is_some(), + "the deadline must be persisted so a restart does not reset the clock" + ); + + // The restarted process is still inside the window, then past it. + assert_eq!( + backend + .remove_expired_key("scheduled-before-restart", &days_from_now(1)) + .await + .expect("check"), + ExpiredKeyRemoval::NotExpired, + "a restart must not make an un-due key removable" + ); + assert_eq!( + backend + .remove_expired_key("scheduled-before-restart", &days_from_now(8)) + .await + .expect("removal"), + ExpiredKeyRemoval::Removed + ); + assert!(!exists(&backend, "scheduled-before-restart").await); +} + +#[tokio::test] +async fn removed_key_material_is_actually_destroyed() { + let dir = TempDir::new().expect("temp dir"); + let backend = local_backend(&dir).await; + create(&backend, "material-gone").await; + + let context = ctx(&[("bucket", "deletion-behavior"), ("object", "doomed.bin")]); + let dek = backend + .generate_data_key(GenerateDataKeyRequest { + key_id: "material-gone".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("generate a data key while the key still exists"); + + // While pending deletion, existing ciphertext must still open — this is the + // whole point of the pending window. + schedule(&backend, "material-gone", 7).await; + let during_window = backend + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context.clone(), + grant_tokens: Vec::new(), + }) + .await + .expect("a pending-deletion key must still decrypt"); + assert_eq!(during_window.plaintext, dek.plaintext_key); + + // After the deadline the material is gone and the ciphertext is dead. + assert_eq!( + backend + .remove_expired_key("material-gone", &days_from_now(8)) + .await + .expect("removal"), + ExpiredKeyRemoval::Removed + ); + assert_key_not_found( + backend + .describe_key(DescribeKeyRequest { + key_id: "material-gone".to_string(), + }) + .await, + "material-gone", + ); + assert!( + backend + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context, + grant_tokens: Vec::new(), + }) + .await + .is_err(), + "ciphertext wrapped by a destroyed key must no longer decrypt" + ); + + // And a fresh backend over the same directory agrees: the removal was + // durable, not just an in-memory state change. + let restarted = local_backend(&dir).await; + assert!(!exists(&restarted, "material-gone").await, "the removal must survive a restart"); +} + +#[tokio::test] +async fn deletion_only_touches_its_own_key() { + let dir = TempDir::new().expect("temp dir"); + let backend = local_backend(&dir).await; + for key_id in ["neighbour-a", "target", "neighbour-b"] { + create(&backend, key_id).await; + } + schedule(&backend, "target", 7).await; + + assert_eq!( + backend + .remove_expired_key("target", &days_from_now(8)) + .await + .expect("removal"), + ExpiredKeyRemoval::Removed + ); + + for neighbour in ["neighbour-a", "neighbour-b"] { + let described = backend + .describe_key(DescribeKeyRequest { + key_id: neighbour.to_string(), + }) + .await + .unwrap_or_else(|error| panic!("{neighbour} must be untouched: {error:?}")); + assert_eq!(described.key_metadata.key_state, KeyState::Enabled); + backend + .generate_data_key(GenerateDataKeyRequest { + key_id: neighbour.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: ctx(&[("bucket", "deletion-behavior")]), + }) + .await + .unwrap_or_else(|error| panic!("{neighbour} must still work: {error:?}")); + } +} + +#[tokio::test] +async fn a_backend_without_deletion_support_reports_the_capability_gap() { + let config = KmsConfig::static_kms(common::STATIC_KEY_ID.to_string(), common::static_secret_key()); + let backend = StaticKmsBackend::new(config).await.expect("static backend should build"); + + assert!( + !backend.capabilities().schedule_deletion, + "the static backend must not advertise deletion scheduling" + ); + assert_unsupported_capability( + backend.remove_expired_key(common::STATIC_KEY_ID, &Zoned::now()).await, + "remove_expired_key", + ); +} + +/// The service manager runs a background sweep for backends that support +/// deletion. It uses wall-clock time, so a real expiry cannot be forced from +/// outside; what *is* checkable from here — and what a spurious-deletion bug +/// would break — is that the sweep leaves un-due keys alone while it runs. +#[tokio::test(start_paused = true)] +async fn the_background_sweep_never_removes_an_un_due_key() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + let key_id = kms.create_key("swept-but-not-due").await; + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await + .expect("schedule deletion"); + + // With the clock paused, tokio auto-advances through the sweep interval, so + // this loop drives many sweeps in a fraction of a second. + for _ in 0..40 { + tokio::time::sleep(Duration::from_secs(60)).await; + } + + let described = manager + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .expect("a key inside its window must survive every sweep"); + assert_eq!(described.key_metadata.key_state, KeyState::PendingDeletion); + assert!( + described.key_metadata.deletion_date.is_some(), + "the sweep must not clear the deadline it is waiting on" + ); + + // And it can still be rescued afterwards. + manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await + .expect("a key the sweep left alone must still be cancellable"); +} diff --git a/crates/kms/tests/behavior_keys.rs b/crates/kms/tests/behavior_keys.rs new file mode 100644 index 000000000..2ddf327fd --- /dev/null +++ b/crates/kms/tests/behavior_keys.rs @@ -0,0 +1,697 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: master key lifecycle through `KmsManager`. +//! +//! The state × operation matrix is the load-bearing part. RustFS deliberately +//! deviates from AWS KMS in one direction: **decryption stays available while a +//! key is Disabled or PendingDeletion**, because refusing it would make every +//! object encrypted under that key unreadable the instant an operator disables +//! it. The rest of the matrix is: +//! +//! | state | encrypt / generate DEK | enable / disable | schedule deletion | cancel deletion | decrypt | +//! |-----------------|------------------------|------------------|-------------------|-----------------|---------| +//! | Enabled | allowed | allowed | allowed | rejected | allowed | +//! | Disabled | rejected | allowed | allowed | rejected | allowed | +//! | PendingDeletion | rejected | rejected | rejected | allowed | allowed | +//! +//! `crates/kms/src/backends/contract_tests.rs` pins the same matrix at the +//! backend trait; this file pins it one layer up, where the metadata cache and +//! the manager's invalidation logic also participate — a cache that served a +//! stale `Enabled` snapshot would break the gate without the backend noticing. +//! +//! Not covered here on purpose: tag / description mutation. Those types are +//! re-exported by this crate but their only entry point lives in the admin +//! handlers, outside this crate's public surface. + +mod common; + +use common::{ + BackendCase, BackendKind, TestKms, assert_invalid_operation, assert_key_already_exists, assert_key_not_found, + assert_unsupported_capability, ctx, for_each_backend, without_probe_key, +}; +use rustfs_kms::{ + CancelKeyDeletionRequest, CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, EncryptRequest, + GenerateDataKeyRequest, KeySpec, KeyState, KeyStatus, KeyUsage, KmsManager, ListKeysRequest, +}; + +async fn describe_state(kms: &KmsManager, key_id: &str) -> KeyState { + kms.describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .expect("describe should succeed") + .key_metadata + .key_state +} + +fn generate_request(key_id: &str) -> GenerateDataKeyRequest { + GenerateDataKeyRequest { + key_id: key_id.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: ctx(&[("bucket", "keys-behavior")]), + } +} + +fn encrypt_request(key_id: &str) -> EncryptRequest { + EncryptRequest { + key_id: key_id.to_string(), + plaintext: b"state-gated plaintext".to_vec(), + encryption_context: ctx(&[("bucket", "keys-behavior")]), + grant_tokens: Vec::new(), + } +} + +#[tokio::test] +async fn created_key_is_enabled_and_fully_described() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + + let created = manager + .create_key(CreateKeyRequest { + key_name: Some("described-key".to_string()), + key_usage: KeyUsage::EncryptDecrypt, + description: Some("a described key".to_string()), + ..Default::default() + }) + .await + .expect("create should succeed"); + + assert_eq!(created.key_id, "described-key", "an explicit key name becomes the key id"); + assert_eq!(created.key_metadata.key_id, created.key_id, "metadata must agree with the id"); + assert_eq!(created.key_metadata.key_state, KeyState::Enabled, "a new key is immediately usable"); + assert_eq!(created.key_metadata.key_usage, KeyUsage::EncryptDecrypt); + assert!(created.key_metadata.deletion_date.is_none(), "a new key has no deletion deadline"); + + let described = manager + .describe_key(DescribeKeyRequest { + key_id: created.key_id.clone(), + }) + .await + .expect("describe should succeed") + .key_metadata; + assert_eq!(described.key_id, created.key_id); + assert_eq!(described.key_state, KeyState::Enabled); + assert_eq!( + described.description, created.key_metadata.description, + "describe must return the description supplied at creation" + ); + assert_eq!( + described.creation_date, created.key_metadata.creation_date, + "the creation timestamp is stable across reads" + ); +} + +#[tokio::test] +async fn auto_generated_key_ids_are_unique() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + + let first = manager + .create_key(CreateKeyRequest::default()) + .await + .expect("first auto-named key"); + let second = manager + .create_key(CreateKeyRequest::default()) + .await + .expect("second auto-named key"); + + assert!(!first.key_id.is_empty(), "an auto-generated key id must not be empty"); + assert_ne!(first.key_id, second.key_id, "auto-generated key ids must not collide"); + for created in [&first, &second] { + assert_eq!( + describe_state(&manager, &created.key_id).await, + KeyState::Enabled, + "auto-named keys are Enabled like named ones" + ); + } +} + +#[tokio::test] +async fn duplicate_key_name_is_rejected_without_disturbing_the_original() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + kms.create_key("duplicate-me").await; + + assert_key_already_exists( + manager + .create_key(CreateKeyRequest { + key_name: Some("duplicate-me".to_string()), + description: Some("an impostor".to_string()), + ..Default::default() + }) + .await, + "duplicate-me", + ); + + // The rejected create must not have overwritten the original's material: + // a DEK generated before the conflict still decrypts afterwards. + let context = ctx(&[("bucket", "duplicate")]); + let dek = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: "duplicate-me".to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("data key generation should still work"); + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob, + encryption_context: context, + grant_tokens: Vec::new(), + }) + .await + .expect("the original key material must be intact"); + assert_eq!(decrypted.plaintext, dek.plaintext_key, "round-trip after a rejected create"); +} + +#[tokio::test] +async fn describing_an_unknown_key_reports_key_not_found() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + assert_key_not_found( + manager + .describe_key(DescribeKeyRequest { + key_id: "no-such-key".to_string(), + }) + .await, + "no-such-key", + ); + assert_key_not_found(manager.generate_data_key(generate_request("no-such-key")).await, "no-such-key"); + assert_key_not_found(manager.encrypt(encrypt_request("no-such-key")).await, "no-such-key"); + }) + .await; +} + +#[tokio::test] +async fn list_keys_reports_created_keys_and_honours_filters() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + for name in ["list-a", "list-b", "list-c"] { + kms.create_key(name).await; + } + + let all = manager + .list_keys(ListKeysRequest::default()) + .await + .expect("list should succeed"); + let mut ids = without_probe_key(all.keys.iter().map(|key| key.key_id.clone())); + ids.sort(); + assert_eq!(ids, vec!["list-a", "list-b", "list-c"], "every created key must be listed"); + + // `limit` caps the page, and a capped page must say so. A client that + // paginates by looking at `truncated` would otherwise stop after the first + // page and silently act on a partial key list — for a KMS, that means + // believing keys do not exist when they do. + let limited = manager + .list_keys(ListKeysRequest { + limit: Some(2), + ..Default::default() + }) + .await + .expect("limited list should succeed"); + assert_eq!(limited.keys.len(), 2, "limit must cap the returned page"); + assert!( + limited.truncated, + "a page that was cut short by `limit` must be reported as truncated; 3 keys exist and only 2 were returned" + ); + assert!( + limited.next_marker.is_some(), + "a truncated page must carry a continuation marker so the caller can fetch the rest" + ); + + // A status filter narrows the result to keys in that state. + manager.disable_key("list-b").await.expect("disable should succeed"); + let disabled = manager + .list_keys(ListKeysRequest { + status_filter: Some(KeyStatus::Disabled), + ..Default::default() + }) + .await + .expect("filtered list should succeed"); + assert_eq!( + disabled.keys.iter().map(|k| k.key_id.as_str()).collect::>(), + vec!["list-b"], + "only the disabled key matches the Disabled filter" + ); + + let active = manager + .list_keys(ListKeysRequest { + status_filter: Some(KeyStatus::Active), + ..Default::default() + }) + .await + .expect("filtered list should succeed"); + let mut active_ids = without_probe_key(active.keys.iter().map(|k| k.key_id.clone())); + active_ids.sort(); + assert_eq!(active_ids, vec!["list-a", "list-c"], "the disabled key drops out of the Active filter"); + + // A usage filter that matches nothing yields an empty page, not an error. + let none = manager + .list_keys(ListKeysRequest { + usage_filter: Some(KeyUsage::SignVerify), + ..Default::default() + }) + .await + .expect("non-matching filter should still succeed"); + assert!(none.keys.is_empty(), "a filter matching nothing returns an empty page"); +} + +#[tokio::test] +async fn disable_and_enable_round_trip_through_the_metadata_cache() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + let key_id = kms.create_key("toggle-me").await; + + // Warm the cache first: a stale cached Enabled snapshot would defeat the + // Disabled gate below without the backend ever being consulted. + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + + manager.disable_key(&key_id).await.expect("disable should succeed"); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::Disabled, + "describe must observe the post-mutation state" + ); + + // Disabling again is idempotent, not an error. + manager.disable_key(&key_id).await.expect("repeat disable is idempotent"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Disabled); + + manager.enable_key(&key_id).await.expect("enable should succeed"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + manager.enable_key(&key_id).await.expect("repeat enable is idempotent"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); +} + +#[tokio::test] +async fn scheduled_deletion_carries_a_deadline_and_can_be_cancelled() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + let key_id = kms.create_key("deletable").await; + + let scheduled = manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await + .expect("scheduling deletion should succeed"); + + assert_eq!(scheduled.key_id, key_id); + assert!( + scheduled.deletion_date.is_some(), + "a scheduled deletion must report when the key will actually go away" + ); + assert_eq!(scheduled.key_metadata.key_state, KeyState::PendingDeletion); + assert!( + scheduled.key_metadata.deletion_date.is_some(), + "the metadata must carry the same deadline" + ); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::PendingDeletion, + "the pending state must be visible to a subsequent describe" + ); + + let cancelled = manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await + .expect("cancelling should succeed"); + assert_eq!(cancelled.key_id, key_id); + assert_eq!(cancelled.key_metadata.key_state, KeyState::Enabled, "cancelling restores an usable key"); + assert!(cancelled.key_metadata.deletion_date.is_none(), "cancelling must clear the deadline"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + + // The key really is usable again, not merely reported as such. + manager + .generate_data_key(generate_request(&key_id)) + .await + .expect("a cancelled key must accept new cryptographic work"); + + // Cancelling a key that is not pending deletion is a state error. + assert_invalid_operation( + manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await, + "not pending deletion", + ); +} + +#[tokio::test] +async fn deletion_pending_window_is_bounded() { + let kms = TestKms::local().await; + let manager = kms.kms().await; + + for (name, days) in [("window-too-short", 6u32), ("window-too-long", 31)] { + let key_id = kms.create_key(name).await; + assert_invalid_operation( + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(days), + force_immediate: None, + confirm_key_id: None, + }) + .await, + "between 7 and 30", + ); + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::Enabled, + "a rejected deletion window must leave the key untouched" + ); + } + + // The documented bounds themselves are accepted. + for (name, days) in [("window-min", 7u32), ("window-max", 30)] { + let key_id = kms.create_key(name).await; + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(days), + force_immediate: None, + confirm_key_id: None, + }) + .await + .unwrap_or_else(|error| panic!("{days} days must be accepted: {error:?}")); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::PendingDeletion); + } +} + +#[tokio::test] +async fn forced_immediate_deletion_removes_the_key() { + let kms = TestKms::local_with(|config| config.allow_immediate_deletion = true).await; + let manager = kms.kms().await; + let key_id = kms.create_key("burn-now").await; + + let deleted = manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: None, + force_immediate: Some(true), + confirm_key_id: Some(key_id.clone()), + }) + .await + .expect("forced deletion should succeed"); + assert!(deleted.deletion_date.is_none(), "an immediate deletion has no future deadline to report"); + + assert_key_not_found(manager.describe_key(DescribeKeyRequest { key_id: key_id.clone() }).await, &key_id); + assert_key_not_found(manager.generate_data_key(generate_request(&key_id)).await, &key_id); + assert!( + !manager + .list_keys(ListKeysRequest::default()) + .await + .expect("list should succeed") + .keys + .iter() + .any(|key| key.key_id == key_id), + "a physically deleted key must disappear from listings" + ); + + // The name is free again, and the replacement is a genuinely new key. + let recreated = kms.create_key(&key_id).await; + assert_eq!(describe_state(&manager, &recreated).await, KeyState::Enabled); +} + +/// The full state × operation matrix, run against every offline backend. +/// +/// Backends that cannot reach a state (the static backend has no lifecycle at +/// all) assert the refusal instead — the capability flags are a two-way +/// contract, not just an advertisement. +#[tokio::test] +async fn key_state_gates_every_operation() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let caps = case.caps().await; + let key_id = case.key_id.clone(); + + // --- Enabled: everything is permitted ----------------------------- + assert_eq!( + describe_state(&manager, &key_id).await, + KeyState::Enabled, + "[{}] the seeded key starts Enabled", + case.kind().name() + ); + let enabled_dek = manager + .generate_data_key(generate_request(&key_id)) + .await + .expect("Enabled must permit data key generation"); + manager + .encrypt(encrypt_request(&key_id)) + .await + .expect("Enabled must permit encryption"); + + // Rotation is capability-gated even in the Enabled state. + if !caps.rotate { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + } + + if !caps.enable_disable { + assert_unsupported_capability(manager.disable_key(&key_id).await, "disable_key"); + assert_unsupported_capability(manager.enable_key(&key_id).await, "enable_key"); + } + if !caps.schedule_deletion { + // A read-only backend refuses deletion outright rather than + // pretending to schedule one. + assert!( + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await + .is_err(), + "[{}] a backend without deletion support must refuse delete_key", + case.kind().name() + ); + } + + if case.kind() == BackendKind::Static { + // No further states are reachable on a read-only backend; the + // decrypt-still-works half of the matrix is checked below instead. + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: enabled_dek.ciphertext_blob.clone(), + encryption_context: ctx(&[("bucket", "keys-behavior")]), + grant_tokens: Vec::new(), + }) + .await + .expect("static backend must decrypt its own envelope"); + assert_eq!(decrypted.plaintext, enabled_dek.plaintext_key); + return; + } + + // --- Disabled: no new crypto, but reads and lifecycle recovery ---- + manager.disable_key(&key_id).await.expect("disable should succeed"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Disabled); + + assert_invalid_operation(manager.generate_data_key(generate_request(&key_id)).await, "is disabled"); + assert_invalid_operation(manager.encrypt(encrypt_request(&key_id)).await, "is disabled"); + if caps.rotate { + assert_invalid_operation(manager.rotate_key(&key_id).await, "is disabled"); + } else { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + } + + // The deliberate deviation from AWS KMS: data written before the key + // was disabled must stay readable. + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: enabled_dek.ciphertext_blob.clone(), + encryption_context: ctx(&[("bucket", "keys-behavior")]), + grant_tokens: Vec::new(), + }) + .await + .expect("a Disabled key must still decrypt existing ciphertext"); + assert_eq!( + decrypted.plaintext, enabled_dek.plaintext_key, + "decryption under a Disabled key must return the original data key" + ); + + // Disabled still permits enabling, disabling, and scheduling deletion. + manager + .disable_key(&key_id) + .await + .expect("disable is idempotent while Disabled"); + manager.enable_key(&key_id).await.expect("Disabled must permit re-enabling"); + manager + .disable_key(&key_id) + .await + .expect("back to Disabled for the next step"); + + // --- PendingDeletion: only cancellation and decryption ------------ + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await + .expect("Disabled must permit scheduling deletion"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::PendingDeletion); + + assert_invalid_operation(manager.generate_data_key(generate_request(&key_id)).await, "pending deletion"); + assert_invalid_operation(manager.encrypt(encrypt_request(&key_id)).await, "pending deletion"); + assert_invalid_operation(manager.enable_key(&key_id).await, "pending deletion"); + assert_invalid_operation(manager.disable_key(&key_id).await, "pending deletion"); + assert_invalid_operation( + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await, + "pending deletion", + ); + if caps.rotate { + assert_invalid_operation(manager.rotate_key(&key_id).await, "pending deletion"); + } else { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + } + + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: enabled_dek.ciphertext_blob.clone(), + encryption_context: ctx(&[("bucket", "keys-behavior")]), + grant_tokens: Vec::new(), + }) + .await + .expect("a PendingDeletion key must still decrypt existing ciphertext"); + assert_eq!(decrypted.plaintext, enabled_dek.plaintext_key); + + // Cancellation is the one way out, and it restores full capability. + manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await + .expect("PendingDeletion must permit cancellation"); + assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled); + manager + .generate_data_key(generate_request(&key_id)) + .await + .expect("a cancelled key is fully usable again"); + }) + .await; +} + +#[tokio::test] +async fn static_backend_refuses_every_lifecycle_mutation() { + let kms = TestKms::static_backend().await; + let manager = kms.kms().await; + let caps = kms.capabilities().await; + + // The capability report is the contract; assert it explicitly so a backend + // that silently gains a capability has to update this test. + assert!(caps.encrypt && caps.decrypt && caps.generate_data_key, "static must do crypto"); + assert!( + !caps.rotate && !caps.enable_disable && !caps.schedule_deletion && !caps.versioning && !caps.physical_delete, + "static must advertise no lifecycle capability: {caps:?}" + ); + + assert_invalid_operation( + manager + .create_key(CreateKeyRequest { + key_name: Some("another-key".to_string()), + ..Default::default() + }) + .await, + "read-only", + ); + // Re-creating the configured key is a conflict, not a generic refusal. + assert_key_already_exists( + manager + .create_key(CreateKeyRequest { + key_name: Some(kms.config().static_config().expect("static config").key_id.clone()), + ..Default::default() + }) + .await, + &kms.config().static_config().expect("static config").key_id, + ); + + let key_id = kms.config().static_config().expect("static config").key_id.clone(); + assert_invalid_operation( + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await, + "read-only", + ); + assert_invalid_operation( + manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await, + "read-only", + ); + assert_unsupported_capability(manager.enable_key(&key_id).await, "enable_key"); + assert_unsupported_capability(manager.disable_key(&key_id).await, "disable_key"); + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + + // Operations aimed at any other key id are "not found", not "read-only": + // the distinction matters to the admin API's status mapping. + assert_key_not_found( + manager + .delete_key(DeleteKeyRequest { + key_id: "other".to_string(), + pending_window_in_days: Some(7), + force_immediate: None, + confirm_key_id: None, + }) + .await, + "other", + ); + assert_key_not_found( + manager + .cancel_key_deletion(CancelKeyDeletionRequest { + key_id: "other".to_string(), + }) + .await, + "other", + ); + assert_key_not_found( + manager + .describe_key(DescribeKeyRequest { + key_id: "other".to_string(), + }) + .await, + "other", + ); + + // Despite refusing every mutation, it must still do its actual job. + let dek = manager + .generate_data_key(generate_request(&key_id)) + .await + .expect("static backend must generate data keys"); + assert_eq!(dek.plaintext_key.len(), 32, "AES-256 data key is 32 bytes"); + let listed = manager + .list_keys(ListKeysRequest::default()) + .await + .expect("list should succeed"); + assert_eq!( + listed.keys.iter().map(|k| k.key_id.as_str()).collect::>(), + vec![key_id.as_str()], + "the static backend lists exactly its one configured key" + ); +} diff --git a/crates/kms/tests/behavior_lifecycle.rs b/crates/kms/tests/behavior_lifecycle.rs new file mode 100644 index 000000000..c0867b209 --- /dev/null +++ b/crates/kms/tests/behavior_lifecycle.rs @@ -0,0 +1,456 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: KMS service lifecycle, configuration, and redaction. +//! +//! Pins the contract the admin API depends on: +//! +//! * an unconfigured manager hands out nothing and refuses to start; +//! * `configure` is for the stopped state only — changing a running service is +//! `reconfigure`, which never exposes a stopped interval; +//! * a configuration is validated *before* anything is published, and a failing +//! persistence callback leaves the previous state completely intact; +//! * the Local backend's identity (directory, master key, dev-mode) is frozen +//! once configured, because changing it silently orphans existing key files; +//! * redacted views never carry key material. + +mod common; + +use std::sync::Arc; +use std::time::Duration; + +use common::{STATIC_KEY_ID, TestKms, assert_configuration_error, assert_no_secret_leak, static_secret_key}; +use rustfs_kms::{BackendConfig, KmsConfig, KmsError, KmsServiceManager, KmsServiceStatus, KmsStartOutcome, LocalConfig}; +use tempfile::TempDir; +use url::Url; + +fn local_dev_config(dir: &TempDir) -> KmsConfig { + KmsConfig::local(dir.path().to_path_buf()).with_insecure_development_defaults() +} + +#[tokio::test] +async fn unconfigured_manager_exposes_nothing_and_refuses_to_start() { + let manager = KmsServiceManager::new(); + + assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured); + assert!(manager.get_config().await.is_none(), "no config before configure"); + assert!(manager.get_redacted_config().await.is_none(), "no redacted config either"); + assert!(manager.get_manager().await.is_none(), "no manager handle before start"); + assert!(manager.get_encryption_service().await.is_none(), "no service handle before start"); + assert!(manager.get_service_version().await.is_none(), "no version before start"); + + // A health check on a service that was never started is a definite "not + // healthy", not an error: callers use it as a readiness probe. + assert!( + !manager.health_check().await.expect("health check must not error when idle"), + "an unstarted service must report unhealthy" + ); + + assert_configuration_error(manager.start().await, "no configuration provided"); + + // The failed start records the reason and still exposes no service. + match manager.get_status().await { + KmsServiceStatus::Error(message) => { + assert!(message.contains("no configuration"), "error status should explain why: {message}") + } + other => panic!("expected Error status after starting without config, got {other:?}"), + } + assert!(manager.get_encryption_service().await.is_none()); + assert!(manager.get_service_version().await.is_none()); +} + +#[tokio::test] +async fn configure_start_stop_restart_state_machine() { + let dir = TempDir::new().expect("temp dir"); + let manager = KmsServiceManager::new(); + let config = local_dev_config(&dir); + + manager.configure(config.clone()).await.expect("configure should succeed"); + assert_eq!(manager.get_status().await, KmsServiceStatus::Configured); + assert!( + manager.get_encryption_service().await.is_none(), + "configured but not started must not expose a service" + ); + assert!(manager.get_config().await.is_some(), "configure publishes the config"); + + manager.start().await.expect("start should succeed"); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, Some(1), "first start is version 1"); + assert!(manager.get_manager().await.is_some()); + assert!(manager.get_encryption_service().await.is_some()); + assert!(manager.health_check().await.expect("health check"), "a started local KMS is healthy"); + + manager.stop().await.expect("stop should succeed"); + assert_eq!( + manager.get_status().await, + KmsServiceStatus::Configured, + "stop falls back to Configured, not NotConfigured, because the config survives" + ); + assert!(manager.get_encryption_service().await.is_none(), "stop withdraws the service"); + assert!(manager.get_service_version().await.is_none(), "stop clears the published version"); + assert!(manager.get_config().await.is_some(), "stop keeps the configuration"); + assert!(!manager.health_check().await.expect("health check"), "a stopped service is unhealthy"); + + // Restart after stop reuses the surviving config and takes the next version. + manager.start().await.expect("restart after stop should succeed"); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!( + manager.get_service_version().await, + Some(2), + "the version counter is monotonic across a stop/start cycle" + ); +} + +#[tokio::test] +async fn start_or_restart_reports_the_action_it_took() { + let dir = TempDir::new().expect("temp dir"); + let manager = KmsServiceManager::new(); + manager + .configure(local_dev_config(&dir)) + .await + .expect("configure should succeed"); + + assert_eq!( + manager.start_or_restart(false).await.expect("first start"), + KmsStartOutcome::Started, + "starting a stopped service reports Started" + ); + assert_eq!(manager.get_service_version().await, Some(1)); + + let service_v1 = manager.get_encryption_service().await.expect("service after start"); + + assert_eq!( + manager.start_or_restart(false).await.expect("non-forced start while running"), + KmsStartOutcome::AlreadyRunning, + "a non-forced start on a running service is a no-op" + ); + assert_eq!(manager.get_service_version().await, Some(1), "the no-op path must not consume a version"); + assert!( + Arc::ptr_eq(&service_v1, &manager.get_encryption_service().await.expect("service still running")), + "a no-op start must not swap the live service instance" + ); + + assert_eq!( + manager.start_or_restart(true).await.expect("forced restart"), + KmsStartOutcome::Restarted, + "a forced start on a running service reports Restarted" + ); + assert_eq!(manager.get_service_version().await, Some(2), "a forced restart takes a new version"); + assert!( + !Arc::ptr_eq(&service_v1, &manager.get_encryption_service().await.expect("service after restart")), + "a forced restart must publish a different instance" + ); + + // The withdrawn instance keeps working for operations that already hold it. + assert!( + service_v1.health_check().await.expect("old handle health check"), + "a replaced service stays usable while callers still hold it" + ); +} + +#[tokio::test] +async fn configure_is_rejected_while_running() { + let dir = TempDir::new().expect("temp dir"); + let manager = KmsServiceManager::new(); + let config = local_dev_config(&dir); + manager.configure(config.clone()).await.expect("configure"); + manager.start().await.expect("start"); + + assert_configuration_error(manager.configure(config.clone()).await, "use reconfigure instead"); + + // The rejected call must not have disturbed the running service. + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, Some(1), "a rejected configure takes no version"); +} + +#[tokio::test] +async fn reconfigure_swaps_the_service_without_a_stopped_interval() { + let dir = TempDir::new().expect("temp dir"); + let manager = KmsServiceManager::new(); + let config = local_dev_config(&dir); + manager.configure(config.clone()).await.expect("configure"); + manager.start().await.expect("start"); + + let service_v1 = manager.get_encryption_service().await.expect("service v1"); + let manager_v1 = manager.get_manager().await.expect("manager v1"); + + let mut next = config.clone(); + next.timeout = Duration::from_secs(45); + manager.reconfigure(next).await.expect("reconfigure should succeed"); + + assert_eq!(manager.get_status().await, KmsServiceStatus::Running, "reconfigure never reports stopped"); + assert_eq!(manager.get_service_version().await, Some(2)); + assert_eq!( + manager.get_config().await.expect("config after reconfigure").timeout, + Duration::from_secs(45), + "the published config must be the new one" + ); + + let service_v2 = manager.get_encryption_service().await.expect("service v2"); + let manager_v2 = manager.get_manager().await.expect("manager v2"); + assert!(!Arc::ptr_eq(&service_v1, &service_v2), "reconfigure publishes a new service"); + assert!(!Arc::ptr_eq(&manager_v1, &manager_v2), "reconfigure publishes a new manager"); + + // Both generations stay functional: that is the whole point of the swap. + assert!(service_v1.health_check().await.expect("v1 health"), "old generation still serves"); + assert!(service_v2.health_check().await.expect("v2 health"), "new generation serves"); +} + +#[tokio::test] +async fn failed_persistence_does_not_publish_the_new_configuration() { + let dir = TempDir::new().expect("temp dir"); + let manager = KmsServiceManager::new(); + + // configure_with_persistence: nothing is published when persistence fails. + let result = manager + .configure_with_persistence(local_dev_config(&dir), || async { + Err(KmsError::backend_error("simulated persistence failure")) + }) + .await; + assert!( + matches!(result, Err(KmsError::BackendError { .. })), + "the persistence error must propagate verbatim, got {result:?}" + ); + assert_eq!( + manager.get_status().await, + KmsServiceStatus::NotConfigured, + "a failed persist must leave the manager unconfigured" + ); + assert!(manager.get_config().await.is_none(), "a failed persist must publish no config"); + + // Now bring it up for real, then fail persistence on a reconfigure. + let config = local_dev_config(&dir); + manager.configure(config.clone()).await.expect("configure"); + manager.start().await.expect("start"); + let service_v1 = manager.get_encryption_service().await.expect("service v1"); + + let mut next = config.clone(); + next.timeout = Duration::from_secs(99); + let result = manager + .reconfigure_with_persistence(next, || async { Err(KmsError::backend_error("simulated persistence failure")) }) + .await; + assert!(result.is_err(), "reconfigure must fail when persistence fails, got {result:?}"); + + assert_eq!(manager.get_status().await, KmsServiceStatus::Running, "the old service keeps running"); + assert_eq!( + manager.get_config().await.expect("config").timeout, + config.timeout, + "the old configuration must still be the published one" + ); + assert!( + Arc::ptr_eq(&service_v1, &manager.get_encryption_service().await.expect("service")), + "a failed reconfigure must not swap the live service" + ); +} + +#[tokio::test] +async fn local_backend_identity_is_frozen_after_configuration() { + let dir = TempDir::new().expect("temp dir"); + let other_dir = TempDir::new().expect("other temp dir"); + let manager = KmsServiceManager::new(); + let config = local_dev_config(&dir); + manager.configure(config.clone()).await.expect("configure"); + manager.start().await.expect("start"); + + // Moving the key directory would orphan every existing key file. + let mut moved = config.clone(); + moved.backend_config = BackendConfig::Local(LocalConfig { + key_dir: other_dir.path().to_path_buf(), + master_key: None, + file_permissions: Some(0o600), + }); + assert_configuration_error(manager.reconfigure(moved).await, "key directory cannot be changed"); + + // Changing the at-rest master key would make stored material undecryptable. + let mut rekeyed = config.clone(); + rekeyed.backend_config = BackendConfig::Local(LocalConfig { + key_dir: dir.path().to_path_buf(), + master_key: Some("a-different-master-key".to_string()), + file_permissions: Some(0o600), + }); + assert_configuration_error(manager.reconfigure(rekeyed).await, "master key cannot be changed"); + + // Flipping dev mode changes the at-rest protection of the same directory. + let mut hardened = config.clone(); + hardened.allow_insecure_dev_defaults = false; + hardened.backend_config = BackendConfig::Local(LocalConfig { + key_dir: dir.path().to_path_buf(), + master_key: Some("a-master-key".to_string()), + file_permissions: Some(0o600), + }); + assert!( + manager.reconfigure(hardened).await.is_err(), + "flipping development mode on a configured local backend must be refused" + ); + + // Switching away from Local entirely is refused for the same reason. + let switched = KmsConfig::static_kms(STATIC_KEY_ID.to_string(), static_secret_key()); + assert_configuration_error(manager.reconfigure(switched).await, "backend cannot be changed"); + + // Every rejection above left the original service untouched. + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!( + manager.get_service_version().await, + Some(1), + "rejected transitions must not consume service versions" + ); +} + +#[tokio::test] +async fn invalid_configurations_are_rejected_before_anything_starts() { + let dir = TempDir::new().expect("temp dir"); + + // A zero timeout or zero retry budget would make every operation fail. + let mut zero_timeout = local_dev_config(&dir); + zero_timeout.timeout = Duration::ZERO; + assert_configuration_error(zero_timeout.validate(), "Timeout must be greater than 0"); + + let mut zero_retries = local_dev_config(&dir); + zero_retries.retry_attempts = 0; + assert_configuration_error(zero_retries.validate(), "Retry attempts must be greater than 0"); + + // Cache enabled with no capacity is contradictory. + let mut empty_cache = local_dev_config(&dir); + empty_cache.enable_cache = true; + empty_cache.cache_config.max_keys = 0; + assert_configuration_error(empty_cache.validate(), "max_keys must be greater than 0"); + + // A relative key directory is ambiguous relative to the server's cwd. + let mut relative = local_dev_config(&dir); + relative.backend_config = BackendConfig::Local(LocalConfig { + key_dir: "relative/kms".into(), + master_key: None, + file_permissions: Some(0o600), + }); + assert_configuration_error(relative.validate(), "must be an absolute path"); + + // Production-shaped Local config: no master key is refused without opt-in. + let production_without_master_key = KmsConfig::local("/var/lib/rustfs/kms-behavior".into()); + assert_configuration_error(production_without_master_key.validate(), "requires a master key"); + + // ...and so is a key directory under the process temp dir. + let temp_backed = KmsConfig { + backend_config: BackendConfig::Local(LocalConfig { + key_dir: dir.path().to_path_buf(), + master_key: Some("a-master-key".to_string()), + file_permissions: Some(0o600), + }), + ..KmsConfig::local(dir.path().to_path_buf()) + }; + assert_configuration_error(temp_backed.validate(), "temp directory"); + + // Static backend: the secret must be base64 and exactly 32 bytes. + assert_configuration_error( + KmsConfig::static_kms("k".to_string(), "not base64!!".to_string()).validate(), + "not valid base64", + ); + assert_configuration_error( + KmsConfig::static_kms("k".to_string(), base64_of(&[0u8; 16])).validate(), + "exactly 32 bytes", + ); + assert_configuration_error( + KmsConfig::static_kms(String::new(), static_secret_key()).validate(), + "key_id cannot be empty", + ); + assert_configuration_error( + KmsConfig::static_kms("k".to_string(), String::new()).validate(), + "secret_key cannot be empty", + ); + + // Vault: plaintext HTTP and the built-in dev token are dev-only. + let http = Url::parse("http://127.0.0.1:8200").expect("url"); + assert_configuration_error(KmsConfig::vault(http.clone(), "a-real-token".to_string()).validate(), "requires HTTPS"); + let https = Url::parse("https://vault.example.com:8200").expect("url"); + assert_configuration_error( + KmsConfig::vault(https.clone(), "dev-token".to_string()).validate(), + "dev-token is not allowed", + ); + // An AppRole with no credential at all cannot authenticate. + let approle = KmsConfig::vault_approle(https, "role".to_string(), String::new()); + assert_configuration_error(approle.validate(), "requires a secret_id"); + + // None of the above may be startable through the manager either. + let manager = KmsServiceManager::new(); + assert_configuration_error(manager.configure(zero_timeout).await, "Timeout must be greater than 0"); + assert_eq!( + manager.get_status().await, + KmsServiceStatus::NotConfigured, + "a rejected configure must not move the manager out of NotConfigured" + ); +} + +#[tokio::test] +async fn redacted_views_never_carry_key_material() { + let secret = static_secret_key(); + let manager = KmsServiceManager::new(); + manager + .configure(KmsConfig::static_kms(STATIC_KEY_ID.to_string(), secret.clone())) + .await + .expect("configure static"); + manager.start().await.expect("start static"); + + let redacted = manager.get_redacted_config().await.expect("redacted config"); + let static_config = redacted.static_config().expect("static config"); + assert!(static_config.secret_key.is_empty(), "the redacted view must zero the secret key"); + assert_eq!(static_config.key_id, STATIC_KEY_ID, "the key id is not secret and must survive"); + + let (status, from_state) = manager.get_redacted_state().await; + assert_eq!(status, KmsServiceStatus::Running, "redacted state carries the live status"); + let from_state = from_state.expect("redacted state config"); + assert!( + from_state.static_config().expect("static config").secret_key.is_empty(), + "get_redacted_state must redact exactly like get_redacted_config" + ); + + // Neither the Debug rendering nor a JSON serialization may carry material. + assert_no_secret_leak(&format!("{redacted:?}"), &[&secret]); + assert_no_secret_leak(&serde_json::to_string(&redacted).expect("redacted config serializes"), &[&secret]); + + // Even the unredacted config must keep the secret out of Debug and JSON: + // `Debug` is what ends up in logs, and `secret_key` is `skip_serializing`. + let live = manager.get_config().await.expect("live config"); + assert_no_secret_leak(&format!("{live:?}"), &[&secret]); + assert_no_secret_leak(&serde_json::to_string(&live).expect("live config serializes"), &[&secret]); + + // A Vault token is subject to the same rule in Debug output. + let vault = KmsConfig::vault( + Url::parse("https://vault.example.com:8200").expect("url"), + "super-secret-vault-token".to_string(), + ); + assert_no_secret_leak(&format!("{vault:?}"), &["super-secret-vault-token"]); +} + +#[tokio::test] +async fn harness_restart_brings_the_service_back_over_the_same_state() { + let mut kms = TestKms::local().await; + assert_eq!(kms.manager().get_service_version().await, Some(1)); + + kms.restart().await; + + assert_eq!(kms.manager().get_status().await, KmsServiceStatus::Running); + assert_eq!( + kms.manager().get_service_version().await, + Some(1), + "a restarted process starts its version counter over" + ); + assert!( + kms.manager().health_check().await.expect("health check"), + "the restarted service is healthy" + ); +} + +fn base64_of(bytes: &[u8]) -> String { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.encode(bytes) +} diff --git a/crates/kms/tests/behavior_objects.rs b/crates/kms/tests/behavior_objects.rs new file mode 100644 index 000000000..b154a6e6d --- /dev/null +++ b/crates/kms/tests/behavior_objects.rs @@ -0,0 +1,927 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: object encryption (SSE-S3, SSE-KMS, SSE-C). +//! +//! This is the surface `rustfs/src/storage/ecfs.rs` calls on every PUT and GET, +//! so the contract is unusually load-bearing: a break here is unreadable +//! objects, not a failed request. The suite covers +//! +//! * round-tripping across sizes and both AEAD algorithms, including the +//! degenerate empty object and block-boundary sizes; +//! * the header projection — `metadata_to_headers` is what actually reaches +//! disk, and `headers_to_metadata` is what a GET has to rebuild from it, so +//! the pair must compose into a working decrypt; +//! * tamper and context-mismatch rejection; +//! * SSE-C, where the key never touches the KMS at all. + +mod common; + +use std::collections::HashMap; + +use common::{ + TestKms, assert_context_mismatch, assert_invalid_key_size, assert_invalid_operation, ctx, discard, flip_middle_bit, payload, +}; +use rustfs_kms::{EncryptionAlgorithm, EncryptionMetadata, KmsError, ObjectEncryptionService}; +use tokio::io::AsyncReadExt as _; + +const BUCKET: &str = "objects-behavior"; + +/// Sizes chosen to straddle the AEAD block boundary and the empty case. +const SIZES: &[usize] = &[0, 1, 15, 16, 17, 4096, 65_537]; + +async fn read_all(mut reader: Box) -> Vec { + let mut out = Vec::new(); + reader.read_to_end(&mut out).await.expect("reading plaintext should succeed"); + out +} + +async fn service_with_key(key_id: &str) -> (TestKms, std::sync::Arc) { + let kms = TestKms::local_with(|config| config.default_key_id = Some(key_id.to_string())).await; + kms.create_key(key_id).await; + let service = kms.service().await; + (kms, service) +} + +#[tokio::test] +async fn objects_round_trip_across_sizes_and_algorithms() { + let (_kms, service) = service_with_key("sse-round-trip").await; + + for algorithm in [EncryptionAlgorithm::Aes256, EncryptionAlgorithm::ChaCha20Poly1305] { + for &size in SIZES { + let object_key = format!("{}/{size}.bin", algorithm.as_str()); + let data = payload(size); + + let encrypted = service + .encrypt_object(BUCKET, &object_key, data.as_slice(), &algorithm, None, None) + .await + .unwrap_or_else(|error| panic!("encrypting {size} bytes with {algorithm:?} failed: {error:?}")); + + assert_eq!( + encrypted.metadata.original_size, size as u64, + "metadata must record the plaintext size for {algorithm:?}/{size}" + ); + assert_eq!( + encrypted.metadata.algorithm, + algorithm.as_str(), + "metadata must record the algorithm actually used" + ); + assert_eq!( + encrypted.metadata.iv.len(), + algorithm.iv_size(), + "the IV must be the algorithm's nonce size" + ); + assert!(encrypted.metadata.tag.is_some(), "an AEAD algorithm must produce a tag"); + assert!( + !encrypted.metadata.encrypted_data_key.is_empty(), + "the wrapped DEK must be stored with the object" + ); + // Only assert this where a collision is not a realistic outcome. A + // 1-byte object matches its own ciphertext once every 256 runs, so + // asserting it there would make the suite flaky rather than strict. + // Small objects are still covered: the tag is checked above and the + // decrypt round-trip below is what actually proves the encryption. + if size >= 8 { + assert_ne!(encrypted.ciphertext, data, "ciphertext must differ from plaintext"); + } + + // The context the server binds must name the object unambiguously. + let context = &encrypted.metadata.encryption_context; + assert_eq!(context.get("bucket").map(String::as_str), Some(BUCKET)); + assert_eq!(context.get("object_key").map(String::as_str), Some(object_key.as_str())); + assert_eq!( + context.get("object").map(String::as_str), + Some(object_key.as_str()), + "the legacy `object` key must stay populated for older readers" + ); + assert_eq!(context.get("algorithm").map(String::as_str), Some(algorithm.as_str())); + + let decrypted = read_all( + service + .decrypt_object(BUCKET, &object_key, encrypted.ciphertext.clone(), &encrypted.metadata, None) + .await + .unwrap_or_else(|error| panic!("decrypting {size} bytes with {algorithm:?} failed: {error:?}")), + ) + .await; + assert_eq!(decrypted, data, "round-trip must return the original {size} bytes"); + } + } +} + +#[tokio::test] +async fn two_objects_never_share_a_data_key() { + let (_kms, service) = service_with_key("sse-per-object").await; + let data = payload(512); + + let first = service + .encrypt_object(BUCKET, "first.bin", data.as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("encrypt first"); + let second = service + .encrypt_object(BUCKET, "second.bin", data.as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("encrypt second"); + + assert_ne!( + first.metadata.encrypted_data_key, second.metadata.encrypted_data_key, + "each object must carry its own wrapped data key" + ); + assert_ne!(first.metadata.iv, second.metadata.iv, "each object must get a fresh IV"); + assert_ne!( + first.ciphertext, second.ciphertext, + "identical plaintext under different objects must not produce identical ciphertext" + ); + + // Re-encrypting the *same* object also produces fresh material: a repeated + // PUT must not reuse the previous version's key or IV. + let again = service + .encrypt_object(BUCKET, "first.bin", data.as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("re-encrypt first"); + assert_ne!( + first.metadata.encrypted_data_key, again.metadata.encrypted_data_key, + "a repeated PUT of the same object must mint a new data key" + ); + assert_ne!(first.ciphertext, again.ciphertext, "a repeated PUT must not be deterministic"); +} + +#[tokio::test] +async fn cross_object_ciphertext_and_metadata_do_not_interchange() { + let (_kms, service) = service_with_key("sse-cross-object").await; + + let alpha = service + .encrypt_object(BUCKET, "alpha.bin", payload(300).as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("encrypt alpha"); + let beta = service + .encrypt_object(BUCKET, "beta.bin", payload(300).as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("encrypt beta"); + + // Alpha's bytes under beta's metadata must fail authentication rather than + // return beta's plaintext or garbage. + assert!( + service + .decrypt_object(BUCKET, "beta.bin", alpha.ciphertext.clone(), &beta.metadata, None) + .await + .is_err(), + "one object's ciphertext must not open under another's metadata" + ); + + // A spliced metadata record — beta's wrapped key grafted onto alpha's + // record — must not decrypt either. + let mut spliced = alpha.metadata.clone(); + spliced.encrypted_data_key = beta.metadata.encrypted_data_key.clone(); + assert!( + service + .decrypt_object(BUCKET, "alpha.bin", alpha.ciphertext.clone(), &spliced, None) + .await + .is_err(), + "grafting another object's wrapped key must not yield a working decrypt" + ); +} + +#[tokio::test] +async fn tampered_ciphertext_and_metadata_are_rejected() { + let (_kms, service) = service_with_key("sse-tamper").await; + let data = payload(1024); + let encrypted = service + .encrypt_object(BUCKET, "victim.bin", data.as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("encrypt"); + + let attempt = |ciphertext: Vec, metadata: EncryptionMetadata| { + let service = service.clone(); + async move { + service + .decrypt_object(BUCKET, "victim.bin", ciphertext, &metadata, None) + .await + } + }; + + assert!( + attempt(flip_middle_bit(&encrypted.ciphertext), encrypted.metadata.clone()) + .await + .is_err(), + "a single flipped ciphertext bit must fail authentication" + ); + assert!( + attempt( + encrypted.ciphertext[..encrypted.ciphertext.len() - 1].to_vec(), + encrypted.metadata.clone() + ) + .await + .is_err(), + "a truncated object must fail authentication" + ); + + let mut bad_iv = encrypted.metadata.clone(); + bad_iv.iv = flip_middle_bit(&encrypted.metadata.iv); + assert!( + attempt(encrypted.ciphertext.clone(), bad_iv).await.is_err(), + "a tampered IV must fail authentication" + ); + + let mut bad_tag = encrypted.metadata.clone(); + bad_tag.tag = Some(flip_middle_bit(encrypted.metadata.tag.as_ref().expect("tag"))); + assert!( + attempt(encrypted.ciphertext.clone(), bad_tag).await.is_err(), + "a tampered tag must fail authentication" + ); + + let mut no_tag = encrypted.metadata.clone(); + no_tag.tag = None; + assert_invalid_operation(discard(attempt(encrypted.ciphertext.clone(), no_tag).await), "Missing authentication tag"); + + let mut bad_algorithm = encrypted.metadata.clone(); + bad_algorithm.algorithm = "ROT13".to_string(); + match discard(attempt(encrypted.ciphertext.clone(), bad_algorithm).await) { + Err(KmsError::UnsupportedAlgorithm { algorithm }) => assert_eq!(algorithm, "ROT13"), + other => panic!("expected UnsupportedAlgorithm, got {other:?}"), + } + + // A rewritten context breaks the AAD, so the object stops opening — this is + // what makes the stored context tamper-evident rather than advisory. + let mut rewritten_context = encrypted.metadata.clone(); + rewritten_context + .encryption_context + .insert("bucket".to_string(), "attacker-bucket".to_string()); + assert!( + attempt(encrypted.ciphertext.clone(), rewritten_context).await.is_err(), + "rewriting the bound context must break decryption" + ); + + // The untouched original still decrypts, proving the failures above are + // caused by the tampering and not by a broken fixture. + let recovered = read_all( + attempt(encrypted.ciphertext.clone(), encrypted.metadata.clone()) + .await + .expect("the pristine object must still decrypt"), + ) + .await; + assert_eq!(recovered, data); +} + +#[tokio::test] +async fn expected_context_validation_catches_a_relocated_object() { + let (_kms, service) = service_with_key("sse-context-check").await; + let encrypted = service + .encrypt_object( + BUCKET, + "docs/report.pdf", + payload(64).as_slice(), + &EncryptionAlgorithm::Aes256, + None, + None, + ) + .await + .expect("encrypt"); + + // Matching expectations pass through. + read_all( + service + .decrypt_object( + BUCKET, + "docs/report.pdf", + encrypted.ciphertext.clone(), + &encrypted.metadata, + Some(&ctx(&[("bucket", BUCKET), ("object_key", "docs/report.pdf")])), + ) + .await + .expect("a matching expected context must be accepted"), + ) + .await; + + // A caller expecting a different object refuses before touching the KMS: + // this is the guard against a ciphertext being served under another key. + assert_context_mismatch(discard( + service + .decrypt_object( + BUCKET, + "docs/report.pdf", + encrypted.ciphertext.clone(), + &encrypted.metadata, + Some(&ctx(&[("object_key", "docs/other.pdf")])), + ) + .await, + )); + assert_context_mismatch(discard( + service + .decrypt_object( + BUCKET, + "docs/report.pdf", + encrypted.ciphertext.clone(), + &encrypted.metadata, + Some(&ctx(&[("bucket", "another-bucket")])), + ) + .await, + )); + // A key that was never bound cannot be satisfied. + assert_context_mismatch(discard( + service + .decrypt_object( + BUCKET, + "docs/report.pdf", + encrypted.ciphertext.clone(), + &encrypted.metadata, + Some(&ctx(&[("never-bound", "value")])), + ) + .await, + )); +} + +/// The projection an object actually survives on: metadata is written to disk +/// as headers and rebuilt from them on the next GET. +#[tokio::test] +async fn metadata_survives_the_header_projection() { + let (_kms, service) = service_with_key("sse-headers").await; + + for algorithm in [EncryptionAlgorithm::Aes256, EncryptionAlgorithm::ChaCha20Poly1305] { + let object_key = format!("headers/{}.bin", algorithm.as_str()); + let data = payload(2048); + let encrypted = service + .encrypt_object(BUCKET, &object_key, data.as_slice(), &algorithm, None, None) + .await + .expect("encrypt"); + + let headers = service.metadata_to_headers(&encrypted.metadata); + + // The S3-visible header must reflect the mode the object was written in. + match algorithm { + EncryptionAlgorithm::Aes256 => assert_eq!( + headers.get("x-amz-server-side-encryption").map(String::as_str), + Some("AES256"), + "AES-256 objects advertise SSE-S3" + ), + _ => { + assert_eq!( + headers.get("x-amz-server-side-encryption").map(String::as_str), + Some("aws:kms"), + "non-AES-256 objects advertise SSE-KMS" + ); + assert_eq!( + headers.get("x-amz-server-side-encryption-aws-kms-key-id").map(String::as_str), + Some(encrypted.metadata.key_id.as_str()), + "SSE-KMS must name its key in the S3 header" + ); + } + } + assert_eq!( + headers.get("x-rustfs-encryption-key-id").map(String::as_str), + Some(encrypted.metadata.key_id.as_str()), + "the internal key-id header must always be present for KMS-backed objects" + ); + for required in [ + "x-rustfs-encryption-iv", + "x-rustfs-encryption-tag", + "x-rustfs-encryption-key", + "x-rustfs-encryption-context", + ] { + assert!(headers.contains_key(required), "header {required} is required to rebuild metadata"); + } + // Nothing plaintext-sensitive may ride along in a header. + assert!( + !headers.values().any(|value| value.contains("BEGIN")), + "headers must not carry key material" + ); + + let rebuilt = service + .headers_to_metadata(&headers) + .expect("headers written by this service must parse back"); + + assert_eq!(rebuilt.key_id, encrypted.metadata.key_id, "key id must survive the projection"); + assert_eq!(rebuilt.iv, encrypted.metadata.iv, "IV must survive the projection"); + assert_eq!(rebuilt.tag, encrypted.metadata.tag, "tag must survive the projection"); + assert_eq!( + rebuilt.encrypted_data_key, encrypted.metadata.encrypted_data_key, + "the wrapped DEK must survive the projection" + ); + assert_eq!( + rebuilt.encryption_context, encrypted.metadata.encryption_context, + "the bound context must survive the projection" + ); + + // The point of the projection: the rebuilt record must open the object. + // Field equality is not enough — `decrypt_object` derives its AEAD + // additional data from the context, so the context has to survive as + // *bytes*, not merely as a map. + let decrypted = read_all( + service + .decrypt_object(BUCKET, &object_key, encrypted.ciphertext.clone(), &rebuilt, None) + .await + .unwrap_or_else(|error| panic!("[{algorithm:?}] rebuilt metadata must decrypt the object: {error:?}")), + ) + .await; + assert_eq!(decrypted, data, "[{algorithm:?}] the header round-trip must preserve the plaintext"); + } +} + +#[tokio::test] +async fn headers_missing_required_fields_are_rejected() { + let (_kms, service) = service_with_key("sse-bad-headers").await; + let encrypted = service + .encrypt_object( + BUCKET, + "bad-headers.bin", + payload(32).as_slice(), + &EncryptionAlgorithm::Aes256, + None, + None, + ) + .await + .expect("encrypt"); + let good = service.metadata_to_headers(&encrypted.metadata); + + let without = |name: &str| { + let mut headers = good.clone(); + headers.remove(name); + headers + }; + + assert!( + service.headers_to_metadata(&HashMap::new()).is_err(), + "an empty header set carries no algorithm and must be rejected" + ); + assert!( + service.headers_to_metadata(&without("x-amz-server-side-encryption")).is_err(), + "the algorithm header is required" + ); + assert!( + service.headers_to_metadata(&without("x-rustfs-encryption-iv")).is_err(), + "the IV header is required" + ); + + // Malformed base64 must be a validation error, not a panic. + for field in ["x-rustfs-encryption-iv", "x-rustfs-encryption-tag", "x-rustfs-encryption-key"] { + let mut headers = good.clone(); + headers.insert(field.to_string(), "!!! not base64 !!!".to_string()); + assert!( + service.headers_to_metadata(&headers).is_err(), + "malformed base64 in {field} must be rejected" + ); + } + + let mut bad_context = good.clone(); + bad_context.insert("x-rustfs-encryption-context".to_string(), "{not json".to_string()); + assert!( + service.headers_to_metadata(&bad_context).is_err(), + "a malformed context header must be rejected" + ); +} + +#[tokio::test] +async fn sse_s3_auto_creates_its_key_but_sse_kms_requires_one() { + // No key exists yet; only the default id is configured. + let kms = TestKms::local_with(|config| config.default_key_id = Some("auto-created".to_string())).await; + let service = kms.service().await; + + let encrypted = service + .encrypt_object(BUCKET, "auto.bin", payload(16).as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("SSE-S3 must auto-create its default key"); + assert_eq!(encrypted.metadata.key_id, "auto-created"); + + // The key really exists now and the object opens. + let manager = kms.kms().await; + manager + .describe_key(rustfs_kms::DescribeKeyRequest { + key_id: "auto-created".to_string(), + }) + .await + .expect("the auto-created key must be describable afterwards"); + read_all( + service + .decrypt_object(BUCKET, "auto.bin", encrypted.ciphertext.clone(), &encrypted.metadata, None) + .await + .expect("the auto-created key must decrypt its object"), + ) + .await; + + // A non-AES-256 algorithm is SSE-KMS: the caller must have provisioned the + // key, because auto-creating a customer-named key would be surprising. + assert_invalid_operation( + discard( + service + .encrypt_object( + BUCKET, + "explicit.bin", + payload(16).as_slice(), + &EncryptionAlgorithm::ChaCha20Poly1305, + Some("never-created"), + None, + ) + .await, + ), + "not found", + ); + + // With no key id and no default at all, the call is a configuration error. + let bare = TestKms::local().await; + let bare_service = bare.service().await; + match bare_service + .encrypt_object(BUCKET, "bare.bin", payload(16).as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + { + Err(KmsError::ConfigurationError { message }) => { + assert!(message.contains("No KMS key ID"), "should explain the missing key id: {message}") + } + other => panic!("expected ConfigurationError, got {other:?}"), + } +} + +#[tokio::test] +async fn caller_supplied_context_is_merged_and_bound() { + let (_kms, service) = service_with_key("sse-extra-context").await; + let extra = ctx(&[("tenant", "acme"), ("classification", "internal")]); + + let encrypted = service + .encrypt_object( + BUCKET, + "tenant.bin", + payload(128).as_slice(), + &EncryptionAlgorithm::Aes256, + None, + Some(&extra), + ) + .await + .expect("encrypt with extra context"); + + for (key, value) in &extra { + assert_eq!( + encrypted.metadata.encryption_context.get(key), + Some(value), + "caller context {key} must be preserved in the stored metadata" + ); + } + // The server-owned keys are still present and win over any caller value. + assert_eq!(encrypted.metadata.encryption_context.get("bucket").map(String::as_str), Some(BUCKET)); + + read_all( + service + .decrypt_object(BUCKET, "tenant.bin", encrypted.ciphertext.clone(), &encrypted.metadata, Some(&extra)) + .await + .expect("the merged context must validate and decrypt"), + ) + .await; + + // Dropping a caller-supplied key from the stored context breaks the AAD. + let mut stripped = encrypted.metadata.clone(); + stripped.encryption_context.remove("tenant"); + assert!( + service + .decrypt_object(BUCKET, "tenant.bin", encrypted.ciphertext.clone(), &stripped, None) + .await + .is_err(), + "removing a bound context entry must break decryption" + ); +} + +#[tokio::test] +async fn sse_c_round_trips_and_rejects_the_wrong_key() { + let (_kms, service) = service_with_key("sse-c-unused").await; + let customer_key = [0x11u8; 32]; + let wrong_key = [0x22u8; 32]; + let data = payload(4096); + + let encrypted = service + .encrypt_object_with_customer_key(BUCKET, "customer.bin", data.as_slice(), &customer_key, None) + .await + .expect("SSE-C encrypt"); + + assert_eq!( + encrypted.metadata.key_id, "sse-c", + "SSE-C objects are marked so a GET knows not to consult the KMS" + ); + assert!( + encrypted.metadata.encrypted_data_key.is_empty(), + "SSE-C stores no wrapped data key: the customer holds the only copy" + ); + assert_eq!(encrypted.metadata.original_size, data.len() as u64); + + let decrypted = read_all( + service + .decrypt_object_with_customer_key( + BUCKET, + "customer.bin", + encrypted.ciphertext.clone(), + &encrypted.metadata, + &customer_key, + ) + .await + .expect("SSE-C decrypt with the right key"), + ) + .await; + assert_eq!(decrypted, data, "SSE-C round-trip must return the original bytes"); + + assert!( + service + .decrypt_object_with_customer_key( + BUCKET, + "customer.bin", + encrypted.ciphertext.clone(), + &encrypted.metadata, + &wrong_key + ) + .await + .is_err(), + "a different customer key must not open the object" + ); + + // Key length is validated on both sides before any crypto happens. + assert_invalid_key_size( + service + .encrypt_object_with_customer_key(BUCKET, "short.bin", data.as_slice(), &[0u8; 16], None) + .await, + 32, + 16, + ); + assert_invalid_key_size( + discard( + service + .decrypt_object_with_customer_key( + BUCKET, + "customer.bin", + encrypted.ciphertext.clone(), + &encrypted.metadata, + &[0u8; 31], + ) + .await, + ), + 32, + 31, + ); +} + +#[tokio::test] +async fn sse_c_validates_the_supplied_key_md5() { + let (_kms, service) = service_with_key("sse-c-md5-unused").await; + let customer_key = [0x33u8; 32]; + let correct_md5 = hex::encode(md5_of(&customer_key)); + + service + .encrypt_object_with_customer_key(BUCKET, "md5-ok.bin", payload(64).as_slice(), &customer_key, Some(&correct_md5)) + .await + .expect("a matching MD5 must be accepted"); + + // Uppercase is accepted: the comparison is case-insensitive on the input. + service + .encrypt_object_with_customer_key( + BUCKET, + "md5-upper.bin", + payload(64).as_slice(), + &customer_key, + Some(&correct_md5.to_uppercase()), + ) + .await + .expect("MD5 comparison must be case-insensitive"); + + match service + .encrypt_object_with_customer_key( + BUCKET, + "md5-bad.bin", + payload(64).as_slice(), + &customer_key, + Some("00000000000000000000000000000000"), + ) + .await + { + Err(KmsError::ValidationError { message }) => { + assert!(message.contains("MD5"), "the error must name the MD5 check: {message}") + } + other => panic!("expected ValidationError for an MD5 mismatch, got {other:?}"), + } +} + +#[tokio::test] +async fn sse_c_and_kms_objects_do_not_cross_paths() { + let (_kms, service) = service_with_key("sse-c-crossover").await; + let customer_key = [0x44u8; 32]; + + let kms_object = service + .encrypt_object(BUCKET, "kms.bin", payload(256).as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("KMS encrypt"); + let sse_c_object = service + .encrypt_object_with_customer_key(BUCKET, "customer.bin", payload(256).as_slice(), &customer_key, None) + .await + .expect("SSE-C encrypt"); + + // A KMS-encrypted object must not be openable through the SSE-C path, even + // with a valid-looking key: the metadata marker is what routes the GET. + assert_invalid_operation( + discard( + service + .decrypt_object_with_customer_key( + BUCKET, + "kms.bin", + kms_object.ciphertext.clone(), + &kms_object.metadata, + &customer_key, + ) + .await, + ), + "not encrypted with SSE-C", + ); + + // And an SSE-C object has no wrapped data key for the KMS path to unwrap. + assert!( + service + .decrypt_object(BUCKET, "customer.bin", sse_c_object.ciphertext.clone(), &sse_c_object.metadata, None) + .await + .is_err(), + "the KMS path must not be able to open an SSE-C object" + ); + + // The SSE-C header projection advertises the customer algorithm, which is + // how `headers_to_metadata` re-identifies the object on the way back. + let headers = service.metadata_to_headers(&sse_c_object.metadata); + assert_eq!( + headers + .get("x-amz-server-side-encryption-customer-algorithm") + .map(String::as_str), + Some("AES256"), + "SSE-C objects must advertise the customer algorithm header" + ); + assert!(!headers.contains_key("x-rustfs-encryption-key-id"), "SSE-C must not claim a KMS key id"); + let rebuilt = service.headers_to_metadata(&headers).expect("SSE-C headers must parse"); + assert_eq!(rebuilt.key_id, "sse-c", "the SSE-C marker must survive the header projection"); + assert!( + rebuilt.encrypted_data_key.is_empty(), + "no wrapped key may materialise out of SSE-C headers" + ); + assert_eq!( + rebuilt.encryption_context, sse_c_object.metadata.encryption_context, + "the SSE-C context must survive the projection" + ); + + let decrypted = read_all( + service + .decrypt_object_with_customer_key(BUCKET, "customer.bin", sse_c_object.ciphertext.clone(), &rebuilt, &customer_key) + .await + .expect("SSE-C must decrypt from a record rebuilt out of its own headers"), + ) + .await; + assert_eq!(decrypted.len(), 256, "the SSE-C header round-trip must preserve the object"); +} + +fn md5_of(bytes: &[u8]) -> Vec { + use md5::Digest as _; + let mut hasher = md5::Md5::new(); + hasher.update(bytes); + hasher.finalize().to_vec() +} + +/// Serialize a context in reverse-sorted key order. +/// +/// Deliberately not the canonical ordering, and derived from the real context +/// rather than hand-written: the service adds its own bucket-path entry, so a +/// literal would silently describe a different map and prove nothing. +fn non_canonical_context_json(context: &HashMap) -> String { + let mut entries: Vec<_> = context.iter().collect(); + entries.sort_by(|left, right| right.0.cmp(left.0)); + let body = entries + .iter() + .map(|(key, value)| { + format!( + "{}:{}", + serde_json::to_string(key).expect("key serializes"), + serde_json::to_string(value).expect("value serializes") + ) + }) + .collect::>() + .join(","); + format!("{{{body}}}") +} + +/// An object sealed before the context was canonicalized must still open. +/// +/// The AAD is the *serialization* of the encryption context, not the map. Any +/// object written while the context was serialized straight from a `HashMap` +/// carries whatever order that map happened to iterate in, and +/// `x-rustfs-encryption-context` is where that exact byte sequence survives. +/// Canonicalizing on the way back in would recompute sorted AAD, fail the AEAD, +/// and make a readable object permanently unreadable — so the stored bytes have +/// to win over anything re-derived from the parsed map. +/// +/// The legacy object is reconstructed here the only way a black-box test can: +/// by rewriting the projected header into a non-sorted ordering *and* pinning +/// the sealed bytes to that same ordering, which is exactly the on-disk state a +/// pre-upgrade write left behind. +#[tokio::test] +async fn an_object_sealed_under_a_non_canonical_context_still_opens() { + let (_kms, service) = service_with_key("sse-legacy-aad").await; + let object_key = "legacy-context.bin"; + let data = payload(512); + + // Several entries, so an ordering difference is observable at all. + let context = ctx(&[("zeta", "26"), ("alpha", "1"), ("mu", "13")]); + let encrypted = service + .encrypt_object(BUCKET, object_key, data.as_slice(), &EncryptionAlgorithm::Aes256, None, Some(&context)) + .await + .expect("encrypting with a multi-entry context should succeed"); + + let headers = service.metadata_to_headers(&encrypted.metadata); + let stored_context = headers + .get("x-rustfs-encryption-context") + .expect("the context must be projected into a header"); + + // What the projection stores must be the bytes the object was sealed + // under, or the two can drift apart without anything failing yet. + let rebuilt = service.headers_to_metadata(&headers).expect("headers must parse back"); + assert_eq!( + rebuilt.context_aad.as_deref(), + Some(stored_context.as_bytes()), + "the rebuilt record must carry the stored context bytes verbatim" + ); + let reopened = read_all( + service + .decrypt_object(BUCKET, object_key, encrypted.ciphertext.clone(), &rebuilt, None) + .await + .expect("an object must open from its own projected headers"), + ) + .await; + assert_eq!(reopened, data); + + // Now the legacy shape: same pairs, different serialization order. An + // object written before canonicalization has exactly this on disk. + let legacy_json = non_canonical_context_json(&encrypted.metadata.encryption_context); + let legacy_json = legacy_json.as_str(); + assert_ne!( + legacy_json, stored_context, + "the legacy ordering must actually differ from the canonical one, or this proves nothing" + ); + let legacy_context: HashMap = serde_json::from_str(legacy_json).expect("legacy context parses"); + assert_eq!(legacy_context, encrypted.metadata.encryption_context, "same pairs, different order"); + + let legacy_metadata = EncryptionMetadata { + context_aad: Some(legacy_json.as_bytes().to_vec()), + encryption_context: legacy_context, + ..encrypted.metadata.clone() + }; + + // Re-projecting a legacy record must not rewrite it into sorted form: that + // would destroy the only copy of the ordering the object needs. + let legacy_headers = service.metadata_to_headers(&legacy_metadata); + assert_eq!( + legacy_headers.get("x-rustfs-encryption-context").map(String::as_str), + Some(legacy_json), + "a re-projection must preserve the original context ordering byte-for-byte" + ); + assert_eq!( + service + .headers_to_metadata(&legacy_headers) + .expect("legacy headers must parse") + .context_aad + .as_deref(), + Some(legacy_json.as_bytes()), + "the legacy ordering must survive a full header round trip" + ); +} + +/// Rewriting the stored context is a tamper, not a legacy read. +/// +/// The flip side of honouring the stored bytes: they are authenticated, so +/// changing them — even to a reordering that parses to an identical map — must +/// fail rather than silently re-deriving a working AAD. +#[tokio::test] +async fn a_rewritten_context_header_fails_authentication() { + let (_kms, service) = service_with_key("sse-tampered-context").await; + let object_key = "tampered-context.bin"; + let data = payload(256); + + let context = ctx(&[("zeta", "26"), ("alpha", "1"), ("mu", "13")]); + let encrypted = service + .encrypt_object(BUCKET, object_key, data.as_slice(), &EncryptionAlgorithm::Aes256, None, Some(&context)) + .await + .expect("encrypt should succeed"); + + let mut headers = service.metadata_to_headers(&encrypted.metadata); + // Same pairs, different serialization: a pure ordering rewrite, so the + // rejection can only come from the AAD bytes and not from a changed map. + headers.insert( + "x-rustfs-encryption-context".to_string(), + non_canonical_context_json(&encrypted.metadata.encryption_context), + ); + + let tampered = service.headers_to_metadata(&headers).expect("tampered headers still parse"); + assert!( + discard( + service + .decrypt_object(BUCKET, object_key, encrypted.ciphertext.clone(), &tampered, None) + .await + ) + .is_err(), + "a context the object was not sealed under must not open it" + ); +} diff --git a/crates/kms/tests/behavior_resilience.rs b/crates/kms/tests/behavior_resilience.rs new file mode 100644 index 000000000..536257978 --- /dev/null +++ b/crates/kms/tests/behavior_resilience.rs @@ -0,0 +1,308 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: how the service reacts to an unreachable backend. +//! +//! The rule that matters operationally is **a bad configuration must never take +//! down a working KMS**. Applying a config that points at a dead Vault is a +//! routine operator mistake; if it stopped the running service, every encrypted +//! object in the deployment would become unreadable until someone noticed. So +//! the candidate is health-checked *before* the swap, and a failing candidate +//! is discarded with the incumbent still serving. +//! +//! Everything here runs offline: an unreachable backend is a loopback port with +//! nothing listening, which is deterministic and needs no external server. +//! +//! Error *classification* and retry accounting for transport faults live in +//! `tests/vault_fault_injection.rs`, which drives the same public API with a +//! metrics recorder attached. This file deliberately does not duplicate it. +//! +//! Not covered offline: throttling (429) and recoverable 5xx responses. Forcing +//! those needs the crate's scripted Vault responder, which is `pub(crate)` and +//! therefore out of reach from an integration test; they are pinned by the +//! in-crate wiring tests in `backends::vault` instead. + +mod common; + +use std::sync::Arc; +use std::time::Duration; + +use common::{TestKms, assert_configuration_error, ctx}; +use rustfs_kms::{ + BackendConfig, DecryptRequest, GenerateDataKeyRequest, KeySpec, KmsBackend as KmsBackendKind, KmsConfig, KmsServiceManager, + KmsServiceStatus, VaultAuthMethod, VaultConfig, +}; + +/// A loopback address with nothing listening on it: reserve a port, then +/// release it so a connection there is refused immediately. +fn dead_address() -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve a loopback port"); + let address = format!("http://{}", listener.local_addr().expect("reserved port addr")); + drop(listener); + address +} + +fn unreachable_vault_config() -> KmsConfig { + KmsConfig { + backend: KmsBackendKind::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig { + address: dead_address(), + auth_method: VaultAuthMethod::Token { + token: "unused-token".to_string(), + }, + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/resilience".to_string(), + tls: None, + })), + allow_insecure_dev_defaults: true, + // Keep the failure fast: one short attempt is enough to prove the point. + timeout: Duration::from_millis(300), + retry_attempts: 1, + ..KmsConfig::default() + } +} + +#[tokio::test] +async fn starting_against_an_unreachable_backend_fails_without_publishing_a_service() { + let manager = KmsServiceManager::new(); + let config = unreachable_vault_config(); + manager + .configure(config.clone()) + .await + .expect("configuring an unreachable backend is allowed: validation is not connectivity"); + assert_eq!(manager.get_status().await, KmsServiceStatus::Configured); + + let error = manager + .start() + .await + .expect_err("starting must fail when the backend is unreachable"); + assert!( + format!("{error}").contains("KMS backend"), + "the failure must name the backend, got {error}" + ); + + match manager.get_status().await { + KmsServiceStatus::Error(message) => assert!(!message.is_empty(), "a failed start must record why"), + other => panic!("a failed start must leave an Error status, got {other:?}"), + } + assert!( + manager.get_encryption_service().await.is_none(), + "a failed start must not publish a half-built service" + ); + assert!(manager.get_manager().await.is_none(), "a failed start must not publish a manager either"); + assert!( + manager.get_service_version().await.is_none(), + "a failed start must not claim a service version" + ); + assert!( + manager.get_config().await.is_some(), + "the configuration survives so an operator can fix and retry" + ); + assert!( + !manager + .health_check() + .await + .expect("health check must not error when nothing runs"), + "a service that never started is unhealthy" + ); +} + +/// The load-bearing case: pointing a *running* KMS at a dead backend must be a +/// rejected reconfigure, not an outage. +#[tokio::test] +async fn a_failing_candidate_never_replaces_a_healthy_service() { + let kms = TestKms::local().await; + let manager = kms.manager().clone(); + let key_id = kms.create_key("survives-bad-config").await; + let context = ctx(&[("bucket", "resilience-behavior")]); + + let incumbent = manager.get_encryption_service().await.expect("service v1"); + let incumbent_manager = manager.get_manager().await.expect("manager v1"); + let dek = incumbent_manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("the incumbent works before the bad reconfigure"); + + // Attempt the bad swap. The Local backend's identity is also frozen, so a + // cross-backend move is refused before connectivity is even attempted — + // assert the refusal, then assert nothing moved. + let error = manager + .reconfigure(unreachable_vault_config()) + .await + .expect_err("a reconfigure onto an unreachable backend must fail"); + assert!(!format!("{error}").is_empty(), "the failure must be reported"); + + assert_eq!( + manager.get_status().await, + KmsServiceStatus::Running, + "the incumbent must still be Running after a rejected reconfigure" + ); + assert_eq!( + manager.get_service_version().await, + Some(1), + "a rejected candidate must not consume a service version" + ); + assert!( + Arc::ptr_eq(&incumbent, &manager.get_encryption_service().await.expect("service")), + "the published service must still be the incumbent instance" + ); + assert!( + manager.get_config().await.expect("config").local_config().is_some(), + "the published configuration must still be the Local one" + ); + + // And it is not merely present — it still does real work, on both old and + // freshly fetched handles. + let decrypted = manager + .get_manager() + .await + .expect("manager") + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context.clone(), + grant_tokens: Vec::new(), + }) + .await + .expect("the surviving service must still decrypt"); + assert_eq!(decrypted.plaintext, dek.plaintext_key); + assert!(manager.health_check().await.expect("health check"), "the survivor is healthy"); +} + +#[tokio::test] +async fn a_vault_backend_reconfigure_onto_a_dead_address_is_rejected() { + // Start from a Vault-shaped (never-started) configuration so the transition + // guard does not short-circuit the connectivity check, and confirm that the + // candidate's health check is what refuses it. + let manager = KmsServiceManager::new(); + manager + .configure(unreachable_vault_config()) + .await + .expect("configure is allowed"); + + // Reconfigure while nothing is running: the candidate must still be + // health-checked, so an unreachable backend cannot be published. + let error = manager + .reconfigure(unreachable_vault_config()) + .await + .expect_err("an unreachable candidate must not be published even from a stopped state"); + assert!( + format!("{error}").contains("reconfigure") || format!("{error}").contains("backend"), + "the failure must point at the backend, got {error}" + ); + assert!( + manager.get_encryption_service().await.is_none(), + "no service may be published by a failed reconfigure" + ); + assert!( + manager.get_service_version().await.is_none(), + "no version may be consumed by a failed reconfigure" + ); +} + +#[tokio::test] +async fn credentials_that_cannot_be_read_fail_closed_at_start() { + // A Vault Agent token sink that is not there: the service must refuse to + // start rather than come up and send unauthenticated requests. + let missing = std::path::PathBuf::from("/nonexistent/rustfs-kms-behavior/vault-token"); + let config = KmsConfig { + backend: KmsBackendKind::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig { + address: dead_address(), + auth_method: VaultAuthMethod::token_file(missing), + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/resilience".to_string(), + tls: None, + })), + allow_insecure_dev_defaults: true, + timeout: Duration::from_millis(300), + retry_attempts: 1, + ..KmsConfig::default() + }; + + let manager = KmsServiceManager::new(); + manager.configure(config).await.expect("configure"); + assert!( + manager.start().await.is_err(), + "a missing credential source must keep the service from starting" + ); + assert!( + manager.get_encryption_service().await.is_none(), + "no service may be published without usable credentials" + ); + + // An empty token-file path is a configuration error, caught before start. + let empty_path = KmsConfig { + backend: KmsBackendKind::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig { + address: "https://vault.example.com:8200".to_string(), + auth_method: VaultAuthMethod::token_file(std::path::PathBuf::new()), + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/resilience".to_string(), + tls: None, + })), + ..KmsConfig::default() + }; + assert_configuration_error(empty_path.validate(), "token file path cannot be empty"); +} + +#[tokio::test] +async fn a_stopped_service_refuses_work_without_losing_its_state() { + // Stopping is not a failure mode, but it is an unavailability the callers + // must handle: handles disappear, the config stays, and a restart recovers. + let kms = TestKms::local().await; + let manager = kms.manager().clone(); + let key_id = kms.create_key("stop-and-recover").await; + let context = ctx(&[("bucket", "resilience-behavior")]); + + let dek = kms + .kms() + .await + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.clone(), + key_spec: KeySpec::Aes256, + encryption_context: context.clone(), + }) + .await + .expect("generate before stopping"); + + manager.stop().await.expect("stop"); + assert!(manager.get_encryption_service().await.is_none(), "a stopped service hands out no handles"); + assert!(!manager.health_check().await.expect("health check"), "a stopped service is unhealthy"); + + // Stopping twice is idempotent, not an error. + manager.stop().await.expect("a second stop must be a no-op"); + + manager.start().await.expect("restart after stop"); + let recovered = kms + .kms() + .await + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob, + encryption_context: context, + grant_tokens: Vec::new(), + }) + .await + .expect("work done before the stop must still be readable after the restart"); + assert_eq!(recovered.plaintext, dek.plaintext_key); +} diff --git a/crates/kms/tests/behavior_rotation.rs b/crates/kms/tests/behavior_rotation.rs new file mode 100644 index 000000000..f9ff7e55e --- /dev/null +++ b/crates/kms/tests/behavior_rotation.rs @@ -0,0 +1,306 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: key rotation and the version history it must preserve. +//! +//! `BackendCapabilities::rotate` is documented as "rotation that retains prior +//! versions for decryption", and `versioning` as "multiple key versions +//! addressable after rotation". Those are the two claims this file exists to +//! hold, because breaking them destroys data silently: a rotation that dropped +//! the outgoing version would leave every object sealed before it permanently +//! unreadable, while every rotation itself still reported success. +//! +//! Only the Vault backends advertise these capabilities, so this file is the +//! working side of a contract the rest of the suite only ever sees refused. +//! Without the Vault lane on (`RUSTFS_KMS_VAULT_TOKEN`) these specs still run, +//! but they only assert the `UnsupportedCapability` half — see `common`. + +mod common; + +use common::{BackendCase, assert_unsupported_capability, ctx, for_each_backend, payload}; +use rustfs_kms::{DecryptRequest, EncryptRequest, GenerateDataKeyRequest, KeySpec}; + +fn context() -> std::collections::HashMap { + ctx(&[("bucket", "rotation-behavior"), ("object", "alpha.bin")]) +} + +fn generate_request(key_id: &str) -> GenerateDataKeyRequest { + GenerateDataKeyRequest { + key_id: key_id.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: context(), + } +} + +/// The core promise: material sealed before a rotation still opens after it. +/// +/// This is the assertion that a "rotation" which merely overwrote the key +/// would fail. Everything else about rotation is recoverable; this is not. +#[tokio::test] +async fn ciphertext_from_before_a_rotation_still_decrypts_after_it() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + let caps = case.caps().await; + let key_id = case.key_id.clone(); + + let before = manager + .generate_data_key(generate_request(&key_id)) + .await + .unwrap_or_else(|error| panic!("[{label}] generate before rotation should succeed: {error:?}")); + + if !caps.rotate { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + return; + } + + manager + .rotate_key(&key_id) + .await + .unwrap_or_else(|error| panic!("[{label}] a backend advertising rotate must rotate: {error:?}")); + + let reopened = manager + .decrypt(DecryptRequest { + ciphertext: before.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| { + panic!("[{label}] rotation must retain the prior version; pre-rotation ciphertext failed to open: {error:?}") + }); + + assert_eq!( + reopened.plaintext, before.plaintext_key, + "[{label}] the pre-rotation data key must come back byte-identical" + ); + }) + .await; +} + +/// A rotation must not stop the key from being used going forward, and the +/// material it produces afterwards must be independent of the old version. +#[tokio::test] +async fn a_rotated_key_keeps_working_and_issues_fresh_material() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + let caps = case.caps().await; + let key_id = case.key_id.clone(); + + if !caps.rotate { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + return; + } + + let before = manager + .generate_data_key(generate_request(&key_id)) + .await + .expect("generate before rotation should succeed"); + + manager.rotate_key(&key_id).await.expect("rotate should succeed"); + + let after = manager + .generate_data_key(generate_request(&key_id)) + .await + .unwrap_or_else(|error| panic!("[{label}] the key must still issue data keys after rotation: {error:?}")); + + assert_ne!( + after.plaintext_key, before.plaintext_key, + "[{label}] a data key issued after rotation must not repeat the earlier one" + ); + assert_ne!( + after.ciphertext_blob, before.ciphertext_blob, + "[{label}] the wrapped blob must differ across a rotation" + ); + + // Both generations must be openable at the same time — this is what + // `versioning` means in practice. + for (name, dek) in [("pre-rotation", &before), ("post-rotation", &after)] { + let opened = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] the {name} data key must stay decryptable: {error:?}")); + assert_eq!(opened.plaintext, dek.plaintext_key, "[{label}] {name} round-trip"); + } + }) + .await; +} + +/// Master-key encryption must survive a rotation on the same terms as data +/// keys: the ciphertext is what a caller stored, and it has to keep opening. +#[tokio::test] +async fn master_key_ciphertext_survives_a_rotation() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + let caps = case.caps().await; + let key_id = case.key_id.clone(); + + if !caps.rotate { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + return; + } + + let plaintext = payload(512); + let sealed = manager + .encrypt(EncryptRequest { + key_id: key_id.clone(), + plaintext: plaintext.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .expect("encrypt before rotation should succeed"); + + manager.rotate_key(&key_id).await.expect("rotate should succeed"); + + let opened = manager + .decrypt(DecryptRequest { + ciphertext: sealed.ciphertext.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| panic!("[{label}] pre-rotation ciphertext must open after rotation: {error:?}")); + + assert_eq!(opened.plaintext, plaintext, "[{label}] the plaintext must survive the rotation"); + }) + .await; +} + +/// Repeated rotations must accumulate versions, not overwrite a single spare. +/// +/// A backend that kept only "current and previous" would pass a single-rotation +/// test and still lose the oldest objects on the second rotation. +#[tokio::test] +async fn every_generation_survives_repeated_rotations() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + let caps = case.caps().await; + let key_id = case.key_id.clone(); + + if !caps.rotate || !caps.versioning { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + return; + } + + let mut generations = Vec::new(); + for round in 0..3 { + let dek = manager + .generate_data_key(generate_request(&key_id)) + .await + .unwrap_or_else(|error| panic!("[{label}] generate in round {round} should succeed: {error:?}")); + generations.push(dek); + manager + .rotate_key(&key_id) + .await + .unwrap_or_else(|error| panic!("[{label}] rotation {round} should succeed: {error:?}")); + } + + for (round, dek) in generations.iter().enumerate() { + let opened = manager + .decrypt(DecryptRequest { + ciphertext: dek.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| { + panic!( + "[{label}] the data key from round {round} was lost after {} rotations: {error:?}", + generations.len() + ) + }); + assert_eq!( + opened.plaintext, dek.plaintext_key, + "[{label}] round {round} must round-trip after every later rotation" + ); + } + }) + .await; +} + +/// Version history must live in the backend, not in process memory. +#[tokio::test] +async fn rotation_history_survives_a_restart() { + for_each_backend(|case: BackendCase| async move { + let mut case = case; + let label = case.kind().name(); + let caps = case.caps().await; + let key_id = case.key_id.clone(); + + { + let manager = case.kms.kms().await; + if !caps.rotate { + assert_unsupported_capability(manager.rotate_key(&key_id).await, "rotate_key"); + return; + } + } + + let before = { + let manager = case.kms.kms().await; + let dek = manager + .generate_data_key(generate_request(&key_id)) + .await + .expect("generate before rotation should succeed"); + manager.rotate_key(&key_id).await.expect("rotate should succeed"); + dek + }; + + case.kms.restart().await; + + let manager = case.kms.kms().await; + let opened = manager + .decrypt(DecryptRequest { + ciphertext: before.ciphertext_blob.clone(), + encryption_context: context(), + grant_tokens: Vec::new(), + }) + .await + .unwrap_or_else(|error| { + panic!("[{label}] a pre-rotation key must still open after a restart — the version history must be durable: {error:?}") + }); + + assert_eq!( + opened.plaintext, before.plaintext_key, + "[{label}] the retained version must survive a restart intact" + ); + }) + .await; +} + +/// Rotating a key that does not exist must fail as a missing key, not be +/// silently treated as a no-op that a caller would read as success. +#[tokio::test] +async fn rotating_an_unknown_key_fails() { + for_each_backend(|case: BackendCase| async move { + let manager = case.kms.kms().await; + let label = case.kind().name(); + let caps = case.caps().await; + + let result = manager.rotate_key("rotation-no-such-key").await; + if !caps.rotate { + assert_unsupported_capability(result, "rotate_key"); + return; + } + assert!(result.is_err(), "[{label}] rotating a key that does not exist must not report success"); + }) + .await; +} diff --git a/crates/kms/tests/behavior_serde.rs b/crates/kms/tests/behavior_serde.rs new file mode 100644 index 000000000..57f812e80 --- /dev/null +++ b/crates/kms/tests/behavior_serde.rs @@ -0,0 +1,323 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Black-box behavior: the wire and on-disk serialization contracts. +//! +//! These names outlive the process that wrote them. Object encryption headers +//! sit in `xl.meta` for the life of an object; persisted `KmsConfig` documents +//! are read back by later versions; backup manifests must stay decodable by a +//! future restore. Renaming any of them is a compatibility break that no +//! behavioral test would catch, because a fresh write and a fresh read agree +//! with each other perfectly. +//! +//! Two techniques, deliberately: +//! +//! * **Enum wire spellings are asserted directly.** They are short, they are +//! the highest-risk rename, and an explicit `assert_eq!` documents the +//! intended spelling at the point of the check. +//! * **Struct shapes are snapshotted as sorted field-name lists**, not as +//! values. Values contain UUIDs, timestamps, and random key material, so a +//! value snapshot would be unstable; the field set is exactly the part that +//! constitutes the contract, and a change to it should be reviewed. + +mod common; + +use std::collections::BTreeSet; + +use common::{STATIC_KEY_ID, TestKms, static_secret_key}; +use rustfs_kms::backup::{AeadAlgorithm, ArtifactKind, CompletenessState, DigestAlgorithm}; +use rustfs_kms::{EncryptionAlgorithm, KeySpec, KeyState, KeyStatus, KeyUsage, KmsBackend, KmsConfig, KmsServiceStatus}; +use serde::Serialize; + +/// Sorted top-level field names of a value's JSON object form. +fn field_names(value: &T) -> Vec { + let json = serde_json::to_value(value).expect("value should serialize"); + match json { + serde_json::Value::Object(map) => map.keys().cloned().collect::>().into_iter().collect(), + other => panic!("expected a JSON object, got {other}"), + } +} + +fn wire(value: &T) -> String { + serde_json::to_string(value).expect("value should serialize") +} + +/// The tags a persisted document is matched on. A rename here silently +/// orphans every previously written record, so each spelling is stated +/// explicitly rather than snapshotted. +#[test] +fn enum_wire_spellings_are_stable() { + // Backend discriminators appear in persisted KMS configuration. + assert_eq!(wire(&KmsBackend::Local), r#""Local""#); + assert_eq!(wire(&KmsBackend::Static), r#""Static""#); + assert_eq!(wire(&KmsBackend::VaultKv2), r#""VaultKV2""#); + assert_eq!(wire(&KmsBackend::VaultTransit), r#""VaultTransit""#); + // The pre-rename spelling must still deserialize. + assert_eq!( + serde_json::from_str::(r#""Vault""#).expect("legacy label"), + KmsBackend::VaultKv2, + "the legacy `Vault` label must keep deserializing to VaultKV2" + ); + + // Key lifecycle vocabulary, persisted in key records and returned by the + // admin API. + assert_eq!(wire(&KeyState::Enabled), r#""Enabled""#); + assert_eq!(wire(&KeyState::Disabled), r#""Disabled""#); + assert_eq!(wire(&KeyState::PendingDeletion), r#""PendingDeletion""#); + assert_eq!(wire(&KeyState::PendingImport), r#""PendingImport""#); + assert_eq!(wire(&KeyState::Unavailable), r#""Unavailable""#); + + assert_eq!(wire(&KeyStatus::Active), r#""Active""#); + assert_eq!(wire(&KeyStatus::Disabled), r#""Disabled""#); + assert_eq!(wire(&KeyStatus::PendingDeletion), r#""PendingDeletion""#); + assert_eq!(wire(&KeyStatus::Deleted), r#""Deleted""#); + + assert_eq!(wire(&KeyUsage::EncryptDecrypt), r#""EncryptDecrypt""#); + assert_eq!(wire(&KeyUsage::SignVerify), r#""SignVerify""#); + + assert_eq!(wire(&KeySpec::Aes256), r#""Aes256""#); + assert_eq!(wire(&KeySpec::Aes128), r#""Aes128""#); + assert_eq!(wire(&KeySpec::ChaCha20), r#""ChaCha20""#); + + // Algorithm names double as S3 header values, so they are externally + // visible as well as persisted. + assert_eq!(wire(&EncryptionAlgorithm::Aes256), r#""AES256""#); + assert_eq!(wire(&EncryptionAlgorithm::ChaCha20Poly1305), r#""ChaCha20Poly1305""#); + assert_eq!(wire(&EncryptionAlgorithm::AwsKms), r#""aws:kms""#); + // The string form used on the wire must match the serde form exactly. + for algorithm in [ + EncryptionAlgorithm::Aes256, + EncryptionAlgorithm::ChaCha20Poly1305, + EncryptionAlgorithm::AwsKms, + ] { + assert_eq!( + wire(&algorithm), + format!("\"{}\"", algorithm.as_str()), + "as_str and the serde spelling must not drift apart" + ); + assert_eq!( + algorithm.as_str().parse::().expect("must parse back"), + algorithm, + "as_str must round-trip through FromStr" + ); + } + + // Service status is returned by the admin status endpoint. + assert_eq!(wire(&KmsServiceStatus::NotConfigured), r#""NotConfigured""#); + assert_eq!(wire(&KmsServiceStatus::Configured), r#""Configured""#); + assert_eq!(wire(&KmsServiceStatus::Running), r#""Running""#); + assert_eq!(wire(&KmsServiceStatus::Error("boom".to_string())), r#"{"Error":"boom"}"#); + + // Backup bundle vocabulary: written into manifests that a future version + // must still decode. + assert_eq!(wire(&ArtifactKind::KeyMaterial), r#""key-material""#); + assert_eq!(wire(&ArtifactKind::KeyMetadata), r#""key-metadata""#); + assert_eq!(wire(&ArtifactKind::MasterKeySalt), r#""master-key-salt""#); + assert_eq!(wire(&ArtifactKind::KmsConfig), r#""kms-config""#); + assert_eq!(wire(&ArtifactKind::Alias), r#""alias""#); + assert_eq!(wire(&ArtifactKind::Policy), r#""policy""#); + assert_eq!(wire(&CompletenessState::InProgress), r#""in-progress""#); + assert_eq!(wire(&CompletenessState::Complete), r#""complete""#); + assert_eq!(wire(&AeadAlgorithm::Aes256Gcm), r#""aes-256-gcm""#); + assert_eq!(wire(&DigestAlgorithm::Sha256), r#""sha-256""#); +} + +/// The header names an encrypted object carries for the rest of its life. +#[tokio::test] +async fn object_encryption_header_names_are_stable() { + let kms = TestKms::local_with(|config| config.default_key_id = Some("serde-key".to_string())).await; + kms.create_key("serde-key").await; + let service = kms.service().await; + let data = b"serde contract".to_vec(); + + let sse_s3 = service + .encrypt_object("bucket", "object", data.as_slice(), &EncryptionAlgorithm::Aes256, None, None) + .await + .expect("SSE-S3 encrypt"); + let sse_kms = service + .encrypt_object( + "bucket", + "object", + data.as_slice(), + &EncryptionAlgorithm::ChaCha20Poly1305, + Some("serde-key"), + None, + ) + .await + .expect("SSE-KMS encrypt"); + let sse_c = service + .encrypt_object_with_customer_key("bucket", "object", data.as_slice(), &[0x5cu8; 32], None) + .await + .expect("SSE-C encrypt"); + + let names = |result: &rustfs_kms::EncryptionMetadata| { + let mut names: Vec = service.metadata_to_headers(result).into_keys().collect(); + names.sort(); + names + }; + + insta::assert_yaml_snapshot!("sse_s3_header_names", names(&sse_s3.metadata)); + insta::assert_yaml_snapshot!("sse_kms_header_names", names(&sse_kms.metadata)); + insta::assert_yaml_snapshot!("sse_c_header_names", names(&sse_c.metadata)); + + // The encryption context is embedded as a JSON object under one header; + // its key set is part of the same contract. + let mut context_keys: Vec = sse_s3.metadata.encryption_context.keys().cloned().collect(); + context_keys.sort(); + insta::assert_yaml_snapshot!("sse_s3_encryption_context_keys", context_keys); +} + +/// The shape of a persisted `KmsConfig` document, per backend. +#[test] +fn persisted_configuration_shape_is_stable() { + let local = KmsConfig::local("/var/lib/rustfs/kms".into()); + insta::assert_yaml_snapshot!("kms_config_field_names", field_names(&local)); + + let local_backend = serde_json::to_value(&local.backend_config).expect("serialize"); + insta::assert_yaml_snapshot!("backend_config_local_shape", shape_of(&local_backend)); + + let static_config = KmsConfig::static_kms(STATIC_KEY_ID.to_string(), static_secret_key()); + let static_backend = serde_json::to_value(&static_config.backend_config).expect("serialize"); + insta::assert_yaml_snapshot!("backend_config_static_shape", shape_of(&static_backend)); + assert!( + !serde_json::to_string(&static_config.backend_config) + .expect("serialize") + .contains(&static_secret_key()), + "the static secret is `skip_serializing` and must never appear in a persisted document" + ); + + let vault = KmsConfig::vault(url::Url::parse("https://vault.example.com:8200").expect("url"), "token".to_string()); + let vault_backend = serde_json::to_value(&vault.backend_config).expect("serialize"); + insta::assert_yaml_snapshot!("backend_config_vault_kv2_shape", shape_of(&vault_backend)); + + let transit = KmsConfig::vault_transit(url::Url::parse("https://vault.example.com:8200").expect("url"), "token".to_string()); + let transit_backend = serde_json::to_value(&transit.backend_config).expect("serialize"); + insta::assert_yaml_snapshot!("backend_config_vault_transit_shape", shape_of(&transit_backend)); +} + +/// Every persisted `KmsConfig` must survive a round trip unchanged, so a +/// document written by this build is readable by it. +#[test] +fn persisted_configuration_round_trips() { + for (label, config) in [ + ("local", KmsConfig::local("/var/lib/rustfs/kms".into())), + ( + "vault-kv2", + KmsConfig::vault(url::Url::parse("https://vault.example.com:8200").expect("url"), "token".to_string()), + ), + ( + "vault-transit", + KmsConfig::vault_transit(url::Url::parse("https://vault.example.com:8200").expect("url"), "token".to_string()), + ), + ] { + let encoded = serde_json::to_string(&config).unwrap_or_else(|error| panic!("{label} should serialize: {error}")); + let decoded: KmsConfig = + serde_json::from_str(&encoded).unwrap_or_else(|error| panic!("{label} should deserialize: {error}")); + assert_eq!(decoded.backend, config.backend, "{label}: backend must survive"); + assert_eq!(decoded.timeout, config.timeout, "{label}: timeout must survive"); + assert_eq!(decoded.retry_attempts, config.retry_attempts, "{label}: retries must survive"); + assert_eq!(decoded.enable_cache, config.enable_cache, "{label}: cache flag must survive"); + assert_eq!(decoded.default_key_id, config.default_key_id, "{label}: default key must survive"); + assert_eq!( + serde_json::to_string(&decoded).expect("re-serialize"), + encoded, + "{label}: re-encoding a decoded document must be byte-identical" + ); + } + + // The Static backend is the documented exception: its secret is dropped on + // serialization, so a round trip cannot restore it. Recording that here + // keeps a future reader from treating it as a bug. + let static_config = KmsConfig::static_kms(STATIC_KEY_ID.to_string(), static_secret_key()); + let decoded: KmsConfig = + serde_json::from_str(&serde_json::to_string(&static_config).expect("serialize")).expect("deserialize"); + assert!( + decoded.static_config().expect("static config").secret_key.is_empty(), + "a persisted static config never carries its secret; the operator re-supplies it" + ); + assert_eq!( + decoded.static_config().expect("static config").key_id, + STATIC_KEY_ID, + "the key id does survive persistence" + ); +} + +/// The manifest a future restore has to decode. +#[tokio::test] +async fn backup_manifest_shape_is_stable() { + use rustfs_kms::LocalConfig; + use rustfs_kms::backends::local::LocalKmsClient; + use rustfs_kms::backup::{BackupKek, LocalBackupExportRequest, export_local_backup}; + + let kms = TestKms::local().await; + kms.create_key("manifest-shape").await; + let client = LocalKmsClient::new(LocalConfig { + key_dir: kms.key_dir().expect("key dir"), + master_key: None, + file_permissions: Some(0o600), + }) + .await + .expect("client"); + + let out = tempfile::TempDir::new().expect("temp dir"); + let manifest = export_local_backup( + &client, + &BackupKek::new("kek", 1, [0x11u8; 32]).expect("kek"), + &LocalBackupExportRequest { + backup_id: "shape".to_string(), + deployment_identity: "shape-deployment".to_string(), + rustfs_version: "0.0.0".to_string(), + snapshot_generation: 1, + destination: out.path().join("bundle"), + sanitized_config: None, + }, + ) + .await + .expect("export"); + + insta::assert_yaml_snapshot!("backup_manifest_field_names", field_names(&manifest)); + + let artifact = manifest.artifacts.first().expect("at least one artifact"); + insta::assert_yaml_snapshot!("backup_artifact_field_names", field_names(artifact)); + insta::assert_yaml_snapshot!("backup_kek_descriptor_field_names", field_names(&manifest.backup_kek)); + insta::assert_yaml_snapshot!( + "backup_local_kdf_field_names", + field_names(manifest.local_kdf.as_ref().expect("local kdf")) + ); +} + +/// Describe a JSON value as a structural fingerprint: object keys are kept, +/// leaf values are replaced by their type name. Stable across runs while still +/// catching a renamed or retyped field. +fn shape_of(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(map) => { + // Sort explicitly rather than relying on `serde_json::Map` being a + // `BTreeMap`: that depends on the `preserve_order` feature, and an + // unordered fingerprint would make these snapshots flaky. + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + let mut out = serde_json::Map::new(); + for key in keys { + out.insert(key.clone(), shape_of(&map[key])); + } + serde_json::Value::Object(out) + } + serde_json::Value::Array(items) => serde_json::Value::Array(items.iter().map(shape_of).take(1).collect()), + serde_json::Value::String(_) => serde_json::Value::String("".to_string()), + serde_json::Value::Number(_) => serde_json::Value::String("".to_string()), + serde_json::Value::Bool(_) => serde_json::Value::String("".to_string()), + serde_json::Value::Null => serde_json::Value::String("".to_string()), + } +} diff --git a/crates/kms/tests/common/mod.rs b/crates/kms/tests/common/mod.rs new file mode 100644 index 000000000..f8123d803 --- /dev/null +++ b/crates/kms/tests/common/mod.rs @@ -0,0 +1,491 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared harness for the `rustfs-kms` black-box behavior suite. +//! +//! Everything here drives the crate through the same public entry points the +//! server uses (`KmsServiceManager` -> `KmsManager` / `ObjectEncryptionService`), +//! so the suite keeps holding after internal refactors. Two rules keep it +//! black-box: +//! +//! * no `pub(crate)` internals, no on-disk key format parsing; +//! * assertions target observable contract — error *variants*, returned values, +//! and state that survives a restart — never implementation details. +//! +//! Every harness instance owns its own `KmsServiceManager`; the process-global +//! singleton is deliberately avoided so tests never cross-talk under nextest. + +#![allow(dead_code)] // each test binary uses a different slice of the harness + +use std::collections::HashMap; +use std::fmt::Debug; +use std::future::Future; +use std::path::PathBuf; +use std::sync::Arc; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use rustfs_kms::backends::BackendCapabilities; +use rustfs_kms::{ + CreateKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus, ObjectEncryptionService, + Result, +}; +use tempfile::TempDir; + +/// Key id configured for the static backend harness. +pub const STATIC_KEY_ID: &str = "behavior-static-key"; + +/// Deterministic 32-byte secret for the static backend, base64 encoded. +/// +/// Fixed rather than random so a failure is reproducible; it is test-only +/// material and never leaves this crate's test binaries. +pub fn static_secret_key() -> String { + BASE64.encode([0x5au8; 32]) +} + +/// Which backend a harness instance is running. +/// +/// Local and Static always run. The two Vault backends are **opt-in**: they +/// need a reachable server, so they join the matrix only when +/// `RUSTFS_KMS_VAULT_TOKEN` is set (see [`live_vault_backends`]). +/// +/// This matters for how a green run should be read. `rotate` and `versioning` +/// are advertised *only* by the Vault backends, so without the Vault lane every +/// capability-gated branch for them in a `for_each_backend` spec runs the +/// `UnsupportedCapability` side and never the working side — a rotation that +/// silently dropped prior key versions would pass. An offline-only run says +/// nothing about whether the Vault backends work. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackendKind { + Local, + Static, + VaultKv2, + VaultTransit, +} + +impl BackendKind { + pub fn name(self) -> &'static str { + match self { + Self::Local => "local", + Self::Static => "static", + Self::VaultKv2 => "vault-kv2", + Self::VaultTransit => "vault-transit", + } + } + + /// Whether this backend keeps its state on an external server that outlives + /// the harness, so key names must not collide between runs. + pub fn is_vault(self) -> bool { + matches!(self, Self::VaultKv2 | Self::VaultTransit) + } +} + +/// Address of the live Vault, defaulting to the usual local dev server. +pub fn vault_address() -> String { + std::env::var("RUSTFS_KMS_VAULT_ADDR").unwrap_or_else(|_| "http://127.0.0.1:8200".to_string()) +} + +/// Token for the live Vault, or `None` when the Vault lane is switched off. +/// +/// Presence of this variable is the single switch that adds the Vault backends +/// to every `for_each_backend` spec. +pub fn vault_token() -> Option { + std::env::var("RUSTFS_KMS_VAULT_TOKEN").ok().filter(|token| !token.is_empty()) +} + +/// The Vault backends to include in the matrix for this run. +pub fn live_vault_backends() -> Vec { + match vault_token() { + Some(_) => vec![BackendKind::VaultKv2, BackendKind::VaultTransit], + None => Vec::new(), + } +} + +/// A key name that cannot collide with another run against the same Vault. +/// +/// Vault state is persistent and shared, unlike the per-test temp directory the +/// local backend gets, so a fixed name would make a rerun collide with its own +/// leftovers and turn every assertion into a function of run order. +pub fn unique_key_name(prefix: &str) -> String { + format!("{prefix}-{}", uuid::Uuid::new_v4().simple()) +} + +/// A running KMS service, reachable only through the crate's public API. +pub struct TestKms { + manager: Arc, + kind: BackendKind, + config: KmsConfig, + /// Held for the harness lifetime so the local key directory outlives a + /// simulated process restart. + _dir: Option, +} + +impl TestKms { + /// Local backend with development defaults and no default key id. + pub async fn local() -> Self { + Self::local_with(|_| {}).await + } + + /// Local backend with `tweak` applied to the configuration before start. + pub async fn local_with(tweak: impl FnOnce(&mut KmsConfig)) -> Self { + let dir = TempDir::new().expect("create temp key dir"); + let mut config = KmsConfig::local(dir.path().to_path_buf()).with_insecure_development_defaults(); + tweak(&mut config); + let manager = start_manager(&config).await; + Self { + manager, + kind: BackendKind::Local, + config, + _dir: Some(dir), + } + } + + /// Vault KV v2 backend against the live server. + /// + /// Panics when the Vault lane is off — callers gate on + /// [`live_vault_backends`] rather than calling this blind. + pub async fn vault_kv2() -> Self { + let token = vault_token().expect("RUSTFS_KMS_VAULT_TOKEN must be set to run the Vault lane"); + let address = vault_address().parse().expect("RUSTFS_KMS_VAULT_ADDR must be a URL"); + // A local dev Vault speaks plain HTTP, which the config guard refuses + // unless development mode is declared explicitly. + let config = KmsConfig::vault(address, token).with_insecure_development_defaults(); + let manager = start_manager(&config).await; + Self { + manager, + kind: BackendKind::VaultKv2, + config, + _dir: None, + } + } + + /// Vault Transit backend against the live server. + pub async fn vault_transit() -> Self { + let token = vault_token().expect("RUSTFS_KMS_VAULT_TOKEN must be set to run the Vault lane"); + let address = vault_address().parse().expect("RUSTFS_KMS_VAULT_ADDR must be a URL"); + let config = KmsConfig::vault_transit(address, token).with_insecure_development_defaults(); + let manager = start_manager(&config).await; + Self { + manager, + kind: BackendKind::VaultTransit, + config, + _dir: None, + } + } + + /// Static single-key backend with a fixed key id and secret. + pub async fn static_backend() -> Self { + let config = KmsConfig::static_kms(STATIC_KEY_ID.to_string(), static_secret_key()); + let manager = start_manager(&config).await; + Self { + manager, + kind: BackendKind::Static, + config, + _dir: None, + } + } + + /// Simulate a process restart: stop the running service and bring a brand + /// new manager up over the same configuration and key directory. + /// + /// A fresh manager (rather than `stop` + `start` on the same one) is what + /// makes this meaningful — it discards every in-memory cache and version + /// counter, so anything that still holds afterwards came off disk. + pub async fn restart(&mut self) { + self.manager.stop().await.expect("stop should succeed"); + self.manager = start_manager(&self.config).await; + } + + pub fn manager(&self) -> &Arc { + &self.manager + } + + pub fn kind(&self) -> BackendKind { + self.kind + } + + pub fn config(&self) -> &KmsConfig { + &self.config + } + + /// Key directory of the local backend, for restart-over-same-state setups. + pub fn key_dir(&self) -> Option { + self.config.local_config().map(|local| local.key_dir.clone()) + } + + pub async fn kms(&self) -> Arc { + self.manager.get_manager().await.expect("KMS manager should be running") + } + + pub async fn service(&self) -> Arc { + self.manager + .get_encryption_service() + .await + .expect("encryption service should be running") + } + + pub async fn capabilities(&self) -> BackendCapabilities { + self.kms().await.backend_capabilities() + } + + /// Create a key and return its id, failing loudly on backends that cannot. + pub async fn create_key(&self, name: &str) -> String { + let response = self + .kms() + .await + .create_key(CreateKeyRequest { + key_name: Some(name.to_string()), + key_usage: KeyUsage::EncryptDecrypt, + description: Some(format!("black-box behavior key {name}")), + ..Default::default() + }) + .await + .unwrap_or_else(|error| panic!("create_key({name}) should succeed on {}: {error:?}", self.kind.name())); + assert_eq!(response.key_id, name, "created key id must be the requested name"); + response.key_id + } +} + +async fn start_manager(config: &KmsConfig) -> Arc { + let manager = Arc::new(KmsServiceManager::new()); + manager.configure(config.clone()).await.expect("configure should succeed"); + manager.start().await.expect("start should succeed"); + assert_eq!( + manager.get_status().await, + KmsServiceStatus::Running, + "manager must report Running right after a successful start" + ); + manager +} + +/// One backend under the shared behavior spec, pre-seeded with a usable key. +pub struct BackendCase { + pub kms: TestKms, + /// A key that exists and is Enabled on this backend. + pub key_id: String, +} + +impl BackendCase { + async fn new(kind: BackendKind) -> Self { + match kind { + BackendKind::Local => { + let kms = TestKms::local().await; + let key_id = kms.create_key("behavior-local-key").await; + Self { kms, key_id } + } + BackendKind::Static => { + let kms = TestKms::static_backend().await; + Self { + kms, + key_id: STATIC_KEY_ID.to_string(), + } + } + BackendKind::VaultKv2 => { + let kms = TestKms::vault_kv2().await; + let key_id = kms.create_key(&unique_key_name("behavior-kv2")).await; + Self { kms, key_id } + } + BackendKind::VaultTransit => { + let kms = TestKms::vault_transit().await; + let key_id = kms.create_key(&unique_key_name("behavior-transit")).await; + Self { kms, key_id } + } + } + } + + pub fn kind(&self) -> BackendKind { + self.kms.kind() + } + + pub async fn caps(&self) -> BackendCapabilities { + self.kms.capabilities().await + } +} + +/// Run one behavior spec against every backend in this run's matrix. +/// +/// Always Local and Static; plus the Vault backends when the Vault lane is on +/// (see [`live_vault_backends`]). +/// +/// The spec is expected to branch on `case.caps()`: a capability a backend +/// advertises must behave correctly, and one it does not advertise must be +/// rejected with `UnsupportedCapability` (or the backend's documented +/// read-only refusal). The contract is deliberately two-directional. +pub async fn for_each_backend(spec: F) +where + F: Fn(BackendCase) -> Fut, + Fut: Future, +{ + let kinds = [BackendKind::Local, BackendKind::Static] + .into_iter() + .chain(live_vault_backends()); + for kind in kinds { + let case = BackendCase::new(kind).await; + spec(case).await; + } +} + +/// Drop the service's own startup probe key from a listing. +/// +/// Starting the service provisions the reserved [`rustfs_kms::probe::PROBE_KEY_ID`] +/// to verify the backend is actually usable, so it exists on every running +/// service and is not something a spec created. Exact-set assertions filter it +/// out: it is startup machinery, not behavior under test, and asserting it in +/// every expected list would couple those specs to the probe's naming. +pub fn without_probe_key(ids: impl IntoIterator) -> Vec { + ids.into_iter().filter(|id| id != rustfs_kms::probe::PROBE_KEY_ID).collect() +} + +/// Build an encryption context from literal pairs. +pub fn ctx(pairs: &[(&str, &str)]) -> HashMap { + pairs.iter().map(|(k, v)| ((*k).to_string(), (*v).to_string())).collect() +} + +/// Deterministic pseudo-random payload of `len` bytes. +/// +/// Avoids a RNG dependency in the assertions while still producing data that a +/// broken cipher cannot accidentally round-trip (unlike an all-zero buffer). +pub fn payload(len: usize) -> Vec { + (0..len).map(|i| ((i * 31 + 17) % 251) as u8).collect() +} + +/// Drop a successful value so a result whose `Ok` type is not `Debug` (an +/// `AsyncRead` trait object, for instance) can still go through the error +/// assertions below. +pub fn discard(result: Result) -> Result<()> { + result.map(|_| ()) +} + +/// Flip one bit in the middle of `bytes`, returning the tampered copy. +pub fn flip_middle_bit(bytes: &[u8]) -> Vec { + assert!(!bytes.is_empty(), "cannot tamper with empty bytes"); + let mut tampered = bytes.to_vec(); + let index = tampered.len() / 2; + tampered[index] ^= 0b0000_1000; + tampered +} + +// --------------------------------------------------------------------------- +// Error-variant assertions +// +// Every failure path is pinned to a KmsError *variant*, never to message text: +// messages are diagnostics and may be reworded, whereas the variant is what +// callers (admin handlers, ecfs) actually match on. +// --------------------------------------------------------------------------- + +#[track_caller] +pub fn assert_key_not_found(result: Result, expected_key_id: &str) { + match result { + Err(KmsError::KeyNotFound { key_id }) => assert!( + key_id.contains(expected_key_id), + "KeyNotFound should name {expected_key_id:?}, got {key_id:?}" + ), + other => panic!("expected KeyNotFound({expected_key_id}), got {other:?}"), + } +} + +#[track_caller] +pub fn assert_key_already_exists(result: Result, expected_key_id: &str) { + match result { + Err(KmsError::KeyAlreadyExists { key_id }) => { + assert_eq!(key_id, expected_key_id, "KeyAlreadyExists must name the conflicting key") + } + other => panic!("expected KeyAlreadyExists({expected_key_id}), got {other:?}"), + } +} + +#[track_caller] +pub fn assert_invalid_operation(result: Result, message_fragment: &str) { + match result { + Err(KmsError::InvalidOperation { message }) => assert!( + message.contains(message_fragment), + "InvalidOperation should mention {message_fragment:?}, got {message:?}" + ), + other => panic!("expected InvalidOperation containing {message_fragment:?}, got {other:?}"), + } +} + +#[track_caller] +pub fn assert_unsupported_capability(result: Result, expected_operation: &str) { + match result { + Err(KmsError::UnsupportedCapability { operation, .. }) => { + assert_eq!(operation, expected_operation, "UnsupportedCapability must name the refused operation") + } + other => panic!("expected UnsupportedCapability({expected_operation}), got {other:?}"), + } +} + +#[track_caller] +pub fn assert_context_mismatch(result: Result) { + match result { + Err(KmsError::ContextMismatch { .. }) => {} + other => panic!("expected ContextMismatch, got {other:?}"), + } +} + +#[track_caller] +pub fn assert_configuration_error(result: Result, message_fragment: &str) { + match result { + Err(KmsError::ConfigurationError { message }) => assert!( + message.contains(message_fragment), + "ConfigurationError should mention {message_fragment:?}, got {message:?}" + ), + other => panic!("expected ConfigurationError containing {message_fragment:?}, got {other:?}"), + } +} + +#[track_caller] +pub fn assert_validation_error(result: Result) { + match result { + Err(KmsError::ValidationError { .. }) => {} + other => panic!("expected ValidationError, got {other:?}"), + } +} + +#[track_caller] +pub fn assert_cryptographic_error(result: Result) { + match result { + Err(KmsError::CryptographicError { .. }) => {} + other => panic!("expected CryptographicError, got {other:?}"), + } +} + +#[track_caller] +pub fn assert_invalid_key_size(result: Result, expected: usize, actual: usize) { + match result { + Err(KmsError::InvalidKeySize { + expected: got_expected, + actual: got_actual, + }) => { + assert_eq!(got_expected, expected, "InvalidKeySize.expected"); + assert_eq!(got_actual, actual, "InvalidKeySize.actual"); + } + other => panic!("expected InvalidKeySize({expected}, {actual}), got {other:?}"), + } +} + +/// Assert that a rendered representation carries none of the given secrets. +/// +/// Used against `Debug` and serde output of configs and responses: the crate's +/// security rule is that key material never reaches a log or an API payload. +#[track_caller] +pub fn assert_no_secret_leak(rendered: &str, secrets: &[&str]) { + for secret in secrets { + assert!( + !rendered.contains(secret), + "rendered output leaked a secret ({} chars of it): {rendered}", + secret.len() + ); + } +} diff --git a/crates/kms/tests/snapshots/behavior_serde__backend_config_local_shape.snap b/crates/kms/tests/snapshots/behavior_serde__backend_config_local_shape.snap new file mode 100644 index 000000000..c1dda50f2 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backend_config_local_shape.snap @@ -0,0 +1,8 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: shape_of(&local_backend) +--- +Local: + file_permissions: "" + key_dir: "" + master_key: "" diff --git a/crates/kms/tests/snapshots/behavior_serde__backend_config_static_shape.snap b/crates/kms/tests/snapshots/behavior_serde__backend_config_static_shape.snap new file mode 100644 index 000000000..711ab77d2 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backend_config_static_shape.snap @@ -0,0 +1,6 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: shape_of(&static_backend) +--- +Static: + key_id: "" diff --git a/crates/kms/tests/snapshots/behavior_serde__backend_config_vault_kv2_shape.snap b/crates/kms/tests/snapshots/behavior_serde__backend_config_vault_kv2_shape.snap new file mode 100644 index 000000000..1e07bc72a --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backend_config_vault_kv2_shape.snap @@ -0,0 +1,14 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: shape_of(&vault_backend) +--- +VaultKV2: + address: "" + auth_method: + Token: + token: "" + key_path_prefix: "" + kv_mount: "" + mount_path: "" + namespace: "" + tls: "" diff --git a/crates/kms/tests/snapshots/behavior_serde__backend_config_vault_transit_shape.snap b/crates/kms/tests/snapshots/behavior_serde__backend_config_vault_transit_shape.snap new file mode 100644 index 000000000..57eab3f4f --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backend_config_vault_transit_shape.snap @@ -0,0 +1,14 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: shape_of(&transit_backend) +--- +VaultTransit: + address: "" + auth_method: + Token: + token: "" + metadata_key_prefix: "" + metadata_kv_mount: "" + mount_path: "" + namespace: "" + tls: "" diff --git a/crates/kms/tests/snapshots/behavior_serde__backup_artifact_field_names.snap b/crates/kms/tests/snapshots/behavior_serde__backup_artifact_field_names.snap new file mode 100644 index 000000000..228f6985f --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backup_artifact_field_names.snap @@ -0,0 +1,9 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: field_names(artifact) +--- +- aead_algorithm +- encrypted_digest +- kind +- len +- path diff --git a/crates/kms/tests/snapshots/behavior_serde__backup_kek_descriptor_field_names.snap b/crates/kms/tests/snapshots/behavior_serde__backup_kek_descriptor_field_names.snap new file mode 100644 index 000000000..b622bcbff --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backup_kek_descriptor_field_names.snap @@ -0,0 +1,7 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: field_names(&manifest.backup_kek) +--- +- aead_algorithm +- kek_id +- kek_version diff --git a/crates/kms/tests/snapshots/behavior_serde__backup_local_kdf_field_names.snap b/crates/kms/tests/snapshots/behavior_serde__backup_local_kdf_field_names.snap new file mode 100644 index 000000000..41b7d05f2 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backup_local_kdf_field_names.snap @@ -0,0 +1,6 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: "field_names(manifest.local_kdf.as_ref().expect(\"local kdf\"))" +--- +- derivation +- protection_modes diff --git a/crates/kms/tests/snapshots/behavior_serde__backup_manifest_field_names.snap b/crates/kms/tests/snapshots/behavior_serde__backup_manifest_field_names.snap new file mode 100644 index 000000000..975d624a3 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__backup_manifest_field_names.snap @@ -0,0 +1,18 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: field_names(&manifest) +--- +- artifacts +- at_rest_protection +- backend +- backup_id +- backup_kek +- completeness +- created_at +- deployment_identity +- format_version +- local_kdf +- manifest_digest +- responsibility +- rustfs_version +- snapshot_generation diff --git a/crates/kms/tests/snapshots/behavior_serde__kms_config_field_names.snap b/crates/kms/tests/snapshots/behavior_serde__kms_config_field_names.snap new file mode 100644 index 000000000..bf3ce5298 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__kms_config_field_names.snap @@ -0,0 +1,12 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: field_names(&local) +--- +- allow_insecure_dev_defaults +- backend +- backend_config +- cache_config +- default_key_id +- enable_cache +- retry_attempts +- timeout diff --git a/crates/kms/tests/snapshots/behavior_serde__sse_c_header_names.snap b/crates/kms/tests/snapshots/behavior_serde__sse_c_header_names.snap new file mode 100644 index 000000000..22edc4d53 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__sse_c_header_names.snap @@ -0,0 +1,11 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: names(&sse_c.metadata) +--- +- x-amz-server-side-encryption +- x-amz-server-side-encryption-customer-algorithm +- x-rustfs-encryption-algorithm +- x-rustfs-encryption-context +- x-rustfs-encryption-iv +- x-rustfs-encryption-key +- x-rustfs-encryption-tag diff --git a/crates/kms/tests/snapshots/behavior_serde__sse_kms_header_names.snap b/crates/kms/tests/snapshots/behavior_serde__sse_kms_header_names.snap new file mode 100644 index 000000000..656f1acf8 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__sse_kms_header_names.snap @@ -0,0 +1,12 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: names(&sse_kms.metadata) +--- +- x-amz-server-side-encryption +- x-amz-server-side-encryption-aws-kms-key-id +- x-rustfs-encryption-algorithm +- x-rustfs-encryption-context +- x-rustfs-encryption-iv +- x-rustfs-encryption-key +- x-rustfs-encryption-key-id +- x-rustfs-encryption-tag diff --git a/crates/kms/tests/snapshots/behavior_serde__sse_s3_encryption_context_keys.snap b/crates/kms/tests/snapshots/behavior_serde__sse_s3_encryption_context_keys.snap new file mode 100644 index 000000000..2c7e889c7 --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__sse_s3_encryption_context_keys.snap @@ -0,0 +1,8 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: context_keys +--- +- algorithm +- bucket +- object +- object_key diff --git a/crates/kms/tests/snapshots/behavior_serde__sse_s3_header_names.snap b/crates/kms/tests/snapshots/behavior_serde__sse_s3_header_names.snap new file mode 100644 index 000000000..b6b3f907a --- /dev/null +++ b/crates/kms/tests/snapshots/behavior_serde__sse_s3_header_names.snap @@ -0,0 +1,11 @@ +--- +source: crates/kms/tests/behavior_serde.rs +expression: names(&sse_s3.metadata) +--- +- x-amz-server-side-encryption +- x-rustfs-encryption-algorithm +- x-rustfs-encryption-context +- x-rustfs-encryption-iv +- x-rustfs-encryption-key +- x-rustfs-encryption-key-id +- x-rustfs-encryption-tag