From 3c00ad604868b1d58f70bfcbfdb4aa7c60b1f026 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 20:23:28 +0800 Subject: [PATCH] fix(kms): make the deletion waiting window non-bypassable (#5535) --- .../src/kms/configured_roundtrip_test.rs | 86 +++--- crates/e2e_test/src/kms/kms_vault_test.rs | 32 ++- crates/kms/src/api_types.rs | 21 +- crates/kms/src/backends/aws.rs | 4 + crates/kms/src/backends/contract_tests.rs | 1 + crates/kms/src/backends/local.rs | 48 +++- crates/kms/src/backends/static_kms.rs | 1 + crates/kms/src/backends/vault.rs | 56 +++- crates/kms/src/backends/vault_transit.rs | 54 +++- crates/kms/src/config.rs | 68 +++++ crates/kms/src/deletion_worker.rs | 1 + crates/kms/src/manager.rs | 259 +++++++++++++++++- crates/kms/src/types.rs | 28 +- docs/operations/kms-backend-security.md | 4 +- rustfs/src/admin/handlers/kms_dynamic.rs | 3 + rustfs/src/admin/handlers/kms_keys.rs | 6 + rustfs/src/init.rs | 4 + 17 files changed, 602 insertions(+), 74 deletions(-) diff --git a/crates/e2e_test/src/kms/configured_roundtrip_test.rs b/crates/e2e_test/src/kms/configured_roundtrip_test.rs index 835ce4e9f..13e96ef0b 100644 --- a/crates/e2e_test/src/kms/configured_roundtrip_test.rs +++ b/crates/e2e_test/src/kms/configured_roundtrip_test.rs @@ -126,7 +126,10 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_ assert_eq!(cancelled["success"], true); assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled"); - let removed = kms_admin_request( + // A default server refuses to skip the waiting window (rustfs/backlog#1585): + // immediate deletion is unrecoverable and takes every object encrypted under + // the key with it, so the endpoint must reject it rather than honour it. + let refused = kms_admin_request( base_url, http::Method::DELETE, "/rustfs/admin/v3/kms/keys/delete", @@ -140,37 +143,49 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_ access_key, secret_key, ) - .await?; - let removed: serde_json::Value = serde_json::from_str(&removed)?; - assert_eq!(removed["success"], true); + .await + .err() + .ok_or("immediate KMS key deletion must be refused on a default server")?; + assert!( + refused.to_string().contains("400 Bad Request"), + "refused immediate deletion must report a client error: {refused}" + ); - let listed = - kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?; - let listed: serde_json::Value = serde_json::from_str(&listed)?; - assert_eq!(listed["success"], true); - let keys = listed["keys"] - .as_array() - .ok_or("list KMS keys response omitted keys after deletion")?; - if let Some(key) = keys.iter().find(|key| key["key_id"] == key_id) { - assert_eq!(key["status"], "PendingDeletion", "a retained force-deleted key must be pending deletion"); - let removed = kms_admin_request( - base_url, - http::Method::DELETE, - "/rustfs/admin/v3/kms/keys/delete", - Some( - &serde_json::json!({ - "key_id": key_id, - "force_immediate": true - }) - .to_string(), - ), - access_key, - secret_key, - ) - .await?; - let removed: serde_json::Value = serde_json::from_str(&removed)?; - assert_eq!(removed["success"], true); - } + // The refused request left the key alone, so the window-bounded path still + // has something to schedule. + let described = kms_admin_request( + base_url, + http::Method::GET, + &format!("/rustfs/admin/v3/kms/keys/{key_id}"), + None, + access_key, + secret_key, + ) + .await?; + let described: serde_json::Value = serde_json::from_str(&described)?; + assert_eq!( + described["key_metadata"]["key_state"], "Enabled", + "a refused immediate deletion must leave the key usable" + ); + + let rescheduled = kms_admin_request( + base_url, + http::Method::DELETE, + "/rustfs/admin/v3/kms/keys/delete", + Some( + &serde_json::json!({ + "key_id": key_id, + "pending_window_in_days": 7 + }) + .to_string(), + ), + access_key, + secret_key, + ) + .await?; + let rescheduled: serde_json::Value = serde_json::from_str(&rescheduled)?; + assert_eq!(rescheduled["success"], true); + assert!(rescheduled["deletion_date"].is_string()); let listed = kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?; @@ -179,10 +194,11 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_ let keys = listed["keys"] .as_array() .ok_or("final list KMS keys response omitted keys after deletion")?; - assert!( - keys.iter().all(|key| key["key_id"] != key_id), - "force-deleted KMS key must no longer appear in list" - ); + let key = keys + .iter() + .find(|key| key["key_id"] == key_id) + .ok_or("a key awaiting its deletion window must still be listed")?; + assert_eq!(key["status"], "PendingDeletion", "a scheduled key must be pending deletion"); Ok(()) } diff --git a/crates/e2e_test/src/kms/kms_vault_test.rs b/crates/e2e_test/src/kms/kms_vault_test.rs index 0a1201bd8..1ca370815 100644 --- a/crates/e2e_test/src/kms/kms_vault_test.rs +++ b/crates/e2e_test/src/kms/kms_vault_test.rs @@ -449,29 +449,31 @@ async fn test_vault_kms_key_crud( info!("✅ Delete verification: Key state correctly changed to: {}", key_state); - // Force Delete - Force immediate deletion for PendingDeletion key - let force_delete_response = crate::common::execute_awscurl( + // Force Delete - a default server refuses to skip the waiting window + // (rustfs/backlog#1585): destroying the key material immediately would take + // every object encrypted under the key with it. + let force_delete_error = crate::common::execute_awscurl( &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"), "DELETE", None, access_key, secret_key, ) - .await?; + .await + .expect_err("Immediate KMS key deletion must be refused on a default server"); + info!("✅ Force Delete: correctly refused for key {}: {}", key_id, force_delete_error); - // Parse and validate the force delete response - let force_delete_result: serde_json::Value = serde_json::from_str(&force_delete_response)?; - assert_eq!(force_delete_result["success"], true, "Force delete operation must return success=true"); - info!("✅ Force Delete: Successfully force deleted key: {}", key_id); + // The refused request must leave the key exactly as it was: still present, + // still pending deletion, still recoverable through cancel-deletion. + let describe_after_refusal = + crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?; + let describe_after_refusal: serde_json::Value = serde_json::from_str(&describe_after_refusal)?; + assert_eq!( + describe_after_refusal["key_metadata"]["key_state"], "PendingDeletion", + "A refused immediate deletion must leave the key pending deletion" + ); - // Verify key no longer exists after force deletion (should return error) - let describe_force_deleted_result = - crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await; - - // After force deletion, key should not be found (GET should fail) - assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found"); - - info!("✅ Force Delete verification: Key was permanently deleted and is no longer accessible"); + info!("✅ Force Delete verification: Key survived the refused immediate deletion"); info!("Vault KMS key CRUD operations completed successfully"); Ok(()) diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index efd782e9b..50f7b98da 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -16,8 +16,8 @@ use crate::config::{ BackendConfig, CacheConfig, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend, - KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, redacted_secret, - redacted_secret_option, + KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, + allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option, }; use crate::service_manager::KmsServiceStatus; use crate::types::{KeyMetadata, KeyUsage}; @@ -506,6 +506,10 @@ impl ConfigureLocalKmsRequest { file_permissions: self.file_permissions, }), allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false), + // Read from server configuration, never from the request body: the + // gate must mean the same thing whether KMS was configured at + // startup or through this endpoint. + allow_immediate_deletion: allow_immediate_deletion_from_env(), timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)), retry_attempts: self.retry_attempts.unwrap_or(3), enable_cache: self.enable_cache.unwrap_or(true), @@ -543,6 +547,10 @@ impl ConfigureVaultKmsRequest { }, })), allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false), + // Read from server configuration, never from the request body: the + // gate must mean the same thing whether KMS was configured at + // startup or through this endpoint. + allow_immediate_deletion: allow_immediate_deletion_from_env(), timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)), retry_attempts: self.retry_attempts.unwrap_or(3), enable_cache: self.enable_cache.unwrap_or(true), @@ -580,6 +588,10 @@ impl ConfigureVaultTransitKmsRequest { }, })), allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false), + // Read from server configuration, never from the request body: the + // gate must mean the same thing whether KMS was configured at + // startup or through this endpoint. + allow_immediate_deletion: allow_immediate_deletion_from_env(), timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)), retry_attempts: self.retry_attempts.unwrap_or(3), enable_cache: self.enable_cache.unwrap_or(true), @@ -603,6 +615,10 @@ impl ConfigureStaticKmsRequest { secret_key: self.secret_key.clone(), }), allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false), + // Read from server configuration, never from the request body: the + // gate must mean the same thing whether KMS was configured at + // startup or through this endpoint. + allow_immediate_deletion: allow_immediate_deletion_from_env(), timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)), retry_attempts: self.retry_attempts.unwrap_or(3), enable_cache: self.enable_cache.unwrap_or(true), @@ -858,6 +874,7 @@ mod tests { tls: None, })), allow_insecure_dev_defaults: true, + allow_immediate_deletion: false, timeout: Duration::from_secs(30), retry_attempts: 3, enable_cache: true, diff --git a/crates/kms/src/backends/aws.rs b/crates/kms/src/backends/aws.rs index 829daaa4f..54321b067 100644 --- a/crates/kms/src/backends/aws.rs +++ b/crates/kms/src/backends/aws.rs @@ -1068,6 +1068,7 @@ mod tests { key_id: "test-key".to_string(), pending_window_in_days: Some(days), force_immediate: None, + confirm_key_id: None, }) .await .expect_err("an out-of-range window must be rejected"); @@ -1086,6 +1087,7 @@ mod tests { key_id: "test-key".to_string(), pending_window_in_days: None, force_immediate: Some(true), + confirm_key_id: None, }) .await .expect_err("immediate deletion must be rejected"); @@ -1228,6 +1230,7 @@ mod tests { key_id: key_id.clone(), pending_window_in_days: Some(7), force_immediate: None, + confirm_key_id: None, }) .await .expect("scheduling deletion must succeed"); @@ -1246,6 +1249,7 @@ mod tests { key_id, pending_window_in_days: Some(7), force_immediate: None, + confirm_key_id: None, }) .await .expect("re-scheduling deletion must succeed"); diff --git a/crates/kms/src/backends/contract_tests.rs b/crates/kms/src/backends/contract_tests.rs index 52d9beb18..32e7998f1 100644 --- a/crates/kms/src/backends/contract_tests.rs +++ b/crates/kms/src/backends/contract_tests.rs @@ -108,6 +108,7 @@ fn schedule_request(key_id: &str) -> DeleteKeyRequest { key_id: key_id.to_string(), pending_window_in_days: Some(7), force_immediate: None, + confirm_key_id: None, } } diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index f2a16e710..aae314d28 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -1617,9 +1617,15 @@ impl KmsBackend for LocalKmsBackend { // Schedule for deletion (default 30 days) ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?; - let days = request.pending_window_in_days.unwrap_or(30); - if !(7..=30).contains(&days) { - return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string())); + // Defensive: KmsManager::delete_key is the enforcement point for the + // waiting window and rejects out-of-range requests before any + // backend runs. This repeats the bound for callers holding a backend + // handle directly (tests, maintenance tasks). + let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS); + if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) { + return Err(KmsError::invalid_parameter(format!( + "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}" + ))); } let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400); @@ -2678,6 +2684,7 @@ mod tests { key_id: "durable-key".to_string(), pending_window_in_days: None, force_immediate: Some(true), + confirm_key_id: None, }) .await .expect("delete key"); @@ -2685,6 +2692,41 @@ mod tests { assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory"); } + /// KmsManager::delete_key is the enforcement point for the waiting window; + /// this pins the backend's defensive copy of the same bound, which is all + /// that stands between a direct backend caller and a one-day window. + #[tokio::test] + async fn delete_key_refuses_a_window_outside_the_supported_range() { + let (client, _temp_dir) = create_test_client().await; + let key_id = "window-bounds-key"; + client.create_key(key_id, "AES_256", None).await.expect("create key"); + let backend = LocalKmsBackend { client }; + + for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] { + let result = backend + .delete_key(DeleteKeyRequest { + key_id: key_id.to_string(), + pending_window_in_days: Some(days), + ..Default::default() + }) + .await; + assert!( + matches!(result, Err(KmsError::InvalidOperation { .. })), + "a {days}-day window must be refused, got {result:?}" + ); + } + + let state = backend + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .expect("describe should succeed") + .key_metadata + .key_state; + assert_eq!(state, KeyState::Enabled, "a refused window must not schedule the key"); + } + #[tokio::test] async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() { use durable_file::{CommitStep, failpoint}; diff --git a/crates/kms/src/backends/static_kms.rs b/crates/kms/src/backends/static_kms.rs index 7234a787b..6f048d41a 100644 --- a/crates/kms/src/backends/static_kms.rs +++ b/crates/kms/src/backends/static_kms.rs @@ -683,6 +683,7 @@ mod tests { key_id: key_id.clone(), pending_window_in_days: Some(7), force_immediate: None, + confirm_key_id: None, }, ) .await; diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index 9c58acc77..03c4f777d 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -1427,11 +1427,15 @@ impl KmsBackend for VaultKmsBackend { // Schedule for deletion (default 30 days) ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?; - let days = request.pending_window_in_days.unwrap_or(30); - if !(7..=30).contains(&days) { - return Err(crate::error::KmsError::invalid_parameter( - "pending_window_in_days must be between 7 and 30".to_string(), - )); + // Defensive: KmsManager::delete_key is the enforcement point for the + // waiting window and rejects out-of-range requests before any + // backend runs. This repeats the bound for callers holding a backend + // handle directly (tests, maintenance tasks). + let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS); + if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) { + return Err(crate::error::KmsError::invalid_parameter(format!( + "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}" + ))); } let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400); @@ -2274,6 +2278,7 @@ mod tests { key_id: key_id.clone(), pending_window_in_days: Some(7), force_immediate: Some(false), + confirm_key_id: None, }) .await .expect("schedule delete"); @@ -2777,6 +2782,7 @@ mod tests { key_id: "wired-key".to_string(), pending_window_in_days: Some(7), force_immediate: Some(false), + confirm_key_id: None, }) .await .expect("the schedule must retry past the lost race and commit"); @@ -2793,6 +2799,45 @@ mod tests { ); } + /// KmsManager::delete_key is the enforcement point for the waiting window; + /// this pins the backend's defensive copy of the same bound, which is all + /// that stands between a direct backend caller and a one-day window. + #[tokio::test] + async fn wired_schedule_deletion_refuses_a_window_outside_the_supported_range() { + for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] { + let vault = ScriptedVault::serve(vec![ + // describe_key: key info plus stored metadata. + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ScriptedResponse::ok(kv2_read_data(&healthy_key_data())), + ]) + .await; + let config = KmsConfig::vault( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build"); + + let result = backend + .delete_key(DeleteKeyRequest { + key_id: "wired-key".to_string(), + pending_window_in_days: Some(days), + ..Default::default() + }) + .await; + assert!( + matches!(result, Err(KmsError::InvalidOperation { .. })), + "a {days}-day window must be refused, got {result:?}" + ); + + let requests = vault.requests(); + assert!( + !requests.iter().any(|line| line.starts_with("POST ")), + "a refused window must not write anything: {requests:?}" + ); + } + } + /// Conflict semantics are re-read *and* re-gate: when the re-read after a /// lost race shows the key was concurrently scheduled for deletion, the /// state gate rejects the retry instead of blindly re-applying it. @@ -2825,6 +2870,7 @@ mod tests { key_id: "wired-key".to_string(), pending_window_in_days: Some(7), force_immediate: Some(false), + confirm_key_id: None, }) .await .expect_err("the retry must re-run the state gate against the fresh record"); diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 52fd323dd..f35e46b38 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -1219,9 +1219,15 @@ impl KmsBackend for VaultTransitKmsBackend { } else { ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?; - let days = request.pending_window_in_days.unwrap_or(30); - if !(7..=30).contains(&days) { - return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30")); + // Defensive: KmsManager::delete_key is the enforcement point for the + // waiting window and rejects out-of-range requests before any + // backend runs. This repeats the bound for callers holding a backend + // handle directly (tests, maintenance tasks). + let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS); + if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) { + return Err(KmsError::invalid_parameter(format!( + "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}" + ))); } let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400); @@ -1819,6 +1825,48 @@ mod tests { assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}"); } + /// KmsManager::delete_key is the enforcement point for the waiting window; + /// this pins the backend's defensive copy of the same bound, which is all + /// that stands between a direct backend caller and a one-day window. + #[tokio::test] + async fn wired_backend_delete_refuses_a_window_outside_the_supported_range() { + for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] { + let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default()); + let vault = ScriptedVault::serve(vec![ + // The state gate reads the transit key, then its metadata record. + ScriptedResponse::ok(transit_key_read_data("wired-key")), + ScriptedResponse::ok(metadata_read_data(&metadata)), + ]) + .await; + let config = KmsConfig::vault_transit( + url::Url::parse(&vault.address).expect("scripted vault address should parse"), + "scripted-token".to_string(), + ) + .with_insecure_development_defaults(); + let backend = VaultTransitKmsBackend::new(config) + .await + .expect("vault transit backend should build"); + + let result = backend + .delete_key(DeleteKeyRequest { + key_id: "wired-key".to_string(), + pending_window_in_days: Some(days), + ..Default::default() + }) + .await; + assert!( + matches!(result, Err(KmsError::InvalidOperation { .. })), + "a {days}-day window must be refused, got {result:?}" + ); + + let requests = vault.requests(); + assert!( + !requests.iter().any(|line| line.starts_with("POST ")), + "a refused window must not write anything: {requests:?}" + ); + } + } + /// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the /// current secret version used as the check-and-set base. fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value { diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 37cde700b..67be8b547 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -24,6 +24,7 @@ use std::time::Duration; use url::Url; pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS"; +pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION"; pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX"; @@ -164,6 +165,24 @@ pub struct KmsConfig { /// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault. #[serde(default)] pub allow_insecure_dev_defaults: bool, + /// Allow `DeleteKey` requests to skip the pending-deletion waiting window and + /// destroy key material right away. + /// + /// Off by default: an immediate deletion is unrecoverable and takes every + /// object encrypted under the key with it, so the waiting window (plus + /// `CancelKeyDeletion`) is the only recovery path. Operators who genuinely + /// need immediate deletion — throwaway test clusters, key material that was + /// never used — must turn it on through server configuration + /// ([`ENV_KMS_ALLOW_IMMEDIATE_DELETION`]); the request must still echo the + /// key id back for confirmation. + /// + /// Not part of the serialized configuration, and not settable through the + /// admin configure API. It is per-server operator state that has to be + /// re-stated to survive a restart: persisting it would carry one operator's + /// one-time enablement into the cluster-wide config that every node reloads, + /// long after the deletion it was turned on for. + #[serde(skip)] + pub allow_immediate_deletion: bool, /// Timeout for a single backend attempt. /// /// This bounds one outbound request, not the whole operation: the operation @@ -187,6 +206,7 @@ impl Default for KmsConfig { default_key_id: None, backend_config: BackendConfig::default(), allow_insecure_dev_defaults: false, + allow_immediate_deletion: false, timeout: Duration::from_secs(30), retry_attempts: 3, enable_cache: true, @@ -714,6 +734,12 @@ impl KmsConfig { self } + /// Explicitly allow deletions that bypass the pending-deletion waiting window. + pub fn with_immediate_deletion_allowed(mut self) -> Self { + self.allow_immediate_deletion = true; + self + } + /// Set operation timeout pub fn with_timeout(mut self, timeout: Duration) -> Self { self.timeout = timeout; @@ -927,6 +953,7 @@ impl KmsConfig { config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache); config.allow_insecure_dev_defaults = get_env_bool(ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS, config.allow_insecure_dev_defaults); + config.allow_immediate_deletion = get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, config.allow_immediate_deletion); // Backend-specific configuration match config.backend { @@ -1042,6 +1069,15 @@ impl KmsConfig { } } +/// Read the immediate-deletion gate from the environment. +/// +/// Callers that assemble a [`KmsConfig`] field by field instead of going +/// through [`KmsConfig::from_env`] use this, so the gate keeps one name, one +/// default, and one place to look it up. +pub fn allow_immediate_deletion_from_env() -> bool { + get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, false) +} + fn vault_tls_config(skip_tls_verify: bool) -> Option { skip_tls_verify.then_some(TlsConfig { ca_cert_path: None, @@ -1802,6 +1838,38 @@ mod tests { assert_eq!(refresh_safety_window_secs, None); } + /// The gate lives in server configuration only: it never rides along in a + /// serialized config, and a stored config that claims it must not be + /// believed. Otherwise one operator's one-time enablement would reach every + /// node that later reloads that config. + #[test] + fn immediate_deletion_gate_is_server_local_and_never_persisted() { + let persisted = + serde_json::to_value(KmsConfig::default().with_immediate_deletion_allowed()).expect("kms config should serialize"); + assert!( + persisted.get("allow_immediate_deletion").is_none(), + "the gate must not be written into a persisted config: {persisted}" + ); + + let mut forged = persisted; + forged + .as_object_mut() + .expect("a persisted config must be a JSON object") + .insert("allow_immediate_deletion".to_string(), serde_json::json!(true)); + let restored: KmsConfig = serde_json::from_value(forged).expect("an unknown gate field must not break loading"); + assert!(!restored.allow_immediate_deletion, "a stored gate must fail closed"); + + with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, Some("true"))], || { + assert!( + allow_immediate_deletion_from_env(), + "the gate must be reachable from server configuration" + ); + }); + with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, None::<&str>)], || { + assert!(!allow_immediate_deletion_from_env()); + }); + } + #[test] fn test_validate_rejects_incomplete_approle() { let mut config = KmsConfig::vault_approle( diff --git a/crates/kms/src/deletion_worker.rs b/crates/kms/src/deletion_worker.rs index 5edd89dc5..b39748afa 100644 --- a/crates/kms/src/deletion_worker.rs +++ b/crates/kms/src/deletion_worker.rs @@ -376,6 +376,7 @@ mod tests { key_id: key_id.to_string(), pending_window_in_days: Some(7), force_immediate: None, + confirm_key_id: None, }) .await .expect("deletion should be scheduled"); diff --git a/crates/kms/src/manager.rs b/crates/kms/src/manager.rs index 44ee80c39..517c82eb3 100644 --- a/crates/kms/src/manager.rs +++ b/crates/kms/src/manager.rs @@ -17,17 +17,19 @@ use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink}; use crate::backends::KmsBackend; use crate::cache::{KmsCache, KmsCacheStats}; -use crate::config::KmsConfig; -use crate::error::Result; +use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, KmsConfig}; +use crate::error::{KmsError, Result}; use crate::types::{ - CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse, - DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse, - GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext, + CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, + DEFAULT_PENDING_DELETION_WINDOW_DAYS, DecryptRequest, DecryptResponse, DeleteKeyRequest, DeleteKeyResponse, + DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyRequest, GenerateDataKeyResponse, + ListKeysRequest, ListKeysResponse, MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext, }; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; use tokio::sync::RwLock; +use tracing::warn; /// KMS Manager coordinates operations between backends and caching #[derive(Clone)] @@ -38,12 +40,18 @@ pub struct KmsManager { enable_cache: bool, backend_kind: &'static str, audit_sink: Option>, + allow_immediate_deletion: bool, } impl KmsManager { /// Create a new KMS manager with the given backend and config pub fn new(backend: Arc, config: KmsConfig) -> Self { let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64))); + if config.allow_immediate_deletion { + warn!( + "KMS immediate key deletion is enabled: a DeleteKey request may destroy key material without any waiting window, and every object encrypted under that key becomes permanently unreadable" + ); + } Self { backend, cache, @@ -51,6 +59,7 @@ impl KmsManager { enable_cache: config.enable_cache, backend_kind: config.backend.as_str(), audit_sink: None, + allow_immediate_deletion: config.allow_immediate_deletion, } } @@ -225,7 +234,8 @@ impl KmsManager { Ok(()) } - /// Delete a key + /// Delete a key, either scheduled behind the waiting window or — when the + /// server allows it — immediately. /// /// Audited as an internal operation; callers serving an authenticated /// request should use [`Self::delete_key_with_context`]. @@ -246,7 +256,13 @@ impl KmsManager { result } + /// This is the single enforcement point for the waiting window: every + /// admin-facing deletion goes through here, so the checks below run before + /// any backend sees the request. The backends repeat the window bound as a + /// defensive assertion for callers that hold a backend handle directly. async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result { + self.check_deletion_request(&request)?; + let response = self.backend.delete_key(request).await?; // Remove from cache if enabled and key is being deleted @@ -258,6 +274,44 @@ impl KmsManager { Ok(response) } + /// Gate a deletion request before it reaches the backend. + /// + /// Immediate deletion is unrecoverable, so it needs both a server-side + /// opt-in and a per-request confirmation that echoes the key id; without + /// either, the request is refused rather than downgraded to a scheduled + /// deletion, so a caller never believes a key is gone when it is not. + fn check_deletion_request(&self, request: &DeleteKeyRequest) -> Result<()> { + if !request.force_immediate.unwrap_or(false) { + let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS); + if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) { + return Err(KmsError::invalid_parameter(format!( + "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}" + ))); + } + return Ok(()); + } + + if !self.allow_immediate_deletion { + return Err(KmsError::invalid_operation(format!( + "immediate deletion of key {} is not allowed; schedule the deletion and wait out the pending window, or set {ENV_KMS_ALLOW_IMMEDIATE_DELETION}=true on the server", + request.key_id + ))); + } + + if request.confirm_key_id.as_deref() != Some(request.key_id.as_str()) { + return Err(KmsError::invalid_operation(format!( + "immediate deletion of key {} requires confirm_key_id to repeat the key id exactly", + request.key_id + ))); + } + + warn!( + key_id = %request.key_id, + "immediate KMS key deletion accepted; key material is destroyed without a waiting window and cannot be recovered" + ); + Ok(()) + } + /// Cancel key deletion /// /// Audited as an internal operation; callers serving an authenticated @@ -605,6 +659,7 @@ mod tests { key_id: AUDITED_KEY_ID.to_string(), pending_window_in_days: None, force_immediate: None, + confirm_key_id: None, }, context, ) @@ -1019,4 +1074,196 @@ mod tests { .await .expect("second data key should decrypt with its own context"); } + + /// Manager over a local backend, with the immediate-deletion gate set as + /// the server operator would set it. + async fn deletion_manager(temp_dir: &tempfile::TempDir, allow_immediate_deletion: bool) -> KmsManager { + let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults(); + config.allow_immediate_deletion = allow_immediate_deletion; + let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend")); + KmsManager::new(backend, config) + } + + async fn create_named_key(manager: &KmsManager, key_name: &str) -> String { + manager + .create_key(CreateKeyRequest { + key_name: Some(key_name.to_string()), + key_usage: KeyUsage::EncryptDecrypt, + ..Default::default() + }) + .await + .expect("Failed to create key") + .key_id + } + + /// Data key generated up front, decrypted again afterwards: a refused + /// deletion must leave the master key material byte-for-byte usable, not + /// merely leave a metadata record behind. + async fn data_key_probe(manager: &KmsManager, key_id: &str) -> (Vec, Vec) { + let generated = manager + .generate_data_key(GenerateDataKeyRequest { + key_id: key_id.to_string(), + key_spec: KeySpec::Aes256, + encryption_context: HashMap::new(), + }) + .await + .expect("Failed to generate data key"); + (generated.plaintext_key, generated.ciphertext_blob) + } + + async fn assert_key_material_intact(manager: &KmsManager, key_id: &str, probe: &(Vec, Vec)) { + let state = manager + .describe_key(DescribeKeyRequest { + key_id: key_id.to_string(), + }) + .await + .expect("a key that was not deleted must still be describable") + .key_metadata + .key_state; + assert_eq!(state, KeyState::Enabled, "a refused deletion must not change the key state"); + + let decrypted = manager + .decrypt(DecryptRequest { + ciphertext: probe.1.clone(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }) + .await + .expect("key material must still decrypt data keys issued before the refused deletion"); + assert_eq!(decrypted.plaintext, probe.0, "decrypted data key must match the original plaintext"); + } + + #[tokio::test] + async fn immediate_deletion_is_refused_under_default_config() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let manager = deletion_manager(&temp_dir, false).await; + let key_id = create_named_key(&manager, "default-config-force-delete").await; + let probe = data_key_probe(&manager, &key_id).await; + + // Confirmation present and correct: the server-side gate alone must + // refuse this, no matter how well-formed the request is. + let error = manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + force_immediate: Some(true), + confirm_key_id: Some(key_id.clone()), + ..Default::default() + }) + .await + .expect_err("immediate deletion must be refused unless the server allows it"); + assert!( + matches!(error, KmsError::InvalidOperation { .. }), + "expected InvalidOperation, got {error:?}" + ); + + assert_key_material_intact(&manager, &key_id, &probe).await; + } + + #[tokio::test] + async fn immediate_deletion_requires_a_matching_confirmation() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let manager = deletion_manager(&temp_dir, true).await; + let key_id = create_named_key(&manager, "confirmation-required").await; + let probe = data_key_probe(&manager, &key_id).await; + + for confirmation in [None, Some(String::new()), Some(format!("{key_id}-typo"))] { + let result = manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + force_immediate: Some(true), + confirm_key_id: confirmation.clone(), + ..Default::default() + }) + .await; + assert!( + matches!(result, Err(KmsError::InvalidOperation { .. })), + "confirmation {confirmation:?} must be refused, got {result:?}" + ); + } + + assert_key_material_intact(&manager, &key_id, &probe).await; + } + + #[tokio::test] + async fn immediate_deletion_succeeds_with_a_matching_confirmation() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let manager = deletion_manager(&temp_dir, true).await; + let key_id = create_named_key(&manager, "confirmed-force-delete").await; + + manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + force_immediate: Some(true), + confirm_key_id: Some(key_id.clone()), + ..Default::default() + }) + .await + .expect("a confirmed immediate deletion must be allowed once the server enables it"); + + let error = manager + .describe_key(DescribeKeyRequest { key_id: key_id.clone() }) + .await + .expect_err("an immediately deleted key must be gone"); + assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}"); + } + + #[tokio::test] + async fn pending_window_outside_the_supported_range_is_refused() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let manager = deletion_manager(&temp_dir, false).await; + let key_id = create_named_key(&manager, "window-bounds").await; + let probe = data_key_probe(&manager, &key_id).await; + + for days in [0, MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] { + let result = manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: Some(days), + ..Default::default() + }) + .await; + assert!( + matches!(result, Err(KmsError::InvalidOperation { .. })), + "a {days}-day window must be refused, got {result:?}" + ); + } + + assert_key_material_intact(&manager, &key_id, &probe).await; + } + + #[tokio::test] + async fn scheduled_deletion_keeps_its_existing_behaviour() { + let temp_dir = tempdir().expect("Failed to create temp dir"); + let manager = deletion_manager(&temp_dir, false).await; + + for (name, days) in [ + ("schedule-default-window", None), + ("schedule-min-window", Some(MIN_PENDING_DELETION_WINDOW_DAYS)), + ("schedule-max-window", Some(MAX_PENDING_DELETION_WINDOW_DAYS)), + ] { + let key_id = create_named_key(&manager, name).await; + let response = manager + .delete_key(DeleteKeyRequest { + key_id: key_id.clone(), + pending_window_in_days: days, + ..Default::default() + }) + .await + .expect("scheduling a deletion inside the window must still succeed"); + assert!(response.deletion_date.is_some(), "a scheduled deletion must report its deadline"); + assert_eq!(response.key_metadata.key_state, KeyState::PendingDeletion); + + manager + .cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() }) + .await + .expect("a scheduled deletion must still be cancellable"); + let state = manager + .describe_key(DescribeKeyRequest { key_id }) + .await + .expect("describe should succeed") + .key_metadata + .key_state; + assert_eq!(state, KeyState::Enabled, "cancelling must restore the key"); + } + } } diff --git a/crates/kms/src/types.rs b/crates/kms/src/types.rs index c0cdb6f9e..514d5fd6f 100644 --- a/crates/kms/src/types.rs +++ b/crates/kms/src/types.rs @@ -893,15 +893,37 @@ impl std::str::FromStr for EncryptionAlgorithm { } } +/// Shortest pending-deletion waiting window a caller may ask for, in days. +/// +/// The window is enforced once, in [`crate::manager::KmsManager::delete_key`]; +/// backends keep the same bound as a defensive check for direct callers. +pub const MIN_PENDING_DELETION_WINDOW_DAYS: u32 = 7; + +/// Longest pending-deletion waiting window a caller may ask for, in days. +pub const MAX_PENDING_DELETION_WINDOW_DAYS: u32 = 30; + +/// Waiting window applied when a delete request does not name one. +pub const DEFAULT_PENDING_DELETION_WINDOW_DAYS: u32 = MAX_PENDING_DELETION_WINDOW_DAYS; + /// Request to delete a key -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct DeleteKeyRequest { /// Key ID to delete pub key_id: String, - /// Number of days to wait before deletion (7-30 days, optional) + /// Number of days to wait before deletion (7-30 days, optional, defaults to 30) pub pending_window_in_days: Option, - /// Force immediate deletion (for development/testing only) + /// Destroy the key material right away instead of scheduling it. + /// + /// Refused unless the server enables `allow_immediate_deletion`; see + /// [`crate::manager::KmsManager::delete_key`] for the gate. pub force_immediate: Option, + /// Key id echoed back by the caller to confirm an immediate deletion. + /// + /// Must equal `key_id` exactly whenever `force_immediate` is set. Optional + /// with a serde default because this type is part of the admin API + /// contract: clients that never ask for immediate deletion are unaffected. + #[serde(default)] + pub confirm_key_id: Option, } /// Response from delete key operation diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 7638c10ca..5f590bf44 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -45,7 +45,7 @@ path "secret/metadata/rustfs/kms/keys/*" { Notes: - The trailing wildcards also cover the per-version material records that rotation creates under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed. -- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. +- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. RustFS refuses `force_immediate` unless the server sets `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, so leaving that gate off keeps the capability unreachable no matter what the Vault policy allows. - Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them. - Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure. @@ -63,7 +63,7 @@ Decryption loads exactly the version recorded in the envelope and fails closed w - Every version record that any stored DEK envelope references must remain readable. Until an object rewrap/migration capability exists, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped. - Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely. -- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. +- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must echo the key id back as `confirm_key_id`. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key. - For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext. ### Upgrade before first rotation (hard constraint) diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index c972ef905..d8f9e565f 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -192,6 +192,9 @@ async fn save_kms_config(config: &KmsConfig) -> Result<(), String> { fn decode_persisted_kms_config(data: &[u8]) -> serde_json::Result<(KmsConfig, bool)> { let mut config: KmsConfig = serde_json::from_slice(data)?; + // The immediate-deletion gate is per-server operator state, never stored, + // so a config loaded from cluster storage still has to pick it up here. + config.allow_immediate_deletion = rustfs_kms::config::allow_immediate_deletion_from_env(); let value: serde_json::Value = serde_json::from_slice(data)?; let is_missing_development_flag = value .as_object() diff --git a/rustfs/src/admin/handlers/kms_keys.rs b/rustfs/src/admin/handlers/kms_keys.rs index 72f9cfc97..0c17045f4 100644 --- a/rustfs/src/admin/handlers/kms_keys.rs +++ b/rustfs/src/admin/handlers/kms_keys.rs @@ -1023,6 +1023,9 @@ pub struct DeleteKmsKeyRequest { pub key_id: String, pub pending_window_in_days: Option, pub force_immediate: Option, + /// Echo of `key_id`, required by the service gate before it will destroy + /// key material immediately. Absent for an ordinary scheduled deletion. + pub confirm_key_id: Option, } #[derive(Debug, Serialize, Deserialize)] @@ -1089,11 +1092,13 @@ impl Operation for DeleteKmsKeyHandler { // Extract pending_window_in_days and force_immediate from query parameters let pending_window_in_days = query_params.get("pending_window_in_days").and_then(|s| s.parse::().ok()); let force_immediate = query_params.get("force_immediate").and_then(|s| s.parse::().ok()); + let confirm_key_id = query_params.get("confirm_key_id").map(|s| s.to_string()); DeleteKmsKeyRequest { key_id: key_id.clone(), pending_window_in_days, force_immediate, + confirm_key_id, } } else { serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))? @@ -1131,6 +1136,7 @@ impl Operation for DeleteKmsKeyHandler { key_id: request.key_id.clone(), pending_window_in_days: request.pending_window_in_days, force_immediate: request.force_immediate, + confirm_key_id: request.confirm_key_id.clone(), }; match manager.delete_key_with_context(kms_request, audit.context()).await { diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 454c8af03..fe5746582 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -291,6 +291,7 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result std::io::Result std::io::Result std::io::Result