fix(kms): make the deletion waiting window non-bypassable (#5535)

This commit is contained in:
Zhengchao An
2026-08-01 20:23:28 +08:00
committed by GitHub
parent 85bc0d3ce2
commit 3c00ad6048
17 changed files with 602 additions and 74 deletions
@@ -126,7 +126,10 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
assert_eq!(cancelled["success"], true);
assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled");
let removed = kms_admin_request(
// A default server refuses to skip the waiting window (rustfs/backlog#1585):
// immediate deletion is unrecoverable and takes every object encrypted under
// the key with it, so the endpoint must reject it rather than honour it.
let refused = kms_admin_request(
base_url,
http::Method::DELETE,
"/rustfs/admin/v3/kms/keys/delete",
@@ -140,37 +143,49 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
access_key,
secret_key,
)
.await?;
let removed: serde_json::Value = serde_json::from_str(&removed)?;
assert_eq!(removed["success"], true);
.await
.err()
.ok_or("immediate KMS key deletion must be refused on a default server")?;
assert!(
refused.to_string().contains("400 Bad Request"),
"refused immediate deletion must report a client error: {refused}"
);
let listed =
kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
let listed: serde_json::Value = serde_json::from_str(&listed)?;
assert_eq!(listed["success"], true);
let keys = listed["keys"]
.as_array()
.ok_or("list KMS keys response omitted keys after deletion")?;
if let Some(key) = keys.iter().find(|key| key["key_id"] == key_id) {
assert_eq!(key["status"], "PendingDeletion", "a retained force-deleted key must be pending deletion");
let removed = kms_admin_request(
base_url,
http::Method::DELETE,
"/rustfs/admin/v3/kms/keys/delete",
Some(
&serde_json::json!({
"key_id": key_id,
"force_immediate": true
})
.to_string(),
),
access_key,
secret_key,
)
.await?;
let removed: serde_json::Value = serde_json::from_str(&removed)?;
assert_eq!(removed["success"], true);
}
// The refused request left the key alone, so the window-bounded path still
// has something to schedule.
let described = kms_admin_request(
base_url,
http::Method::GET,
&format!("/rustfs/admin/v3/kms/keys/{key_id}"),
None,
access_key,
secret_key,
)
.await?;
let described: serde_json::Value = serde_json::from_str(&described)?;
assert_eq!(
described["key_metadata"]["key_state"], "Enabled",
"a refused immediate deletion must leave the key usable"
);
let rescheduled = kms_admin_request(
base_url,
http::Method::DELETE,
"/rustfs/admin/v3/kms/keys/delete",
Some(
&serde_json::json!({
"key_id": key_id,
"pending_window_in_days": 7
})
.to_string(),
),
access_key,
secret_key,
)
.await?;
let rescheduled: serde_json::Value = serde_json::from_str(&rescheduled)?;
assert_eq!(rescheduled["success"], true);
assert!(rescheduled["deletion_date"].is_string());
let listed =
kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
@@ -179,10 +194,11 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
let keys = listed["keys"]
.as_array()
.ok_or("final list KMS keys response omitted keys after deletion")?;
assert!(
keys.iter().all(|key| key["key_id"] != key_id),
"force-deleted KMS key must no longer appear in list"
);
let key = keys
.iter()
.find(|key| key["key_id"] == key_id)
.ok_or("a key awaiting its deletion window must still be listed")?;
assert_eq!(key["status"], "PendingDeletion", "a scheduled key must be pending deletion");
Ok(())
}
+17 -15
View File
@@ -449,29 +449,31 @@ async fn test_vault_kms_key_crud(
info!("✅ Delete verification: Key state correctly changed to: {}", key_state);
// Force Delete - Force immediate deletion for PendingDeletion key
let force_delete_response = crate::common::execute_awscurl(
// Force Delete - a default server refuses to skip the waiting window
// (rustfs/backlog#1585): destroying the key material immediately would take
// every object encrypted under the key with it.
let force_delete_error = crate::common::execute_awscurl(
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
"DELETE",
None,
access_key,
secret_key,
)
.await?;
.await
.expect_err("Immediate KMS key deletion must be refused on a default server");
info!("✅ Force Delete: correctly refused for key {}: {}", key_id, force_delete_error);
// Parse and validate the force delete response
let force_delete_result: serde_json::Value = serde_json::from_str(&force_delete_response)?;
assert_eq!(force_delete_result["success"], true, "Force delete operation must return success=true");
info!("✅ Force Delete: Successfully force deleted key: {}", key_id);
// The refused request must leave the key exactly as it was: still present,
// still pending deletion, still recoverable through cancel-deletion.
let describe_after_refusal =
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
let describe_after_refusal: serde_json::Value = serde_json::from_str(&describe_after_refusal)?;
assert_eq!(
describe_after_refusal["key_metadata"]["key_state"], "PendingDeletion",
"A refused immediate deletion must leave the key pending deletion"
);
// Verify key no longer exists after force deletion (should return error)
let describe_force_deleted_result =
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;
// After force deletion, key should not be found (GET should fail)
assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
info!("✅ Force Delete verification: Key was permanently deleted and is no longer accessible");
info!("✅ Force Delete verification: Key survived the refused immediate deletion");
info!("Vault KMS key CRUD operations completed successfully");
Ok(())
+19 -2
View File
@@ -16,8 +16,8 @@
use crate::config::{
BackendConfig, CacheConfig, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend,
KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, redacted_secret,
redacted_secret_option,
KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig,
allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option,
};
use crate::service_manager::KmsServiceStatus;
use crate::types::{KeyMetadata, KeyUsage};
@@ -506,6 +506,10 @@ impl ConfigureLocalKmsRequest {
file_permissions: self.file_permissions,
}),
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
// Read from server configuration, never from the request body: the
// gate must mean the same thing whether KMS was configured at
// startup or through this endpoint.
allow_immediate_deletion: allow_immediate_deletion_from_env(),
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
retry_attempts: self.retry_attempts.unwrap_or(3),
enable_cache: self.enable_cache.unwrap_or(true),
@@ -543,6 +547,10 @@ impl ConfigureVaultKmsRequest {
},
})),
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
// Read from server configuration, never from the request body: the
// gate must mean the same thing whether KMS was configured at
// startup or through this endpoint.
allow_immediate_deletion: allow_immediate_deletion_from_env(),
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
retry_attempts: self.retry_attempts.unwrap_or(3),
enable_cache: self.enable_cache.unwrap_or(true),
@@ -580,6 +588,10 @@ impl ConfigureVaultTransitKmsRequest {
},
})),
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
// Read from server configuration, never from the request body: the
// gate must mean the same thing whether KMS was configured at
// startup or through this endpoint.
allow_immediate_deletion: allow_immediate_deletion_from_env(),
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
retry_attempts: self.retry_attempts.unwrap_or(3),
enable_cache: self.enable_cache.unwrap_or(true),
@@ -603,6 +615,10 @@ impl ConfigureStaticKmsRequest {
secret_key: self.secret_key.clone(),
}),
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
// Read from server configuration, never from the request body: the
// gate must mean the same thing whether KMS was configured at
// startup or through this endpoint.
allow_immediate_deletion: allow_immediate_deletion_from_env(),
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
retry_attempts: self.retry_attempts.unwrap_or(3),
enable_cache: self.enable_cache.unwrap_or(true),
@@ -858,6 +874,7 @@ mod tests {
tls: None,
})),
allow_insecure_dev_defaults: true,
allow_immediate_deletion: false,
timeout: Duration::from_secs(30),
retry_attempts: 3,
enable_cache: true,
+4
View File
@@ -1068,6 +1068,7 @@ mod tests {
key_id: "test-key".to_string(),
pending_window_in_days: Some(days),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect_err("an out-of-range window must be rejected");
@@ -1086,6 +1087,7 @@ mod tests {
key_id: "test-key".to_string(),
pending_window_in_days: None,
force_immediate: Some(true),
confirm_key_id: None,
})
.await
.expect_err("immediate deletion must be rejected");
@@ -1228,6 +1230,7 @@ mod tests {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect("scheduling deletion must succeed");
@@ -1246,6 +1249,7 @@ mod tests {
key_id,
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect("re-scheduling deletion must succeed");
@@ -108,6 +108,7 @@ fn schedule_request(key_id: &str) -> DeleteKeyRequest {
key_id: key_id.to_string(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
}
}
+45 -3
View File
@@ -1617,9 +1617,15 @@ impl KmsBackend for LocalKmsBackend {
// Schedule for deletion (default 30 days)
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
// Defensive: KmsManager::delete_key is the enforcement point for the
// waiting window and rejects out-of-range requests before any
// backend runs. This repeats the bound for callers holding a backend
// handle directly (tests, maintenance tasks).
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -2678,6 +2684,7 @@ mod tests {
key_id: "durable-key".to_string(),
pending_window_in_days: None,
force_immediate: Some(true),
confirm_key_id: None,
})
.await
.expect("delete key");
@@ -2685,6 +2692,41 @@ mod tests {
assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory");
}
/// KmsManager::delete_key is the enforcement point for the waiting window;
/// this pins the backend's defensive copy of the same bound, which is all
/// that stands between a direct backend caller and a one-day window.
#[tokio::test]
async fn delete_key_refuses_a_window_outside_the_supported_range() {
let (client, _temp_dir) = create_test_client().await;
let key_id = "window-bounds-key";
client.create_key(key_id, "AES_256", None).await.expect("create key");
let backend = LocalKmsBackend { client };
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let result = backend
.delete_key(DeleteKeyRequest {
key_id: key_id.to_string(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
}
let state = backend
.describe_key(DescribeKeyRequest {
key_id: key_id.to_string(),
})
.await
.expect("describe should succeed")
.key_metadata
.key_state;
assert_eq!(state, KeyState::Enabled, "a refused window must not schedule the key");
}
#[tokio::test]
async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() {
use durable_file::{CommitStep, failpoint};
+1
View File
@@ -683,6 +683,7 @@ mod tests {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
},
)
.await;
+51 -5
View File
@@ -1427,11 +1427,15 @@ impl KmsBackend for VaultKmsBackend {
// Schedule for deletion (default 30 days)
ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(crate::error::KmsError::invalid_parameter(
"pending_window_in_days must be between 7 and 30".to_string(),
));
// Defensive: KmsManager::delete_key is the enforcement point for the
// waiting window and rejects out-of-range requests before any
// backend runs. This repeats the bound for callers holding a backend
// handle directly (tests, maintenance tasks).
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(crate::error::KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -2274,6 +2278,7 @@ mod tests {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: Some(false),
confirm_key_id: None,
})
.await
.expect("schedule delete");
@@ -2777,6 +2782,7 @@ mod tests {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(7),
force_immediate: Some(false),
confirm_key_id: None,
})
.await
.expect("the schedule must retry past the lost race and commit");
@@ -2793,6 +2799,45 @@ mod tests {
);
}
/// KmsManager::delete_key is the enforcement point for the waiting window;
/// this pins the backend's defensive copy of the same bound, which is all
/// that stands between a direct backend caller and a one-day window.
#[tokio::test]
async fn wired_schedule_deletion_refuses_a_window_outside_the_supported_range() {
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let vault = ScriptedVault::serve(vec![
// describe_key: key info plus stored metadata.
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
])
.await;
let config = KmsConfig::vault(
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
"scripted-token".to_string(),
)
.with_insecure_development_defaults();
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
let result = backend
.delete_key(DeleteKeyRequest {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
let requests = vault.requests();
assert!(
!requests.iter().any(|line| line.starts_with("POST ")),
"a refused window must not write anything: {requests:?}"
);
}
}
/// Conflict semantics are re-read *and* re-gate: when the re-read after a
/// lost race shows the key was concurrently scheduled for deletion, the
/// state gate rejects the retry instead of blindly re-applying it.
@@ -2825,6 +2870,7 @@ mod tests {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(7),
force_immediate: Some(false),
confirm_key_id: None,
})
.await
.expect_err("the retry must re-run the state gate against the fresh record");
+51 -3
View File
@@ -1219,9 +1219,15 @@ impl KmsBackend for VaultTransitKmsBackend {
} else {
ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"));
// Defensive: KmsManager::delete_key is the enforcement point for the
// waiting window and rejects out-of-range requests before any
// backend runs. This repeats the bound for callers holding a backend
// handle directly (tests, maintenance tasks).
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -1819,6 +1825,48 @@ mod tests {
assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}");
}
/// KmsManager::delete_key is the enforcement point for the waiting window;
/// this pins the backend's defensive copy of the same bound, which is all
/// that stands between a direct backend caller and a one-day window.
#[tokio::test]
async fn wired_backend_delete_refuses_a_window_outside_the_supported_range() {
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let vault = ScriptedVault::serve(vec![
// The state gate reads the transit key, then its metadata record.
ScriptedResponse::ok(transit_key_read_data("wired-key")),
ScriptedResponse::ok(metadata_read_data(&metadata)),
])
.await;
let config = KmsConfig::vault_transit(
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
"scripted-token".to_string(),
)
.with_insecure_development_defaults();
let backend = VaultTransitKmsBackend::new(config)
.await
.expect("vault transit backend should build");
let result = backend
.delete_key(DeleteKeyRequest {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
let requests = vault.requests();
assert!(
!requests.iter().any(|line| line.starts_with("POST ")),
"a refused window must not write anything: {requests:?}"
);
}
}
/// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the
/// current secret version used as the check-and-set base.
fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value {
+68
View File
@@ -24,6 +24,7 @@ use std::time::Duration;
use url::Url;
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
@@ -164,6 +165,24 @@ pub struct KmsConfig {
/// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault.
#[serde(default)]
pub allow_insecure_dev_defaults: bool,
/// Allow `DeleteKey` requests to skip the pending-deletion waiting window and
/// destroy key material right away.
///
/// Off by default: an immediate deletion is unrecoverable and takes every
/// object encrypted under the key with it, so the waiting window (plus
/// `CancelKeyDeletion`) is the only recovery path. Operators who genuinely
/// need immediate deletion — throwaway test clusters, key material that was
/// never used — must turn it on through server configuration
/// ([`ENV_KMS_ALLOW_IMMEDIATE_DELETION`]); the request must still echo the
/// key id back for confirmation.
///
/// Not part of the serialized configuration, and not settable through the
/// admin configure API. It is per-server operator state that has to be
/// re-stated to survive a restart: persisting it would carry one operator's
/// one-time enablement into the cluster-wide config that every node reloads,
/// long after the deletion it was turned on for.
#[serde(skip)]
pub allow_immediate_deletion: bool,
/// Timeout for a single backend attempt.
///
/// This bounds one outbound request, not the whole operation: the operation
@@ -187,6 +206,7 @@ impl Default for KmsConfig {
default_key_id: None,
backend_config: BackendConfig::default(),
allow_insecure_dev_defaults: false,
allow_immediate_deletion: false,
timeout: Duration::from_secs(30),
retry_attempts: 3,
enable_cache: true,
@@ -714,6 +734,12 @@ impl KmsConfig {
self
}
/// Explicitly allow deletions that bypass the pending-deletion waiting window.
pub fn with_immediate_deletion_allowed(mut self) -> Self {
self.allow_immediate_deletion = true;
self
}
/// Set operation timeout
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
@@ -927,6 +953,7 @@ impl KmsConfig {
config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache);
config.allow_insecure_dev_defaults =
get_env_bool(ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS, config.allow_insecure_dev_defaults);
config.allow_immediate_deletion = get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, config.allow_immediate_deletion);
// Backend-specific configuration
match config.backend {
@@ -1042,6 +1069,15 @@ impl KmsConfig {
}
}
/// Read the immediate-deletion gate from the environment.
///
/// Callers that assemble a [`KmsConfig`] field by field instead of going
/// through [`KmsConfig::from_env`] use this, so the gate keeps one name, one
/// default, and one place to look it up.
pub fn allow_immediate_deletion_from_env() -> bool {
get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, false)
}
fn vault_tls_config(skip_tls_verify: bool) -> Option<TlsConfig> {
skip_tls_verify.then_some(TlsConfig {
ca_cert_path: None,
@@ -1802,6 +1838,38 @@ mod tests {
assert_eq!(refresh_safety_window_secs, None);
}
/// The gate lives in server configuration only: it never rides along in a
/// serialized config, and a stored config that claims it must not be
/// believed. Otherwise one operator's one-time enablement would reach every
/// node that later reloads that config.
#[test]
fn immediate_deletion_gate_is_server_local_and_never_persisted() {
let persisted =
serde_json::to_value(KmsConfig::default().with_immediate_deletion_allowed()).expect("kms config should serialize");
assert!(
persisted.get("allow_immediate_deletion").is_none(),
"the gate must not be written into a persisted config: {persisted}"
);
let mut forged = persisted;
forged
.as_object_mut()
.expect("a persisted config must be a JSON object")
.insert("allow_immediate_deletion".to_string(), serde_json::json!(true));
let restored: KmsConfig = serde_json::from_value(forged).expect("an unknown gate field must not break loading");
assert!(!restored.allow_immediate_deletion, "a stored gate must fail closed");
with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, Some("true"))], || {
assert!(
allow_immediate_deletion_from_env(),
"the gate must be reachable from server configuration"
);
});
with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, None::<&str>)], || {
assert!(!allow_immediate_deletion_from_env());
});
}
#[test]
fn test_validate_rejects_incomplete_approle() {
let mut config = KmsConfig::vault_approle(
+1
View File
@@ -376,6 +376,7 @@ mod tests {
key_id: key_id.to_string(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect("deletion should be scheduled");
+253 -6
View File
@@ -17,17 +17,19 @@
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
use crate::backends::KmsBackend;
use crate::cache::{KmsCache, KmsCacheStats};
use crate::config::KmsConfig;
use crate::error::Result;
use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, KmsConfig};
use crate::error::{KmsError, Result};
use crate::types::{
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse,
DEFAULT_PENDING_DELETION_WINDOW_DAYS, DecryptRequest, DecryptResponse, DeleteKeyRequest, DeleteKeyResponse,
DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyRequest, GenerateDataKeyResponse,
ListKeysRequest, ListKeysResponse, MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext,
};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::RwLock;
use tracing::warn;
/// KMS Manager coordinates operations between backends and caching
#[derive(Clone)]
@@ -38,12 +40,18 @@ pub struct KmsManager {
enable_cache: bool,
backend_kind: &'static str,
audit_sink: Option<Arc<dyn KmsAuditSink>>,
allow_immediate_deletion: bool,
}
impl KmsManager {
/// Create a new KMS manager with the given backend and config
pub fn new(backend: Arc<dyn KmsBackend>, config: KmsConfig) -> Self {
let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64)));
if config.allow_immediate_deletion {
warn!(
"KMS immediate key deletion is enabled: a DeleteKey request may destroy key material without any waiting window, and every object encrypted under that key becomes permanently unreadable"
);
}
Self {
backend,
cache,
@@ -51,6 +59,7 @@ impl KmsManager {
enable_cache: config.enable_cache,
backend_kind: config.backend.as_str(),
audit_sink: None,
allow_immediate_deletion: config.allow_immediate_deletion,
}
}
@@ -225,7 +234,8 @@ impl KmsManager {
Ok(())
}
/// Delete a key
/// Delete a key, either scheduled behind the waiting window or — when the
/// server allows it — immediately.
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::delete_key_with_context`].
@@ -246,7 +256,13 @@ impl KmsManager {
result
}
/// This is the single enforcement point for the waiting window: every
/// admin-facing deletion goes through here, so the checks below run before
/// any backend sees the request. The backends repeat the window bound as a
/// defensive assertion for callers that hold a backend handle directly.
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
self.check_deletion_request(&request)?;
let response = self.backend.delete_key(request).await?;
// Remove from cache if enabled and key is being deleted
@@ -258,6 +274,44 @@ impl KmsManager {
Ok(response)
}
/// Gate a deletion request before it reaches the backend.
///
/// Immediate deletion is unrecoverable, so it needs both a server-side
/// opt-in and a per-request confirmation that echoes the key id; without
/// either, the request is refused rather than downgraded to a scheduled
/// deletion, so a caller never believes a key is gone when it is not.
fn check_deletion_request(&self, request: &DeleteKeyRequest) -> Result<()> {
if !request.force_immediate.unwrap_or(false) {
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
return Ok(());
}
if !self.allow_immediate_deletion {
return Err(KmsError::invalid_operation(format!(
"immediate deletion of key {} is not allowed; schedule the deletion and wait out the pending window, or set {ENV_KMS_ALLOW_IMMEDIATE_DELETION}=true on the server",
request.key_id
)));
}
if request.confirm_key_id.as_deref() != Some(request.key_id.as_str()) {
return Err(KmsError::invalid_operation(format!(
"immediate deletion of key {} requires confirm_key_id to repeat the key id exactly",
request.key_id
)));
}
warn!(
key_id = %request.key_id,
"immediate KMS key deletion accepted; key material is destroyed without a waiting window and cannot be recovered"
);
Ok(())
}
/// Cancel key deletion
///
/// Audited as an internal operation; callers serving an authenticated
@@ -605,6 +659,7 @@ mod tests {
key_id: AUDITED_KEY_ID.to_string(),
pending_window_in_days: None,
force_immediate: None,
confirm_key_id: None,
},
context,
)
@@ -1019,4 +1074,196 @@ mod tests {
.await
.expect("second data key should decrypt with its own context");
}
/// Manager over a local backend, with the immediate-deletion gate set as
/// the server operator would set it.
async fn deletion_manager(temp_dir: &tempfile::TempDir, allow_immediate_deletion: bool) -> KmsManager {
let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
config.allow_immediate_deletion = allow_immediate_deletion;
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
KmsManager::new(backend, config)
}
async fn create_named_key(manager: &KmsManager, key_name: &str) -> String {
manager
.create_key(CreateKeyRequest {
key_name: Some(key_name.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
..Default::default()
})
.await
.expect("Failed to create key")
.key_id
}
/// Data key generated up front, decrypted again afterwards: a refused
/// deletion must leave the master key material byte-for-byte usable, not
/// merely leave a metadata record behind.
async fn data_key_probe(manager: &KmsManager, key_id: &str) -> (Vec<u8>, Vec<u8>) {
let generated = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: key_id.to_string(),
key_spec: KeySpec::Aes256,
encryption_context: HashMap::new(),
})
.await
.expect("Failed to generate data key");
(generated.plaintext_key, generated.ciphertext_blob)
}
async fn assert_key_material_intact(manager: &KmsManager, key_id: &str, probe: &(Vec<u8>, Vec<u8>)) {
let state = manager
.describe_key(DescribeKeyRequest {
key_id: key_id.to_string(),
})
.await
.expect("a key that was not deleted must still be describable")
.key_metadata
.key_state;
assert_eq!(state, KeyState::Enabled, "a refused deletion must not change the key state");
let decrypted = manager
.decrypt(DecryptRequest {
ciphertext: probe.1.clone(),
encryption_context: HashMap::new(),
grant_tokens: Vec::new(),
})
.await
.expect("key material must still decrypt data keys issued before the refused deletion");
assert_eq!(decrypted.plaintext, probe.0, "decrypted data key must match the original plaintext");
}
#[tokio::test]
async fn immediate_deletion_is_refused_under_default_config() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, false).await;
let key_id = create_named_key(&manager, "default-config-force-delete").await;
let probe = data_key_probe(&manager, &key_id).await;
// Confirmation present and correct: the server-side gate alone must
// refuse this, no matter how well-formed the request is.
let error = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
..Default::default()
})
.await
.expect_err("immediate deletion must be refused unless the server allows it");
assert!(
matches!(error, KmsError::InvalidOperation { .. }),
"expected InvalidOperation, got {error:?}"
);
assert_key_material_intact(&manager, &key_id, &probe).await;
}
#[tokio::test]
async fn immediate_deletion_requires_a_matching_confirmation() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, true).await;
let key_id = create_named_key(&manager, "confirmation-required").await;
let probe = data_key_probe(&manager, &key_id).await;
for confirmation in [None, Some(String::new()), Some(format!("{key_id}-typo"))] {
let result = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: confirmation.clone(),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"confirmation {confirmation:?} must be refused, got {result:?}"
);
}
assert_key_material_intact(&manager, &key_id, &probe).await;
}
#[tokio::test]
async fn immediate_deletion_succeeds_with_a_matching_confirmation() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, true).await;
let key_id = create_named_key(&manager, "confirmed-force-delete").await;
manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
..Default::default()
})
.await
.expect("a confirmed immediate deletion must be allowed once the server enables it");
let error = manager
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
.await
.expect_err("an immediately deleted key must be gone");
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
}
#[tokio::test]
async fn pending_window_outside_the_supported_range_is_refused() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, false).await;
let key_id = create_named_key(&manager, "window-bounds").await;
let probe = data_key_probe(&manager, &key_id).await;
for days in [0, MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let result = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
}
assert_key_material_intact(&manager, &key_id, &probe).await;
}
#[tokio::test]
async fn scheduled_deletion_keeps_its_existing_behaviour() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let manager = deletion_manager(&temp_dir, false).await;
for (name, days) in [
("schedule-default-window", None),
("schedule-min-window", Some(MIN_PENDING_DELETION_WINDOW_DAYS)),
("schedule-max-window", Some(MAX_PENDING_DELETION_WINDOW_DAYS)),
] {
let key_id = create_named_key(&manager, name).await;
let response = manager
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
pending_window_in_days: days,
..Default::default()
})
.await
.expect("scheduling a deletion inside the window must still succeed");
assert!(response.deletion_date.is_some(), "a scheduled deletion must report its deadline");
assert_eq!(response.key_metadata.key_state, KeyState::PendingDeletion);
manager
.cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() })
.await
.expect("a scheduled deletion must still be cancellable");
let state = manager
.describe_key(DescribeKeyRequest { key_id })
.await
.expect("describe should succeed")
.key_metadata
.key_state;
assert_eq!(state, KeyState::Enabled, "cancelling must restore the key");
}
}
}
+25 -3
View File
@@ -893,15 +893,37 @@ impl std::str::FromStr for EncryptionAlgorithm {
}
}
/// Shortest pending-deletion waiting window a caller may ask for, in days.
///
/// The window is enforced once, in [`crate::manager::KmsManager::delete_key`];
/// backends keep the same bound as a defensive check for direct callers.
pub const MIN_PENDING_DELETION_WINDOW_DAYS: u32 = 7;
/// Longest pending-deletion waiting window a caller may ask for, in days.
pub const MAX_PENDING_DELETION_WINDOW_DAYS: u32 = 30;
/// Waiting window applied when a delete request does not name one.
pub const DEFAULT_PENDING_DELETION_WINDOW_DAYS: u32 = MAX_PENDING_DELETION_WINDOW_DAYS;
/// Request to delete a key
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DeleteKeyRequest {
/// Key ID to delete
pub key_id: String,
/// Number of days to wait before deletion (7-30 days, optional)
/// Number of days to wait before deletion (7-30 days, optional, defaults to 30)
pub pending_window_in_days: Option<u32>,
/// Force immediate deletion (for development/testing only)
/// Destroy the key material right away instead of scheduling it.
///
/// Refused unless the server enables `allow_immediate_deletion`; see
/// [`crate::manager::KmsManager::delete_key`] for the gate.
pub force_immediate: Option<bool>,
/// Key id echoed back by the caller to confirm an immediate deletion.
///
/// Must equal `key_id` exactly whenever `force_immediate` is set. Optional
/// with a serde default because this type is part of the admin API
/// contract: clients that never ask for immediate deletion are unaffected.
#[serde(default)]
pub confirm_key_id: Option<String>,
}
/// Response from delete key operation
+2 -2
View File
@@ -45,7 +45,7 @@ path "secret/metadata/rustfs/kms/keys/*" {
Notes:
- The trailing wildcards also cover the per-version material records that rotation creates under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed.
- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys.
- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. RustFS refuses `force_immediate` unless the server sets `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, so leaving that gate off keeps the capability unreachable no matter what the Vault policy allows.
- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them.
- Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure.
@@ -63,7 +63,7 @@ Decryption loads exactly the version recorded in the envelope and fails closed w
- Every version record that any stored DEK envelope references must remain readable. Until an object rewrap/migration capability exists, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped.
- Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely.
- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them.
- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must echo the key id back as `confirm_key_id`. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key.
- For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext.
### Upgrade before first rotation (hard constraint)
+3
View File
@@ -192,6 +192,9 @@ async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
fn decode_persisted_kms_config(data: &[u8]) -> serde_json::Result<(KmsConfig, bool)> {
let mut config: KmsConfig = serde_json::from_slice(data)?;
// The immediate-deletion gate is per-server operator state, never stored,
// so a config loaded from cluster storage still has to pick it up here.
config.allow_immediate_deletion = rustfs_kms::config::allow_immediate_deletion_from_env();
let value: serde_json::Value = serde_json::from_slice(data)?;
let is_missing_development_flag = value
.as_object()
+6
View File
@@ -1023,6 +1023,9 @@ pub struct DeleteKmsKeyRequest {
pub key_id: String,
pub pending_window_in_days: Option<u32>,
pub force_immediate: Option<bool>,
/// Echo of `key_id`, required by the service gate before it will destroy
/// key material immediately. Absent for an ordinary scheduled deletion.
pub confirm_key_id: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -1089,11 +1092,13 @@ impl Operation for DeleteKmsKeyHandler {
// Extract pending_window_in_days and force_immediate from query parameters
let pending_window_in_days = query_params.get("pending_window_in_days").and_then(|s| s.parse::<u32>().ok());
let force_immediate = query_params.get("force_immediate").and_then(|s| s.parse::<bool>().ok());
let confirm_key_id = query_params.get("confirm_key_id").map(|s| s.to_string());
DeleteKmsKeyRequest {
key_id: key_id.clone(),
pending_window_in_days,
force_immediate,
confirm_key_id,
}
} else {
serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?
@@ -1131,6 +1136,7 @@ impl Operation for DeleteKmsKeyHandler {
key_id: request.key_id.clone(),
pending_window_in_days: request.pending_window_in_days,
force_immediate: request.force_immediate,
confirm_key_id: request.confirm_key_id.clone(),
};
match manager.delete_key_with_context(kms_request, audit.context()).await {
+4
View File
@@ -291,6 +291,7 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
file_permissions: Some(0o600),
}),
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
default_key_id: cfg.kms_default_key_id.clone(),
timeout: std::time::Duration::from_secs(30),
retry_attempts: 3,
@@ -328,6 +329,7 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
tls: None,
})),
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
default_key_id: cfg.kms_default_key_id.clone(),
timeout: std::time::Duration::from_secs(30),
retry_attempts: 3,
@@ -363,6 +365,7 @@ fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustf
..rustfs_kms::config::VaultTransitConfig::default()
})),
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
default_key_id: cfg.kms_default_key_id.clone(),
timeout: std::time::Duration::from_secs(30),
retry_attempts: 3,
@@ -417,6 +420,7 @@ fn build_static_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::
default_key_id: cfg.kms_default_key_id.clone().or(Some(key_id)),
backend_config: rustfs_kms::config::BackendConfig::Static(static_config),
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
..Default::default()
};