fix(kms): make the deletion waiting window non-bypassable (#5535)

This commit is contained in:
Zhengchao An
2026-08-01 20:23:28 +08:00
committed by GitHub
parent 85bc0d3ce2
commit 3c00ad6048
17 changed files with 602 additions and 74 deletions
+4
View File
@@ -1068,6 +1068,7 @@ mod tests {
key_id: "test-key".to_string(),
pending_window_in_days: Some(days),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect_err("an out-of-range window must be rejected");
@@ -1086,6 +1087,7 @@ mod tests {
key_id: "test-key".to_string(),
pending_window_in_days: None,
force_immediate: Some(true),
confirm_key_id: None,
})
.await
.expect_err("immediate deletion must be rejected");
@@ -1228,6 +1230,7 @@ mod tests {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect("scheduling deletion must succeed");
@@ -1246,6 +1249,7 @@ mod tests {
key_id,
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
})
.await
.expect("re-scheduling deletion must succeed");
@@ -108,6 +108,7 @@ fn schedule_request(key_id: &str) -> DeleteKeyRequest {
key_id: key_id.to_string(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
}
}
+45 -3
View File
@@ -1617,9 +1617,15 @@ impl KmsBackend for LocalKmsBackend {
// Schedule for deletion (default 30 days)
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
// Defensive: KmsManager::delete_key is the enforcement point for the
// waiting window and rejects out-of-range requests before any
// backend runs. This repeats the bound for callers holding a backend
// handle directly (tests, maintenance tasks).
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -2678,6 +2684,7 @@ mod tests {
key_id: "durable-key".to_string(),
pending_window_in_days: None,
force_immediate: Some(true),
confirm_key_id: None,
})
.await
.expect("delete key");
@@ -2685,6 +2692,41 @@ mod tests {
assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory");
}
/// KmsManager::delete_key is the enforcement point for the waiting window;
/// this pins the backend's defensive copy of the same bound, which is all
/// that stands between a direct backend caller and a one-day window.
#[tokio::test]
async fn delete_key_refuses_a_window_outside_the_supported_range() {
let (client, _temp_dir) = create_test_client().await;
let key_id = "window-bounds-key";
client.create_key(key_id, "AES_256", None).await.expect("create key");
let backend = LocalKmsBackend { client };
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let result = backend
.delete_key(DeleteKeyRequest {
key_id: key_id.to_string(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
}
let state = backend
.describe_key(DescribeKeyRequest {
key_id: key_id.to_string(),
})
.await
.expect("describe should succeed")
.key_metadata
.key_state;
assert_eq!(state, KeyState::Enabled, "a refused window must not schedule the key");
}
#[tokio::test]
async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() {
use durable_file::{CommitStep, failpoint};
+1
View File
@@ -683,6 +683,7 @@ mod tests {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: None,
confirm_key_id: None,
},
)
.await;
+51 -5
View File
@@ -1427,11 +1427,15 @@ impl KmsBackend for VaultKmsBackend {
// Schedule for deletion (default 30 days)
ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(crate::error::KmsError::invalid_parameter(
"pending_window_in_days must be between 7 and 30".to_string(),
));
// Defensive: KmsManager::delete_key is the enforcement point for the
// waiting window and rejects out-of-range requests before any
// backend runs. This repeats the bound for callers holding a backend
// handle directly (tests, maintenance tasks).
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(crate::error::KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -2274,6 +2278,7 @@ mod tests {
key_id: key_id.clone(),
pending_window_in_days: Some(7),
force_immediate: Some(false),
confirm_key_id: None,
})
.await
.expect("schedule delete");
@@ -2777,6 +2782,7 @@ mod tests {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(7),
force_immediate: Some(false),
confirm_key_id: None,
})
.await
.expect("the schedule must retry past the lost race and commit");
@@ -2793,6 +2799,45 @@ mod tests {
);
}
/// KmsManager::delete_key is the enforcement point for the waiting window;
/// this pins the backend's defensive copy of the same bound, which is all
/// that stands between a direct backend caller and a one-day window.
#[tokio::test]
async fn wired_schedule_deletion_refuses_a_window_outside_the_supported_range() {
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let vault = ScriptedVault::serve(vec![
// describe_key: key info plus stored metadata.
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
])
.await;
let config = KmsConfig::vault(
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
"scripted-token".to_string(),
)
.with_insecure_development_defaults();
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
let result = backend
.delete_key(DeleteKeyRequest {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
let requests = vault.requests();
assert!(
!requests.iter().any(|line| line.starts_with("POST ")),
"a refused window must not write anything: {requests:?}"
);
}
}
/// Conflict semantics are re-read *and* re-gate: when the re-read after a
/// lost race shows the key was concurrently scheduled for deletion, the
/// state gate rejects the retry instead of blindly re-applying it.
@@ -2825,6 +2870,7 @@ mod tests {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(7),
force_immediate: Some(false),
confirm_key_id: None,
})
.await
.expect_err("the retry must re-run the state gate against the fresh record");
+51 -3
View File
@@ -1219,9 +1219,15 @@ impl KmsBackend for VaultTransitKmsBackend {
} else {
ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
let days = request.pending_window_in_days.unwrap_or(30);
if !(7..=30).contains(&days) {
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"));
// Defensive: KmsManager::delete_key is the enforcement point for the
// waiting window and rejects out-of-range requests before any
// backend runs. This repeats the bound for callers holding a backend
// handle directly (tests, maintenance tasks).
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
return Err(KmsError::invalid_parameter(format!(
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
)));
}
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -1819,6 +1825,48 @@ mod tests {
assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}");
}
/// KmsManager::delete_key is the enforcement point for the waiting window;
/// this pins the backend's defensive copy of the same bound, which is all
/// that stands between a direct backend caller and a one-day window.
#[tokio::test]
async fn wired_backend_delete_refuses_a_window_outside_the_supported_range() {
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
let vault = ScriptedVault::serve(vec![
// The state gate reads the transit key, then its metadata record.
ScriptedResponse::ok(transit_key_read_data("wired-key")),
ScriptedResponse::ok(metadata_read_data(&metadata)),
])
.await;
let config = KmsConfig::vault_transit(
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
"scripted-token".to_string(),
)
.with_insecure_development_defaults();
let backend = VaultTransitKmsBackend::new(config)
.await
.expect("vault transit backend should build");
let result = backend
.delete_key(DeleteKeyRequest {
key_id: "wired-key".to_string(),
pending_window_in_days: Some(days),
..Default::default()
})
.await;
assert!(
matches!(result, Err(KmsError::InvalidOperation { .. })),
"a {days}-day window must be refused, got {result:?}"
);
let requests = vault.requests();
assert!(
!requests.iter().any(|line| line.starts_with("POST ")),
"a refused window must not write anything: {requests:?}"
);
}
}
/// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the
/// current secret version used as the check-and-set base.
fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value {