mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-02 03:19:19 +00:00
fix(kms): make the deletion waiting window non-bypassable (#5535)
This commit is contained in:
@@ -126,7 +126,10 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
|
||||
assert_eq!(cancelled["success"], true);
|
||||
assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled");
|
||||
|
||||
let removed = kms_admin_request(
|
||||
// A default server refuses to skip the waiting window (rustfs/backlog#1585):
|
||||
// immediate deletion is unrecoverable and takes every object encrypted under
|
||||
// the key with it, so the endpoint must reject it rather than honour it.
|
||||
let refused = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
@@ -140,37 +143,49 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let removed: serde_json::Value = serde_json::from_str(&removed)?;
|
||||
assert_eq!(removed["success"], true);
|
||||
.await
|
||||
.err()
|
||||
.ok_or("immediate KMS key deletion must be refused on a default server")?;
|
||||
assert!(
|
||||
refused.to_string().contains("400 Bad Request"),
|
||||
"refused immediate deletion must report a client error: {refused}"
|
||||
);
|
||||
|
||||
let listed =
|
||||
kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
|
||||
let listed: serde_json::Value = serde_json::from_str(&listed)?;
|
||||
assert_eq!(listed["success"], true);
|
||||
let keys = listed["keys"]
|
||||
.as_array()
|
||||
.ok_or("list KMS keys response omitted keys after deletion")?;
|
||||
if let Some(key) = keys.iter().find(|key| key["key_id"] == key_id) {
|
||||
assert_eq!(key["status"], "PendingDeletion", "a retained force-deleted key must be pending deletion");
|
||||
let removed = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
Some(
|
||||
&serde_json::json!({
|
||||
"key_id": key_id,
|
||||
"force_immediate": true
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let removed: serde_json::Value = serde_json::from_str(&removed)?;
|
||||
assert_eq!(removed["success"], true);
|
||||
}
|
||||
// The refused request left the key alone, so the window-bounded path still
|
||||
// has something to schedule.
|
||||
let described = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::GET,
|
||||
&format!("/rustfs/admin/v3/kms/keys/{key_id}"),
|
||||
None,
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let described: serde_json::Value = serde_json::from_str(&described)?;
|
||||
assert_eq!(
|
||||
described["key_metadata"]["key_state"], "Enabled",
|
||||
"a refused immediate deletion must leave the key usable"
|
||||
);
|
||||
|
||||
let rescheduled = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
Some(
|
||||
&serde_json::json!({
|
||||
"key_id": key_id,
|
||||
"pending_window_in_days": 7
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let rescheduled: serde_json::Value = serde_json::from_str(&rescheduled)?;
|
||||
assert_eq!(rescheduled["success"], true);
|
||||
assert!(rescheduled["deletion_date"].is_string());
|
||||
|
||||
let listed =
|
||||
kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
|
||||
@@ -179,10 +194,11 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
|
||||
let keys = listed["keys"]
|
||||
.as_array()
|
||||
.ok_or("final list KMS keys response omitted keys after deletion")?;
|
||||
assert!(
|
||||
keys.iter().all(|key| key["key_id"] != key_id),
|
||||
"force-deleted KMS key must no longer appear in list"
|
||||
);
|
||||
let key = keys
|
||||
.iter()
|
||||
.find(|key| key["key_id"] == key_id)
|
||||
.ok_or("a key awaiting its deletion window must still be listed")?;
|
||||
assert_eq!(key["status"], "PendingDeletion", "a scheduled key must be pending deletion");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -449,29 +449,31 @@ async fn test_vault_kms_key_crud(
|
||||
|
||||
info!("✅ Delete verification: Key state correctly changed to: {}", key_state);
|
||||
|
||||
// Force Delete - Force immediate deletion for PendingDeletion key
|
||||
let force_delete_response = crate::common::execute_awscurl(
|
||||
// Force Delete - a default server refuses to skip the waiting window
|
||||
// (rustfs/backlog#1585): destroying the key material immediately would take
|
||||
// every object encrypted under the key with it.
|
||||
let force_delete_error = crate::common::execute_awscurl(
|
||||
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
|
||||
"DELETE",
|
||||
None,
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
.await
|
||||
.expect_err("Immediate KMS key deletion must be refused on a default server");
|
||||
info!("✅ Force Delete: correctly refused for key {}: {}", key_id, force_delete_error);
|
||||
|
||||
// Parse and validate the force delete response
|
||||
let force_delete_result: serde_json::Value = serde_json::from_str(&force_delete_response)?;
|
||||
assert_eq!(force_delete_result["success"], true, "Force delete operation must return success=true");
|
||||
info!("✅ Force Delete: Successfully force deleted key: {}", key_id);
|
||||
// The refused request must leave the key exactly as it was: still present,
|
||||
// still pending deletion, still recoverable through cancel-deletion.
|
||||
let describe_after_refusal =
|
||||
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
|
||||
let describe_after_refusal: serde_json::Value = serde_json::from_str(&describe_after_refusal)?;
|
||||
assert_eq!(
|
||||
describe_after_refusal["key_metadata"]["key_state"], "PendingDeletion",
|
||||
"A refused immediate deletion must leave the key pending deletion"
|
||||
);
|
||||
|
||||
// Verify key no longer exists after force deletion (should return error)
|
||||
let describe_force_deleted_result =
|
||||
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;
|
||||
|
||||
// After force deletion, key should not be found (GET should fail)
|
||||
assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
|
||||
|
||||
info!("✅ Force Delete verification: Key was permanently deleted and is no longer accessible");
|
||||
info!("✅ Force Delete verification: Key survived the refused immediate deletion");
|
||||
|
||||
info!("Vault KMS key CRUD operations completed successfully");
|
||||
Ok(())
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
|
||||
use crate::config::{
|
||||
BackendConfig, CacheConfig, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend,
|
||||
KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, redacted_secret,
|
||||
redacted_secret_option,
|
||||
KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig,
|
||||
allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option,
|
||||
};
|
||||
use crate::service_manager::KmsServiceStatus;
|
||||
use crate::types::{KeyMetadata, KeyUsage};
|
||||
@@ -506,6 +506,10 @@ impl ConfigureLocalKmsRequest {
|
||||
file_permissions: self.file_permissions,
|
||||
}),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -543,6 +547,10 @@ impl ConfigureVaultKmsRequest {
|
||||
},
|
||||
})),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -580,6 +588,10 @@ impl ConfigureVaultTransitKmsRequest {
|
||||
},
|
||||
})),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -603,6 +615,10 @@ impl ConfigureStaticKmsRequest {
|
||||
secret_key: self.secret_key.clone(),
|
||||
}),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -858,6 +874,7 @@ mod tests {
|
||||
tls: None,
|
||||
})),
|
||||
allow_insecure_dev_defaults: true,
|
||||
allow_immediate_deletion: false,
|
||||
timeout: Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
enable_cache: true,
|
||||
|
||||
@@ -1068,6 +1068,7 @@ mod tests {
|
||||
key_id: "test-key".to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect_err("an out-of-range window must be rejected");
|
||||
@@ -1086,6 +1087,7 @@ mod tests {
|
||||
key_id: "test-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect_err("immediate deletion must be rejected");
|
||||
@@ -1228,6 +1230,7 @@ mod tests {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("scheduling deletion must succeed");
|
||||
@@ -1246,6 +1249,7 @@ mod tests {
|
||||
key_id,
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("re-scheduling deletion must succeed");
|
||||
|
||||
@@ -108,6 +108,7 @@ fn schedule_request(key_id: &str) -> DeleteKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1617,9 +1617,15 @@ impl KmsBackend for LocalKmsBackend {
|
||||
// Schedule for deletion (default 30 days)
|
||||
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
|
||||
// Defensive: KmsManager::delete_key is the enforcement point for the
|
||||
// waiting window and rejects out-of-range requests before any
|
||||
// backend runs. This repeats the bound for callers holding a backend
|
||||
// handle directly (tests, maintenance tasks).
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
@@ -2678,6 +2684,7 @@ mod tests {
|
||||
key_id: "durable-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("delete key");
|
||||
@@ -2685,6 +2692,41 @@ mod tests {
|
||||
assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory");
|
||||
}
|
||||
|
||||
/// KmsManager::delete_key is the enforcement point for the waiting window;
|
||||
/// this pins the backend's defensive copy of the same bound, which is all
|
||||
/// that stands between a direct backend caller and a one-day window.
|
||||
#[tokio::test]
|
||||
async fn delete_key_refuses_a_window_outside_the_supported_range() {
|
||||
let (client, _temp_dir) = create_test_client().await;
|
||||
let key_id = "window-bounds-key";
|
||||
client.create_key(key_id, "AES_256", None).await.expect("create key");
|
||||
let backend = LocalKmsBackend { client };
|
||||
|
||||
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let result = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let state = backend
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("describe should succeed")
|
||||
.key_metadata
|
||||
.key_state;
|
||||
assert_eq!(state, KeyState::Enabled, "a refused window must not schedule the key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() {
|
||||
use durable_file::{CommitStep, failpoint};
|
||||
|
||||
@@ -683,6 +683,7 @@ mod tests {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1427,11 +1427,15 @@ impl KmsBackend for VaultKmsBackend {
|
||||
// Schedule for deletion (default 30 days)
|
||||
ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(crate::error::KmsError::invalid_parameter(
|
||||
"pending_window_in_days must be between 7 and 30".to_string(),
|
||||
));
|
||||
// Defensive: KmsManager::delete_key is the enforcement point for the
|
||||
// waiting window and rejects out-of-range requests before any
|
||||
// backend runs. This repeats the bound for callers holding a backend
|
||||
// handle directly (tests, maintenance tasks).
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(crate::error::KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
}
|
||||
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
@@ -2274,6 +2278,7 @@ mod tests {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(false),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("schedule delete");
|
||||
@@ -2777,6 +2782,7 @@ mod tests {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(false),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("the schedule must retry past the lost race and commit");
|
||||
@@ -2793,6 +2799,45 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// KmsManager::delete_key is the enforcement point for the waiting window;
|
||||
/// this pins the backend's defensive copy of the same bound, which is all
|
||||
/// that stands between a direct backend caller and a one-day window.
|
||||
#[tokio::test]
|
||||
async fn wired_schedule_deletion_refuses_a_window_outside_the_supported_range() {
|
||||
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
|
||||
|
||||
let result = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
|
||||
let requests = vault.requests();
|
||||
assert!(
|
||||
!requests.iter().any(|line| line.starts_with("POST ")),
|
||||
"a refused window must not write anything: {requests:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Conflict semantics are re-read *and* re-gate: when the re-read after a
|
||||
/// lost race shows the key was concurrently scheduled for deletion, the
|
||||
/// state gate rejects the retry instead of blindly re-applying it.
|
||||
@@ -2825,6 +2870,7 @@ mod tests {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(false),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect_err("the retry must re-run the state gate against the fresh record");
|
||||
|
||||
@@ -1219,9 +1219,15 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
} else {
|
||||
ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"));
|
||||
// Defensive: KmsManager::delete_key is the enforcement point for the
|
||||
// waiting window and rejects out-of-range requests before any
|
||||
// backend runs. This repeats the bound for callers holding a backend
|
||||
// handle directly (tests, maintenance tasks).
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
}
|
||||
|
||||
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
@@ -1819,6 +1825,48 @@ mod tests {
|
||||
assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}");
|
||||
}
|
||||
|
||||
/// KmsManager::delete_key is the enforcement point for the waiting window;
|
||||
/// this pins the backend's defensive copy of the same bound, which is all
|
||||
/// that stands between a direct backend caller and a one-day window.
|
||||
#[tokio::test]
|
||||
async fn wired_backend_delete_refuses_a_window_outside_the_supported_range() {
|
||||
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// The state gate reads the transit key, then its metadata record.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
ScriptedResponse::ok(metadata_read_data(&metadata)),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault_transit(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultTransitKmsBackend::new(config)
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let result = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
|
||||
let requests = vault.requests();
|
||||
assert!(
|
||||
!requests.iter().any(|line| line.starts_with("POST ")),
|
||||
"a refused window must not write anything: {requests:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the
|
||||
/// current secret version used as the check-and-set base.
|
||||
fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value {
|
||||
|
||||
@@ -24,6 +24,7 @@ use std::time::Duration;
|
||||
use url::Url;
|
||||
|
||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||
@@ -164,6 +165,24 @@ pub struct KmsConfig {
|
||||
/// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault.
|
||||
#[serde(default)]
|
||||
pub allow_insecure_dev_defaults: bool,
|
||||
/// Allow `DeleteKey` requests to skip the pending-deletion waiting window and
|
||||
/// destroy key material right away.
|
||||
///
|
||||
/// Off by default: an immediate deletion is unrecoverable and takes every
|
||||
/// object encrypted under the key with it, so the waiting window (plus
|
||||
/// `CancelKeyDeletion`) is the only recovery path. Operators who genuinely
|
||||
/// need immediate deletion — throwaway test clusters, key material that was
|
||||
/// never used — must turn it on through server configuration
|
||||
/// ([`ENV_KMS_ALLOW_IMMEDIATE_DELETION`]); the request must still echo the
|
||||
/// key id back for confirmation.
|
||||
///
|
||||
/// Not part of the serialized configuration, and not settable through the
|
||||
/// admin configure API. It is per-server operator state that has to be
|
||||
/// re-stated to survive a restart: persisting it would carry one operator's
|
||||
/// one-time enablement into the cluster-wide config that every node reloads,
|
||||
/// long after the deletion it was turned on for.
|
||||
#[serde(skip)]
|
||||
pub allow_immediate_deletion: bool,
|
||||
/// Timeout for a single backend attempt.
|
||||
///
|
||||
/// This bounds one outbound request, not the whole operation: the operation
|
||||
@@ -187,6 +206,7 @@ impl Default for KmsConfig {
|
||||
default_key_id: None,
|
||||
backend_config: BackendConfig::default(),
|
||||
allow_insecure_dev_defaults: false,
|
||||
allow_immediate_deletion: false,
|
||||
timeout: Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
enable_cache: true,
|
||||
@@ -714,6 +734,12 @@ impl KmsConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Explicitly allow deletions that bypass the pending-deletion waiting window.
|
||||
pub fn with_immediate_deletion_allowed(mut self) -> Self {
|
||||
self.allow_immediate_deletion = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set operation timeout
|
||||
pub fn with_timeout(mut self, timeout: Duration) -> Self {
|
||||
self.timeout = timeout;
|
||||
@@ -927,6 +953,7 @@ impl KmsConfig {
|
||||
config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache);
|
||||
config.allow_insecure_dev_defaults =
|
||||
get_env_bool(ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS, config.allow_insecure_dev_defaults);
|
||||
config.allow_immediate_deletion = get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, config.allow_immediate_deletion);
|
||||
|
||||
// Backend-specific configuration
|
||||
match config.backend {
|
||||
@@ -1042,6 +1069,15 @@ impl KmsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the immediate-deletion gate from the environment.
|
||||
///
|
||||
/// Callers that assemble a [`KmsConfig`] field by field instead of going
|
||||
/// through [`KmsConfig::from_env`] use this, so the gate keeps one name, one
|
||||
/// default, and one place to look it up.
|
||||
pub fn allow_immediate_deletion_from_env() -> bool {
|
||||
get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, false)
|
||||
}
|
||||
|
||||
fn vault_tls_config(skip_tls_verify: bool) -> Option<TlsConfig> {
|
||||
skip_tls_verify.then_some(TlsConfig {
|
||||
ca_cert_path: None,
|
||||
@@ -1802,6 +1838,38 @@ mod tests {
|
||||
assert_eq!(refresh_safety_window_secs, None);
|
||||
}
|
||||
|
||||
/// The gate lives in server configuration only: it never rides along in a
|
||||
/// serialized config, and a stored config that claims it must not be
|
||||
/// believed. Otherwise one operator's one-time enablement would reach every
|
||||
/// node that later reloads that config.
|
||||
#[test]
|
||||
fn immediate_deletion_gate_is_server_local_and_never_persisted() {
|
||||
let persisted =
|
||||
serde_json::to_value(KmsConfig::default().with_immediate_deletion_allowed()).expect("kms config should serialize");
|
||||
assert!(
|
||||
persisted.get("allow_immediate_deletion").is_none(),
|
||||
"the gate must not be written into a persisted config: {persisted}"
|
||||
);
|
||||
|
||||
let mut forged = persisted;
|
||||
forged
|
||||
.as_object_mut()
|
||||
.expect("a persisted config must be a JSON object")
|
||||
.insert("allow_immediate_deletion".to_string(), serde_json::json!(true));
|
||||
let restored: KmsConfig = serde_json::from_value(forged).expect("an unknown gate field must not break loading");
|
||||
assert!(!restored.allow_immediate_deletion, "a stored gate must fail closed");
|
||||
|
||||
with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, Some("true"))], || {
|
||||
assert!(
|
||||
allow_immediate_deletion_from_env(),
|
||||
"the gate must be reachable from server configuration"
|
||||
);
|
||||
});
|
||||
with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, None::<&str>)], || {
|
||||
assert!(!allow_immediate_deletion_from_env());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_incomplete_approle() {
|
||||
let mut config = KmsConfig::vault_approle(
|
||||
|
||||
@@ -376,6 +376,7 @@ mod tests {
|
||||
key_id: key_id.to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("deletion should be scheduled");
|
||||
|
||||
+253
-6
@@ -17,17 +17,19 @@
|
||||
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
|
||||
use crate::backends::KmsBackend;
|
||||
use crate::cache::{KmsCache, KmsCacheStats};
|
||||
use crate::config::KmsConfig;
|
||||
use crate::error::Result;
|
||||
use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, KmsConfig};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::types::{
|
||||
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
|
||||
DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
|
||||
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
|
||||
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse,
|
||||
DEFAULT_PENDING_DELETION_WINDOW_DAYS, DecryptRequest, DecryptResponse, DeleteKeyRequest, DeleteKeyResponse,
|
||||
DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyRequest, GenerateDataKeyResponse,
|
||||
ListKeysRequest, ListKeysResponse, MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::warn;
|
||||
|
||||
/// KMS Manager coordinates operations between backends and caching
|
||||
#[derive(Clone)]
|
||||
@@ -38,12 +40,18 @@ pub struct KmsManager {
|
||||
enable_cache: bool,
|
||||
backend_kind: &'static str,
|
||||
audit_sink: Option<Arc<dyn KmsAuditSink>>,
|
||||
allow_immediate_deletion: bool,
|
||||
}
|
||||
|
||||
impl KmsManager {
|
||||
/// Create a new KMS manager with the given backend and config
|
||||
pub fn new(backend: Arc<dyn KmsBackend>, config: KmsConfig) -> Self {
|
||||
let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64)));
|
||||
if config.allow_immediate_deletion {
|
||||
warn!(
|
||||
"KMS immediate key deletion is enabled: a DeleteKey request may destroy key material without any waiting window, and every object encrypted under that key becomes permanently unreadable"
|
||||
);
|
||||
}
|
||||
Self {
|
||||
backend,
|
||||
cache,
|
||||
@@ -51,6 +59,7 @@ impl KmsManager {
|
||||
enable_cache: config.enable_cache,
|
||||
backend_kind: config.backend.as_str(),
|
||||
audit_sink: None,
|
||||
allow_immediate_deletion: config.allow_immediate_deletion,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,7 +234,8 @@ impl KmsManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete a key
|
||||
/// Delete a key, either scheduled behind the waiting window or — when the
|
||||
/// server allows it — immediately.
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::delete_key_with_context`].
|
||||
@@ -246,7 +256,13 @@ impl KmsManager {
|
||||
result
|
||||
}
|
||||
|
||||
/// This is the single enforcement point for the waiting window: every
|
||||
/// admin-facing deletion goes through here, so the checks below run before
|
||||
/// any backend sees the request. The backends repeat the window bound as a
|
||||
/// defensive assertion for callers that hold a backend handle directly.
|
||||
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
self.check_deletion_request(&request)?;
|
||||
|
||||
let response = self.backend.delete_key(request).await?;
|
||||
|
||||
// Remove from cache if enabled and key is being deleted
|
||||
@@ -258,6 +274,44 @@ impl KmsManager {
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Gate a deletion request before it reaches the backend.
|
||||
///
|
||||
/// Immediate deletion is unrecoverable, so it needs both a server-side
|
||||
/// opt-in and a per-request confirmation that echoes the key id; without
|
||||
/// either, the request is refused rather than downgraded to a scheduled
|
||||
/// deletion, so a caller never believes a key is gone when it is not.
|
||||
fn check_deletion_request(&self, request: &DeleteKeyRequest) -> Result<()> {
|
||||
if !request.force_immediate.unwrap_or(false) {
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !self.allow_immediate_deletion {
|
||||
return Err(KmsError::invalid_operation(format!(
|
||||
"immediate deletion of key {} is not allowed; schedule the deletion and wait out the pending window, or set {ENV_KMS_ALLOW_IMMEDIATE_DELETION}=true on the server",
|
||||
request.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
if request.confirm_key_id.as_deref() != Some(request.key_id.as_str()) {
|
||||
return Err(KmsError::invalid_operation(format!(
|
||||
"immediate deletion of key {} requires confirm_key_id to repeat the key id exactly",
|
||||
request.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
warn!(
|
||||
key_id = %request.key_id,
|
||||
"immediate KMS key deletion accepted; key material is destroyed without a waiting window and cannot be recovered"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Cancel key deletion
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
@@ -605,6 +659,7 @@ mod tests {
|
||||
key_id: AUDITED_KEY_ID.to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
@@ -1019,4 +1074,196 @@ mod tests {
|
||||
.await
|
||||
.expect("second data key should decrypt with its own context");
|
||||
}
|
||||
|
||||
/// Manager over a local backend, with the immediate-deletion gate set as
|
||||
/// the server operator would set it.
|
||||
async fn deletion_manager(temp_dir: &tempfile::TempDir, allow_immediate_deletion: bool) -> KmsManager {
|
||||
let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
config.allow_immediate_deletion = allow_immediate_deletion;
|
||||
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
|
||||
KmsManager::new(backend, config)
|
||||
}
|
||||
|
||||
async fn create_named_key(manager: &KmsManager, key_name: &str) -> String {
|
||||
manager
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(key_name.to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("Failed to create key")
|
||||
.key_id
|
||||
}
|
||||
|
||||
/// Data key generated up front, decrypted again afterwards: a refused
|
||||
/// deletion must leave the master key material byte-for-byte usable, not
|
||||
/// merely leave a metadata record behind.
|
||||
async fn data_key_probe(manager: &KmsManager, key_id: &str) -> (Vec<u8>, Vec<u8>) {
|
||||
let generated = manager
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: HashMap::new(),
|
||||
})
|
||||
.await
|
||||
.expect("Failed to generate data key");
|
||||
(generated.plaintext_key, generated.ciphertext_blob)
|
||||
}
|
||||
|
||||
async fn assert_key_material_intact(manager: &KmsManager, key_id: &str, probe: &(Vec<u8>, Vec<u8>)) {
|
||||
let state = manager
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("a key that was not deleted must still be describable")
|
||||
.key_metadata
|
||||
.key_state;
|
||||
assert_eq!(state, KeyState::Enabled, "a refused deletion must not change the key state");
|
||||
|
||||
let decrypted = manager
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: probe.1.clone(),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect("key material must still decrypt data keys issued before the refused deletion");
|
||||
assert_eq!(decrypted.plaintext, probe.0, "decrypted data key must match the original plaintext");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_is_refused_under_default_config() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, false).await;
|
||||
let key_id = create_named_key(&manager, "default-config-force-delete").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
// Confirmation present and correct: the server-side gate alone must
|
||||
// refuse this, no matter how well-formed the request is.
|
||||
let error = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("immediate deletion must be refused unless the server allows it");
|
||||
assert!(
|
||||
matches!(error, KmsError::InvalidOperation { .. }),
|
||||
"expected InvalidOperation, got {error:?}"
|
||||
);
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_requires_a_matching_confirmation() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, true).await;
|
||||
let key_id = create_named_key(&manager, "confirmation-required").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
for confirmation in [None, Some(String::new()), Some(format!("{key_id}-typo"))] {
|
||||
let result = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: confirmation.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"confirmation {confirmation:?} must be refused, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_succeeds_with_a_matching_confirmation() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, true).await;
|
||||
let key_id = create_named_key(&manager, "confirmed-force-delete").await;
|
||||
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("a confirmed immediate deletion must be allowed once the server enables it");
|
||||
|
||||
let error = manager
|
||||
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
|
||||
.await
|
||||
.expect_err("an immediately deleted key must be gone");
|
||||
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_window_outside_the_supported_range_is_refused() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, false).await;
|
||||
let key_id = create_named_key(&manager, "window-bounds").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
for days in [0, MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let result = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduled_deletion_keeps_its_existing_behaviour() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, false).await;
|
||||
|
||||
for (name, days) in [
|
||||
("schedule-default-window", None),
|
||||
("schedule-min-window", Some(MIN_PENDING_DELETION_WINDOW_DAYS)),
|
||||
("schedule-max-window", Some(MAX_PENDING_DELETION_WINDOW_DAYS)),
|
||||
] {
|
||||
let key_id = create_named_key(&manager, name).await;
|
||||
let response = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: days,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("scheduling a deletion inside the window must still succeed");
|
||||
assert!(response.deletion_date.is_some(), "a scheduled deletion must report its deadline");
|
||||
assert_eq!(response.key_metadata.key_state, KeyState::PendingDeletion);
|
||||
|
||||
manager
|
||||
.cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() })
|
||||
.await
|
||||
.expect("a scheduled deletion must still be cancellable");
|
||||
let state = manager
|
||||
.describe_key(DescribeKeyRequest { key_id })
|
||||
.await
|
||||
.expect("describe should succeed")
|
||||
.key_metadata
|
||||
.key_state;
|
||||
assert_eq!(state, KeyState::Enabled, "cancelling must restore the key");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+25
-3
@@ -893,15 +893,37 @@ impl std::str::FromStr for EncryptionAlgorithm {
|
||||
}
|
||||
}
|
||||
|
||||
/// Shortest pending-deletion waiting window a caller may ask for, in days.
|
||||
///
|
||||
/// The window is enforced once, in [`crate::manager::KmsManager::delete_key`];
|
||||
/// backends keep the same bound as a defensive check for direct callers.
|
||||
pub const MIN_PENDING_DELETION_WINDOW_DAYS: u32 = 7;
|
||||
|
||||
/// Longest pending-deletion waiting window a caller may ask for, in days.
|
||||
pub const MAX_PENDING_DELETION_WINDOW_DAYS: u32 = 30;
|
||||
|
||||
/// Waiting window applied when a delete request does not name one.
|
||||
pub const DEFAULT_PENDING_DELETION_WINDOW_DAYS: u32 = MAX_PENDING_DELETION_WINDOW_DAYS;
|
||||
|
||||
/// Request to delete a key
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct DeleteKeyRequest {
|
||||
/// Key ID to delete
|
||||
pub key_id: String,
|
||||
/// Number of days to wait before deletion (7-30 days, optional)
|
||||
/// Number of days to wait before deletion (7-30 days, optional, defaults to 30)
|
||||
pub pending_window_in_days: Option<u32>,
|
||||
/// Force immediate deletion (for development/testing only)
|
||||
/// Destroy the key material right away instead of scheduling it.
|
||||
///
|
||||
/// Refused unless the server enables `allow_immediate_deletion`; see
|
||||
/// [`crate::manager::KmsManager::delete_key`] for the gate.
|
||||
pub force_immediate: Option<bool>,
|
||||
/// Key id echoed back by the caller to confirm an immediate deletion.
|
||||
///
|
||||
/// Must equal `key_id` exactly whenever `force_immediate` is set. Optional
|
||||
/// with a serde default because this type is part of the admin API
|
||||
/// contract: clients that never ask for immediate deletion are unaffected.
|
||||
#[serde(default)]
|
||||
pub confirm_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Response from delete key operation
|
||||
|
||||
@@ -45,7 +45,7 @@ path "secret/metadata/rustfs/kms/keys/*" {
|
||||
Notes:
|
||||
|
||||
- The trailing wildcards also cover the per-version material records that rotation creates under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed.
|
||||
- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys.
|
||||
- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. RustFS refuses `force_immediate` unless the server sets `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, so leaving that gate off keeps the capability unreachable no matter what the Vault policy allows.
|
||||
- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them.
|
||||
- Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure.
|
||||
|
||||
@@ -63,7 +63,7 @@ Decryption loads exactly the version recorded in the envelope and fails closed w
|
||||
|
||||
- Every version record that any stored DEK envelope references must remain readable. Until an object rewrap/migration capability exists, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped.
|
||||
- Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely.
|
||||
- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them.
|
||||
- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must echo the key id back as `confirm_key_id`. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key.
|
||||
- For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext.
|
||||
|
||||
### Upgrade before first rotation (hard constraint)
|
||||
|
||||
@@ -192,6 +192,9 @@ async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
|
||||
|
||||
fn decode_persisted_kms_config(data: &[u8]) -> serde_json::Result<(KmsConfig, bool)> {
|
||||
let mut config: KmsConfig = serde_json::from_slice(data)?;
|
||||
// The immediate-deletion gate is per-server operator state, never stored,
|
||||
// so a config loaded from cluster storage still has to pick it up here.
|
||||
config.allow_immediate_deletion = rustfs_kms::config::allow_immediate_deletion_from_env();
|
||||
let value: serde_json::Value = serde_json::from_slice(data)?;
|
||||
let is_missing_development_flag = value
|
||||
.as_object()
|
||||
|
||||
@@ -1023,6 +1023,9 @@ pub struct DeleteKmsKeyRequest {
|
||||
pub key_id: String,
|
||||
pub pending_window_in_days: Option<u32>,
|
||||
pub force_immediate: Option<bool>,
|
||||
/// Echo of `key_id`, required by the service gate before it will destroy
|
||||
/// key material immediately. Absent for an ordinary scheduled deletion.
|
||||
pub confirm_key_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
@@ -1089,11 +1092,13 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
// Extract pending_window_in_days and force_immediate from query parameters
|
||||
let pending_window_in_days = query_params.get("pending_window_in_days").and_then(|s| s.parse::<u32>().ok());
|
||||
let force_immediate = query_params.get("force_immediate").and_then(|s| s.parse::<bool>().ok());
|
||||
let confirm_key_id = query_params.get("confirm_key_id").map(|s| s.to_string());
|
||||
|
||||
DeleteKmsKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days,
|
||||
force_immediate,
|
||||
confirm_key_id,
|
||||
}
|
||||
} else {
|
||||
serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?
|
||||
@@ -1131,6 +1136,7 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
key_id: request.key_id.clone(),
|
||||
pending_window_in_days: request.pending_window_in_days,
|
||||
force_immediate: request.force_immediate,
|
||||
confirm_key_id: request.confirm_key_id.clone(),
|
||||
};
|
||||
|
||||
match manager.delete_key_with_context(kms_request, audit.context()).await {
|
||||
|
||||
@@ -291,6 +291,7 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
||||
file_permissions: Some(0o600),
|
||||
}),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
@@ -328,6 +329,7 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
||||
tls: None,
|
||||
})),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
@@ -363,6 +365,7 @@ fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustf
|
||||
..rustfs_kms::config::VaultTransitConfig::default()
|
||||
})),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
@@ -417,6 +420,7 @@ fn build_static_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::
|
||||
default_key_id: cfg.kms_default_key_id.clone().or(Some(key_id)),
|
||||
backend_config: rustfs_kms::config::BackendConfig::Static(static_config),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user