mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 18:46:17 +00:00
feat(kms): report the configuration references that block a key deletion (#5598)
* feat(kms): report configuration references that block a key deletion Adds a KeyImpactReport that states which configuration still points at a key, how exhaustively the sources were read, and which sources were not consulted at all. The report deliberately carries no in-use or safe-to-delete claim: it covers the configuration layer only, so an empty reference list means nothing was found in the scanned sources, never that the key is unreferenced. Immediate deletion destroys key material without ever reaching the deletion worker, so it never passed the worker's reference gate. The manager now consults the same checker on that path and refuses with a typed KeyStillReferenced error. This only ever adds a refusal; the scheduled deletion path and the worker's blocking behaviour are unchanged. * test(kms): cover the immediate-deletion reference refusal * feat(kms): surface configuration references on the admin key endpoints DeleteKey and DescribeKey now return an impact section listing the configuration that points at the key, so an operator scheduling a deletion sees what will refuse to destroy the material instead of learning it from a server-side log once the window has run out. The section is reported, never acted on: scheduling still succeeds while references exist, and the deletion worker's gate remains the only thing that decides whether material is destroyed. An immediate deletion that the manager refuses for an outstanding reference now answers 409. * test(kms): pin the impact wire shape and the unreferenced force-delete path * fix(kms): make the DescribeKey impact section opt-in Collecting the section lists every bucket, and DescribeKey is polled, so carrying that fan-out on the default read path trades a hot path's cost for a diagnostic. It is now collected only for impact=true; without the parameter the endpoint does exactly the work it did before and returns no impact field. A value that is neither true nor false is refused rather than read as off, so a typo cannot answer a request for the section with a response that merely lacks one. DeleteKey still reports unconditionally: that is the request whose consequences the caller cannot otherwise see, and it is not polled. * fix(kms): box the query-parse refusal now that responses carry impact The delete response grew an impact section, which pushed it past the size clippy accepts inline in a Result. It is a full response body rather than an error code, so it is boxed at the one place that returns it as an error; the wire shape and the public field type are unchanged.
This commit is contained in:
@@ -19,12 +19,13 @@ use crate::admin::auth::{validate_admin_request, validate_admin_request_with_kms
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{current_kms_runtime_service_manager, current_or_init_kms_runtime_service_manager};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::kms_deletion_gate::current_key_impact;
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use base64::Engine;
|
||||
use hyper::{HeaderMap, Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_kms::{KmsAuditOperation, KmsError, types::*};
|
||||
use rustfs_kms::{KeyImpactReport, KmsAuditOperation, KmsError, types::*};
|
||||
use rustfs_policy::policy::action::{Action, KmsAction};
|
||||
use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
|
||||
@@ -372,12 +373,13 @@ impl Operation for DescribeKeyHandler {
|
||||
mod tests {
|
||||
use super::{
|
||||
CancelKmsKeyDeletionRequest, CreateKeyApiRequest, CreateKmsKeyRequest, DeleteKmsKeyRequest, DeleteKmsKeyResponse,
|
||||
GenerateDataKeyApiRequest, delete_key_error_status, delete_request_from_query, extract_key_id, kms_create_key_actions,
|
||||
kms_delete_key_actions, kms_describe_key_actions, kms_generate_data_key_actions, kms_list_keys_actions, scoped_key_id,
|
||||
DescribeKmsKeyResponse, GenerateDataKeyApiRequest, delete_key_error_status, delete_request_from_query, extract_key_id,
|
||||
key_impact_if_requested, kms_create_key_actions, kms_delete_key_actions, kms_describe_key_actions,
|
||||
kms_generate_data_key_actions, kms_list_keys_actions, scoped_key_id, wants_key_impact,
|
||||
};
|
||||
use http::Uri;
|
||||
use hyper::StatusCode;
|
||||
use rustfs_kms::KmsError;
|
||||
use rustfs_kms::{KeyImpactReport, KeyReference, KeyReferenceKind, KmsError, ReferenceScope};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
|
||||
use rustfs_policy::policy::{Args, Policy};
|
||||
use std::collections::HashMap;
|
||||
@@ -465,7 +467,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_query(query: &str) -> Result<DeleteKmsKeyRequest, DeleteKmsKeyResponse> {
|
||||
fn delete_query(query: &str) -> Result<DeleteKmsKeyRequest, Box<DeleteKmsKeyResponse>> {
|
||||
let uri: Uri = format!("/rustfs/admin/v3/kms/keys/delete?{query}")
|
||||
.parse()
|
||||
.expect("uri should parse");
|
||||
@@ -557,6 +559,15 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A key the deployment still points at is refused with 409, not 400: the
|
||||
/// request is well formed and the key exists, and what has to change to
|
||||
/// make it succeed is the configuration, not the request.
|
||||
#[test]
|
||||
fn a_still_referenced_key_reports_a_conflict() {
|
||||
let error = KmsError::key_still_referenced("key-a", vec!["bucket:sse-bucket".to_string()]);
|
||||
assert_eq!(delete_key_error_status(&error), StatusCode::CONFLICT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_key_id_reads_the_body_first_and_falls_back_to_the_query() {
|
||||
let uri: Uri = "/rustfs/admin/v3/kms/keys/delete?keyId=query-key"
|
||||
@@ -795,6 +806,162 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn referenced_impact() -> KeyImpactReport {
|
||||
let mut impact = KeyImpactReport::configuration_layer("key-a");
|
||||
impact.push_reference(KeyReference {
|
||||
kind: KeyReferenceKind::BucketDefaultEncryption,
|
||||
id: "sse-bucket".to_string(),
|
||||
detail: "bucket default encryption names this key".to_string(),
|
||||
});
|
||||
impact
|
||||
}
|
||||
|
||||
/// Scheduling a deletion for a key that bucket configuration still points
|
||||
/// at succeeds — it destroys nothing and stays cancellable — but the
|
||||
/// caller has to be told, in the same response, what will refuse the
|
||||
/// destruction once the window runs out.
|
||||
#[test]
|
||||
fn a_scheduled_deletion_succeeds_and_still_names_what_will_block_it() {
|
||||
let response = DeleteKmsKeyResponse {
|
||||
success: true,
|
||||
message: "key deleted successfully".to_string(),
|
||||
key_id: "key-a".to_string(),
|
||||
deletion_date: Some("2026-01-01T00:00:00Z".to_string()),
|
||||
impact: Some(referenced_impact()),
|
||||
};
|
||||
|
||||
let json = serde_json::to_value(&response).expect("response should serialize");
|
||||
assert_eq!(
|
||||
json["success"], true,
|
||||
"an outstanding reference must not change the outcome of a schedule"
|
||||
);
|
||||
assert_eq!(json["impact"]["references"][0]["kind"], "bucket-default-encryption");
|
||||
assert_eq!(json["impact"]["references"][0]["id"], "sse-bucket");
|
||||
}
|
||||
|
||||
/// The impact section is exhaustive only over what it read, and says so.
|
||||
/// A caller must be able to tell an empty reference list apart from a
|
||||
/// key that nothing uses, because this report cannot distinguish them.
|
||||
#[test]
|
||||
fn the_impact_section_states_its_coverage_and_claims_no_key_is_unused() {
|
||||
let empty = DescribeKmsKeyResponse {
|
||||
success: true,
|
||||
message: "Key described successfully".to_string(),
|
||||
key_metadata: None,
|
||||
impact: Some(KeyImpactReport::configuration_layer("key-a")),
|
||||
};
|
||||
|
||||
let json = serde_json::to_value(&empty).expect("response should serialize");
|
||||
assert_eq!(json["impact"]["references"].as_array().expect("references is an array").len(), 0);
|
||||
assert_eq!(json["impact"]["completeness"], "exact");
|
||||
assert_eq!(json["impact"]["coverage"]["scanned"][0], "bucket-default-encryption");
|
||||
assert_eq!(
|
||||
json["impact"]["coverage"]["not_scanned"][0], "object-envelopes",
|
||||
"an empty reference list is only honest next to the scopes it did not read"
|
||||
);
|
||||
|
||||
let referenced = serde_json::to_value(DeleteKmsKeyResponse {
|
||||
success: true,
|
||||
message: String::new(),
|
||||
key_id: "key-a".to_string(),
|
||||
deletion_date: None,
|
||||
impact: Some(referenced_impact()),
|
||||
})
|
||||
.expect("response should serialize");
|
||||
|
||||
for body in [json.to_string(), referenced.to_string()] {
|
||||
for forbidden in ["in_use", "unused", "unreferenced", "safe_to_delete", "deletable"] {
|
||||
assert!(!body.contains(forbidden), "the impact section must not carry a `{forbidden}` claim");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The impact section is a report, never an input to a decision here. The
|
||||
/// deletion worker's fail-closed gate stays the only thing that decides
|
||||
/// whether material is destroyed; a handler that branched on this report
|
||||
/// would be treating "nothing found in the configuration layer" as
|
||||
/// "nothing uses this key", which it is not.
|
||||
#[test]
|
||||
fn handlers_report_the_key_impact_without_acting_on_it() {
|
||||
let src = include_str!("kms_keys.rs");
|
||||
|
||||
// Deleting is the request whose consequences the caller cannot
|
||||
// otherwise see, and it is not polled, so it always reports.
|
||||
let delete = operation_block(src, "DeleteKmsKeyHandler");
|
||||
assert!(
|
||||
delete.contains("let impact = current_key_impact("),
|
||||
"the delete path must report the configuration impact unconditionally"
|
||||
);
|
||||
|
||||
// Describing is polled, so the same collection is opt-in there.
|
||||
let describe = operation_block(src, "DescribeKmsKeyHandler");
|
||||
assert!(
|
||||
describe.contains("key_impact_if_requested(wants_impact,"),
|
||||
"the describe path must collect the impact only when the request asked for it"
|
||||
);
|
||||
assert!(
|
||||
!describe.contains("current_key_impact("),
|
||||
"the describe path must not reach the collection except through the opt-in"
|
||||
);
|
||||
|
||||
// Neither handler may read what the report says. Constructing it and
|
||||
// handing it to the response is the whole of their business: a handler
|
||||
// that inspected it would be treating "nothing found in the
|
||||
// configuration layer" as "nothing uses this key", which it is not.
|
||||
for (handler, block) in [("DeleteKmsKeyHandler", delete), ("DescribeKmsKeyHandler", describe)] {
|
||||
for inspection in [
|
||||
"impact.blocks_destruction",
|
||||
"impact.references",
|
||||
"impact.completeness",
|
||||
"impact.coverage",
|
||||
"impact.key_id",
|
||||
"match impact",
|
||||
] {
|
||||
assert!(!block.contains(inspection), "{handler} must not read the impact report (`{inspection}`)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn describe_uri(query: &str) -> Uri {
|
||||
format!("/rustfs/admin/v3/kms/keys/key-a{query}")
|
||||
.parse()
|
||||
.expect("uri should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_impact_section_is_opt_in_and_refuses_an_unreadable_opt_in() {
|
||||
assert_eq!(
|
||||
wants_key_impact(&describe_uri("")),
|
||||
Ok(false),
|
||||
"the default read path must not ask for it"
|
||||
);
|
||||
assert_eq!(wants_key_impact(&describe_uri("?impact=false")), Ok(false));
|
||||
assert_eq!(wants_key_impact(&describe_uri("?impact=true")), Ok(true));
|
||||
|
||||
// A typo must not be read as "off": the caller asked for the section,
|
||||
// and an absent section would be indistinguishable from an empty one.
|
||||
for query in ["?impact=maybe", "?impact=1", "?impact=", "?impact=TRUE"] {
|
||||
let error = wants_key_impact(&describe_uri(query)).expect_err("a malformed opt-in must be refused");
|
||||
assert!(error.contains("expected 'true' or 'false'"), "unhelpful message for {query}: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A report can only come from the collection, so `None` is proof that the
|
||||
/// default read path never reached the bucket listing behind it.
|
||||
#[tokio::test]
|
||||
async fn the_default_describe_path_never_collects_the_impact() {
|
||||
assert!(key_impact_if_requested(false, "key-a", None).await.is_none());
|
||||
|
||||
let collected = key_impact_if_requested(true, "key-a", None)
|
||||
.await
|
||||
.expect("an opted-in describe must carry the section");
|
||||
assert_eq!(collected.key_id, "key-a");
|
||||
assert_eq!(
|
||||
collected.coverage.not_scanned,
|
||||
vec![ReferenceScope::ObjectEnvelopes, ReferenceScope::InProgressMultipartUploads]
|
||||
);
|
||||
}
|
||||
|
||||
fn operation_block<'a>(src: &'a str, handler: &str) -> &'a str {
|
||||
let marker = format!("impl Operation for {handler}");
|
||||
let block = src.split_once(&marker).expect("handler impl should exist").1;
|
||||
@@ -1135,6 +1302,17 @@ pub struct DeleteKmsKeyResponse {
|
||||
pub message: String,
|
||||
pub key_id: String,
|
||||
pub deletion_date: Option<String>,
|
||||
/// Configuration that still points at the key when the request was
|
||||
/// handled. A scheduled deletion succeeds regardless — it destroys
|
||||
/// nothing and stays cancellable — but the deletion worker will refuse to
|
||||
/// destroy the material while any of these references remain, so an
|
||||
/// operator has to be able to see them at the moment they schedule it
|
||||
/// rather than only in a server-side log once the window has run out.
|
||||
///
|
||||
/// `None` when the request never got far enough to identify a key or
|
||||
/// reach the KMS service. See [`KeyImpactReport`] for what an empty
|
||||
/// reference list does and does not mean.
|
||||
pub impact: Option<KeyImpactReport>,
|
||||
}
|
||||
|
||||
const IMMEDIATE_DELETION_QUERY_RETIRED: &str = "immediate deletion is no longer accepted as a query parameter; send it as a JSON body with force_immediate and confirm_key_id set to the key id";
|
||||
@@ -1149,13 +1327,22 @@ const IMMEDIATE_DELETION_QUERY_RETIRED: &str = "immediate deletion is no longer
|
||||
/// accident. An immediate-deletion attempt made this way is refused rather
|
||||
/// than downgraded to a scheduled deletion, so the caller cannot mistake one
|
||||
/// outcome for the other.
|
||||
fn delete_request_from_query(uri: &hyper::Uri) -> Result<DeleteKmsKeyRequest, DeleteKmsKeyResponse> {
|
||||
///
|
||||
/// The refusal is boxed because it is a full response body, not an error code:
|
||||
/// carrying one inline would make every `Ok` of this function as wide as the
|
||||
/// widest failure it can describe.
|
||||
fn delete_request_from_query(uri: &hyper::Uri) -> Result<DeleteKmsKeyRequest, Box<DeleteKmsKeyResponse>> {
|
||||
let query_params = extract_query_params(uri);
|
||||
let refuse = |key_id: &str, message: &str| DeleteKmsKeyResponse {
|
||||
success: false,
|
||||
message: message.to_string(),
|
||||
key_id: key_id.to_string(),
|
||||
deletion_date: None,
|
||||
let refuse = |key_id: &str, message: &str| {
|
||||
Box::new(DeleteKmsKeyResponse {
|
||||
success: false,
|
||||
message: message.to_string(),
|
||||
key_id: key_id.to_string(),
|
||||
deletion_date: None,
|
||||
// The request never reached the KMS service, so nothing was
|
||||
// collected; this is not a report that found no references.
|
||||
impact: None,
|
||||
})
|
||||
};
|
||||
|
||||
let Some(key_id) = query_params.get("keyId") else {
|
||||
@@ -1193,6 +1380,10 @@ fn delete_key_error_status(error: &KmsError) -> StatusCode {
|
||||
| KmsError::MaterialCorrupt { .. }
|
||||
| KmsError::MaterialAuthenticationFailed { .. }
|
||||
| KmsError::UnsupportedFormatVersion { .. } => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
// The request was well formed and the key exists; it is the state of
|
||||
// the deployment around it that refuses, and that state can change
|
||||
// without the caller changing anything about the request.
|
||||
KmsError::KeyStillReferenced { .. } => StatusCode::CONFLICT,
|
||||
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
@@ -1255,6 +1446,7 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
message: "kms service manager is not initialized".to_string(),
|
||||
key_id: request.key_id,
|
||||
deletion_date: None,
|
||||
impact: None,
|
||||
};
|
||||
let data =
|
||||
serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
@@ -1269,6 +1461,7 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
message: "kms service is not running".to_string(),
|
||||
key_id: request.key_id,
|
||||
deletion_date: None,
|
||||
impact: None,
|
||||
};
|
||||
let data =
|
||||
serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
@@ -1284,6 +1477,13 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
confirm_key_id: request.confirm_key_id.clone(),
|
||||
};
|
||||
|
||||
// Collected before the request is carried out, so the response
|
||||
// describes the deployment the caller is acting on. It is reported,
|
||||
// never acted on here: the deletion worker re-runs this check against
|
||||
// live configuration before it destroys anything, which is the gate
|
||||
// that decides the outcome.
|
||||
let impact = current_key_impact(&request.key_id, manager.get_default_key_id().map(String::as_str)).await;
|
||||
|
||||
match manager.delete_key_with_context(kms_request, audit.context()).await {
|
||||
Ok(kms_response) => {
|
||||
info!(
|
||||
@@ -1300,6 +1500,7 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
message: "key deleted successfully".to_string(),
|
||||
key_id: kms_response.key_id,
|
||||
deletion_date: kms_response.deletion_date,
|
||||
impact: Some(impact),
|
||||
};
|
||||
|
||||
let data =
|
||||
@@ -1327,6 +1528,7 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
message: format!("Failed to delete key: {e}"),
|
||||
key_id: request.key_id,
|
||||
deletion_date: None,
|
||||
impact: Some(impact),
|
||||
};
|
||||
|
||||
let data =
|
||||
@@ -1640,6 +1842,42 @@ pub struct DescribeKmsKeyResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
pub key_metadata: Option<KeyMetadata>,
|
||||
/// Configuration that currently points at the key, so the blast radius of
|
||||
/// a deletion can be read off without scheduling one first.
|
||||
///
|
||||
/// Present only when the request asked for it with `impact=true`; `None`
|
||||
/// otherwise, and whenever the key could not be described at all. Absent
|
||||
/// therefore means "not collected", never "nothing references this key" —
|
||||
/// see [`KeyImpactReport`] for what a collected one does and does not say.
|
||||
pub impact: Option<KeyImpactReport>,
|
||||
}
|
||||
|
||||
/// Whether the caller asked for the configuration impact section.
|
||||
///
|
||||
/// Off unless asked for. Collecting the section lists every bucket, and this
|
||||
/// endpoint is polled, so the default read path must not carry that fan-out;
|
||||
/// the delete path reports unconditionally instead, because that is the
|
||||
/// request whose consequences the operator cannot otherwise see.
|
||||
///
|
||||
/// A value that is neither `true` nor `false` is refused rather than read as
|
||||
/// "off". Silently downgrading a typo would answer a request for the section
|
||||
/// with a response that has none, and an absent section must never be
|
||||
/// mistaken for one that found nothing.
|
||||
fn wants_key_impact(uri: &hyper::Uri) -> Result<bool, String> {
|
||||
match extract_query_params(uri).get("impact") {
|
||||
None => Ok(false),
|
||||
Some(value) => value
|
||||
.parse::<bool>()
|
||||
.map_err(|_| format!("invalid value for 'impact': expected 'true' or 'false', got '{value}'")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration impact of the described key, collected only when requested.
|
||||
async fn key_impact_if_requested(requested: bool, key_id: &str, default_key_id: Option<&str>) -> Option<KeyImpactReport> {
|
||||
if !requested {
|
||||
return None;
|
||||
}
|
||||
Some(current_key_impact(key_id, default_key_id).await)
|
||||
}
|
||||
|
||||
/// Describe a KMS key
|
||||
@@ -1677,6 +1915,7 @@ impl Operation for DescribeKmsKeyHandler {
|
||||
success: false,
|
||||
message: "missing required parameter: 'keyId'".to_string(),
|
||||
key_metadata: None,
|
||||
impact: None,
|
||||
};
|
||||
let data =
|
||||
serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
@@ -1685,11 +1924,31 @@ impl Operation for DescribeKmsKeyHandler {
|
||||
return Ok(S3Response::with_headers((StatusCode::BAD_REQUEST, Body::from(data)), headers));
|
||||
};
|
||||
|
||||
// Validated before the key is looked up, so a malformed opt-in fails as
|
||||
// the input error it is instead of riding along on the key's outcome.
|
||||
let wants_impact = match wants_key_impact(&req.uri) {
|
||||
Ok(wants_impact) => wants_impact,
|
||||
Err(message) => {
|
||||
let response = DescribeKmsKeyResponse {
|
||||
success: false,
|
||||
message,
|
||||
key_metadata: None,
|
||||
impact: None,
|
||||
};
|
||||
let data =
|
||||
serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_TYPE, "application/json".parse().expect("operation should succeed"));
|
||||
return Ok(S3Response::with_headers((StatusCode::BAD_REQUEST, Body::from(data)), headers));
|
||||
}
|
||||
};
|
||||
|
||||
let Some(service_manager) = kms_service_manager_from_context() else {
|
||||
let response = DescribeKmsKeyResponse {
|
||||
success: false,
|
||||
message: "kms service manager is not initialized".to_string(),
|
||||
key_metadata: None,
|
||||
impact: None,
|
||||
};
|
||||
let data =
|
||||
serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
@@ -1703,6 +1962,7 @@ impl Operation for DescribeKmsKeyHandler {
|
||||
success: false,
|
||||
message: "kms service is not running".to_string(),
|
||||
key_metadata: None,
|
||||
impact: None,
|
||||
};
|
||||
let data =
|
||||
serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
|
||||
@@ -1726,10 +1986,16 @@ impl Operation for DescribeKmsKeyHandler {
|
||||
state = "completed",
|
||||
"admin kms keys state"
|
||||
);
|
||||
// Read-only, and opt-in: the same configuration-layer facts the
|
||||
// delete path reports, so the blast radius of a deletion can be
|
||||
// inspected without scheduling one first.
|
||||
let impact =
|
||||
key_impact_if_requested(wants_impact, key_id, manager.get_default_key_id().map(String::as_str)).await;
|
||||
let response = DescribeKmsKeyResponse {
|
||||
success: true,
|
||||
message: "Key described successfully".to_string(),
|
||||
key_metadata: Some(kms_response.key_metadata),
|
||||
impact,
|
||||
};
|
||||
|
||||
let data =
|
||||
@@ -1768,6 +2034,7 @@ impl Operation for DescribeKmsKeyHandler {
|
||||
success: false,
|
||||
message: format!("Failed to describe key: {e}"),
|
||||
key_metadata: None,
|
||||
impact: None,
|
||||
};
|
||||
|
||||
let data =
|
||||
|
||||
+162
-21
@@ -17,18 +17,27 @@
|
||||
//! must not be removed. Object-level references (existing envelopes) are out
|
||||
//! of scope here; those objects stay decryptable only until the key is gone,
|
||||
//! which is why the pending-deletion window exists.
|
||||
//!
|
||||
//! The same collection also backs the impact section of the admin key
|
||||
//! responses, so an operator sees the references that will block a deletion at
|
||||
//! the moment they schedule it rather than only in a server-side log once the
|
||||
//! window has run out. Both consumers read the same collection: the report
|
||||
//! cannot describe a deployment the gate does not enforce.
|
||||
//!
|
||||
//! Cost is bounded by the number of buckets — one bucket listing plus one
|
||||
//! cached metadata lookup each. Nothing here lists objects or versions.
|
||||
|
||||
use crate::runtime_sources::current_object_store_handle;
|
||||
use crate::storage_api::kms::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage_api::kms::{ECStore, StorageError, get_bucket_sse_config};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_kms::DeletionReferenceChecker;
|
||||
use rustfs_kms::{DeletionReferenceChecker, KeyImpactReport, KeyReference, KeyReferenceKind};
|
||||
use s3s::dto::ServerSideEncryptionConfiguration;
|
||||
use std::sync::Arc;
|
||||
use tracing::warn;
|
||||
|
||||
/// Reference reported when bucket configuration cannot be inspected at all.
|
||||
const BUCKET_CONFIG_UNAVAILABLE: &str = "bucket-encryption-config:unavailable";
|
||||
/// Source name reported when bucket configuration cannot be inspected at all.
|
||||
const BUCKET_ENCRYPTION_SOURCE: &str = "bucket-encryption-config";
|
||||
|
||||
/// Blocks deletion of keys referenced by any bucket's SSE configuration
|
||||
/// (default bucket KMS key). Registered on the KMS service manager at startup
|
||||
@@ -38,33 +47,73 @@ pub(crate) struct BucketEncryptionReferenceChecker;
|
||||
#[async_trait]
|
||||
impl DeletionReferenceChecker for BucketEncryptionReferenceChecker {
|
||||
async fn references(&self, key_id: &str) -> Vec<String> {
|
||||
collect_references(key_id, current_object_store_handle()).await
|
||||
// Bucket configuration only, and no service default key: the deletion
|
||||
// worker checks the default key itself before it consults a checker,
|
||||
// so this reports exactly the reference set it always has.
|
||||
collect_key_impact(key_id, None, current_object_store_handle())
|
||||
.await
|
||||
.references
|
||||
.iter()
|
||||
.map(blocking_reference)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_references(key_id: &str, store: Option<Arc<ECStore>>) -> Vec<String> {
|
||||
/// Configuration-layer impact of deleting `key_id`.
|
||||
///
|
||||
/// Exhaustive over what it covers, and explicit about what it does not:
|
||||
/// object envelopes written under the key are not enumerated, so an empty
|
||||
/// reference list is never a statement that the key is unused.
|
||||
pub(crate) async fn current_key_impact(key_id: &str, default_key_id: Option<&str>) -> KeyImpactReport {
|
||||
collect_key_impact(key_id, default_key_id, current_object_store_handle()).await
|
||||
}
|
||||
|
||||
async fn collect_key_impact(key_id: &str, default_key_id: Option<&str>, store: Option<Arc<ECStore>>) -> KeyImpactReport {
|
||||
let mut report = KeyImpactReport::configuration_layer(key_id);
|
||||
|
||||
if default_key_id == Some(key_id) {
|
||||
report.push_reference(KeyReference {
|
||||
kind: KeyReferenceKind::ServiceDefaultKey,
|
||||
id: key_id.to_string(),
|
||||
detail: "the KMS service is configured to use this key as its default key".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Fail closed: destroying key material is irreversible while the deletion
|
||||
// worker retries every sweep, so an uninspectable configuration must block
|
||||
// the removal rather than wave it through. This also covers the worker
|
||||
// racing server startup, before the object store is published.
|
||||
let Some(store) = store else {
|
||||
warn!(key_id, "KMS deletion reference check: object store not ready; blocking removal");
|
||||
return vec![BUCKET_CONFIG_UNAVAILABLE.to_string()];
|
||||
report.push_reference(unreadable_source(
|
||||
"object store is not ready, so bucket encryption configuration could not be read",
|
||||
));
|
||||
return report;
|
||||
};
|
||||
let buckets = match store.list_bucket(&BucketOptions::default()).await {
|
||||
Ok(buckets) => buckets,
|
||||
Err(error) => {
|
||||
warn!(key_id, %error, "KMS deletion reference check: listing buckets failed; blocking removal");
|
||||
return vec![BUCKET_CONFIG_UNAVAILABLE.to_string()];
|
||||
report.push_reference(unreadable_source(format!("buckets could not be listed: {error}")));
|
||||
return report;
|
||||
}
|
||||
};
|
||||
|
||||
let mut references = Vec::new();
|
||||
for bucket in &buckets {
|
||||
let lookup = get_bucket_sse_config(&bucket.name).await;
|
||||
references.extend(bucket_reference(&bucket.name, lookup, key_id));
|
||||
if let Some(reference) = bucket_reference(&bucket.name, lookup, key_id) {
|
||||
report.push_reference(reference);
|
||||
}
|
||||
}
|
||||
report
|
||||
}
|
||||
|
||||
fn unreadable_source(detail: impl Into<String>) -> KeyReference {
|
||||
KeyReference {
|
||||
kind: KeyReferenceKind::UnreadableSource,
|
||||
id: BUCKET_ENCRYPTION_SOURCE.to_string(),
|
||||
detail: detail.into(),
|
||||
}
|
||||
references
|
||||
}
|
||||
|
||||
/// `Some(reference)` when the bucket's encryption configuration references
|
||||
@@ -74,9 +123,13 @@ fn bucket_reference(
|
||||
bucket: &str,
|
||||
lookup: Result<ServerSideEncryptionConfiguration, StorageError>,
|
||||
key_id: &str,
|
||||
) -> Option<String> {
|
||||
) -> Option<KeyReference> {
|
||||
match lookup {
|
||||
Ok(config) if sse_config_references_key(&config, key_id) => Some(format!("bucket:{bucket}")),
|
||||
Ok(config) if sse_config_references_key(&config, key_id) => Some(KeyReference {
|
||||
kind: KeyReferenceKind::BucketDefaultEncryption,
|
||||
id: bucket.to_string(),
|
||||
detail: "bucket default encryption names this key, so new objects are written under it".to_string(),
|
||||
}),
|
||||
Ok(_) => None,
|
||||
Err(StorageError::ConfigNotFound) => None,
|
||||
Err(error) => {
|
||||
@@ -86,11 +139,29 @@ fn bucket_reference(
|
||||
%error,
|
||||
"KMS deletion reference check: unreadable bucket encryption config; blocking removal"
|
||||
);
|
||||
Some(format!("bucket:{bucket}:encryption-config-unreadable"))
|
||||
Some(KeyReference {
|
||||
kind: KeyReferenceKind::UnreadableResource,
|
||||
id: bucket.to_string(),
|
||||
detail: format!("bucket encryption configuration could not be read: {error}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reference identifier handed to the deletion worker.
|
||||
///
|
||||
/// The worker only needs an identifier to log and to count as non-empty, so
|
||||
/// these strings stay exactly as they were before the structured report
|
||||
/// existed; a reference is a reference regardless of how it renders.
|
||||
fn blocking_reference(reference: &KeyReference) -> String {
|
||||
match reference.kind {
|
||||
KeyReferenceKind::BucketDefaultEncryption => format!("bucket:{}", reference.id),
|
||||
KeyReferenceKind::UnreadableResource => format!("bucket:{}:encryption-config-unreadable", reference.id),
|
||||
KeyReferenceKind::UnreadableSource => format!("{}:unavailable", reference.id),
|
||||
KeyReferenceKind::ServiceDefaultKey => format!("kms-service-default-key:{}", reference.id),
|
||||
}
|
||||
}
|
||||
|
||||
fn sse_config_references_key(config: &ServerSideEncryptionConfiguration, key_id: &str) -> bool {
|
||||
config.rules.iter().any(|rule| {
|
||||
rule.apply_server_side_encryption_by_default
|
||||
@@ -103,6 +174,7 @@ fn sse_config_references_key(config: &ServerSideEncryptionConfiguration, key_id:
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_kms::ReferenceCompleteness;
|
||||
use s3s::dto::{ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionRule};
|
||||
|
||||
fn sse_kms_config(key_id: Option<&str>) -> ServerSideEncryptionConfiguration {
|
||||
@@ -119,10 +191,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn referencing_bucket_blocks_deletion() {
|
||||
assert_eq!(
|
||||
bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1"),
|
||||
Some("bucket:sse-bucket".to_string())
|
||||
);
|
||||
let reference = bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1")
|
||||
.expect("a bucket that names the key must be reported");
|
||||
assert_eq!(reference.kind, KeyReferenceKind::BucketDefaultEncryption);
|
||||
assert_eq!(reference.id, "sse-bucket");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -135,13 +207,82 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn unreadable_config_blocks_deletion() {
|
||||
let reference = bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1");
|
||||
assert_eq!(reference, Some("bucket:broken:encryption-config-unreadable".to_string()));
|
||||
let reference = bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1")
|
||||
.expect("an unreadable bucket must be reported");
|
||||
assert_eq!(reference.kind, KeyReferenceKind::UnreadableResource);
|
||||
assert_eq!(reference.id, "broken");
|
||||
}
|
||||
|
||||
/// The deletion worker's gate is the only thing standing between an
|
||||
/// expired key and destroyed material; reshaping the collection it reads
|
||||
/// must not change a single identifier it receives. The report is the
|
||||
/// second reading of that same collection, so it must never look clear
|
||||
/// where the gate objects.
|
||||
#[test]
|
||||
fn worker_reference_identifiers_are_unchanged() {
|
||||
let cases = [
|
||||
(
|
||||
bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1"),
|
||||
"bucket:sse-bucket",
|
||||
),
|
||||
(
|
||||
bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1"),
|
||||
"bucket:broken:encryption-config-unreadable",
|
||||
),
|
||||
(
|
||||
Some(unreadable_source("object store is not ready")),
|
||||
"bucket-encryption-config:unavailable",
|
||||
),
|
||||
];
|
||||
|
||||
for (reference, expected) in cases {
|
||||
let reference = reference.expect("case must produce a reference");
|
||||
assert_eq!(blocking_reference(&reference), expected);
|
||||
|
||||
let mut report = KeyImpactReport::configuration_layer("kms-key-1");
|
||||
report.push_reference(reference);
|
||||
assert!(report.blocks_destruction(), "{expected} must read as blocking in the report too");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_object_store_blocks_deletion() {
|
||||
let references = collect_references("kms-key-1", None).await;
|
||||
assert_eq!(references, vec![BUCKET_CONFIG_UNAVAILABLE.to_string()]);
|
||||
let references: Vec<String> = collect_key_impact("kms-key-1", None, None)
|
||||
.await
|
||||
.references
|
||||
.iter()
|
||||
.map(blocking_reference)
|
||||
.collect();
|
||||
assert_eq!(references, vec!["bucket-encryption-config:unavailable".to_string()]);
|
||||
}
|
||||
|
||||
/// An unreachable object store must never render as "we looked and found
|
||||
/// nothing": the report says the configuration could not be read.
|
||||
#[tokio::test]
|
||||
async fn missing_object_store_reports_unavailable_completeness() {
|
||||
let report = collect_key_impact("kms-key-1", None, None).await;
|
||||
|
||||
assert_eq!(report.completeness, ReferenceCompleteness::Unavailable);
|
||||
assert!(report.blocks_destruction());
|
||||
assert_eq!(report.references.len(), 1);
|
||||
assert_eq!(report.references[0].kind, KeyReferenceKind::UnreadableSource);
|
||||
}
|
||||
|
||||
/// The service default key is a configuration reference in its own right,
|
||||
/// and it is decidable without touching the object store.
|
||||
#[tokio::test]
|
||||
async fn service_default_key_is_reported_before_any_store_lookup() {
|
||||
let report = collect_key_impact("kms-key-1", Some("kms-key-1"), None).await;
|
||||
|
||||
assert_eq!(report.references[0].kind, KeyReferenceKind::ServiceDefaultKey);
|
||||
assert_eq!(report.references[0].id, "kms-key-1");
|
||||
|
||||
let other = collect_key_impact("kms-key-1", Some("kms-key-2"), None).await;
|
||||
assert!(
|
||||
!other
|
||||
.references
|
||||
.iter()
|
||||
.any(|reference| reference.kind == KeyReferenceKind::ServiceDefaultKey)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user