From 076390ed7633b46fccf4d94e7c2616c617029ef0 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sun, 2 Aug 2026 19:06:57 +0800 Subject: [PATCH] test(kms): pin AWS timeout and contract divergence --- crates/kms/src/backends/aws.rs | 82 ++++++++++++++++++++++++- docs/operations/kms-backend-security.md | 2 + 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/crates/kms/src/backends/aws.rs b/crates/kms/src/backends/aws.rs index 464f6dbab..7853aca26 100644 --- a/crates/kms/src/backends/aws.rs +++ b/crates/kms/src/backends/aws.rs @@ -822,7 +822,7 @@ impl KmsBackend for AwsKmsBackend { mod tests { use super::*; use aws_sdk_kms::config::{BehaviorVersion, Credentials, Region}; - use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient}; + use aws_smithy_http_client::test_util::{NeverClient, ReplayEvent, StaticReplayClient}; use aws_smithy_types::body::SdkBody; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; @@ -1008,6 +1008,34 @@ mod tests { assert_eq!(http_client.actual_requests().count(), 3, "both throttled attempts should be replayed"); } + /// A connector that never responds must be cut off by the backend's + /// per-attempt timeout rather than hanging the KMS operation indefinitely. + #[tokio::test(start_paused = true)] + async fn stalled_aws_request_is_cut_off_by_the_attempt_timeout() { + let never_client = NeverClient::new(); + let sdk_config = aws_sdk_kms::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(Credentials::new("AKIDTEST", "secret", None, None, "scripted")) + .http_client(never_client.clone()) + .retry_config(aws_sdk_kms::config::retry::RetryConfig::disabled()) + .build(); + let mut kms_config = KmsConfig::aws(Some("us-east-1".to_string())); + kms_config.timeout = std::time::Duration::from_millis(5_000); + kms_config.retry_attempts = 1; + let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config); + + let error = backend + .describe_key(DescribeKeyRequest { + key_id: "stalled-key".to_string(), + }) + .await + .expect_err("a stalled AWS request must be cut off by the attempt timeout"); + + assert!(matches!(error, KmsError::OperationTimedOut { .. }), "unexpected error: {error:?}"); + assert_eq!(never_client.num_calls(), 1, "one configured attempt must reach the connector"); + } + /// Access denial is deterministic: replaying it cannot help and would only /// multiply the audit trail of denied calls. #[tokio::test(start_paused = true)] @@ -1120,6 +1148,58 @@ mod tests { assert_eq!(http_client.actual_requests().count(), 0, "no key may be created in AWS"); } + /// AWS intentionally does not use the shared lifecycle contract driver. + /// The driver requires disabled/pending keys to decrypt, expects cancelling + /// deletion to re-enable a key, and creates keys by caller-assigned name; + /// AWS rejects the decryption assumption, leaves a cancelled key + /// disabled, and cannot honour the third. + #[tokio::test] + async fn aws_backend_shared_contract_exemption_is_pinned() { + let (_http, backend) = scripted_backend(vec![error_event(400, "DisabledException", "key is disabled")]); + let error = backend + .decrypt(DecryptRequest { + ciphertext: b"blob".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }) + .await + .expect_err("AWS must reject decrypt with a disabled key"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "unexpected error: {error:?}"); + + let (_http, backend) = scripted_backend(vec![error_event(400, "KMSInvalidStateException", "key is pending deletion")]); + let error = backend + .decrypt(DecryptRequest { + ciphertext: b"blob".to_vec(), + encryption_context: HashMap::new(), + grant_tokens: Vec::new(), + }) + .await + .expect_err("AWS must reject decrypt with a pending-deletion key"); + assert!(matches!(error, KmsError::InvalidOperation { .. }), "unexpected error: {error:?}"); + + let (_http, backend) = scripted_backend(vec![ + ok_event(serde_json::json!({})), + ok_event(key_metadata_json("test-key", "Disabled")), + ]); + let response = backend + .cancel_key_deletion(CancelKeyDeletionRequest { + key_id: "test-key".to_string(), + }) + .await + .expect("AWS cancellation should complete"); + assert_eq!(response.key_metadata.key_state, KeyState::Disabled); + + let (_http, backend) = scripted_backend(Vec::new()); + let error = backend + .create_key(CreateKeyRequest { + key_name: Some("contract-key".to_string()), + ..Default::default() + }) + .await + .expect_err("AWS cannot create a key under a caller-assigned name"); + assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "unexpected error: {error:?}"); + } + /// AWS rejects `Limit: 0` outright, so the request cannot be forwarded as /// written; clamping it up to one would answer a caller that asked for no /// keys with a key. The empty page is served locally instead. diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index f178b0cc3..742a15398 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -202,6 +202,8 @@ AWS owns key state, backing-key rotation, and the deletion window, and this back Two consequences follow from that last row: **SSE-S3 key auto-creation and the synthetic KMS probe are unavailable on this backend**, because both address a key by a name they choose. Pre-create keys in AWS and reference them by AWS key id or ARN. +The AWS backend is intentionally exempt from `backends::contract_tests::assert_state_machine_contract`. That shared driver assumes that disabled and pending-deletion keys still decrypt, that cancelling deletion returns a key to `Enabled`, and that creation accepts a caller-assigned key name. AWS rejects decryption for the first case, leaves a cancelled key `Disabled`, and assigns key identifiers itself, so running the driver would encode the wrong behavior. The exemption is pinned by the offline `aws_backend_shared_contract_exemption_is_pinned` test in `crates/kms/src/backends/aws.rs`; if AWS changes any of these semantics, integrate the backend into the shared driver and remove this exemption rather than weakening the shared assertions. + Key versions are opaque. AWS addresses backing keys internally and picks the right one to decrypt with, so RustFS reports `key_version` as 1 and cannot enumerate versions. Rotation uses `RotateKeyOnDemand`, which retains prior backing keys for decryption; AWS's separate automatic yearly rotation is neither enabled nor reported on by RustFS. The KMS admin API accepts the AWS backend as `"backend_type": "AWS"` (aliases `aws`, `aws-kms`, `aws_kms`, `AwsKms`) on `/v3/kms/configure` and `/v3/kms/reconfigure`. The body carries `region` (**required**), and optionally `endpoint_url`, `default_key_id`, and the shared timeout/retry/cache settings. It accepts no credential fields at all — unknown fields are rejected — because every node resolves credentials through its own provider chain.