fix(s3): strip every encryption marker from a copy destination (#5597)

fix(sse): strip inherited SSE key-id and algorithm on copy

strip_managed_encryption_metadata cleared the MinIO spellings of the
managed-SSE key id and seal algorithm but not their RustFS-native
counterparts, so a CopyObject destination kept the source object's
x-rustfs-encryption-key-id and x-rustfs-encryption-algorithm.

When the destination resolves to no server-side encryption, nothing
rewrites those keys. is_object_encryption_marker treats any remaining
x-rustfs-encryption-* key as proof the payload is encrypted, so the
plaintext destination reports ObjectInfo::is_encrypted, the reader takes
its encrypted branch, and the read fails closed with "encrypted object
metadata is incomplete" because the actual key material was stripped.

Add both constants to the strip list so a destination inherits no
encryption marker it has no material for.

.github/workflows/minio-interop.yml

@@ -18,7 +18,14 @@
 # This is NOT a PR gate. The fixtures are real MinIO backend trees generated on
 # the fly (they are gitignored, never committed), so the job regenerates them
 # each run with Docker and then runs the `#[ignore]` reader tests in
-# crates/ecstore/tests/minio_generated_read_test.rs.
+# rustfs/src/storage/minio_generated_read_test.rs.
+#
+# Scope: end-to-end MinIO-to-RustFS SSE interop is NOT implemented yet. Both
+# envelope parsers reject MinIO's own wrapped-DEK shape — see
+# `is_data_key_envelope` in crates/kms/src/encryption/dek.rs and the
+# `deny_unknown_fields` `LocalSseDekEnvelope` in rustfs/src/storage/sse.rs — and
+# closing that gap is tracked in rustfs/backlog#1638. Treat this job as the
+# harness for #1638, not as standing evidence that a MinIO migration reads back.
 #
 # Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
 # unlike the self-hosted fleet, whose pods drift in Docker/pip availability
@@ -55,6 +62,19 @@ jobs:
     env:
       # Fixed 32-byte test KMS key baked into the fixture lab; not a secret.
       RUSTFS_MINIO_STATIC_KMS_KEY_B64: IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
+      # Single definition of "the interop tests", shared by the guard step and
+      # the run step so the two cannot drift apart.
+      #
+      # These used to live in crates/ecstore/tests/minio_generated_read_test.rs
+      # and were selected with `-p rustfs-ecstore -E
+      # 'binary(minio_generated_read_test)'`. #5435 moved them into the `rustfs`
+      # crate as a `#[cfg(test)] mod`, which deleted that test binary; the
+      # selector was never updated and has selected zero interop tests ever
+      # since (cargo-nextest 0.9.140 now rejects it outright: "operator didn't
+      # match any binary names", exit 94).
+      INTEROP_PACKAGE: rustfs
+      INTEROP_FEATURES: rio-v2
+      INTEROP_FILTER: "test(minio_generated_read_test::)"
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -71,8 +91,29 @@ jobs:
       - name: Generate real MinIO fixtures via Docker
         run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh
 
+      # `binary(...)` at least dies loudly when nothing matches, but `test(...)`
+      # is a perfectly valid filterset that matches zero tests, so the next
+      # rename or module move would leave this job selecting nothing and
+      # reporting success without executing a single interop assertion. Count
+      # the selection and fail with a reason instead.
+      #
+      # Count only `filter-match.status == "matches"`: the top-level
+      # `test-count` in the JSON is the package total and ignores `-E` entirely.
+      - name: Assert the interop selector still matches tests
+        run: |
+          set -euo pipefail
+          count="$(cargo nextest list --run-ignored all \
+            -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
+            -E "$INTEROP_FILTER" --message-format json \
+            | python3 -c 'import json,sys; d=json.load(sys.stdin); print(sum(1 for s in d.get("rust-suites", {}).values() for t in s.get("testcases", {}).values() if t.get("filter-match", {}).get("status") == "matches"))')"
+          echo "interop tests selected: ${count}"
+          if [ "${count}" -eq 0 ]; then
+            echo "::error::Selector '${INTEROP_FILTER}' in package '${INTEROP_PACKAGE}' matched 0 tests. The MinIO interop reader tests have moved or been renamed again; fix the selector instead of letting this job pass without running them. Context: rustfs/backlog#1638."
+            exit 1
+          fi
+
       - name: Run MinIO interop reader tests
         run: |
-          cargo nextest run --run-ignored ignored-only \
-            -p rustfs-ecstore --features rio-v2 \
-            -E 'binary(minio_generated_read_test)'
+          cargo nextest run --run-ignored ignored-only --no-tests=fail \
+            -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
+            -E "$INTEROP_FILTER"

_typos.toml

@@ -47,6 +47,9 @@ consts = "consts"
 Hashi = "Hashi"  # HashiCorp
 # Accept alternate spelling used in parser/XML comments.
 unparseable = "unparseable"
+# Disaster-recovery objectives: recovery time and recovery point.
+RTO = "RTO"
+rto = "rto"
 
 [files]
 extend-exclude = []

crates/e2e_test/src/kms/configured_roundtrip_test.rs

@@ -126,6 +126,51 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
     assert_eq!(cancelled["success"], true);
     assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled");
 
+    // A window outside 7-30 days is refused at the endpoint, whatever the
+    // backend: the bound is enforced once in the service, so no backend can
+    // stretch or skip it (rustfs/backlog#1585).
+    for days in [6, 31] {
+        let refused = kms_admin_request(
+            base_url,
+            http::Method::DELETE,
+            "/rustfs/admin/v3/kms/keys/delete",
+            Some(
+                &serde_json::json!({
+                    "key_id": key_id,
+                    "pending_window_in_days": days
+                })
+                .to_string(),
+            ),
+            access_key,
+            secret_key,
+        )
+        .await
+        .err()
+        .ok_or_else(|| format!("a {days}-day deletion window must be refused"))?;
+        assert!(
+            refused.to_string().contains("400 Bad Request"),
+            "a {days}-day deletion window must report a client error: {refused}"
+        );
+    }
+
+    // Immediate deletion is no longer reachable through the query string, so it
+    // fails before the service gate is even consulted.
+    let refused = kms_admin_request(
+        base_url,
+        http::Method::DELETE,
+        &format!("/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
+        None,
+        access_key,
+        secret_key,
+    )
+    .await
+    .err()
+    .ok_or("immediate KMS key deletion must not be reachable through the query string")?;
+    assert!(
+        refused.to_string().contains("400 Bad Request"),
+        "a query-string immediate deletion must report a client error: {refused}"
+    );
+
     // A default server refuses to skip the waiting window (rustfs/backlog#1585):
     // immediate deletion is unrecoverable and takes every object encrypted under
     // the key with it, so the endpoint must reject it rather than honour it.
@@ -151,7 +196,7 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
         "refused immediate deletion must report a client error: {refused}"
     );
 
-    // The refused request left the key alone, so the window-bounded path still
+    // The refused requests left the key alone, so the window-bounded path still
     // has something to schedule.
     let described = kms_admin_request(
         base_url,

crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs

@@ -0,0 +1,483 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Negative authorization matrix for per-key KMS access control.
+//!
+//! Every case here is an end-to-end denial that the pre-`kms` resource server
+//! allowed, so a regression that reopens one of them fails this file rather than
+//! only a unit test. The matrix varies one dimension at a time:
+//!
+//! - **wrong identity**: a caller holding S3 rights but no `kms` grant at all
+//! - **wrong key**: a caller scoped to key A naming key B
+//! - **wrong action**: a caller holding `kms:GenerateDataKey` but not `kms:Decrypt`
+//!   (and, on the admin plane, `kms:DisableKey` but not `kms:RotateKey`)
+//! - **wrong context**: an explicit `Deny` beating a wildcard `Allow`, and SSE-S3
+//!   staying exempt from `kms` authorization
+//!
+//! Each matrix opens with a positive control. Without it a denial proves nothing:
+//! an identity whose policy has not propagated yet is denied everything.
+
+use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
+use crate::common::{admin_ok, admin_request, init_logging};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::config::{Config, Credentials, Region};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::ServerSideEncryption;
+use serial_test::serial;
+use std::time::Duration;
+use tracing::info;
+
+type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
+
+const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
+const OTHER_KEY: &str = "kms-matrix-other-key";
+const BUCKET: &str = "kms-authz-matrix";
+const SECRET: &str = "kms-matrix-secret";
+const PAYLOAD: &[u8] = b"kms authorization matrix payload";
+
+/// How long an identity change may take to reach the request path.
+const IAM_PROPAGATION: Duration = Duration::from_secs(20);
+
+fn s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
+    let config = Config::builder()
+        .credentials_provider(Credentials::new(access_key, secret_key, None, None, "kms-authz-matrix"))
+        .region(Region::new("us-east-1"))
+        .endpoint_url(url)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+/// Start a server whose SSE-KMS data path authorizes against the named key.
+///
+/// The enforcement switch defaults to off for compatibility, so it has to be set
+/// explicitly; without it every negative case below would silently pass as an allow.
+async fn start_enforcing_server(env: &mut LocalKMSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
+    create_key_with_specific_id(&env.kms_keys_dir, ALLOWED_KEY).await?;
+    create_key_with_specific_id(&env.kms_keys_dir, OTHER_KEY).await?;
+
+    let key_dir = env.kms_keys_dir.clone();
+    let args = vec![
+        "--kms-enable",
+        "--kms-backend",
+        "local",
+        "--kms-key-dir",
+        key_dir.as_str(),
+        "--kms-default-key-id",
+        ALLOWED_KEY,
+    ];
+
+    let mut envs = vec![("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")];
+    envs.extend_from_slice(extra_env);
+
+    env.base_env.start_rustfs_server_with_env(args, &envs).await?;
+    Ok(())
+}
+
+/// Create `user` with `policy_document` attached under a canned policy of the same name.
+async fn provision_user(env: &LocalKMSTestEnvironment, user: &str, policy_document: &str) -> TestResult {
+    admin_ok(
+        &env.base_env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={user}"),
+        Some(policy_document.to_string()),
+    )
+    .await?;
+    provision_user_with_policy(env, user, user).await
+}
+
+/// Create `user` and attach an existing policy (built-in or canned) by name.
+async fn provision_user_with_policy(env: &LocalKMSTestEnvironment, user: &str, policy_name: &str) -> TestResult {
+    admin_ok(
+        &env.base_env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+        Some(serde_json::json!({ "secretKey": SECRET, "status": "enabled" }).to_string()),
+    )
+    .await?;
+    admin_ok(
+        &env.base_env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={user}&isGroup=false"),
+        None,
+    )
+    .await?;
+    Ok(())
+}
+
+/// The S3 half of every data-path policy below: full object access, no KMS grant.
+fn s3_full_access_statement() -> serde_json::Value {
+    serde_json::json!({
+        "Effect": "Allow",
+        "Action": ["s3:*"],
+        "Resource": ["arn:aws:s3:::*"]
+    })
+}
+
+fn policy_document(statements: Vec<serde_json::Value>) -> String {
+    serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
+}
+
+async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
+    client
+        .put_object()
+        .bucket(BUCKET)
+        .key(key)
+        .body(ByteStream::from_static(PAYLOAD))
+        .server_side_encryption(ServerSideEncryption::AwsKms)
+        .ssekms_key_id(kms_key_id)
+        .send()
+        .await
+        .map(|_| ())
+        .map_err(aws_sdk_s3::Error::from)
+}
+
+/// Assert the operation failed with `AccessDenied` rather than any other error.
+///
+/// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
+/// would hide both a leak of key state and an outage masquerading as a denial.
+fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
+    let error = result.expect_err(&format!("{what} must be denied"));
+    assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
+}
+
+/// Retry an SSE-KMS write until the identity's policy has reached the request path.
+async fn wait_for_sse_kms_write(client: &Client, key: &str, kms_key_id: &str) -> TestResult {
+    let deadline = tokio::time::Instant::now() + IAM_PROPAGATION;
+    loop {
+        match put_sse_kms(client, key, kms_key_id).await {
+            Ok(()) => return Ok(()),
+            Err(error) if tokio::time::Instant::now() >= deadline => {
+                return Err(format!("positive control never became authorized: {error:?}").into());
+            }
+            Err(_) => tokio::time::sleep(Duration::from_millis(500)).await,
+        }
+    }
+}
+
+/// Retry an admin call until it stops returning 403, i.e. the policy is live.
+async fn wait_for_admin_success(
+    env: &LocalKMSTestEnvironment,
+    user: &str,
+    method: http::Method,
+    path: &str,
+    body: Option<String>,
+) -> TestResult {
+    let deadline = tokio::time::Instant::now() + IAM_PROPAGATION;
+    loop {
+        let (status, response) = admin_request(&env.base_env.url, method.clone(), path, body.clone(), user, SECRET).await?;
+        if status.is_success() {
+            return Ok(());
+        }
+        if tokio::time::Instant::now() >= deadline {
+            return Err(format!("positive control never became authorized: {method} {path} -> {status} {response}").into());
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+}
+
+async fn assert_admin_denied(
+    env: &LocalKMSTestEnvironment,
+    user: &str,
+    method: http::Method,
+    path: &str,
+    body: Option<String>,
+    what: &str,
+) -> TestResult {
+    let (status, response) = admin_request(&env.base_env.url, method, path, body, user, SECRET).await?;
+    assert_eq!(status.as_u16(), 403, "{what} must be denied, got {status}: {response}");
+    assert!(response.contains("AccessDenied"), "{what} must carry AccessDenied: {response}");
+    Ok(())
+}
+
+fn disable_body(key_id: &str) -> String {
+    serde_json::json!({ "key_id": key_id }).to_string()
+}
+
+/// Data-path matrix: SSE-KMS writes and reads are authorized against the resolved key.
+#[tokio::test]
+#[serial]
+async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
+    init_logging();
+
+    let mut env = LocalKMSTestEnvironment::new().await?;
+    start_enforcing_server(&mut env, &[("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true")]).await?;
+    env.base_env.create_test_bucket(BUCKET).await?;
+
+    // Scoped to ALLOWED_KEY only.
+    provision_user(
+        &env,
+        "kmsmatrixscoped",
+        &policy_document(vec![
+            s3_full_access_statement(),
+            serde_json::json!({
+                "Effect": "Allow",
+                "Action": ["kms:GenerateDataKey", "kms:Decrypt"],
+                "Resource": [format!("arn:aws:kms:::key/{ALLOWED_KEY}")]
+            }),
+        ]),
+    )
+    .await?;
+    // S3 rights only: the identity shape that existed before per-key authorization.
+    provision_user(&env, "kmsmatrixs3only", &policy_document(vec![s3_full_access_statement()])).await?;
+    // May wrap a data key but may never unwrap one.
+    provision_user(
+        &env,
+        "kmsmatrixwriter",
+        &policy_document(vec![
+            s3_full_access_statement(),
+            serde_json::json!({
+                "Effect": "Allow",
+                "Action": ["kms:GenerateDataKey"],
+                "Resource": ["arn:aws:kms:::*"]
+            }),
+        ]),
+    )
+    .await?;
+    // Wildcard allow, explicit deny on one key.
+    provision_user(
+        &env,
+        "kmsmatrixdenied",
+        &policy_document(vec![
+            s3_full_access_statement(),
+            serde_json::json!({
+                "Effect": "Allow",
+                "Action": ["kms:*"],
+                "Resource": ["arn:aws:kms:::*"]
+            }),
+            serde_json::json!({
+                "Effect": "Deny",
+                "Action": ["kms:*"],
+                "Resource": [format!("arn:aws:kms:::key/{OTHER_KEY}")]
+            }),
+        ]),
+    )
+    .await?;
+
+    let scoped = s3_client(&env.base_env.url, "kmsmatrixscoped", SECRET);
+    let s3_only = s3_client(&env.base_env.url, "kmsmatrixs3only", SECRET);
+    let writer = s3_client(&env.base_env.url, "kmsmatrixwriter", SECRET);
+    let denied = s3_client(&env.base_env.url, "kmsmatrixdenied", SECRET);
+
+    // --- positive control -----------------------------------------------------
+    wait_for_sse_kms_write(&scoped, "scoped/allowed", ALLOWED_KEY).await?;
+    let read = scoped.get_object().bucket(BUCKET).key("scoped/allowed").send().await?;
+    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), PAYLOAD);
+    info!("positive control: scoped identity may write and read under its own key");
+
+    // --- wrong key ------------------------------------------------------------
+    assert_access_denied(
+        put_sse_kms(&scoped, "scoped/other", OTHER_KEY).await,
+        "SSE-KMS write under a key outside the identity's scope",
+    );
+
+    // --- wrong identity -------------------------------------------------------
+    assert_access_denied(
+        put_sse_kms(&s3_only, "s3only/allowed", ALLOWED_KEY).await,
+        "SSE-KMS write by an identity holding no kms grant",
+    );
+    // The object the scoped identity wrote is readable by its owner only.
+    assert_access_denied(
+        s3_only
+            .get_object()
+            .bucket(BUCKET)
+            .key("scoped/allowed")
+            .send()
+            .await
+            .map(|_| ())
+            .map_err(aws_sdk_s3::Error::from),
+        "SSE-KMS read by an identity holding no kms grant",
+    );
+
+    // --- wrong action ---------------------------------------------------------
+    wait_for_sse_kms_write(&writer, "writer/allowed", ALLOWED_KEY).await?;
+    assert_access_denied(
+        writer
+            .get_object()
+            .bucket(BUCKET)
+            .key("writer/allowed")
+            .send()
+            .await
+            .map(|_| ())
+            .map_err(aws_sdk_s3::Error::from),
+        "SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
+    );
+
+    // --- wrong context: explicit Deny beats a wildcard Allow -------------------
+    wait_for_sse_kms_write(&denied, "denied/allowed", ALLOWED_KEY).await?;
+    assert_access_denied(
+        put_sse_kms(&denied, "denied/other", OTHER_KEY).await,
+        "SSE-KMS write under a key covered by an explicit Deny",
+    );
+
+    // --- wrong context: SSE-S3 is out of scope --------------------------------
+    // SSE-S3 wraps its data key with a server-owned key the caller never names, so
+    // it must stay reachable for an identity with no kms grant at all.
+    s3_only
+        .put_object()
+        .bucket(BUCKET)
+        .key("s3only/sse-s3")
+        .body(ByteStream::from_static(PAYLOAD))
+        .server_side_encryption(ServerSideEncryption::Aes256)
+        .send()
+        .await?;
+    let sse_s3_read = s3_only.get_object().bucket(BUCKET).key("s3only/sse-s3").send().await?;
+    assert_eq!(sse_s3_read.body.collect().await?.into_bytes().as_ref(), PAYLOAD);
+
+    // ... and so must an unencrypted object.
+    s3_only
+        .put_object()
+        .bucket(BUCKET)
+        .key("s3only/plain")
+        .body(ByteStream::from_static(PAYLOAD))
+        .send()
+        .await?;
+    s3_only.get_object().bucket(BUCKET).key("s3only/plain").send().await?;
+
+    Ok(())
+}
+
+/// Admin-plane matrix: KMS key endpoints are authorized against the key they name.
+///
+/// Runs without the SSE enforcement switch: admin scoping is unconditional, and
+/// leaving the switch off proves the two planes are independent.
+#[tokio::test]
+#[serial]
+async fn kms_admin_per_key_authorization_negative_matrix() -> TestResult {
+    init_logging();
+
+    let mut env = LocalKMSTestEnvironment::new().await?;
+    start_enforcing_server(&mut env, &[]).await?;
+
+    // Built-in role templates, attached by name.
+    provision_user_with_policy(&env, "kmsmatrixkeyadmin", "KMSKeyAdministrator").await?;
+    provision_user_with_policy(&env, "kmsmatrixauditor", "KMSAuditor").await?;
+    // A narrowed copy of the administrator template, scoped to one key.
+    provision_user(
+        &env,
+        "kmsmatrixscopedadmin",
+        &policy_document(vec![serde_json::json!({
+            "Effect": "Allow",
+            "Action": ["kms:DisableKey", "kms:EnableKey"],
+            "Resource": [format!("arn:aws:kms:::key/{ALLOWED_KEY}")]
+        })]),
+    )
+    .await?;
+
+    // --- positive control -----------------------------------------------------
+    wait_for_admin_success(
+        &env,
+        "kmsmatrixkeyadmin",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/disable",
+        Some(disable_body(OTHER_KEY)),
+    )
+    .await?;
+    admin_request(
+        &env.base_env.url,
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/enable",
+        Some(disable_body(OTHER_KEY)),
+        "kmsmatrixkeyadmin",
+        SECRET,
+    )
+    .await?;
+
+    // --- wrong action: the administrator template withholds service-wide powers -
+    assert_admin_denied(
+        &env,
+        "kmsmatrixkeyadmin",
+        http::Method::GET,
+        "/rustfs/admin/v3/kms/config",
+        None,
+        "KMSKeyAdministrator reading the KMS backend configuration (kms:Configure)",
+    )
+    .await?;
+    assert_admin_denied(
+        &env,
+        "kmsmatrixkeyadmin",
+        http::Method::GET,
+        "/rustfs/admin/v3/kms/backup",
+        None,
+        "KMSKeyAdministrator exporting a backup bundle (kms:Backup)",
+    )
+    .await?;
+    // Separation of duties: managing a key never implies using it.
+    assert_admin_denied(
+        &env,
+        "kmsmatrixkeyadmin",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/generate-data-key",
+        Some(serde_json::json!({ "key_id": ALLOWED_KEY }).to_string()),
+        "KMSKeyAdministrator generating a data key (kms:GenerateDataKey)",
+    )
+    .await?;
+
+    // --- wrong action: the auditor template is read-only ----------------------
+    wait_for_admin_success(&env, "kmsmatrixauditor", http::Method::GET, "/rustfs/admin/v3/kms/keys", None).await?;
+    assert_admin_denied(
+        &env,
+        "kmsmatrixauditor",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/disable",
+        Some(disable_body(ALLOWED_KEY)),
+        "KMSAuditor disabling a key (kms:DisableKey)",
+    )
+    .await?;
+
+    // --- wrong key ------------------------------------------------------------
+    wait_for_admin_success(
+        &env,
+        "kmsmatrixscopedadmin",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/disable",
+        Some(disable_body(ALLOWED_KEY)),
+    )
+    .await?;
+    assert_admin_denied(
+        &env,
+        "kmsmatrixscopedadmin",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/disable",
+        Some(disable_body(OTHER_KEY)),
+        "key-scoped administrator disabling a key outside its scope",
+    )
+    .await?;
+    // --- wrong action, same key ----------------------------------------------
+    assert_admin_denied(
+        &env,
+        "kmsmatrixscopedadmin",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/rotate",
+        Some(disable_body(ALLOWED_KEY)),
+        "key-scoped administrator rotating a key it may only enable and disable",
+    )
+    .await?;
+
+    // --- wrong identity -------------------------------------------------------
+    provision_user(&env, "kmsmatrixnokms", &policy_document(vec![s3_full_access_statement()])).await?;
+    assert_admin_denied(
+        &env,
+        "kmsmatrixnokms",
+        http::Method::POST,
+        "/rustfs/admin/v3/kms/keys/disable",
+        Some(disable_body(ALLOWED_KEY)),
+        "identity holding no kms grant disabling a key",
+    )
+    .await?;
+
+    Ok(())
+}

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -417,6 +417,22 @@ async fn test_vault_kms_key_crud(
 
     info!("✅ Read: Successfully listed keys, found test key");
 
+    // A waiting window outside 7-30 days is refused at the endpoint for this
+    // backend too: the bound is enforced once in the service (rustfs/backlog#1585).
+    for days in [6, 31] {
+        let window_error = crate::common::execute_awscurl(
+            &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&pending_window_in_days={days}"),
+            "DELETE",
+            None,
+            access_key,
+            secret_key,
+        )
+        .await
+        .err()
+        .ok_or_else(|| format!("A {days}-day deletion window must be refused"))?;
+        info!("✅ Delete window {} correctly refused: {}", days, window_error);
+    }
+
     // Delete
     let delete_response = crate::common::execute_awscurl(
         &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}"),
@@ -449,9 +465,10 @@ async fn test_vault_kms_key_crud(
 
     info!("✅ Delete verification: Key state correctly changed to: {}", key_state);
 
-    // Force Delete - a default server refuses to skip the waiting window
-    // (rustfs/backlog#1585): destroying the key material immediately would take
-    // every object encrypted under the key with it.
+    // Force Delete - the query string can no longer ask for immediate deletion,
+    // and a default server refuses it in any case (rustfs/backlog#1585):
+    // destroying the key material immediately would take every object encrypted
+    // under the key with it.
     let force_delete_error = crate::common::execute_awscurl(
         &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
         "DELETE",

crates/e2e_test/src/kms/mod.rs

@@ -53,3 +53,6 @@ mod copy_object_version_restore_sse_test;
 
 #[cfg(test)]
 mod configured_roundtrip_test;
+
+#[cfg(test)]
+mod kms_authorization_negative_matrix_test;

crates/e2e_test/src/replication_extension_test.rs

@@ -1507,6 +1507,10 @@ async fn build_sse_replication_pair(
             ("RUSTFS_KMS_KEY_DIR", source_kms_key_dir.as_str()),
             ("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID),
             ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
+            // Per-key KMS authorization is on so this contract is pinned in the
+            // configuration replication will eventually ship with: the replication
+            // worker carries no request identity and must stay exempt.
+            ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
         ]);
     }
     source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
@@ -1519,6 +1523,7 @@ async fn build_sse_replication_pair(
             ("RUSTFS_KMS_KEY_DIR", target_kms_key_dir.as_str()),
             ("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID),
             ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
+            ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
         ]);
     }
     target_env

crates/ecstore/src/api/mod.rs

@@ -173,8 +173,9 @@ pub mod bucket {
 
     pub mod replication {
         pub use crate::bucket::replication::replication_pool::{
-            DurableMrfBacklogSummary, DurableMrfBucketBacklog, MrfBacklogObservabilitySummary, MrfBucketBacklogObservability,
-            durable_mrf_backlog_summary_snapshot, mrf_backlog_observability_snapshot,
+            DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
+            MrfBucketBacklogObservability, durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot,
+            mrf_backlog_observability_snapshot,
         };
         pub use crate::bucket::replication::{
             BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
@@ -183,13 +184,13 @@ pub mod bucket {
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
             ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
-            ReplicationType, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus, VersionPurgeStatusType,
-            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
-            get_global_replication_stats, init_background_replication, read_durable_mrf_backlog, replication_state_to_filemeta,
-            replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
-            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
-            should_use_existing_delete_replication_source, unsupported_replication_config_field,
-            validate_replication_config_target_arns, version_purge_status_to_filemeta,
+            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
+            VersionPurgeStatusType, delete_replication_state_from_config, delete_replication_version_id,
+            get_global_replication_pool, get_global_replication_stats, init_background_replication, read_durable_mrf_backlog,
+            replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
+            resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
+            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
+            unsupported_replication_config_field, validate_replication_config_target_arns, version_purge_status_to_filemeta,
         };
     }
 

crates/ecstore/src/bucket/replication/mod.rs

@@ -78,7 +78,7 @@ pub use replication_queue_boundary::{
 };
 pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
 pub use replication_scanner_bridge::ReplicationScannerBridge;
-pub use replication_state::ReplicationStats;
+pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
 pub use replication_stats_boundary::BucketStats;
 pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
 pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -86,12 +86,26 @@ pub struct DurableMrfBucketBacklog {
     pub bytes: u64,
 }
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct DurableMrfTargetBacklog {
+    pub bucket: String,
+    pub target_arn: String,
+    pub count: u64,
+    pub bytes: u64,
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct DurableMrfBacklogSummary {
     pub available: bool,
     pub buckets: Vec<DurableMrfBucketBacklog>,
 }
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+struct DurableMrfBacklogSnapshot {
+    summary: DurableMrfBacklogSummary,
+    targets: Vec<DurableMrfTargetBacklog>,
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct MrfBucketBacklogObservability {
     pub bucket: String,
@@ -110,6 +124,8 @@ pub struct MrfBacklogObservabilitySummary {
 
 static DURABLE_MRF_BACKLOG_SUMMARY: LazyLock<StdRwLock<DurableMrfBacklogSummary>> =
     LazyLock::new(|| StdRwLock::new(DurableMrfBacklogSummary::default()));
+static DURABLE_MRF_TARGET_BACKLOG: LazyLock<StdRwLock<Vec<DurableMrfTargetBacklog>>> =
+    LazyLock::new(|| StdRwLock::new(Vec::new()));
 static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTracker>> =
     LazyLock::new(|| StdRwLock::new(MrfBacklogObservabilityTracker::default()));
 
@@ -117,13 +133,15 @@ static MRF_BACKLOG_OBSERVABILITY: LazyLock<StdRwLock<MrfBacklogObservabilityTrac
 struct DurableMrfBacklogTracker {
     available: bool,
     buckets: HashMap<String, DurableMrfBucketBacklog>,
+    targets: HashMap<(String, String), DurableMrfTargetBacklog>,
 }
 
 impl DurableMrfBacklogTracker {
-    fn add_entry(&mut self, bucket_name: String, entry_size: i64) {
-        let Ok(size) = u64::try_from(entry_size) else {
+    fn add_entry(&mut self, entry: &MrfReplicateEntry) {
+        let Ok(size) = u64::try_from(entry.size) else {
             self.available = false;
             self.buckets.clear();
+            self.targets.clear();
             return;
         };
 
@@ -131,6 +149,7 @@ impl DurableMrfBacklogTracker {
             return;
         }
 
+        let bucket_name = entry.bucket.clone();
         let bucket = match self.buckets.entry(bucket_name) {
             Entry::Occupied(entry) => entry.into_mut(),
             Entry::Vacant(entry) => {
@@ -143,16 +162,39 @@ impl DurableMrfBacklogTracker {
         };
         bucket.count = bucket.count.saturating_add(1);
         bucket.bytes = bucket.bytes.saturating_add(size);
+
+        for target_arn in &entry.target_arns {
+            if target_arn.is_empty() {
+                continue;
+            }
+            let key = (entry.bucket.clone(), target_arn.clone());
+            let target = match self.targets.entry(key) {
+                Entry::Occupied(entry) => entry.into_mut(),
+                Entry::Vacant(entry) => {
+                    let (bucket, target_arn) = entry.key().clone();
+                    entry.insert(DurableMrfTargetBacklog {
+                        bucket,
+                        target_arn,
+                        ..Default::default()
+                    })
+                }
+            };
+            target.count = target.count.saturating_add(1);
+            target.bytes = target.bytes.saturating_add(size);
+        }
     }
 
-    fn into_summary(self) -> DurableMrfBacklogSummary {
+    fn into_snapshot(self) -> DurableMrfBacklogSnapshot {
         if !self.available {
-            return DurableMrfBacklogSummary::default();
+            return DurableMrfBacklogSnapshot::default();
         }
 
-        DurableMrfBacklogSummary {
-            available: true,
-            buckets: self.buckets.into_values().collect(),
+        DurableMrfBacklogSnapshot {
+            summary: DurableMrfBacklogSummary {
+                available: true,
+                buckets: self.buckets.into_values().collect(),
+            },
+            targets: self.targets.into_values().collect(),
         }
     }
 }
@@ -218,7 +260,21 @@ impl MrfBacklogObservabilityTracker {
     }
 }
 
-fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSummary
+fn durable_mrf_backlog_summary_from_entries<'a>(
+    entries: impl IntoIterator<Item = &'a MrfReplicateEntry>,
+) -> DurableMrfBacklogSnapshot {
+    let mut tracker = DurableMrfBacklogTracker {
+        available: true,
+        ..Default::default()
+    };
+    for entry in entries {
+        tracker.add_entry(entry);
+    }
+    tracker.into_snapshot()
+}
+
+#[cfg(test)]
+fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSnapshot
 where
     I: IntoIterator<Item = (String, i64)>,
 {
@@ -227,16 +283,38 @@ where
         ..Default::default()
     };
     for (bucket_name, entry_size) in entries {
-        tracker.add_entry(bucket_name, entry_size);
+        tracker.add_entry(&MrfReplicateEntry {
+            bucket: bucket_name,
+            object: String::new(),
+            version_id: None,
+            retry_count: 0,
+            size: entry_size,
+            op: MrfOpKind::Object,
+            delete_marker_version_id: None,
+            delete_marker: false,
+            delete_marker_mtime: None,
+            target_arns: Vec::new(),
+        });
+    }
+    tracker.into_snapshot()
+}
+
+fn set_durable_mrf_backlog_snapshot(snapshot: DurableMrfBacklogSnapshot) {
+    match DURABLE_MRF_BACKLOG_SUMMARY.write() {
+        Ok(mut guard) => *guard = snapshot.summary,
+        Err(poisoned) => *poisoned.into_inner() = snapshot.summary,
+    }
+    match DURABLE_MRF_TARGET_BACKLOG.write() {
+        Ok(mut guard) => *guard = snapshot.targets,
+        Err(poisoned) => *poisoned.into_inner() = snapshot.targets,
     }
-    tracker.into_summary()
 }
 
 fn set_durable_mrf_backlog_summary(summary: DurableMrfBacklogSummary) {
-    match DURABLE_MRF_BACKLOG_SUMMARY.write() {
-        Ok(mut guard) => *guard = summary,
-        Err(poisoned) => *poisoned.into_inner() = summary,
-    }
+    set_durable_mrf_backlog_snapshot(DurableMrfBacklogSnapshot {
+        summary,
+        targets: Vec::new(),
+    });
 }
 
 pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
@@ -246,6 +324,13 @@ pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
     }
 }
 
+pub fn durable_mrf_target_backlog_snapshot() -> Vec<DurableMrfTargetBacklog> {
+    match DURABLE_MRF_TARGET_BACKLOG.read() {
+        Ok(guard) => guard.clone(),
+        Err(poisoned) => poisoned.into_inner().clone(),
+    }
+}
+
 pub fn mrf_backlog_observability_snapshot() -> MrfBacklogObservabilitySummary {
     match MRF_BACKLOG_OBSERVABILITY.read() {
         Ok(guard) => guard.snapshot(),
@@ -675,6 +760,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica task
     pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
+        let target_arns = ri.dsc.replicate_target_arns();
         // If object is large, queue it to a static set of large workers
         if should_queue_large_object(ri.size) {
             use std::collections::hash_map::DefaultHasher;
@@ -691,10 +777,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
                 if let Some(worker) = lrg_workers.get(index) {
                     self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
+                    self.stats.inc_target_q(&ri.bucket, &target_arns, ri.size);
                     if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
                         return ReplicationQueueAdmission::Queued;
                     }
                     self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
+                    self.stats.dec_target_q(&ri.bucket, &target_arns, ri.size);
 
                     // Try to add more workers if possible
                     let max_l_workers = *self.max_l_workers.read().await;
@@ -723,10 +811,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         };
 
         self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
+        self.stats.inc_target_q(&ri.bucket, &target_arns, ri.size);
         if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
             return ReplicationQueueAdmission::Queued;
         }
         self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
+        self.stats.dec_target_q(&ri.bucket, &target_arns, ri.size);
 
         // Queue to MRF if all workers are busy.
         let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
@@ -740,6 +830,11 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues a replica delete task
     pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission {
+        let target_arns = if doi.target_arn.is_empty() {
+            Vec::new()
+        } else {
+            vec![doi.target_arn.clone()]
+        };
         let ch = self
             .worker_queue_channel(&doi.op_type, &doi.bucket, &doi.delete_object.object_name, 0)
             .await;
@@ -749,10 +844,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         };
 
         self.stats.inc_q(&doi.bucket, 0, true, doi.op_type);
+        self.stats.inc_target_q(&doi.bucket, &target_arns, 0);
         if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
             return ReplicationQueueAdmission::Queued;
         }
         self.stats.dec_q(&doi.bucket, 0, true, doi.op_type);
+        self.stats.dec_target_q(&doi.bucket, &target_arns, 0);
 
         let admission = self.queue_mrf_save_admission(doi.to_mrf_entry(), "delete").await;
 
@@ -771,9 +868,11 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         let bucket = entry.bucket.clone();
         let size = entry.size;
         let is_delete = matches!(entry.op, MrfOpKind::Delete);
+        let target_arns = entry.target_arns.clone();
         let admission = queue_mrf_save_entry(&self.mrf_save_tx, entry, queue_type).await;
         if admission == ReplicationQueueAdmission::Queued {
             self.stats.inc_q(&bucket, size, is_delete, ReplicationType::Heal);
+            self.stats.inc_target_q(&bucket, &target_arns, size);
         }
         admission
     }
@@ -827,9 +926,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     return;
                 }
             };
-            set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
-                entries.iter().map(|entry| (entry.bucket.clone(), entry.size)),
-            ));
+            set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
 
             let total = entries.len();
             let mut queued_count = 0usize;
@@ -1018,7 +1115,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                                 }
                                 continue;
                             }
-                            durable_tracker.add_entry(e.bucket.clone(), e.size);
+                            durable_tracker.add_entry(&e);
                             observe_mrf_pending(&e);
                             pending.push(e);
                             dirty = true;
@@ -1029,7 +1126,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                             if pending.len() - flushed_len >= 1000
                                 && let Some(duration_millis) = flush_mrf_to_disk(&pending, &storage).await
                             {
-                                set_durable_mrf_backlog_summary(durable_tracker.clone().into_summary());
+                                set_durable_mrf_backlog_snapshot(durable_tracker.clone().into_snapshot());
                                 observe_mrf_pending_flushed(&pending[flushed_len..], duration_millis);
                                 dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
                                 flushed_len = pending.len();
@@ -1039,7 +1136,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                         None => {
                             // Channel closed (pool shutting down) — final flush.
                             if dirty && let Some(duration_millis) = flush_mrf_to_disk(&pending, &storage).await {
-                                set_durable_mrf_backlog_summary(durable_tracker.clone().into_summary());
+                                set_durable_mrf_backlog_snapshot(durable_tracker.clone().into_snapshot());
                                 observe_mrf_pending_flushed(&pending[flushed_len..], duration_millis);
                                 dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
                             }
@@ -1048,7 +1145,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     },
                     _ = interval.tick() => {
                         if dirty && let Some(duration_millis) = flush_mrf_to_disk(&pending, &storage).await {
-                            set_durable_mrf_backlog_summary(durable_tracker.clone().into_summary());
+                            set_durable_mrf_backlog_snapshot(durable_tracker.clone().into_snapshot());
                             observe_mrf_pending_flushed(&pending[flushed_len..], duration_millis);
                             dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
                             flushed_len = pending.len();
@@ -1451,6 +1548,7 @@ struct ReplicationBacklogGuard {
     size: i64,
     is_delete_marker: bool,
     op_type: ReplicationType,
+    target_arns: Vec<String>,
 }
 
 impl ReplicationBacklogGuard {
@@ -1461,16 +1559,23 @@ impl ReplicationBacklogGuard {
             size: object.size,
             is_delete_marker: object.delete_marker,
             op_type: object.op_type,
+            target_arns: object.dsc.replicate_target_arns(),
         }
     }
 
     fn for_delete(stats: Arc<ReplicationStats>, delete: &DeletedObjectReplicationInfo) -> Self {
+        let target_arns = if delete.target_arn.is_empty() {
+            Vec::new()
+        } else {
+            vec![delete.target_arn.clone()]
+        };
         Self {
             stats,
             bucket: delete.bucket.clone(),
             size: 0,
             is_delete_marker: true,
             op_type: delete.op_type,
+            target_arns,
         }
     }
 }
@@ -1478,6 +1583,7 @@ impl ReplicationBacklogGuard {
 impl Drop for ReplicationBacklogGuard {
     fn drop(&mut self) {
         self.stats.dec_q(&self.bucket, self.size, self.is_delete_marker, self.op_type);
+        self.stats.dec_target_q(&self.bucket, &self.target_arns, self.size);
     }
 }
 
@@ -1524,6 +1630,7 @@ async fn queue_mrf_save_entry(
 fn dec_mrf_entries(stats: &ReplicationStats, entries: &[MrfReplicateEntry]) {
     for entry in entries {
         stats.dec_q(&entry.bucket, entry.size, matches!(entry.op, MrfOpKind::Delete), ReplicationType::Heal);
+        stats.dec_target_q(&entry.bucket, &entry.target_arns, entry.size);
     }
 }
 
@@ -1915,6 +2022,7 @@ async fn queue_replicate_deletes(batch: ReplicationHealResyncDeletes) -> Replica
 
 #[cfg(test)]
 mod tests {
+    use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
     use super::super::replication_resync_boundary::{decode_mrf_file, encode_mrf_file, encode_resync_file};
     use super::super::replication_storage_boundary::{
         DeletedObject, FileInfo, GetObjectReader, HTTPRangeSpec, ListOperations, ObjectIO, ObjectOperations, PutObjReader,
@@ -2260,6 +2368,22 @@ mod tests {
         (stats.replication_stats.q_stat.curr.count, stats.replication_stats.q_stat.curr.bytes)
     }
 
+    fn current_target_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, target_arn: &str) -> Option<(u64, u64)> {
+        pool.stats
+            .runtime_target_backlog_snapshot()
+            .into_iter()
+            .find(|target| target.bucket == bucket && target.target_arn == target_arn)
+            .map(|target| (target.count, target.bytes))
+    }
+
+    fn test_replicate_decision(target_arns: &[&str]) -> ReplicateDecision {
+        let mut decision = ReplicateDecision::default();
+        for target_arn in target_arns {
+            decision.set(ReplicateTargetDecision::new((*target_arn).to_string(), true, false));
+        }
+        decision
+    }
+
     async fn wait_for_current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, expected: (i64, i64)) {
         tokio::time::timeout(Duration::from_secs(10), async {
             loop {
@@ -2293,6 +2417,35 @@ mod tests {
         assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
     }
 
+    #[tokio::test]
+    async fn regular_worker_admission_counts_target_backlog_before_receive() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, _rx) = mpsc::channel(1);
+        pool.workers.write().await.push(tx);
+
+        let admission = pool
+            .queue_replica_task(ReplicateObjectInfo {
+                bucket: "target-admission-bucket".to_string(),
+                name: "object".to_string(),
+                size: 4096,
+                op_type: ReplicationType::Object,
+                dsc: test_replicate_decision(&["arn:rustfs:replication:target-b", "arn:rustfs:replication:target-a"]),
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        assert_eq!(current_queue(&pool, "target-admission-bucket").await, (1, 4096));
+        assert_eq!(
+            current_target_queue(&pool, "target-admission-bucket", "arn:rustfs:replication:target-a"),
+            Some((1, 4096))
+        );
+        assert_eq!(
+            current_target_queue(&pool, "target-admission-bucket", "arn:rustfs:replication:target-b"),
+            Some((1, 4096))
+        );
+    }
+
     #[tokio::test]
     async fn large_worker_admission_counts_channel_backlog_before_receive() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -2336,6 +2489,33 @@ mod tests {
         assert_eq!(current_queue(&pool, "delete-admission-bucket").await, (1, 0));
     }
 
+    #[tokio::test]
+    async fn delete_admission_counts_target_backlog_before_receive() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, _rx) = mpsc::channel(1);
+        pool.workers.write().await.push(tx);
+
+        let admission = pool
+            .queue_replica_delete_task(DeletedObjectReplicationInfo {
+                bucket: "delete-target-admission-bucket".to_string(),
+                target_arn: "arn:rustfs:replication:target-a".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "deleted-object".to_string(),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        assert_eq!(current_queue(&pool, "delete-target-admission-bucket").await, (1, 0));
+        assert_eq!(
+            current_target_queue(&pool, "delete-target-admission-bucket", "arn:rustfs:replication:target-a"),
+            Some((1, 0))
+        );
+    }
+
     #[tokio::test]
     async fn regular_worker_drains_current_backlog_after_processing() {
         let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
@@ -2702,6 +2882,7 @@ mod tests {
                 name: "fallback-object".to_string(),
                 size: 2048,
                 op_type: ReplicationType::Object,
+                dsc: test_replicate_decision(&["arn:rustfs:replication:target-a"]),
                 ..Default::default()
             })
             .await;
@@ -2710,6 +2891,10 @@ mod tests {
         let queued = pool.stats.get_latest_replication_stats("runtime-backlog").await;
         assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
         assert_eq!(queued.replication_stats.q_stat.curr.bytes, 2048);
+        assert_eq!(
+            current_target_queue(&pool, "runtime-backlog", "arn:rustfs:replication:target-a"),
+            Some((1, 2048))
+        );
     }
 
     #[test]
@@ -2745,6 +2930,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
         let second = MrfReplicateEntry {
             object: "second".to_string(),
@@ -2794,6 +2980,7 @@ mod tests {
                 delete_marker_version_id: None,
                 delete_marker: false,
                 delete_marker_mtime: None,
+                target_arns: Vec::new(),
             },
             "test",
         )
@@ -2824,6 +3011,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
         observe_mrf_pending(&entry);
 
@@ -2845,12 +3033,14 @@ mod tests {
     async fn replication_backlog_guard_decrements_on_drop() {
         let stats = Arc::new(ReplicationStats::new());
         stats.inc_q("guard-bucket", 256, false, ReplicationType::Object);
+        stats.inc_target_q("guard-bucket", &["arn:rustfs:replication:target-a".to_string()], 256);
 
         {
             let object = ReplicateObjectInfo {
                 bucket: "guard-bucket".to_string(),
                 size: 256,
                 op_type: ReplicationType::Object,
+                dsc: test_replicate_decision(&["arn:rustfs:replication:target-a"]),
                 ..Default::default()
             };
             let _guard = ReplicationBacklogGuard::for_object(stats.clone(), &object);
@@ -2859,6 +3049,30 @@ mod tests {
         let queued = stats.get_latest_replication_stats("guard-bucket").await;
         assert_eq!(queued.replication_stats.q_stat.curr.count, 0);
         assert_eq!(queued.replication_stats.q_stat.curr.bytes, 0);
+        assert!(stats.runtime_target_backlog_snapshot().is_empty());
+    }
+
+    #[test]
+    fn dec_mrf_entries_decrements_target_backlog() {
+        let stats = ReplicationStats::new();
+        let entry = MrfReplicateEntry {
+            bucket: "mrf-target-drain-bucket".to_string(),
+            object: "object".to_string(),
+            version_id: None,
+            retry_count: 1,
+            size: 1024,
+            op: MrfOpKind::Object,
+            delete_marker_version_id: None,
+            delete_marker: false,
+            delete_marker_mtime: None,
+            target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
+        };
+
+        stats.inc_q(&entry.bucket, entry.size, false, ReplicationType::Heal);
+        stats.inc_target_q(&entry.bucket, &entry.target_arns, entry.size);
+        dec_mrf_entries(&stats, std::slice::from_ref(&entry));
+
+        assert!(stats.runtime_target_backlog_snapshot().is_empty());
     }
 
     #[test]
@@ -2873,6 +3087,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
         let second = MrfReplicateEntry {
             object: "second".to_string(),
@@ -2984,6 +3199,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
 
         let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
@@ -3017,6 +3233,7 @@ mod tests {
             delete_marker_version_id: Some(dm_vid),
             delete_marker: true,
             delete_marker_mtime: Some(mtime_nanos),
+            target_arns: Vec::new(),
         };
 
         let encoded = encode_mrf_file(std::slice::from_ref(&entry)).expect("encode");
@@ -3050,6 +3267,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
 
         let encoded = encode_mrf_file(&[entry]).expect("encode");
@@ -3078,6 +3296,7 @@ mod tests {
                 delete_marker_version_id: None,
                 delete_marker: false,
                 delete_marker_mtime: None,
+                target_arns: Vec::new(),
             },
             MrfReplicateEntry {
                 bucket: "b".to_string(),
@@ -3089,6 +3308,7 @@ mod tests {
                 delete_marker_version_id: Some(del_dm_vid),
                 delete_marker: true,
                 delete_marker_mtime: None,
+                target_arns: Vec::new(),
             },
         ];
 
@@ -3118,6 +3338,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
         assert_eq!(obj_entry.op, MrfOpKind::Object);
 
@@ -3132,6 +3353,7 @@ mod tests {
             delete_marker_version_id: Some(Uuid::new_v4()),
             delete_marker: true,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
         assert_eq!(del_entry.op, MrfOpKind::Delete);
 
@@ -3147,6 +3369,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         };
         assert_eq!(legacy_entry.op, MrfOpKind::Object, "legacy default must be Object");
     }
@@ -3196,6 +3419,7 @@ mod tests {
         // The "deleteMarkerMtime" key was absent in old files — #[serde(default)] must fill in
         // None so replay falls back to the current time (backlog#867 backward compatibility).
         assert_eq!(entry.delete_marker_mtime, None, "missing deleteMarkerMtime key must default to None");
+        assert!(entry.target_arns.is_empty(), "old MRF entries must not be attributed to a target");
     }
 
     #[test]
@@ -3210,6 +3434,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         }];
         let encoded = encode_mrf_file(&entries).expect("durable MRF backlog should encode");
 
@@ -3226,9 +3451,10 @@ mod tests {
 
     #[test]
     fn durable_mrf_summary_aggregates_entries_by_bucket_for_obs() {
-        let summary =
+        let snapshot =
             durable_mrf_backlog_summary_from_sizes([("b1".to_string(), 1024), ("b1".to_string(), 512), ("b2".to_string(), 0)]);
 
+        let summary = snapshot.summary;
         assert!(summary.available);
         let buckets = summary
             .buckets
@@ -3239,13 +3465,82 @@ mod tests {
         assert_eq!(buckets["b1"].bytes, 1536);
         assert_eq!(buckets["b2"].count, 1);
         assert_eq!(buckets["b2"].bytes, 0);
+        assert!(snapshot.targets.is_empty());
+    }
+
+    #[test]
+    fn durable_mrf_summary_aggregates_target_backlog_without_attributing_legacy_entries() {
+        let entries = vec![
+            MrfReplicateEntry {
+                bucket: "b1".to_string(),
+                object: "object-a".to_string(),
+                version_id: None,
+                retry_count: 0,
+                size: 1024,
+                op: MrfOpKind::Object,
+                delete_marker_version_id: None,
+                delete_marker: false,
+                delete_marker_mtime: None,
+                target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
+            },
+            MrfReplicateEntry {
+                bucket: "b1".to_string(),
+                object: "object-b".to_string(),
+                version_id: None,
+                retry_count: 0,
+                size: 512,
+                op: MrfOpKind::Object,
+                delete_marker_version_id: None,
+                delete_marker: false,
+                delete_marker_mtime: None,
+                target_arns: vec!["arn:target-a".to_string()],
+            },
+            MrfReplicateEntry {
+                bucket: "b1".to_string(),
+                object: "legacy-object".to_string(),
+                version_id: None,
+                retry_count: 0,
+                size: 256,
+                op: MrfOpKind::Object,
+                delete_marker_version_id: None,
+                delete_marker: false,
+                delete_marker_mtime: None,
+                target_arns: Vec::new(),
+            },
+        ];
+
+        let snapshot = durable_mrf_backlog_summary_from_entries(&entries);
+
+        let summary = snapshot.summary;
+        assert!(summary.available);
+        let buckets = summary
+            .buckets
+            .into_iter()
+            .map(|bucket| (bucket.bucket.clone(), bucket))
+            .collect::<HashMap<_, _>>();
+        assert_eq!(buckets["b1"].count, 3);
+        assert_eq!(buckets["b1"].bytes, 1792);
+
+        let targets = snapshot
+            .targets
+            .into_iter()
+            .map(|target| ((target.bucket.clone(), target.target_arn.clone()), target))
+            .collect::<HashMap<_, _>>();
+        let target_a = &targets[&("b1".to_string(), "arn:target-a".to_string())];
+        assert_eq!(target_a.count, 2);
+        assert_eq!(target_a.bytes, 1536);
+        let target_b = &targets[&("b1".to_string(), "arn:target-b".to_string())];
+        assert_eq!(target_b.count, 1);
+        assert_eq!(target_b.bytes, 1024);
     }
 
     #[test]
     fn durable_mrf_summary_marks_invalid_sizes_unavailable() {
         let invalid = durable_mrf_backlog_summary_from_sizes([("bucket".to_string(), -1)]);
-        assert!(!invalid.available);
-        assert!(invalid.buckets.is_empty());
+        let summary = invalid.summary;
+        assert!(!summary.available);
+        assert!(summary.buckets.is_empty());
+        assert!(invalid.targets.is_empty());
     }
 
     #[test]
@@ -3264,6 +3559,7 @@ mod tests {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: Vec::new(),
         }])
         .expect("invalid persisted entry should still encode for boundary testing");
         let invalid = durable_mrf_backlog_from_read(Ok(negative));

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -22,9 +22,9 @@ use super::replication_stats_boundary::{
     QueueCache, ReplicationMetricScope, SRMetricsSummary, XferStats,
 };
 use super::runtime_boundary as runtime_sources;
-use std::collections::HashMap;
+use std::collections::{HashMap, hash_map::Entry};
 use std::sync::atomic::{AtomicI64, Ordering};
-use std::sync::{Arc, Mutex as StdMutex};
+use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
 use std::time::{Duration, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
@@ -153,6 +153,83 @@ pub struct ReplicationStats {
     pub most_recent_stats: Arc<Mutex<HashMap<String, BucketStats>>>,
 }
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct RuntimeReplicationTargetBacklog {
+    pub bucket: String,
+    pub target_arn: String,
+    pub count: u64,
+    pub bytes: u64,
+}
+
+type TargetQueueKey = (String, String);
+type TargetQueueCache = HashMap<TargetQueueKey, InQueueMetric>;
+
+struct TargetQueueCacheSlot {
+    owner: Weak<StdMutex<QueueCache>>,
+    metrics: TargetQueueCache,
+}
+
+impl TargetQueueCacheSlot {
+    fn new(owner: &Arc<StdMutex<QueueCache>>) -> Self {
+        Self {
+            owner: Arc::downgrade(owner),
+            metrics: TargetQueueCache::default(),
+        }
+    }
+
+    fn belongs_to(&self, owner: &Arc<StdMutex<QueueCache>>) -> bool {
+        self.owner.upgrade().is_some_and(|current| Arc::ptr_eq(&current, owner))
+    }
+}
+
+// Keep runtime target counters outside ReplicationStats to preserve its public struct shape.
+static TARGET_QUEUE_CACHES: LazyLock<StdMutex<Vec<TargetQueueCacheSlot>>> = LazyLock::new(|| StdMutex::new(Vec::new()));
+
+fn i64_to_u64_floor_zero(value: i64) -> u64 {
+    u64::try_from(value.max(0)).unwrap_or(0)
+}
+
+fn normalized_target_arns(target_arns: &[String]) -> Vec<&str> {
+    let mut target_arns = target_arns
+        .iter()
+        .map(String::as_str)
+        .filter(|target_arn| !target_arn.is_empty())
+        .collect::<Vec<_>>();
+    target_arns.sort_unstable();
+    target_arns.dedup();
+    target_arns
+}
+
+fn target_queue_cache_snapshot(cache: &TargetQueueCache) -> Vec<RuntimeReplicationTargetBacklog> {
+    cache
+        .iter()
+        .filter_map(|((bucket, target_arn), metric)| {
+            let count = i64_to_u64_floor_zero(metric.curr.get_current_count());
+            let bytes = i64_to_u64_floor_zero(metric.curr.get_current_bytes());
+            (count > 0 || bytes > 0).then(|| RuntimeReplicationTargetBacklog {
+                bucket: bucket.clone(),
+                target_arn: target_arn.clone(),
+                count,
+                bytes,
+            })
+        })
+        .collect()
+}
+
+fn prune_stale_target_queue_caches(caches: &mut Vec<TargetQueueCacheSlot>) {
+    caches.retain(|slot| slot.owner.strong_count() > 0);
+}
+
+fn with_target_queue_caches<T>(f: impl FnOnce(&mut Vec<TargetQueueCacheSlot>) -> T) -> T {
+    match TARGET_QUEUE_CACHES.lock() {
+        Ok(mut caches) => f(&mut caches),
+        Err(poisoned) => {
+            let mut caches = poisoned.into_inner();
+            f(&mut caches)
+        }
+    }
+}
+
 impl ReplicationStats {
     pub fn new() -> Self {
         Self {
@@ -684,6 +761,68 @@ impl ReplicationStats {
         }
     }
 
+    pub(crate) fn inc_target_q(&self, bucket: &str, target_arns: &[String], size: i64) {
+        let target_arns = normalized_target_arns(target_arns);
+        if target_arns.is_empty() {
+            return;
+        }
+        with_target_queue_caches(|caches| {
+            prune_stale_target_queue_caches(caches);
+            let slot_index = match caches.iter().position(|slot| slot.belongs_to(&self.q_cache)) {
+                Some(index) => index,
+                None => {
+                    caches.push(TargetQueueCacheSlot::new(&self.q_cache));
+                    caches.len() - 1
+                }
+            };
+            let slot = &mut caches[slot_index];
+            let bucket = bucket.to_string();
+            for target_arn in target_arns {
+                let metric = match slot.metrics.entry((bucket.clone(), target_arn.to_string())) {
+                    Entry::Occupied(entry) => entry.into_mut(),
+                    Entry::Vacant(entry) => entry.insert(InQueueMetric::default()),
+                };
+                metric.curr.add_current(size, 1);
+            }
+        });
+    }
+
+    pub(crate) fn dec_target_q(&self, bucket: &str, target_arns: &[String], size: i64) {
+        let target_arns = normalized_target_arns(target_arns);
+        if target_arns.is_empty() {
+            return;
+        }
+        with_target_queue_caches(|caches| {
+            prune_stale_target_queue_caches(caches);
+            if let Some(slot) = caches.iter_mut().find(|slot| slot.belongs_to(&self.q_cache)) {
+                let bucket = bucket.to_string();
+                for target_arn in target_arns {
+                    if let Some(metric) = slot.metrics.get_mut(&(bucket.clone(), target_arn.to_string())) {
+                        metric.curr.subtract_current(size, 1);
+                    }
+                }
+                slot.metrics
+                    .retain(|_, metric| metric.curr.get_current_count() > 0 || metric.curr.get_current_bytes() > 0);
+            }
+        });
+    }
+
+    pub fn runtime_target_backlog_snapshot(&self) -> Vec<RuntimeReplicationTargetBacklog> {
+        let mut snapshot = with_target_queue_caches(|caches| {
+            caches
+                .iter()
+                .find(|slot| slot.belongs_to(&self.q_cache))
+                .map(|slot| target_queue_cache_snapshot(&slot.metrics))
+                .unwrap_or_default()
+        });
+        snapshot.sort_by(|left, right| {
+            left.bucket
+                .cmp(&right.bucket)
+                .then_with(|| left.target_arn.cmp(&right.target_arn))
+        });
+        snapshot
+    }
+
     /// Increase proxy metrics
     pub async fn inc_proxy(&self, bucket: &str, api: &str, is_err: bool) {
         let mut p_cache = self.p_cache.lock().await;
@@ -707,6 +846,94 @@ impl Default for ReplicationStats {
 mod tests {
     use super::*;
 
+    #[test]
+    fn runtime_target_backlog_snapshot_tracks_targets() {
+        let stats = ReplicationStats::new();
+        stats.inc_target_q(
+            "photos",
+            &[
+                "arn:rustfs:replication:target-b".to_string(),
+                "arn:rustfs:replication:target-a".to_string(),
+                "arn:rustfs:replication:target-a".to_string(),
+            ],
+            1024,
+        );
+
+        let snapshot = stats.runtime_target_backlog_snapshot();
+
+        assert_eq!(
+            snapshot,
+            vec![
+                RuntimeReplicationTargetBacklog {
+                    bucket: "photos".to_string(),
+                    target_arn: "arn:rustfs:replication:target-a".to_string(),
+                    count: 1,
+                    bytes: 1024,
+                },
+                RuntimeReplicationTargetBacklog {
+                    bucket: "photos".to_string(),
+                    target_arn: "arn:rustfs:replication:target-b".to_string(),
+                    count: 1,
+                    bytes: 1024,
+                },
+            ]
+        );
+    }
+
+    #[test]
+    fn runtime_target_backlog_ignores_empty_targets() {
+        let stats = ReplicationStats::new();
+        stats.inc_target_q("photos", &["".to_string()], 1024);
+
+        assert!(stats.runtime_target_backlog_snapshot().is_empty());
+    }
+
+    #[test]
+    fn runtime_target_backlog_is_scoped_to_stats_instance() {
+        let first = ReplicationStats::new();
+        let second = ReplicationStats::new();
+        first.inc_target_q("photos", &["arn:rustfs:replication:target-a".to_string()], 1024);
+
+        assert!(second.runtime_target_backlog_snapshot().is_empty());
+        assert_eq!(first.runtime_target_backlog_snapshot()[0].count, 1);
+    }
+
+    #[test]
+    fn runtime_target_backlog_decrements_with_saturation() {
+        let stats = ReplicationStats::new();
+        let target_arns = ["arn:rustfs:replication:target-a".to_string()];
+        stats.inc_target_q("photos", &target_arns, 1024);
+        stats.dec_target_q("photos", &target_arns, 2048);
+
+        assert!(stats.runtime_target_backlog_snapshot().is_empty());
+    }
+
+    #[test]
+    fn runtime_target_backlog_prunes_stale_sidecar_on_next_access() {
+        {
+            let stats = ReplicationStats::new();
+            stats.inc_target_q("photos", &["arn:rustfs:replication:target-a".to_string()], 1024);
+            assert!(
+                TARGET_QUEUE_CACHES
+                    .lock()
+                    .expect("target queue cache mutex")
+                    .iter()
+                    .any(|slot| slot.owner.strong_count() > 0)
+            );
+        }
+
+        let stats = ReplicationStats::new();
+        stats.inc_target_q("photos", &["arn:rustfs:replication:target-b".to_string()], 1024);
+
+        assert!(
+            TARGET_QUEUE_CACHES
+                .lock()
+                .expect("target queue cache mutex")
+                .iter()
+                .all(|slot| slot.owner.strong_count() > 0)
+        );
+    }
+
     #[tokio::test]
     async fn test_replication_stats_new() {
         let stats = ReplicationStats::new();

crates/kms/examples/kms_dr_drill.rs

@@ -0,0 +1,187 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Operator entry point for the KMS disaster-recovery drill.
+//!
+//! Runs one rehearsal inside a scratch workspace and writes the evidence
+//! bundle. Everything it touches lives under that workspace: it never reads,
+//! writes, or restores into a running deployment's key directory. See
+//! `docs/operations/kms-disaster-recovery-drill.md` for the procedure and for
+//! how to size the dataset so the measured recovery time transfers.
+//!
+//! Exit status is the verdict: 0 when every check held, 1 otherwise, so a
+//! scheduled drill fails its job instead of quietly filing a bad report.
+
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
+use rustfs_kms::backup::{BackupKek, DrillDataset, DrillDisaster, DrillRequest, DrillVerdict, run_local_drill};
+use std::path::PathBuf;
+use std::process::ExitCode;
+use zeroize::Zeroizing;
+
+const ENV_WORKSPACE: &str = "RUSTFS_KMS_DRILL_WORKSPACE";
+const ENV_DRILL_ID: &str = "RUSTFS_KMS_DRILL_ID";
+const ENV_DISASTER: &str = "RUSTFS_KMS_DRILL_DISASTER";
+const ENV_DEPLOYMENT: &str = "RUSTFS_KMS_DRILL_DEPLOYMENT";
+const ENV_MASTER_KEY: &str = "RUSTFS_KMS_DRILL_MASTER_KEY";
+const ENV_KEYS: &str = "RUSTFS_KMS_DRILL_KEYS";
+const ENV_OBJECTS_PER_KEY: &str = "RUSTFS_KMS_DRILL_OBJECTS_PER_KEY";
+const ENV_OBJECT_BYTES: &str = "RUSTFS_KMS_DRILL_OBJECT_BYTES";
+const ENV_EVIDENCE: &str = "RUSTFS_KMS_DRILL_EVIDENCE";
+const ENV_FILE_PERMISSIONS: &str = "RUSTFS_KMS_DRILL_FILE_PERMISSIONS";
+
+// Deliberately the same names the admin backup API reads: drilling with the
+// KEK the real backups are sealed under is what proves that KEK is still
+// retrievable, which is the part of a recovery no bundle can attest to.
+const ENV_KEK: &str = "RUSTFS_KMS_BACKUP_KEK";
+const ENV_KEK_ID: &str = "RUSTFS_KMS_BACKUP_KEK_ID";
+const ENV_KEK_VERSION: &str = "RUSTFS_KMS_BACKUP_KEK_VERSION";
+
+fn usage() -> String {
+    format!(
+        "KMS disaster-recovery drill.\n\
+         \n\
+         Required:\n  \
+           {ENV_WORKSPACE}      absolute path to a scratch directory the drill owns\n  \
+           {ENV_MASTER_KEY}     at-rest master key for the throwaway rehearsal deployment\n  \
+           {ENV_KEK}            base64 of the 32-byte backup KEK\n  \
+           {ENV_KEK_ID}         identifier recorded in the bundle manifest\n\
+         \n\
+         Optional:\n  \
+           {ENV_KEK_VERSION}    backup KEK version (default 1)\n  \
+           {ENV_DRILL_ID}       drill identifier and key-id prefix (default kms-dr-drill)\n  \
+           {ENV_DEPLOYMENT}     deployment identity recorded in the bundle (default kms-dr-drill)\n  \
+           {ENV_DISASTER}       key-directory-lost | master-key-salt-lost | key-record-corrupted\n  \
+           {ENV_KEYS}           master keys to rehearse with (default 4)\n  \
+           {ENV_OBJECTS_PER_KEY} objects sealed per key (default 2)\n  \
+           {ENV_OBJECT_BYTES}   plaintext bytes per object (default 4096)\n  \
+           {ENV_FILE_PERMISSIONS} octal key-file mode (default 600)\n  \
+           {ENV_EVIDENCE}       evidence output path (default <workspace>/evidence.json)"
+    )
+}
+
+fn required(name: &str) -> Result<String, String> {
+    std::env::var(name)
+        .ok()
+        .filter(|value| !value.trim().is_empty())
+        .ok_or_else(|| format!("{name} is required\n\n{}", usage()))
+}
+
+fn optional_usize(name: &str, default: usize) -> Result<usize, String> {
+    match std::env::var(name) {
+        Ok(value) => value
+            .trim()
+            .parse()
+            .map_err(|_| format!("{name} must be an unsigned integer")),
+        Err(_) => Ok(default),
+    }
+}
+
+fn disaster_from_env() -> Result<DrillDisaster, String> {
+    match std::env::var(ENV_DISASTER).ok().as_deref().map(str::trim) {
+        None | Some("") | Some("key-directory-lost") => Ok(DrillDisaster::KeyDirectoryLost),
+        Some("master-key-salt-lost") => Ok(DrillDisaster::MasterKeySaltLost),
+        Some("key-record-corrupted") => Ok(DrillDisaster::KeyRecordCorrupted),
+        Some(other) => Err(format!(
+            "{ENV_DISASTER}={other:?} is not a known disaster; use key-directory-lost, master-key-salt-lost, or key-record-corrupted"
+        )),
+    }
+}
+
+fn kek_from_env() -> Result<BackupKek, String> {
+    let raw = Zeroizing::new(required(ENV_KEK)?);
+    let decoded = Zeroizing::new(BASE64.decode(raw.trim()).map_err(|_| format!("{ENV_KEK} must be base64"))?);
+    if decoded.len() != 32 {
+        return Err(format!("{ENV_KEK} must decode to exactly 32 bytes"));
+    }
+    let mut material = [0u8; 32];
+    material.copy_from_slice(&decoded);
+    let version = optional_usize(ENV_KEK_VERSION, 1)?;
+    let version = u32::try_from(version).map_err(|_| format!("{ENV_KEK_VERSION} is out of range"))?;
+    BackupKek::new(required(ENV_KEK_ID)?, version, material).map_err(|error| error.to_string())
+}
+
+async fn run() -> Result<DrillVerdict, String> {
+    let workspace = PathBuf::from(required(ENV_WORKSPACE)?);
+    if !workspace.is_absolute() {
+        return Err(format!("{ENV_WORKSPACE} must be an absolute path"));
+    }
+    let drill_id = std::env::var(ENV_DRILL_ID)
+        .ok()
+        .filter(|value| !value.trim().is_empty())
+        .unwrap_or_else(|| "kms-dr-drill".to_string());
+    let file_permissions = match std::env::var(ENV_FILE_PERMISSIONS) {
+        Ok(value) => Some(u32::from_str_radix(value.trim(), 8).map_err(|_| format!("{ENV_FILE_PERMISSIONS} must be octal"))?),
+        Err(_) => Some(0o600),
+    };
+
+    let request = DrillRequest {
+        deployment_identity: std::env::var(ENV_DEPLOYMENT)
+            .ok()
+            .filter(|value| !value.trim().is_empty())
+            .unwrap_or_else(|| drill_id.clone()),
+        drill_id,
+        workspace: workspace.clone(),
+        rustfs_version: env!("CARGO_PKG_VERSION").to_string(),
+        // A drill always restores into a target that has observed nothing, so
+        // any generation is monotonic; the number is recorded as evidence of
+        // which snapshot the rehearsal covered.
+        snapshot_generation: 1,
+        dataset: DrillDataset {
+            keys: optional_usize(ENV_KEYS, 4)?,
+            objects_per_key: optional_usize(ENV_OBJECTS_PER_KEY, 2)?,
+            object_bytes: optional_usize(ENV_OBJECT_BYTES, 4096)?,
+        },
+        disaster: disaster_from_env()?,
+        master_key: required(ENV_MASTER_KEY)?,
+        file_permissions,
+    };
+
+    let kek = kek_from_env()?;
+    let evidence = run_local_drill(&kek, &request).await.map_err(|error| error.to_string())?;
+
+    let evidence_path = std::env::var(ENV_EVIDENCE)
+        .ok()
+        .filter(|value| !value.trim().is_empty())
+        .map_or_else(|| workspace.join("evidence.json"), PathBuf::from);
+    let encoded = evidence.encode().map_err(|error| error.to_string())?;
+    std::fs::write(&evidence_path, &encoded).map_err(|error| format!("cannot write {}: {error}", evidence_path.display()))?;
+
+    eprintln!("verdict:  {:?}", evidence.verdict);
+    eprintln!("disaster: {:?}", evidence.disaster);
+    eprintln!(
+        "objects:  {}/{} historical objects decrypted after the restore",
+        evidence.envelope_probes.iter().filter(|probe| probe.verified).count(),
+        evidence.envelope_probes.len()
+    );
+    eprintln!("rpo:      {} ms of work past the snapshot fence was lost", evidence.rpo.rpo_window_millis);
+    eprintln!("rto:      {} ms of recovery work", evidence.rto_millis);
+    for finding in &evidence.findings {
+        eprintln!("finding:  {finding}");
+    }
+    eprintln!("evidence: {}", evidence_path.display());
+
+    Ok(evidence.verdict)
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+    match run().await {
+        Ok(DrillVerdict::Passed) => ExitCode::SUCCESS,
+        Ok(DrillVerdict::Failed) => ExitCode::FAILURE,
+        Err(message) => {
+            eprintln!("{message}");
+            ExitCode::FAILURE
+        }
+    }
+}

crates/kms/src/api_types.rs

@@ -15,9 +15,10 @@
 //! API types for KMS dynamic configuration
 
 use crate::config::{
-    BackendConfig, CacheConfig, DEFAULT_CACHE_TTL, DEFAULT_MAX_CACHED_KEYS, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
-    DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend, KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod,
-    VaultConfig, VaultTransitConfig, allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option,
+    AwsKmsConfig, BackendConfig, CacheConfig, DEFAULT_CACHE_TTL, DEFAULT_MAX_CACHED_KEYS,
+    DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend, KmsConfig, LocalConfig,
+    StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, allow_immediate_deletion_from_env,
+    redacted_secret, redacted_secret_option,
 };
 use crate::service_manager::KmsServiceStatus;
 use crate::types::{KeyMetadata, KeyUsage};
@@ -165,6 +166,47 @@ pub struct ConfigureStaticKmsRequest {
     pub allow_insecure_dev_defaults: Option<bool>,
 }
 
+/// Request to configure KMS with the AWS KMS backend.
+///
+/// Accepts no credential material by design: every node resolves AWS
+/// credentials through the standard `aws-config` provider chain, so nothing
+/// secret is submitted here, persisted with the cluster configuration, or
+/// echoed back by the status API.
+///
+/// Keys are not created by this path. The backend refuses caller-named key
+/// creation because AWS assigns identifiers itself, so `default_key_id` must
+/// name a key that already exists in AWS, by key id or ARN.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ConfigureAwsKmsRequest {
+    /// AWS region hosting the keys.
+    ///
+    /// Mandatory here, unlike the environment-variable path: this
+    /// configuration is persisted once and replayed on every node, so leaving
+    /// the region to each node's ambient provider chain would let nodes
+    /// silently address different regions — and therefore different keys —
+    /// while reporting the same configuration.
+    pub region: String,
+    /// Endpoint override for local emulators and private endpoints. Unset in
+    /// production, where the SDK derives the regional endpoint; a plaintext
+    /// endpoint stays gated behind the development opt-in.
+    pub endpoint_url: Option<String>,
+    /// Default master key ID for auto-encryption, as an AWS key id or ARN
+    pub default_key_id: Option<String>,
+    /// Operation timeout in seconds
+    pub timeout_seconds: Option<u64>,
+    /// Number of retry attempts
+    pub retry_attempts: Option<u32>,
+    /// Enable caching
+    pub enable_cache: Option<bool>,
+    /// Maximum number of keys to cache
+    pub max_cached_keys: Option<usize>,
+    /// Cache TTL in seconds
+    pub cache_ttl_seconds: Option<u64>,
+    /// Allow development-only insecure defaults
+    pub allow_insecure_dev_defaults: Option<bool>,
+}
+
 impl Drop for ConfigureStaticKmsRequest {
     fn drop(&mut self) {
         use zeroize::Zeroize;
@@ -211,6 +253,9 @@ pub enum ConfigureKmsRequest {
     /// Configure with Static single-key backend
     #[serde(rename = "Static", alias = "static")]
     Static(ConfigureStaticKmsRequest),
+    /// Configure with the AWS KMS backend
+    #[serde(rename = "AWS", alias = "AwsKms", alias = "aws", alias = "aws-kms", alias = "aws_kms")]
+    Aws(ConfigureAwsKmsRequest),
 }
 
 /// KMS configuration response
@@ -636,6 +681,33 @@ impl ConfigureStaticKmsRequest {
     }
 }
 
+impl ConfigureAwsKmsRequest {
+    /// Convert to KmsConfig
+    pub fn to_kms_config(&self) -> KmsConfig {
+        KmsConfig {
+            backend: KmsBackend::Aws,
+            default_key_id: self.default_key_id.clone(),
+            backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
+                region: Some(self.region.clone()),
+                endpoint_url: self.endpoint_url.clone(),
+            })),
+            allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
+            // Read from server configuration, never from the request body: the
+            // gate must mean the same thing whether KMS was configured at
+            // startup or through this endpoint.
+            allow_immediate_deletion: allow_immediate_deletion_from_env(),
+            timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
+            retry_attempts: self.retry_attempts.unwrap_or(3),
+            enable_cache: self.enable_cache.unwrap_or(true),
+            cache_config: CacheConfig {
+                max_keys: self.max_cached_keys.unwrap_or(DEFAULT_MAX_CACHED_KEYS),
+                ttl: self.cache_ttl_seconds.map_or(DEFAULT_CACHE_TTL, Duration::from_secs),
+                ..CacheConfig::default()
+            },
+        }
+    }
+}
+
 impl ConfigureKmsRequest {
     /// Convert to KmsConfig
     pub fn to_kms_config(&self) -> KmsConfig {
@@ -644,6 +716,7 @@ impl ConfigureKmsRequest {
             ConfigureKmsRequest::VaultKv2(req) => req.to_kms_config(),
             ConfigureKmsRequest::VaultTransit(req) => req.to_kms_config(),
             ConfigureKmsRequest::Static(req) => req.to_kms_config(),
+            ConfigureKmsRequest::Aws(req) => req.to_kms_config(),
         }
     }
 }
@@ -829,6 +902,108 @@ mod tests {
         assert!(request.to_kms_config().validate().is_ok());
     }
 
+    #[test]
+    fn test_deserialize_aws_configure_request_accepts_type_aliases() {
+        for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
+            let raw = serde_json::json!({
+                "backend_type": backend_type,
+                "region": "eu-central-1",
+                "default_key_id": "arn:aws:kms:eu-central-1:111122223333:key/1234abcd"
+            });
+
+            let request: ConfigureKmsRequest = serde_json::from_value(raw).unwrap_or_else(|e| panic!("{backend_type}: {e}"));
+            let config = request.to_kms_config();
+            assert_eq!(config.backend, KmsBackend::Aws, "backend_type={backend_type}");
+            let aws = config.aws_kms_config().expect("aws backend config");
+            assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
+            assert_eq!(aws.endpoint_url, None);
+            assert!(config.validate().is_ok(), "backend_type={backend_type}");
+        }
+    }
+
+    /// A cluster-persisted AWS configuration must pin its own region: a
+    /// request that leaves it to each node's ambient provider chain is refused
+    /// rather than accepted into a configuration every node interprets
+    /// differently.
+    #[test]
+    fn test_aws_configure_request_requires_an_explicit_region() {
+        let missing = serde_json::json!({
+            "backend_type": "AWS",
+            "default_key_id": "arn:aws:kms:eu-central-1:111122223333:key/1234abcd"
+        });
+        let err = serde_json::from_value::<ConfigureKmsRequest>(missing).expect_err("a region-less AWS request must be refused");
+        assert!(err.to_string().contains("region"), "{err}");
+
+        let empty = serde_json::json!({ "backend_type": "AWS", "region": "" });
+        let request: ConfigureKmsRequest = serde_json::from_value(empty).expect("an empty region deserializes");
+        assert!(request.to_kms_config().validate().is_err(), "an empty region must not validate");
+    }
+
+    #[test]
+    fn test_aws_configure_request_rejects_plaintext_endpoint_without_opt_in() {
+        let raw = serde_json::json!({
+            "backend_type": "AWS",
+            "region": "us-east-1",
+            "endpoint_url": "http://localhost:4566"
+        });
+        let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("aws request should deserialize");
+        assert!(request.to_kms_config().validate().is_err());
+
+        let opt_in = serde_json::json!({
+            "backend_type": "AWS",
+            "region": "us-east-1",
+            "endpoint_url": "http://localhost:4566",
+            "allow_insecure_dev_defaults": true
+        });
+        let request: ConfigureKmsRequest = serde_json::from_value(opt_in).expect("aws request should deserialize");
+        assert!(request.to_kms_config().validate().is_ok());
+    }
+
+    /// The AWS summary carries only non-credential settings, because the
+    /// backend never holds AWS credential material to begin with.
+    #[test]
+    fn test_aws_status_summary_reports_only_non_credential_settings() {
+        let config = ConfigureAwsKmsRequest {
+            region: "us-east-1".to_string(),
+            endpoint_url: None,
+            default_key_id: Some("arn:aws:kms:us-east-1:111122223333:key/1234abcd".to_string()),
+            timeout_seconds: None,
+            retry_attempts: None,
+            enable_cache: None,
+            max_cached_keys: None,
+            cache_ttl_seconds: None,
+            allow_insecure_dev_defaults: None,
+        }
+        .to_kms_config();
+
+        let summary = KmsConfigSummary::from(&config);
+        assert_eq!(summary.backend_type, KmsBackend::Aws);
+        match &summary.backend_summary {
+            BackendSummary::Aws { region, endpoint_url } => {
+                assert_eq!(region.as_deref(), Some("us-east-1"));
+                assert_eq!(endpoint_url.as_deref(), None);
+            }
+            other => panic!("expected aws summary, got {other:?}"),
+        }
+
+        let response = KmsStatusResponse {
+            status: KmsServiceStatus::Running,
+            backend_type: Some(config.backend),
+            healthy: Some(true),
+            config_summary: Some(summary),
+        };
+        let rendered = format!(
+            "{}\n{response:?}",
+            serde_json::to_string(&response).expect("kms status response should serialize")
+        );
+        for credential_field in ["access_key", "secret_key", "session_token", "has_stored_credentials"] {
+            assert!(
+                !rendered.contains(credential_field),
+                "aws status output must not describe credential material: {rendered}"
+            );
+        }
+    }
+
     #[test]
     fn test_configure_request_rejects_unknown_fields() {
         let raw = serde_json::json!({
@@ -853,6 +1028,17 @@ mod tests {
 
         let err = serde_json::from_value::<ConfigureKmsRequest>(raw).expect_err("unknown auth field should fail");
         assert!(err.to_string().contains("unknown field"));
+
+        // AWS credentials belong to the provider chain: a request that tries to
+        // smuggle them in must be refused, not silently ignored.
+        let raw = serde_json::json!({
+            "backend_type": "AWS",
+            "region": "us-east-1",
+            "secret_access_key": "AKIA-not-accepted-here"
+        });
+
+        let err = serde_json::from_value::<ConfigureKmsRequest>(raw).expect_err("unknown aws field should fail");
+        assert!(err.to_string().contains("unknown field"));
     }
 
     #[test]

crates/kms/src/audit.rs

@@ -156,6 +156,7 @@ pub fn error_class(error: &KmsError) -> &'static str {
         KmsError::UnsupportedCapability { .. } => "unsupported_capability",
         KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
         KmsError::BaselineVersionLost { .. } => "baseline_version_lost",
+        KmsError::KeyStillReferenced { .. } => "key_still_referenced",
     }
 }
 

crates/kms/src/backends/aws.rs

@@ -67,7 +67,7 @@ use aws_smithy_types::Blob;
 use jiff::Zoned;
 use tokio_util::sync::CancellationToken;
 
-use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend};
+use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
 use crate::config::{BackendConfig, KmsConfig};
 use crate::error::{KmsError, Result};
 use crate::policy::{self, AttemptError, ErrorClass, OpClass, RetryPolicy, classify_status};
@@ -577,6 +577,13 @@ impl KmsBackend for AwsKmsBackend {
     /// and creation date every caller relies on costs one `DescribeKey` per
     /// listed key. The page size is bounded by the caller's `limit`.
     async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
+        // AWS rejects a `Limit` of zero, and clamping it up to one would return
+        // a key to a caller that asked for none; the empty page is answered
+        // here instead.
+        if list_keys_page_size(request.limit).is_none() {
+            return Ok(empty_key_page());
+        }
+
         let limit = request
             .limit
             .map(|limit| i32::try_from(limit).unwrap_or(i32::MAX).clamp(1, 1000));
@@ -1113,6 +1120,26 @@ mod tests {
         assert_eq!(http_client.actual_requests().count(), 0, "no key may be created in AWS");
     }
 
+    /// AWS rejects `Limit: 0` outright, so the request cannot be forwarded as
+    /// written; clamping it up to one would answer a caller that asked for no
+    /// keys with a key. The empty page is served locally instead.
+    #[tokio::test]
+    async fn zero_limit_list_returns_an_empty_page_without_calling_aws() {
+        let (http_client, backend) = scripted_backend(Vec::new());
+        let response = backend
+            .list_keys(ListKeysRequest {
+                limit: Some(0),
+                ..Default::default()
+            })
+            .await
+            .expect("a zero-limit list must succeed");
+
+        assert!(response.keys.is_empty());
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+        assert_eq!(http_client.actual_requests().count(), 0, "no request should reach AWS");
+    }
+
     /// Signing keys are outside the envelope-encryption surface this backend
     /// serves; creating one is refused rather than silently downgraded.
     #[tokio::test]

crates/kms/src/backends/local.rs

@@ -16,7 +16,7 @@
 
 use crate::backends::{
     BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
-    ensure_tag_keys_are_mutable,
+    ensure_tag_keys_are_mutable, paginate_keys,
 };
 use crate::config::KmsConfig;
 use crate::config::LocalConfig;
@@ -1214,6 +1214,32 @@ impl LocalKmsClient {
         Ok(master_key.into())
     }
 
+    /// Every key identifier in the key directory, sorted.
+    ///
+    /// `read_dir` order is arbitrary and may differ between calls over the same
+    /// directory, so it cannot carry a pagination cursor. Sorting gives the key
+    /// set a stable total order that a marker can point into.
+    async fn sorted_key_ids(&self) -> Result<Vec<String>> {
+        let mut key_ids = Vec::new();
+        let mut entries = fs::read_dir(&self.config.key_dir).await?;
+        while let Some(entry) = entries.next_entry().await? {
+            let path = entry.path();
+            if path.extension().is_some_and(|ext| ext == "key")
+                && let Some(key_id) = path.file_stem().and_then(|stem| stem.to_str())
+            {
+                key_ids.push(key_id.to_string());
+            }
+        }
+        key_ids.sort_unstable();
+        Ok(key_ids)
+    }
+
+    /// One page of the key set, ordered by key identifier.
+    ///
+    /// Paging is real here rather than a first-page-only approximation:
+    /// callers that must see every key — the deletion sweep above all, which
+    /// only destroys expired material for keys it actually lists — depend on
+    /// `truncated` and `next_marker` to reach past the first page.
     pub(crate) async fn list_keys(
         &self,
         request: &ListKeysRequest,
@@ -1221,44 +1247,43 @@ impl LocalKmsClient {
     ) -> Result<ListKeysResponse> {
         debug!("Listing keys");
 
-        let mut keys = Vec::new();
-        let limit = request.limit.unwrap_or(100) as usize;
-        let mut count = 0;
+        let key_ids = self.sorted_key_ids().await?;
+        let page = paginate_keys(&key_ids, request, String::as_str);
 
-        let mut entries = fs::read_dir(&self.config.key_dir).await?;
+        // Only the page is read from disk, so the cost of a list stays bounded
+        // by the requested limit rather than by the size of the key set.
+        let mut keys = Vec::with_capacity(page.items.len());
+        for key_id in page.items {
+            // A key that vanished or cannot be decoded is dropped from the page
+            // instead of failing it: concurrent removal is normal, and the
+            // cursor is derived from the identifier list, so the listing still
+            // advances past it.
+            let key_info = match self.describe_key(key_id, None).await {
+                Ok(key_info) => key_info,
+                Err(error) => {
+                    debug!(key_id, %error, "skipping unreadable key while listing");
+                    continue;
+                }
+            };
 
-        while let Some(entry) = entries.next_entry().await? {
-            if count >= limit {
-                break;
-            }
-
-            let path = entry.path();
-            if path.extension().is_some_and(|ext| ext == "key")
-                && let Some(stem) = path.file_stem()
-                && let Some(key_id) = stem.to_str()
-                && let Ok(key_info) = self.describe_key(key_id, None).await
+            if let Some(ref status_filter) = request.status_filter
+                && &key_info.status != status_filter
             {
-                // Apply filters
-                if let Some(ref status_filter) = request.status_filter
-                    && &key_info.status != status_filter
-                {
-                    continue;
-                }
-                if let Some(ref usage_filter) = request.usage_filter
-                    && &key_info.usage != usage_filter
-                {
-                    continue;
-                }
-
-                keys.push(key_info);
-                count += 1;
+                continue;
             }
+            if let Some(ref usage_filter) = request.usage_filter
+                && &key_info.usage != usage_filter
+            {
+                continue;
+            }
+
+            keys.push(key_info);
         }
 
         Ok(ListKeysResponse {
             keys,
-            next_marker: None, // Simple implementation without pagination
-            truncated: false,
+            next_marker: page.next_marker,
+            truncated: page.truncated,
         })
     }
 
@@ -3087,4 +3112,131 @@ mod tests {
             .expect("repeat removal");
         assert_eq!(outcome, crate::backends::ExpiredKeyRemoval::Removed);
     }
+
+    // -- Listing and pagination ---------------------------------------------
+
+    fn page_request(limit: u32, marker: Option<&str>) -> ListKeysRequest {
+        ListKeysRequest {
+            limit: Some(limit),
+            marker: marker.map(str::to_string),
+            usage_filter: None,
+            status_filter: None,
+        }
+    }
+
+    async fn create_keys(client: &LocalKmsClient, key_ids: &[String]) {
+        for key_id in key_ids {
+            client
+                .create_key(key_id, "AES_256", None)
+                .await
+                .expect("key should be created");
+        }
+    }
+
+    /// Paging must reach every key exactly once. A backend that reports the
+    /// first page as the whole key set strands everything behind it — the
+    /// deletion sweep would never destroy expired material past page one.
+    #[tokio::test]
+    async fn list_keys_pages_through_the_whole_key_set() {
+        let (client, _temp_dir) = create_test_client().await;
+        let expected: Vec<String> = (0..7).map(|index| format!("page-key-{index:02}")).collect();
+        create_keys(&client, &expected).await;
+
+        let mut seen = Vec::new();
+        let mut marker: Option<String> = None;
+        // Bounded so a listing that cannot advance fails the assertion below
+        // instead of hanging the test run.
+        for _ in 0..expected.len() + 1 {
+            let response = client
+                .list_keys(&page_request(3, marker.as_deref()), None)
+                .await
+                .expect("list should succeed");
+            assert!(response.keys.len() <= 3, "a page must not exceed the requested limit");
+            seen.extend(response.keys.iter().map(|key| key.key_id.clone()));
+            if !response.truncated {
+                assert!(response.next_marker.is_none(), "a final page must not offer a cursor");
+                break;
+            }
+            marker = Some(response.next_marker.expect("a truncated page must offer a cursor"));
+        }
+
+        assert_eq!(seen, expected, "paging must visit every key exactly once, in identifier order");
+    }
+
+    /// Exact-limit boundary: a page that ends on the last key is complete.
+    #[tokio::test]
+    async fn list_keys_page_ending_on_the_last_key_is_not_truncated() {
+        let (client, _temp_dir) = create_test_client().await;
+        let key_ids: Vec<String> = (0..3).map(|index| format!("exact-key-{index}")).collect();
+        create_keys(&client, &key_ids).await;
+
+        let response = client
+            .list_keys(&page_request(3, None), None)
+            .await
+            .expect("list should succeed");
+        assert_eq!(response.keys.len(), 3);
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+
+        // One key short of the set, the same listing is truncated.
+        let response = client
+            .list_keys(&page_request(2, None), None)
+            .await
+            .expect("list should succeed");
+        assert!(response.truncated);
+        assert_eq!(response.next_marker.as_deref(), Some("exact-key-1"));
+    }
+
+    /// The cursor is an identifier, not an index, so it keeps working after the
+    /// key it names is destroyed — which is exactly what the deletion sweep
+    /// does to the keys it retires between pages.
+    #[tokio::test]
+    async fn list_keys_resumes_after_a_deleted_marker_key() {
+        let (client, _temp_dir) = create_test_client().await;
+        let key_ids: Vec<String> = ["marker-a", "marker-b", "marker-c"].iter().map(|id| id.to_string()).collect();
+        create_keys(&client, &key_ids).await;
+
+        fs::remove_file(client.master_key_path("marker-b").expect("key path"))
+            .await
+            .expect("key file should be removable");
+
+        let response = client
+            .list_keys(&page_request(10, Some("marker-b")), None)
+            .await
+            .expect("list should succeed");
+        let listed: Vec<&str> = response.keys.iter().map(|key| key.key_id.as_str()).collect();
+        assert_eq!(listed, vec!["marker-c"], "a vanished marker must not restart the listing");
+        assert!(!response.truncated);
+    }
+
+    /// A zero limit means zero keys, and no cursor to loop on.
+    #[tokio::test]
+    async fn list_keys_with_zero_limit_returns_an_empty_page() {
+        let (client, _temp_dir) = create_test_client().await;
+        create_keys(&client, &["zero-limit-key".to_string()]).await;
+
+        let response = client
+            .list_keys(&page_request(0, None), None)
+            .await
+            .expect("a zero-limit list must succeed");
+        assert!(response.keys.is_empty());
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+    }
+
+    /// A limit past the end of the key set returns everything, once.
+    #[tokio::test]
+    async fn list_keys_limit_beyond_the_key_set_returns_one_complete_page() {
+        let (client, _temp_dir) = create_test_client().await;
+        let key_ids: Vec<String> = (0..3).map(|index| format!("huge-limit-key-{index}")).collect();
+        create_keys(&client, &key_ids).await;
+
+        let response = client
+            .list_keys(&page_request(u32::MAX, None), None)
+            .await
+            .expect("list should succeed");
+        assert_eq!(response.keys.len(), 3);
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+    }
 }

crates/kms/src/backends/mod.rs

@@ -126,6 +126,96 @@ pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item =
     Ok(())
 }
 
+/// Page size used when a [`ListKeysRequest`] does not ask for one.
+pub(crate) const DEFAULT_LIST_KEYS_PAGE_SIZE: u32 = 100;
+
+/// One page of a key set the backend has to slice itself.
+pub(crate) struct KeyPage<'a, T> {
+    /// The identifiers this page covers, in listing order.
+    pub(crate) items: &'a [T],
+    /// Where the next page resumes; `None` when this page is the last one.
+    pub(crate) next_marker: Option<String>,
+    /// Whether keys remain beyond this page.
+    pub(crate) truncated: bool,
+}
+
+/// Cut the page `request` asks for out of `sorted`.
+///
+/// `sorted` must be ordered by the identifier `key_id_of` returns, and the
+/// marker is an *exclusive lower bound* on that identifier rather than an index
+/// into the sequence. That is what makes paging survive concurrent mutation:
+/// the next page resumes at the first identifier greater than the marker, so a
+/// key added or removed elsewhere in the ordering — including the marker key
+/// itself, which the deletion sweep routinely destroys — cannot make the
+/// listing skip keys or restart from the beginning.
+///
+/// A `limit` of zero is honoured as written: the caller asked for no keys and
+/// gets an empty, non-truncated page (see [`list_keys_page_size`]).
+///
+/// Filters are applied by the caller to `items` after this slice, so a filtered
+/// page can be shorter than `limit` — and even empty — while more keys remain.
+/// Callers must page until `truncated` is false rather than until a page comes
+/// back short.
+pub(crate) fn paginate_keys<'a, T>(sorted: &'a [T], request: &ListKeysRequest, key_id_of: impl Fn(&T) -> &str) -> KeyPage<'a, T> {
+    let Some(limit) = list_keys_page_size(request.limit) else {
+        return KeyPage {
+            items: &[],
+            next_marker: None,
+            truncated: false,
+        };
+    };
+
+    let start = match request.marker.as_deref() {
+        Some(marker) => sorted.partition_point(|item| key_id_of(item) <= marker),
+        None => 0,
+    };
+    // `partition_point` never exceeds the length, so both bounds stay in range
+    // however large `limit` is.
+    let end = start.saturating_add(limit).min(sorted.len());
+    let items = &sorted[start..end];
+    let truncated = end < sorted.len();
+
+    KeyPage {
+        items,
+        // Resuming from the last identifier on the page, not from an index,
+        // keeps the cursor meaningful after the key it names disappears.
+        next_marker: if truncated {
+            items.last().map(|item| key_id_of(item).to_string())
+        } else {
+            None
+        },
+        truncated,
+    }
+}
+
+/// Resolve the page size of a [`ListKeysRequest`]; `None` means the caller
+/// asked for no keys at all.
+///
+/// `Some(0)` is a well-formed request for an empty page — the reading rustfs
+/// already gives `max-keys=0` on the S3 listing path — not a malformed one and
+/// not an omitted value. Rounding it up to a default would hand back a full
+/// page of keys to a caller that explicitly asked for none.
+pub(crate) fn list_keys_page_size(limit: Option<u32>) -> Option<usize> {
+    match limit.unwrap_or(DEFAULT_LIST_KEYS_PAGE_SIZE) {
+        0 => None,
+        size => Some(size as usize),
+    }
+}
+
+/// The response to a request for zero keys.
+///
+/// `truncated` is false even when keys exist: a zero-length page carries no
+/// identifier to resume from, so claiming more results would hand back a cursor
+/// the caller can never advance and turn a `while truncated` loop into a
+/// non-terminating one.
+pub(crate) fn empty_key_page() -> ListKeysResponse {
+    ListKeysResponse {
+        keys: Vec::new(),
+        next_marker: None,
+        truncated: false,
+    }
+}
+
 /// Simplified KMS backend interface for manager
 #[async_trait]
 pub trait KmsBackend: Send + Sync {
@@ -546,4 +636,84 @@ mod tests {
 
         insta::assert_json_snapshot!("static_backend_capabilities", capabilities_snapshot(backend.capabilities()));
     }
+
+    // -- Pagination boundaries ----------------------------------------------
+
+    fn key_ids(count: usize) -> Vec<String> {
+        (0..count).map(|index| format!("key-{index:02}")).collect()
+    }
+
+    fn page_request(limit: Option<u32>, marker: Option<&str>) -> ListKeysRequest {
+        ListKeysRequest {
+            limit,
+            marker: marker.map(str::to_string),
+            usage_filter: None,
+            status_filter: None,
+        }
+    }
+
+    fn page_of(keys: &[String], limit: Option<u32>, marker: Option<&str>) -> (Vec<String>, Option<String>, bool) {
+        let page = paginate_keys(keys, &page_request(limit, marker), String::as_str);
+        (page.items.to_vec(), page.next_marker, page.truncated)
+    }
+
+    /// Zero keys requested, zero keys returned — and no cursor, so a caller
+    /// looping on `truncated` terminates instead of asking forever. Slicing a
+    /// zero-length page out of a non-empty key set must not reach for the
+    /// element before the page either.
+    #[test]
+    fn zero_limit_returns_an_empty_untruncated_page() {
+        let keys = key_ids(3);
+        assert_eq!(page_of(&keys, Some(0), None), (Vec::new(), None, false));
+        assert_eq!(page_of(&keys, Some(0), Some("key-01")), (Vec::new(), None, false));
+        // Also at the ends of the key set, where a page has no predecessor.
+        assert_eq!(page_of(&[], Some(0), None), (Vec::new(), None, false));
+        assert_eq!(page_of(&keys, Some(0), Some("key-02")), (Vec::new(), None, false));
+
+        assert_eq!(list_keys_page_size(Some(0)), None);
+        assert_eq!(list_keys_page_size(None), Some(DEFAULT_LIST_KEYS_PAGE_SIZE as usize));
+        assert_eq!(list_keys_page_size(Some(7)), Some(7));
+    }
+
+    /// A limit past the end of the key set is not an overflow.
+    #[test]
+    fn oversized_limit_returns_the_whole_key_set_once() {
+        let keys = key_ids(3);
+        assert_eq!(page_of(&keys, Some(u32::MAX), None), (keys.clone(), None, false));
+        assert_eq!(page_of(&keys, Some(u32::MAX), Some("key-01")), (vec![keys[2].clone()], None, false));
+    }
+
+    /// The cursor is an identifier, so a marker naming a key that no longer
+    /// exists resumes after where it would have been instead of restarting.
+    #[test]
+    fn marker_for_a_removed_key_resumes_after_it() {
+        let keys = vec!["key-00".to_string(), "key-02".to_string()];
+        assert_eq!(page_of(&keys, Some(10), Some("key-01")), (vec!["key-02".to_string()], None, false));
+        // A marker past every key ends the listing rather than wrapping.
+        assert_eq!(page_of(&keys, Some(10), Some("key-99")), (Vec::new(), None, false));
+        // A marker before every key yields the whole set.
+        assert_eq!(page_of(&keys, Some(10), Some("key")), (keys.clone(), None, false));
+    }
+
+    /// Truncation flips exactly at the page boundary, and paging covers the
+    /// key set once end to end.
+    #[test]
+    fn pages_tile_the_key_set_exactly_at_the_limit_boundary() {
+        let keys = key_ids(4);
+        assert_eq!(page_of(&keys, Some(4), None), (keys.clone(), None, false));
+        assert_eq!(page_of(&keys, Some(3), None), (keys[..3].to_vec(), Some("key-02".to_string()), true));
+
+        let mut seen = Vec::new();
+        let mut marker = None;
+        loop {
+            let (items, next_marker, truncated) = page_of(&keys, Some(2), marker.as_deref());
+            seen.extend(items);
+            if !truncated {
+                assert!(next_marker.is_none(), "a final page must not offer a cursor");
+                break;
+            }
+            marker = Some(next_marker.expect("a truncated page must offer a cursor"));
+        }
+        assert_eq!(seen, keys, "paging must tile the key set exactly once");
+    }
 }

crates/kms/src/backends/static_kms.rs

@@ -19,9 +19,12 @@
 //!
 //! ## Ciphertext format
 //!
-//! encrypted_data(plaintext_len+16) || nonce (12 bytes)
+//! A JSON-serialized `DataKeyEnvelope` carrying the AES-256-GCM ciphertext, the
+//! 12-byte nonce and the authenticated encryption context. This is a
+//! RustFS-internal format; it is not interchangeable with MinIO's KMS
+//! ciphertext (see the note on `StaticConfig`).
 
-use crate::backends::{BackendCapabilities, KmsBackend};
+use crate::backends::{BackendCapabilities, KmsBackend, empty_key_page, list_keys_page_size};
 use crate::config::{BackendConfig, KmsConfig};
 use crate::encryption::DataKeyEnvelope;
 use crate::error::{KmsError, Result};
@@ -262,6 +265,12 @@ impl StaticKmsBackend {
 
     /// List the single configured key, honouring the pagination marker.
     pub(crate) fn list_configured_key(&self, request: &ListKeysRequest) -> Result<ListKeysResponse> {
+        // A caller asking for no keys gets none, even from a backend whose
+        // whole key set is one key.
+        if list_keys_page_size(request.limit).is_none() {
+            return Ok(empty_key_page());
+        }
+
         let key_info = KeyInfo {
             key_id: self.key_id.clone(),
             description: Some("Static single-key KMS backend".to_string()),
@@ -657,6 +666,23 @@ mod tests {
         assert!(!response.truncated);
     }
 
+    /// A zero limit means zero keys, even for a backend whose whole key set is
+    /// a single configured key.
+    #[tokio::test]
+    async fn zero_limit_list_returns_an_empty_page() {
+        let (backend, _key_id, _key) = create_test_backend().await;
+
+        let response = backend
+            .list_configured_key(&ListKeysRequest {
+                limit: Some(0),
+                ..Default::default()
+            })
+            .expect("a zero-limit list must succeed");
+        assert!(response.keys.is_empty());
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+    }
+
     #[tokio::test]
     async fn lifecycle_mutations_are_unsupported_at_the_product_surface() {
         let (backend, key_id, _key) = create_test_backend().await;

crates/kms/src/backends/vault.rs

@@ -19,8 +19,8 @@ use crate::backends::vault_credentials::{
     token_source_for,
 };
 use crate::backends::{
-    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
-    ensure_tag_keys_are_mutable,
+    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, empty_key_page, ensure_key_state_permits,
+    ensure_key_status_permits, ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys,
 };
 use crate::config::{KmsConfig, VaultConfig};
 use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -80,6 +80,19 @@ struct VaultKeyData {
     /// persistence landed, so it must stay optional for backward compatibility.
     #[serde(default)]
     deletion_date: Option<Zoned>,
+    /// When the key's material last became current through a rotation.
+    ///
+    /// Written by [`VaultKmsClient::rotate_key`] as part of the same
+    /// check-and-set that switches the current version, so it can only be set on
+    /// a rotation that actually committed. `None` means "no rotation time on
+    /// record", which covers two cases that are deliberately not distinguished
+    /// here: a key that was never rotated, and a key rotated by a build that
+    /// predates this field. Nothing is back-filled — inventing a timestamp for
+    /// the second case would report a rotation that this node never observed.
+    /// See [`crate::deletion_worker`] for why collapsing the two is safe for the
+    /// rotation-age gauge.
+    #[serde(default)]
+    rotated_at: Option<Zoned>,
     /// Encrypted key material (base64 encoded)
     encrypted_key_material: String,
     /// Version that pre-versioning envelopes (no `master_key_version`) resolve to.
@@ -966,7 +979,7 @@ impl VaultKmsClient {
                         description: existing.description,
                         metadata: existing.metadata,
                         created_at: existing.created_at,
-                        rotated_at: None,
+                        rotated_at: existing.rotated_at,
                         created_by: None,
                         deletion_date: existing.deletion_date,
                     })
@@ -993,6 +1006,7 @@ impl VaultKmsClient {
             metadata: HashMap::new(),
             tags: HashMap::new(),
             deletion_date: None,
+            rotated_at: None,
             encrypted_key_material: encrypted_material,
             baseline_version: None,
         };
@@ -1039,7 +1053,7 @@ impl VaultKmsClient {
             metadata: key_data.metadata,
             tags: key_data.tags,
             created_at: key_data.created_at,
-            rotated_at: None,
+            rotated_at: key_data.rotated_at,
             created_by: None,
         })
     }
@@ -1051,37 +1065,42 @@ impl VaultKmsClient {
     ) -> Result<ListKeysResponse> {
         debug!("Listing keys with limit: {:?}", request.limit);
 
-        let all_keys = self.list_vault_keys().await?;
-        let limit = request.limit.unwrap_or(100) as usize;
-
-        // Simple pagination implementation
-        let start_idx = request
-            .marker
-            .as_ref()
-            .and_then(|m| all_keys.iter().position(|k| k == m))
-            .map(|idx| idx + 1)
-            .unwrap_or(0);
-
-        let end_idx = std::cmp::min(start_idx + limit, all_keys.len());
-        let keys_page = &all_keys[start_idx..end_idx];
-
-        let mut key_infos = Vec::new();
-        for key_id in keys_page {
-            if let Ok(key_info) = self.describe_key(key_id, None).await {
-                key_infos.push(key_info);
-            }
+        // A caller asking for no keys is answered without reaching Vault.
+        if list_keys_page_size(request.limit).is_none() {
+            return Ok(empty_key_page());
         }
 
-        let next_marker = if end_idx < all_keys.len() {
-            Some(all_keys[end_idx - 1].clone())
-        } else {
-            None
-        };
+        let mut all_keys = self.list_vault_keys().await?;
+        // Vault's own LIST ordering is not part of its contract, so the sort is
+        // what makes the marker a stable cursor across calls.
+        all_keys.sort_unstable();
+        let page = paginate_keys(&all_keys, request, String::as_str);
+
+        let mut key_infos = Vec::with_capacity(page.items.len());
+        for key_id in page.items {
+            // A key that disappeared between the listing and the read is
+            // dropped from the page rather than failing it; the cursor comes
+            // from the identifier list, so the listing still advances past it.
+            let Ok(key_info) = self.describe_key(key_id, None).await else {
+                continue;
+            };
+            if request
+                .status_filter
+                .as_ref()
+                .is_some_and(|status| status != &key_info.status)
+            {
+                continue;
+            }
+            if request.usage_filter.as_ref().is_some_and(|usage| usage != &key_info.usage) {
+                continue;
+            }
+            key_infos.push(key_info);
+        }
 
         Ok(ListKeysResponse {
             keys: key_infos,
-            next_marker,
-            truncated: end_idx < all_keys.len(),
+            next_marker: page.next_marker,
+            truncated: page.truncated,
         })
     }
 
@@ -1263,8 +1282,15 @@ impl VaultKmsClient {
 
         // Step 3: switch the current pointer. The top-level copy of the material is
         // the fast path for new encryptions and must always match `version`.
+        //
+        // The rotation timestamp rides along on this same write: it marks the
+        // moment the new material became current, and persisting it here means it
+        // commits if and only if the rotation does. A rotation that fails after
+        // freezing the version record leaves the key unrotated and unstamped, so
+        // the recorded time never runs ahead of the current version.
         key_data.version = new_version;
         key_data.encrypted_key_material = new_material;
+        key_data.rotated_at = Some(Zoned::now());
         self.cas_store_key_data(key_id, &key_data, cas).await?;
 
         info!(key_id, version = new_version, "Vault KMS master key rotated");
@@ -1278,7 +1304,9 @@ impl VaultKmsClient {
             description: key_data.description.clone(),
             metadata: key_data.metadata.clone(),
             created_at: key_data.created_at.clone(),
-            rotated_at: Some(Zoned::now()),
+            // The persisted value, not a fresh `now()`: what the caller is told
+            // must be what a later describe of the same key reports.
+            rotated_at: key_data.rotated_at.clone(),
             created_by: None,
             deletion_date: key_data.deletion_date.clone(),
         })
@@ -1732,6 +1760,7 @@ mod tests {
             metadata: HashMap::new(),
             tags: HashMap::new(),
             deletion_date: None,
+            rotated_at: None,
             encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
             baseline_version: None,
         }
@@ -1751,6 +1780,35 @@ mod tests {
         })
     }
 
+    /// A caller asking for no keys gets an empty page, and the page arithmetic
+    /// never reaches for the element before an empty page. The scripted key
+    /// listing stays unused: a request for zero keys has nothing to ask Vault.
+    #[tokio::test]
+    async fn zero_limit_list_returns_an_empty_page_without_calling_vault() {
+        let (vault, client) =
+            scripted_client(vec![ScriptedResponse::ok(serde_json::json!({ "keys": ["key-a", "key-b"] }))]).await;
+
+        let response = client
+            .list_keys(
+                &ListKeysRequest {
+                    limit: Some(0),
+                    ..Default::default()
+                },
+                None,
+            )
+            .await
+            .expect("a zero-limit list must succeed");
+
+        assert!(response.keys.is_empty());
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+        assert!(
+            vault.requests().is_empty(),
+            "a request for no keys must not reach Vault: {:?}",
+            vault.requests()
+        );
+    }
+
     #[tokio::test]
     async fn wired_read_retries_transient_status_then_succeeds() {
         let (vault, client) = scripted_client(vec![
@@ -2048,6 +2106,7 @@ mod tests {
             encrypted_key_material: general_purpose::STANDARD.encode([0x42u8; 32]),
             baseline_version: Some(1),
             deletion_date: None,
+            rotated_at: None,
         };
 
         let mut value = serde_json::to_value(&key_data).expect("serialize key data");
@@ -2411,6 +2470,7 @@ mod tests {
             metadata: HashMap::new(),
             tags: HashMap::new(),
             deletion_date: Some(deadline.clone()),
+            rotated_at: None,
             encrypted_key_material: "material".to_string(),
             baseline_version: None,
         };
@@ -3416,6 +3476,141 @@ mod tests {
         );
     }
 
+    /// The rotation-age gauge ages a key from `rotated_at`, falling back to
+    /// `created_at`. A rotation that commits without recording its time makes a
+    /// key rotated many times read exactly like one that was never rotated, so
+    /// the commit must carry the timestamp and a later describe must report it.
+    /// Reverting either half turns this test red.
+    #[tokio::test]
+    async fn wired_rotate_persists_rotation_time_and_describe_reports_it() {
+        let created_at = Zoned::now() - Duration::from_secs(365 * 86400);
+        let mut key_data = healthy_key_data();
+        key_data.created_at = created_at.clone();
+        key_data.version = 2;
+        key_data.baseline_version = Some(1);
+        key_data.description = Some("payload key".to_string());
+        key_data.metadata.insert("owner".to_string(), "platform".to_string());
+        key_data.tags.insert("env".to_string(), "prod".to_string());
+
+        let (vault, client) = scripted_client(vec![
+            ScriptedResponse::ok(kv2_metadata_read_data(3)),
+            ScriptedResponse::ok(kv2_read_data(&key_data)),
+            ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })),
+            // Version 3's material record, then the pointer switch.
+            ScriptedResponse::ok(kv2_write_ack()),
+            ScriptedResponse::ok(kv2_write_ack()),
+        ])
+        .await;
+
+        let rotated = client.rotate_key("wired-key", None).await.expect("rotate a healthy key");
+        let reported = rotated.rotated_at.clone().expect("a committed rotation must report its time");
+
+        let bodies = vault.request_bodies();
+        let committed = parse_write_body(&bodies[4]);
+        let persisted: VaultKeyData =
+            serde_json::from_value(committed["data"].clone()).expect("the committed record must deserialize");
+        assert_eq!(
+            persisted.rotated_at.as_ref().map(Zoned::timestamp),
+            Some(reported.timestamp()),
+            "the rotation must persist the time it reports: {committed}"
+        );
+
+        // The rest of the record rides through the read-modify-write untouched;
+        // a rotation that dropped any of it would corrupt the key.
+        assert_eq!(persisted.version, 3, "{committed}");
+        assert_eq!(persisted.created_at.timestamp(), created_at.timestamp(), "{committed}");
+        assert_eq!(persisted.baseline_version, Some(1), "{committed}");
+        assert_eq!(persisted.description.as_deref(), Some("payload key"), "{committed}");
+        assert_eq!(persisted.metadata.get("owner").map(String::as_str), Some("platform"), "{committed}");
+        assert_eq!(persisted.tags.get("env").map(String::as_str), Some("prod"), "{committed}");
+
+        // Describing the committed record must report the rotation, not the
+        // creation a year earlier that the gauge would otherwise fall back to.
+        let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&persisted))]).await;
+        let described = client
+            .describe_key("wired-key", None)
+            .await
+            .expect("describe the rotated key");
+        assert_eq!(
+            described.rotated_at.as_ref().map(Zoned::timestamp),
+            Some(reported.timestamp()),
+            "describe must report the persisted rotation time"
+        );
+        assert_ne!(
+            described.rotated_at.as_ref().map(Zoned::timestamp),
+            Some(created_at.timestamp()),
+            "a rotated key must not be aged from its creation"
+        );
+    }
+
+    /// A record written before rotation timestamps were persisted carries no
+    /// rotation time. Reporting one anyway — the current time, the read time —
+    /// would tell the rotation-age gauge the key was just rotated and silence a
+    /// genuinely overdue key, so the absence has to travel as `None`.
+    #[tokio::test]
+    async fn wired_describe_key_invents_no_rotation_time_for_legacy_records() {
+        let mut key_data = healthy_key_data();
+        key_data.created_at = Zoned::now() - Duration::from_secs(365 * 86400);
+        key_data.version = 4;
+        key_data.baseline_version = Some(1);
+
+        let mut record = kv2_read_data(&key_data);
+        record["data"]
+            .as_object_mut()
+            .expect("key record must be a JSON object")
+            .remove("rotated_at")
+            .expect("current records must carry the field");
+
+        let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(record)]).await;
+        let described = client
+            .describe_key("wired-key", None)
+            .await
+            .expect("a record without the field must still describe");
+        assert!(
+            described.rotated_at.is_none(),
+            "an unstamped record must not be reported as freshly rotated, got {:?}",
+            described.rotated_at
+        );
+        assert_eq!(
+            described.created_at.timestamp(),
+            key_data.created_at.timestamp(),
+            "the rest of the legacy record must survive the read"
+        );
+        assert_eq!(described.version, 4);
+    }
+
+    /// The persisted KV2 record round-trips its rotation time, and records
+    /// written before the field existed keep deserializing (as None) with the
+    /// rest of their contents intact.
+    #[test]
+    fn vault_key_data_rotated_at_round_trips_and_stays_backward_compatible() {
+        let rotated_at = Zoned::now();
+        let mut key_data = healthy_key_data();
+        key_data.rotated_at = Some(rotated_at.clone());
+        key_data.baseline_version = Some(1);
+        key_data.version = 2;
+        key_data.tags.insert("env".to_string(), "prod".to_string());
+
+        let mut value = serde_json::to_value(&key_data).expect("serialize");
+        let restored: VaultKeyData = serde_json::from_value(value.clone()).expect("round trip");
+        assert_eq!(
+            restored.rotated_at.as_ref().map(Zoned::timestamp),
+            Some(rotated_at.timestamp()),
+            "the rotation time must survive the KV2 round trip"
+        );
+
+        value
+            .as_object_mut()
+            .expect("record must be a JSON object")
+            .remove("rotated_at")
+            .expect("current records must carry the field");
+        let legacy: VaultKeyData = serde_json::from_value(value).expect("legacy record must deserialize");
+        assert!(legacy.rotated_at.is_none());
+        assert_eq!(legacy.version, 2);
+        assert_eq!(legacy.baseline_version, Some(1));
+        assert_eq!(legacy.tags.get("env").map(String::as_str), Some("prod"));
+    }
+
     /// Reading a pre-versioning envelope against a key whose baseline was erased
     /// resolves to the current version, whose material never wrapped it. The
     /// unwrap therefore fails (AES-GCM cannot yield plaintext under the wrong
@@ -3571,4 +3766,226 @@ mod tests {
             "a successful read must not pay for the lost-baseline diagnosis"
         );
     }
+
+    /// The Vault-side state of one KV2 key: the top-level record plus the
+    /// immutable version records rotations froze.
+    ///
+    /// The scripted responder serves canned responses, so a multi-operation
+    /// scenario has to carry the state between operations itself. Rotations
+    /// fold what they *wrote* back into this state (see [`Self::apply_writes`]),
+    /// which is what makes the rotation regressions below real: the material a
+    /// later decrypt resolves is the material the rotation persisted, not a
+    /// fixture the test invented.
+    struct KeyState {
+        key_data: VaultKeyData,
+        version_records: Vec<VaultKeyVersionRecord>,
+    }
+
+    impl KeyState {
+        /// A never-rotated key: no version records exist yet.
+        fn new(key_data: VaultKeyData) -> Self {
+            Self {
+                key_data,
+                version_records: Vec::new(),
+            }
+        }
+
+        fn version_record(&self, version: u32) -> &VaultKeyVersionRecord {
+            self.version_records
+                .iter()
+                .find(|record| record.version == version)
+                .unwrap_or_else(|| panic!("no version record was frozen for version {version}"))
+        }
+
+        /// The versions-directory listing; a key with no records has no
+        /// directory at all.
+        fn versions_listing(&self) -> ScriptedResponse {
+            if self.version_records.is_empty() {
+                return ScriptedResponse::error(404, "not found");
+            }
+            let keys: Vec<String> = self.version_records.iter().map(|record| record.version.to_string()).collect();
+            ScriptedResponse::ok(serde_json::json!({ "keys": keys }))
+        }
+
+        /// Fold the writes an operation made into the state, so the next
+        /// operation reads exactly what Vault would now hold.
+        fn apply_writes(&mut self, requests: &[String], bodies: &[String]) {
+            for (line, body) in requests.iter().zip(bodies) {
+                let Some(path) = line.strip_prefix("POST ") else {
+                    continue;
+                };
+                let data = parse_write_body(body)["data"].clone();
+                if path.contains("/versions/") {
+                    let record: VaultKeyVersionRecord = serde_json::from_value(data).expect("version record write body");
+                    self.version_records.retain(|existing| existing.version != record.version);
+                    self.version_records.push(record);
+                } else {
+                    self.key_data = serde_json::from_value(data).expect("key record write body");
+                }
+            }
+        }
+    }
+
+    /// Encrypt against a scripted Vault serving `state`.
+    async fn encrypt_scripted(state: &KeyState, plaintext: &[u8]) -> EncryptResponse {
+        let (_vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&state.key_data))]).await;
+        client
+            .encrypt(
+                &EncryptRequest {
+                    key_id: "wired-key".to_string(),
+                    plaintext: plaintext.to_vec(),
+                    encryption_context: HashMap::new(),
+                    grant_tokens: Vec::new(),
+                },
+                None,
+            )
+            .await
+            .expect("encrypt must produce an envelope")
+    }
+
+    /// Rotate against a scripted Vault seeded with `state`, and return the state
+    /// Vault holds afterwards.
+    ///
+    /// The first rotation freezes the baseline before creating the next version
+    /// (four writes); later rotations skip that step (two writes).
+    async fn rotate_scripted(state: &KeyState) -> KeyState {
+        let writes = if state.key_data.baseline_version.is_none() { 4 } else { 2 };
+        let mut responses = vec![
+            ScriptedResponse::ok(kv2_metadata_read_data(1)),
+            ScriptedResponse::ok(kv2_read_data(&state.key_data)),
+            state.versions_listing(),
+        ];
+        responses.extend((0..writes).map(|_| ScriptedResponse::ok(kv2_write_ack())));
+        let (vault, client) = scripted_client(responses).await;
+
+        let rotated = client.rotate_key("wired-key", None).await.expect("rotation must commit");
+        assert_eq!(rotated.version, state.key_data.version + 1, "a rotation must advance the version");
+
+        let mut next = KeyState {
+            key_data: state.key_data.clone(),
+            version_records: state.version_records.clone(),
+        };
+        next.apply_writes(&vault.requests(), &vault.request_bodies());
+        next
+    }
+
+    /// Decrypt against a scripted Vault serving `state`, scripting the
+    /// version-record read the envelope's own version calls for. Returns the
+    /// plaintext together with the requests the decrypt made.
+    async fn decrypt_scripted(state: &KeyState, ciphertext: &[u8]) -> (Vec<u8>, Vec<String>) {
+        let envelope: DataKeyEnvelope = serde_json::from_slice(ciphertext).expect("envelope must parse");
+        let mut responses = vec![ScriptedResponse::ok(kv2_read_data(&state.key_data))];
+        if let Some(version) = envelope.master_key_version
+            && version != state.key_data.version
+        {
+            responses.push(ScriptedResponse::ok(kv2_read_version_record_data(state.version_record(version))));
+        }
+        let (vault, client) = scripted_client(responses).await;
+
+        let plaintext = client
+            .decrypt(
+                &DecryptRequest {
+                    ciphertext: ciphertext.to_vec(),
+                    encryption_context: HashMap::new(),
+                    grant_tokens: Vec::new(),
+                },
+                None,
+            )
+            .await
+            .expect("the envelope must decrypt against the rotated key");
+        (plaintext, vault.requests())
+    }
+
+    /// The forward half of the rotation contract: data written before a rotation
+    /// stays readable after it, with no live Vault involved.
+    ///
+    /// The negative half (a regressed pointer must fail closed) is covered by
+    /// `wired_decrypt_fails_closed_when_current_version_regressed`; this pins the
+    /// path that must keep working, which the fail-closed guards could otherwise
+    /// tighten into a rotation that orphans every existing object.
+    #[tokio::test]
+    async fn wired_kv2_envelope_from_before_rotation_still_decrypts() {
+        const PLAINTEXT: &[u8] = b"written-before-the-rotation";
+        let state_v1 = KeyState::new(healthy_key_data());
+
+        let encrypted_v1 = encrypt_scripted(&state_v1, PLAINTEXT).await;
+        let envelope_v1: DataKeyEnvelope = serde_json::from_slice(&encrypted_v1.ciphertext).expect("envelope must parse");
+        assert_eq!(envelope_v1.master_key_version, Some(1));
+
+        let state_v2 = rotate_scripted(&state_v1).await;
+        assert_eq!(state_v2.key_data.version, 2);
+        assert_eq!(state_v2.key_data.baseline_version, Some(1));
+        assert_ne!(
+            state_v2.key_data.encrypted_key_material, state_v1.key_data.encrypted_key_material,
+            "the rotation must have replaced the current material, or the decrypt below proves nothing"
+        );
+
+        let (plaintext, requests) = decrypt_scripted(&state_v2, &encrypted_v1.ciphertext).await;
+        assert_eq!(
+            plaintext, PLAINTEXT,
+            "the pre-rotation envelope must yield its original plaintext, not merely avoid an error"
+        );
+        assert_eq!(
+            requests,
+            vec![
+                "GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string(),
+                "GET /v1/secret/data/rustfs/kms/keys/wired-key/versions/1".to_string(),
+            ],
+            "the old envelope must resolve through the immutable v1 record"
+        );
+
+        // The rotation is not cosmetic: new writes go to the rotated version and
+        // still round-trip, so both generations are live at once.
+        let encrypted_v2 = encrypt_scripted(&state_v2, b"written-after-the-rotation").await;
+        let envelope_v2: DataKeyEnvelope = serde_json::from_slice(&encrypted_v2.ciphertext).expect("envelope must parse");
+        assert_eq!(envelope_v2.master_key_version, Some(2), "new envelopes must carry the rotated version");
+        assert_eq!(encrypted_v2.key_version, 2);
+        let (plaintext_v2, requests_v2) = decrypt_scripted(&state_v2, &encrypted_v2.ciphertext).await;
+        assert_eq!(plaintext_v2, b"written-after-the-rotation".to_vec());
+        assert_eq!(
+            requests_v2.len(),
+            1,
+            "an envelope on the current version must not read a version record: {requests_v2:?}"
+        );
+    }
+
+    /// Old envelopes must survive more than one generation: the baseline is
+    /// frozen once and every intermediate version keeps its own record, so both
+    /// a pre-rotation envelope and one written between the two rotations still
+    /// decrypt after the second.
+    #[tokio::test]
+    async fn wired_kv2_envelopes_survive_consecutive_rotations() {
+        let state_v1 = KeyState::new(healthy_key_data());
+        let encrypted_v1 = encrypt_scripted(&state_v1, b"generation-1").await;
+
+        let state_v2 = rotate_scripted(&state_v1).await;
+        let encrypted_v2 = encrypt_scripted(&state_v2, b"generation-2").await;
+        assert_eq!(encrypted_v2.key_version, 2);
+
+        let state_v3 = rotate_scripted(&state_v2).await;
+        assert_eq!(state_v3.key_data.version, 3);
+        assert_eq!(
+            state_v3.key_data.baseline_version,
+            Some(1),
+            "the baseline is frozen once and carried through later rotations"
+        );
+        let mut recorded: Vec<u32> = state_v3.version_records.iter().map(|record| record.version).collect();
+        recorded.sort_unstable();
+        assert_eq!(recorded, vec![1, 2, 3], "every version that ever wrapped a DEK must keep a record");
+
+        for (ciphertext, expected, version) in [
+            (&encrypted_v1.ciphertext, b"generation-1".as_slice(), 1u32),
+            (&encrypted_v2.ciphertext, b"generation-2".as_slice(), 2),
+        ] {
+            let (plaintext, requests) = decrypt_scripted(&state_v3, ciphertext).await;
+            assert_eq!(
+                plaintext, expected,
+                "an envelope from version {version} must survive two rotations intact"
+            );
+            assert!(
+                requests[1].ends_with(&format!("/versions/{version}")),
+                "the decrypt must resolve the version that wrapped it: {requests:?}"
+            );
+        }
+    }
 }

crates/kms/src/backends/vault_transit.rs

@@ -19,8 +19,8 @@ use crate::backends::vault_credentials::{
     token_source_for,
 };
 use crate::backends::{
-    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
-    ensure_tag_keys_are_mutable,
+    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, empty_key_page, ensure_key_state_permits,
+    ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys,
 };
 use crate::config::{KmsConfig, VaultTransitConfig};
 use crate::encryption::{DataKeyEnvelope, generate_key_material};
@@ -852,7 +852,12 @@ impl VaultTransitKmsClient {
         request: &ListKeysRequest,
         _context: Option<&OperationContext>,
     ) -> Result<ListKeysResponse> {
-        let all_keys = self
+        // A caller asking for no keys is answered without reaching Vault.
+        if list_keys_page_size(request.limit).is_none() {
+            return Ok(empty_key_page());
+        }
+
+        let mut all_keys = self
             .run("vault_transit_list_keys", OpClass::ReadIdempotent, move || async move {
                 let vault = self.vault().map_err(AttemptError::fatal)?;
                 key::list(&vault.client, &self.config.mount_path).await.map_err(|e| {
@@ -861,36 +866,27 @@ impl VaultTransitKmsClient {
             })
             .await?
             .keys;
+        // Vault's own LIST ordering is not part of its contract, so the sort is
+        // what makes the marker a stable cursor across calls.
+        all_keys.sort_unstable();
+        let page = paginate_keys(&all_keys, request, String::as_str);
 
-        let mut filtered = Vec::new();
-        for key_id in all_keys {
-            let key_info = self.key_info(&key_id).await?;
+        // Reading metadata only for the page keeps a list bounded by the
+        // requested limit instead of by the size of the transit mount.
+        let mut keys = Vec::with_capacity(page.items.len());
+        for key_id in page.items {
+            let key_info = self.key_info(key_id).await?;
             let usage_matches = request.usage_filter.as_ref().is_none_or(|usage| usage == &key_info.usage);
             let status_matches = request.status_filter.as_ref().is_none_or(|status| status == &key_info.status);
             if usage_matches && status_matches {
-                filtered.push(key_info);
+                keys.push(key_info);
             }
         }
 
-        let start_idx = request
-            .marker
-            .as_ref()
-            .and_then(|marker| filtered.iter().position(|info| &info.key_id == marker))
-            .map(|idx| idx + 1)
-            .unwrap_or(0);
-        let limit = request.limit.unwrap_or(100) as usize;
-        let end_idx = std::cmp::min(start_idx + limit, filtered.len());
-        let keys = filtered[start_idx..end_idx].to_vec();
-        let next_marker = if end_idx < filtered.len() {
-            Some(filtered[end_idx - 1].key_id.clone())
-        } else {
-            None
-        };
-
         Ok(ListKeysResponse {
             keys,
-            next_marker,
-            truncated: end_idx < filtered.len(),
+            next_marker: page.next_marker,
+            truncated: page.truncated,
         })
     }
 
@@ -1453,6 +1449,35 @@ mod tests {
         serde_json::to_value(&response).expect("serialize transit key read response")
     }
 
+    /// A caller asking for no keys gets an empty page, and the page arithmetic
+    /// never reaches for the element before an empty page. The scripted key
+    /// listing stays unused: a request for zero keys has nothing to ask Vault.
+    #[tokio::test]
+    async fn zero_limit_list_returns_an_empty_page_without_calling_vault() {
+        let (vault, client) =
+            scripted_client(vec![ScriptedResponse::ok(serde_json::json!({ "keys": ["key-a", "key-b"] }))]).await;
+
+        let response = client
+            .list_keys(
+                &ListKeysRequest {
+                    limit: Some(0),
+                    ..Default::default()
+                },
+                None,
+            )
+            .await
+            .expect("a zero-limit list must succeed");
+
+        assert!(response.keys.is_empty());
+        assert!(!response.truncated);
+        assert!(response.next_marker.is_none());
+        assert!(
+            vault.requests().is_empty(),
+            "a request for no keys must not reach Vault: {:?}",
+            vault.requests()
+        );
+    }
+
     #[tokio::test]
     async fn wired_transit_encrypt_retries_transient_status() {
         let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
@@ -2258,4 +2283,89 @@ mod tests {
             "the sweep must not touch the transit key after backing off: {requests:?}"
         );
     }
+
+    /// The forward half of the rotation contract on the Transit path: a data key
+    /// generated before a rotation must still be decryptable after it.
+    ///
+    /// Transit key material never leaves Vault, so an offline responder cannot
+    /// prove the cryptographic round trip — `test_transit_old_ciphertext_decrypts_after_rotate`
+    /// keeps that against a live Vault. What this pins is the wiring the client
+    /// owns and could regress on its own: the historical `vault:v1:` ciphertext
+    /// is forwarded to Vault byte for byte after the rotation bumped the
+    /// recorded version, and nothing on the decrypt path gates on that version.
+    #[tokio::test]
+    async fn wired_transit_pre_rotation_data_key_is_decrypted_unchanged() {
+        const CIPHERTEXT_V1: &str = "vault:v1:scripted-pre-rotation";
+        const RECOVERED_DEK: [u8; 32] = [0x37u8; 32];
+        let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
+        let (vault, client) = scripted_client(vec![
+            // generate_data_key: state gate reads the metadata record, then the
+            // transit encrypt returns first-version ciphertext.
+            ScriptedResponse::ok(metadata_read_data(&metadata)),
+            ScriptedResponse::ok(serde_json::json!({ "ciphertext": CIPHERTEXT_V1 })),
+            // rotate: the state gate hits the metadata cache, the rotation
+            // commits, and the versioned read+write records the version bump.
+            ScriptedResponse::ok(serde_json::json!({})),
+            ScriptedResponse::ok(kv2_metadata_read_data(1)),
+            ScriptedResponse::ok(metadata_read_data(&metadata)),
+            ScriptedResponse::ok(kv2_write_ack()),
+            // decrypt of the pre-rotation envelope; Vault owns the transit
+            // crypto, so the recovered material is the responder's to hand back.
+            ScriptedResponse::ok(serde_json::json!({ "plaintext": BASE64.encode(RECOVERED_DEK) })),
+        ])
+        .await;
+
+        let data_key = client
+            .generate_data_key(
+                &GenerateKeyRequest {
+                    master_key_id: "wired-key".to_string(),
+                    key_spec: "AES_256".to_string(),
+                    key_length: Some(32),
+                    encryption_context: HashMap::new(),
+                    grant_tokens: Vec::new(),
+                },
+                None,
+            )
+            .await
+            .expect("generate_data_key must produce an envelope");
+        let envelope: DataKeyEnvelope = serde_json::from_slice(&data_key.ciphertext).expect("envelope must parse");
+        assert_eq!(envelope.encrypted_key, CIPHERTEXT_V1.as_bytes());
+
+        let rotated = client.rotate_key("wired-key", None).await.expect("rotation must commit");
+        assert_eq!(rotated.version, 2, "the rotation must record the version bump");
+
+        let plaintext = client
+            .decrypt(
+                &DecryptRequest {
+                    ciphertext: data_key.ciphertext.clone(),
+                    encryption_context: HashMap::new(),
+                    grant_tokens: Vec::new(),
+                },
+                None,
+            )
+            .await
+            .expect("a pre-rotation data key must stay decryptable");
+        assert_eq!(
+            plaintext,
+            RECOVERED_DEK.to_vec(),
+            "the decrypt must hand back the recovered material, not merely avoid an error"
+        );
+
+        let requests = vault.requests();
+        assert_eq!(requests.len(), 7, "{requests:?}");
+        assert_eq!(requests[6], "POST /v1/transit/decrypt/wired-key", "{requests:?}");
+
+        // The version the rotation recorded, and the ciphertext the decrypt sent:
+        // the client must forward the historical version verbatim instead of
+        // re-stamping it to (or pinning the request at) the current one.
+        let bodies = vault.request_bodies();
+        let recorded: serde_json::Value = serde_json::from_str(&bodies[5]).expect("metadata write body must be JSON");
+        assert_eq!(recorded["data"]["current_version"], serde_json::json!(2), "{recorded}");
+        let decrypt_body: serde_json::Value = serde_json::from_str(&bodies[6]).expect("decrypt body must be JSON");
+        assert_eq!(
+            decrypt_body["ciphertext"],
+            serde_json::json!(CIPHERTEXT_V1),
+            "the pre-rotation ciphertext must reach Vault unchanged: {decrypt_body}"
+        );
+    }
 }

crates/kms/src/backup/mod.rs

@@ -21,7 +21,8 @@
 //! [`vault_restore`] orchestrates the consumer side for the Vault backends,
 //! whose cryptographic root is restored by Vault's own disaster-recovery
 //! flow. All are crate-internal APIs; the admin API builds on these pieces in
-//! follow-up changes.
+//! follow-up changes. [`drill`] rehearses the whole loop end to end and turns
+//! it into machine-readable evidence.
 //!
 //! # Bundle model
 //!
@@ -51,6 +52,7 @@
 //! format version.
 
 mod capability;
+pub mod drill;
 mod dry_run;
 mod error;
 pub mod local_export;
@@ -59,6 +61,10 @@ mod manifest;
 pub mod vault_restore;
 
 pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
+pub use drill::{
+    DRILL_EVIDENCE_FORMAT_VERSION, DrillBundleEvidence, DrillDataset, DrillDisaster, DrillEvidence, DrillPhase, DrillPhaseTiming,
+    DrillRecoveryEvidence, DrillRequest, DrillRpoEvidence, DrillVerdict, EnvelopeProbe, run_local_drill,
+};
 pub use dry_run::{
     ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport,
 };

crates/kms/src/backup/vault_restore.rs

@@ -114,10 +114,13 @@ use vaultrs::api::transit::responses::ReadKeyData;
 use vaultrs::error::ClientError;
 use vaultrs::{kv2, transit::key};
 
+// The bundle layout names are pub(crate) so the drill harness
+// (`crate::backup::drill`) addresses the exact paths a Vault bundle uses
+// instead of copies that could drift.
 /// Bundle-relative directory holding one KV record artifact per key.
-const VAULT_RECORD_ARTIFACT_DIR: &str = "vault/records";
+pub(crate) const VAULT_RECORD_ARTIFACT_DIR: &str = "vault/records";
 /// Bundle-relative suffix of a KV record artifact.
-const VAULT_RECORD_ARTIFACT_SUFFIX: &str = ".json.enc";
+pub(crate) const VAULT_RECORD_ARTIFACT_SUFFIX: &str = ".json.enc";
 /// KV path segment (under the bundle's path prefix) of the restore commit
 /// marker. The leading dot keeps it out of the key id space the backends
 /// address; a bundle naming a key id equal to it is rejected.

crates/kms/src/config.rs

@@ -297,8 +297,15 @@ impl Default for LocalConfig {
 
 /// Static single-key KMS backend configuration
 ///
-/// Uses a pre-configured AES-256 key to derive data encryption keys via
-/// HMAC-SHA256 + AES-256-GCM, matching the MinIO builtin/static KMS wire format.
+/// The configured 32-byte key is used directly as the AES-256-GCM key that
+/// wraps data encryption keys — there is no HMAC-SHA256 derivation step — and
+/// each wrapped DEK is serialized as a RustFS `DataKeyEnvelope` JSON blob.
+///
+/// This mirrors the *concept* of MinIO's builtin/static single-key KMS, but is
+/// not wire-compatible with it: MinIO wraps DEKs in a different (`{"aead": ...}`)
+/// blob that this backend neither produces nor accepts, so KMS ciphertext
+/// written by MinIO cannot be opened here. Reading MinIO-written SSE objects is
+/// tracked separately in rustfs/backlog#1638.
 #[derive(Clone, Default, Serialize, Deserialize)]
 pub struct StaticConfig {
     /// Key identifier (name) for the single configured key

crates/kms/src/deletion_worker.rs

@@ -36,6 +36,11 @@ use tracing::{debug, info, warn};
 /// How often the worker looks for expired pending deletions.
 pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
 
+/// Keys requested per `ListKeys` call while sweeping. The sweep follows
+/// `truncated`/`next_marker` until the key set is exhausted, so this bounds the
+/// working set of one call, not how many keys a sweep can reach.
+const SWEEP_PAGE_SIZE: u32 = 100;
+
 // ---------------------------------------------------------------------------
 // Metrics
 //
@@ -102,6 +107,17 @@ impl KeyCensus {
             KeyStatus::PendingDeletion => self.pending_deletion += 1,
             KeyStatus::Deleted => self.tombstones += 1,
             KeyStatus::Active | KeyStatus::Disabled => {
+                // A missing rotation time means either "never rotated" or "the
+                // build that rotated it did not record when". Both fall back to
+                // creation, and the two are not worth separate series: for the
+                // first, creation *is* the correct baseline; for the second it
+                // is an upper bound on the true age, so the gauge can only
+                // overstate how overdue a key is, never hide an overdue one.
+                // The overstatement is self-healing — the next rotation stamps
+                // the record — and erring loud is the right direction for a
+                // rotation-readiness alert. Backends never fabricate a
+                // timestamp to close the gap, so nothing here can turn an
+                // unstamped key into a freshly rotated one.
                 let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
                 self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
             }
@@ -146,6 +162,17 @@ fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
 /// referencing configuration lives (for example bucket encryption settings in
 /// the server) and are injected via
 /// [`crate::service_manager::KmsServiceManager::set_deletion_reference_checker`].
+///
+/// # Blocks only
+///
+/// This gate is one-directional and must stay that way: a checker may add a
+/// reason to keep material, never a reason to destroy it. Nothing that
+/// enumerates key usage — least of all a scan whose coverage depends on how
+/// far a background sweep got — may ever be wired in as a condition that
+/// releases a removal this gate is holding. One incomplete scan would then be
+/// enough to destroy the only copy of a key that still has data behind it,
+/// which is why [`crate::key_impact::KeyImpactReport`] exposes no clearance
+/// and is consumed only where a refusal is being decided.
 #[async_trait]
 pub trait DeletionReferenceChecker: Send + Sync {
     /// Identifiers of configuration still referencing `key_id` (bucket names,
@@ -240,7 +267,7 @@ impl DeletionWorker {
         let mut marker: Option<String> = None;
         let listed_everything = loop {
             let request = ListKeysRequest {
-                limit: Some(100),
+                limit: Some(SWEEP_PAGE_SIZE),
                 marker: marker.clone(),
                 usage_filter: None,
                 status_filter: None,
@@ -270,6 +297,13 @@ impl DeletionWorker {
                 break true;
             }
             match response.next_marker {
+                // A backend that hands back the cursor it was just given cannot
+                // advance. Following it would re-list the same page forever, so
+                // the sweep stops and reports the key set as partially seen.
+                Some(ref next_marker) if marker.as_ref() == Some(next_marker) => {
+                    warn!(marker = %next_marker, "KMS deletion sweep listing did not advance");
+                    break false;
+                }
                 Some(next_marker) => marker = Some(next_marker),
                 // Truncated without a marker: the rest of the key set is out
                 // of reach, so the census is incomplete.
@@ -426,6 +460,41 @@ mod tests {
         assert_eq!(report, SweepReport::default());
     }
 
+    /// Schedule `count` keys for deletion and report their identifiers.
+    async fn schedule_keys(backend: &LocalKmsBackend, count: usize) -> Vec<String> {
+        let mut key_ids = Vec::with_capacity(count);
+        for index in 0..count {
+            let key_id = create_key(backend, &format!("expiring-{index:04}")).await;
+            schedule(backend, &key_id).await;
+            key_ids.push(key_id);
+        }
+        key_ids
+    }
+
+    /// A deployment with more keys than fit in one listing page must still have
+    /// every expired key destroyed: the sweep has to follow the cursor instead
+    /// of stopping at the first page.
+    #[tokio::test]
+    async fn sweep_removes_expired_keys_beyond_the_first_page() {
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let backend = local_backend(&temp_dir).await;
+        let total = SWEEP_PAGE_SIZE as usize + 5;
+        let key_ids = schedule_keys(&backend, total).await;
+
+        let report = worker(backend.clone()).sweep(&after_window()).await;
+        assert_eq!(report.failed, 0);
+        assert_eq!(
+            report.removed.len(),
+            total,
+            "every expired key must be swept, not just the ones on the first page"
+        );
+        // The keys sorting last are the ones a first-page-only sweep leaves
+        // behind for good.
+        for key_id in key_ids.iter().rev().take(5) {
+            assert_key_gone(&backend, key_id).await;
+        }
+    }
+
     #[tokio::test]
     async fn cancelled_deletion_always_beats_the_sweep() {
         let temp_dir = tempfile::tempdir().expect("temp dir");
@@ -757,4 +826,27 @@ mod tests {
             }
         }
     }
+
+    /// The census counts the whole key set, not the first page of it. A sweep
+    /// that stops after one page would publish a gauge that understates every
+    /// large deployment by exactly the keys it never listed.
+    #[test]
+    fn lifecycle_gauges_count_keys_beyond_the_first_page() {
+        let total = SWEEP_PAGE_SIZE as usize + 5;
+        let (snapshot, ()) = record_metrics(|| {
+            Box::pin(async move {
+                let temp_dir = tempfile::tempdir().expect("temp dir");
+                let backend = local_backend(&temp_dir).await;
+                schedule_keys(&backend, total).await;
+
+                // Well inside the seven-day window: every key is observed as
+                // pending rather than removed.
+                let report = worker(backend.clone()).sweep(&Zoned::now()).await;
+                assert_eq!(report.skipped, total, "every scheduled key must be inspected");
+                assert!(report.removed.is_empty());
+            })
+        });
+
+        assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(total as f64));
+    }
 }

crates/kms/src/error.rs

@@ -144,6 +144,16 @@ pub enum KmsError {
         "Baseline version lost for key {key_id}: master key version records exist (oldest {oldest_version}) but the key record carries no baseline version, so data keys written before versioned rotation can no longer be resolved to the master key version that wrapped them. A node older than versioned rotation rewrote the key record and dropped the field. Finish upgrading every node, restore baseline_version to {oldest_version} on the key record, then retry"
     )]
     BaselineVersionLost { key_id: String, oldest_version: u32 },
+
+    /// Configuration still points at the key, so its material must not be
+    /// destroyed. Distinct from the generic invalid-operation errors so that
+    /// callers can tell "this key is still wired into the deployment" apart
+    /// from a malformed request and act on the listed references.
+    #[error(
+        "Key {key_id} is still referenced by configuration and its material must not be destroyed: {}. Remove or repoint the listed configuration, then retry",
+        .references.join(", ")
+    )]
+    KeyStillReferenced { key_id: String, references: Vec<String> },
 }
 
 impl KmsError {
@@ -252,6 +262,14 @@ impl KmsError {
         Self::OperationCancelled { message: message.into() }
     }
 
+    /// Create a still-referenced error
+    pub fn key_still_referenced<S: Into<String>>(key_id: S, references: Vec<String>) -> Self {
+        Self::KeyStillReferenced {
+            key_id: key_id.into(),
+            references,
+        }
+    }
+
     /// Create a material missing error
     pub fn material_missing<S: Into<String>>(key_id: S) -> Self {
         Self::MaterialMissing { key_id: key_id.into() }

crates/kms/src/key_impact.rs

@@ -0,0 +1,290 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! What still points at a KMS key, as far as the server can prove it.
+//!
+//! # This report never says a key is unused
+//!
+//! There is deliberately no `in_use`, no `unreferenced`, and no
+//! `safe_to_delete`: the report states which sources were consulted
+//! ([`ReferenceCoverage`]), how exhaustively they could be read
+//! ([`ReferenceCompleteness`]), and what was found. An empty
+//! [`KeyImpactReport::references`] therefore means "nothing was found in the
+//! scanned sources", never "nothing references this key" — object envelopes
+//! written under a key are not, and cannot cheaply be, enumerated here.
+//! Collapsing that distinction into a boolean would hand callers a green
+//! checkmark backed by an unscanned half of the problem, so the distinction
+//! lives in the type rather than in prose a UI can skip.
+//!
+//! # It may only ever add a reason to refuse
+//!
+//! A report is an input to refusing destruction, never to permitting it.
+//! [`KeyImpactReport::blocks_destruction`] is phrased so its `false` case
+//! carries no authority: it means this report found no reason to refuse, not
+//! that any other gate has been satisfied. The deletion worker's own
+//! [`crate::DeletionReferenceChecker`] stays the gate that decides whether
+//! expired material is destroyed.
+
+use serde::{Deserialize, Serialize};
+
+/// A place where references to a key can live.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum ReferenceScope {
+    /// A bucket's default server-side encryption configuration.
+    BucketDefaultEncryption,
+    /// The KMS service's configured default key.
+    ServiceDefaultKey,
+    /// Data-key envelopes stored on object versions.
+    ObjectEnvelopes,
+    /// Session envelopes of multipart uploads that have not completed yet.
+    InProgressMultipartUploads,
+}
+
+/// Why an entry appears in [`KeyImpactReport::references`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum KeyReferenceKind {
+    /// A bucket's default encryption configuration names the key.
+    BucketDefaultEncryption,
+    /// The key is the KMS service's configured default key.
+    ServiceDefaultKey,
+    /// A whole source in [`ReferenceCoverage::scanned`] could not be
+    /// enumerated. Reported as a reference, not as an absence: a source that
+    /// cannot be read may hold references, and destroying material on the
+    /// strength of an unanswered question is the one outcome that cannot be
+    /// undone.
+    UnreadableSource,
+    /// One resource inside an otherwise readable source could not be
+    /// inspected. Reported for the same reason as [`Self::UnreadableSource`].
+    UnreadableResource,
+}
+
+/// One thing that points at a key, or one place that could not be checked.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KeyReference {
+    /// Machine-readable category.
+    pub kind: KeyReferenceKind,
+    /// Identifier of the referencing resource: a bucket name for bucket
+    /// configuration, the key id for the service default key, the affected
+    /// source or resource for the unreadable kinds. Never key material.
+    pub id: String,
+    /// Human-readable detail. Identifiers only; never secrets or material.
+    pub detail: String,
+}
+
+/// Which sources a report consulted, and which it did not look at at all.
+///
+/// `not_scanned` is mandatory rather than implied: a caller reading an empty
+/// reference list has to be able to see, from the report alone, what was left
+/// out of it.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct ReferenceCoverage {
+    /// Sources this report enumerated.
+    pub scanned: Vec<ReferenceScope>,
+    /// Sources this report does not cover at all.
+    pub not_scanned: Vec<ReferenceScope>,
+}
+
+/// How far a report's reference list can be trusted.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum ReferenceCompleteness {
+    /// Every reference within [`ReferenceCoverage::scanned`] was enumerated.
+    /// Reserved for configuration-layer facts, which are finite, cheap to
+    /// read, and therefore exhaustively decidable.
+    Exact,
+    /// The list holds what a snapshot happened to observe within the scanned
+    /// scopes. Absence of a reference is not evidence that none exists.
+    ObservedOnly,
+    /// At least one scanned source could not be read, so the list is not a
+    /// statement about the key at all.
+    Unavailable,
+}
+
+/// What the server can currently say about who points at a key.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KeyImpactReport {
+    /// Key the report is about.
+    pub key_id: String,
+    /// How far [`Self::references`] can be trusted.
+    pub completeness: ReferenceCompleteness,
+    /// Which sources were consulted and which were not.
+    pub coverage: ReferenceCoverage,
+    /// References found, plus one entry per source that could not be read.
+    pub references: Vec<KeyReference>,
+}
+
+impl KeyImpactReport {
+    /// An empty report over the configuration layer: bucket default encryption
+    /// settings and the service default key.
+    ///
+    /// Both are finite and cheap to enumerate, so a report that reads them all
+    /// stays [`ReferenceCompleteness::Exact`]; object-level scopes are
+    /// declared as not scanned and stay that way.
+    pub fn configuration_layer(key_id: impl Into<String>) -> Self {
+        Self {
+            key_id: key_id.into(),
+            completeness: ReferenceCompleteness::Exact,
+            coverage: ReferenceCoverage {
+                scanned: vec![ReferenceScope::BucketDefaultEncryption, ReferenceScope::ServiceDefaultKey],
+                not_scanned: vec![ReferenceScope::ObjectEnvelopes, ReferenceScope::InProgressMultipartUploads],
+            },
+            references: Vec::new(),
+        }
+    }
+
+    /// Record one reference.
+    ///
+    /// An unreadable source or resource downgrades the report to
+    /// [`ReferenceCompleteness::Unavailable`] here rather than at the call
+    /// site, so no producer can report a partially read source as `Exact`.
+    pub fn push_reference(&mut self, reference: KeyReference) {
+        if matches!(reference.kind, KeyReferenceKind::UnreadableSource | KeyReferenceKind::UnreadableResource) {
+            self.completeness = ReferenceCompleteness::Unavailable;
+        }
+        self.references.push(reference);
+    }
+
+    /// Whether this report on its own is reason to refuse destroying the key's
+    /// material right now.
+    ///
+    /// The two answers are not symmetric. `true` is a decision: something
+    /// points at the key, or a source that might could not be read. `false`
+    /// only means this report contributes no objection — it is never a
+    /// clearance, and must not be used to skip, shorten, or satisfy any other
+    /// check on the deletion path.
+    pub fn blocks_destruction(&self) -> bool {
+        // The completeness test is redundant while every producer records an
+        // unreadable source as a reference, and stays here so that a future
+        // producer which forgets to cannot turn an unanswered question into a
+        // silent clearance.
+        !self.references.is_empty() || matches!(self.completeness, ReferenceCompleteness::Unavailable)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn bucket_reference(bucket: &str) -> KeyReference {
+        KeyReference {
+            kind: KeyReferenceKind::BucketDefaultEncryption,
+            id: bucket.to_string(),
+            detail: format!("bucket {bucket} encrypts new objects with this key by default"),
+        }
+    }
+
+    #[test]
+    fn a_fresh_configuration_report_declares_the_object_layer_unscanned() {
+        let report = KeyImpactReport::configuration_layer("kms-key-1");
+
+        assert_eq!(report.completeness, ReferenceCompleteness::Exact);
+        assert!(report.references.is_empty());
+        assert_eq!(
+            report.coverage.not_scanned,
+            vec![ReferenceScope::ObjectEnvelopes, ReferenceScope::InProgressMultipartUploads],
+            "an empty reference list is only readable next to what was left unscanned"
+        );
+        assert!(!report.blocks_destruction());
+    }
+
+    #[test]
+    fn any_reference_blocks_destruction() {
+        let mut report = KeyImpactReport::configuration_layer("kms-key-1");
+        report.push_reference(bucket_reference("sse-bucket"));
+
+        assert!(report.blocks_destruction());
+        assert_eq!(
+            report.completeness,
+            ReferenceCompleteness::Exact,
+            "a fully readable configuration layer stays exact even when it holds references"
+        );
+    }
+
+    #[test]
+    fn an_unreadable_source_is_never_reported_as_an_absence() {
+        for kind in [KeyReferenceKind::UnreadableSource, KeyReferenceKind::UnreadableResource] {
+            let mut report = KeyImpactReport::configuration_layer("kms-key-1");
+            report.push_reference(KeyReference {
+                kind,
+                id: "bucket-default-encryption".to_string(),
+                detail: "configuration could not be read".to_string(),
+            });
+
+            assert_eq!(
+                report.completeness,
+                ReferenceCompleteness::Unavailable,
+                "{kind:?} must downgrade completeness"
+            );
+            assert!(report.blocks_destruction(), "{kind:?} must block destruction");
+        }
+    }
+
+    #[test]
+    fn unavailable_completeness_blocks_even_without_references() {
+        // Guards the fail-closed fallback for a producer that marks a report
+        // unavailable without recording why.
+        let report = KeyImpactReport {
+            completeness: ReferenceCompleteness::Unavailable,
+            ..KeyImpactReport::configuration_layer("kms-key-1")
+        };
+
+        assert!(report.references.is_empty());
+        assert!(report.blocks_destruction());
+    }
+
+    #[test]
+    fn report_round_trips_through_json() {
+        let mut report = KeyImpactReport::configuration_layer("kms-key-1");
+        report.push_reference(bucket_reference("sse-bucket"));
+        report.push_reference(KeyReference {
+            kind: KeyReferenceKind::ServiceDefaultKey,
+            id: "kms-key-1".to_string(),
+            detail: "configured as the KMS service default key".to_string(),
+        });
+
+        let json = serde_json::to_string(&report).expect("serialization should succeed");
+        let decoded: KeyImpactReport = serde_json::from_str(&json).expect("deserialization should succeed");
+        assert_eq!(decoded, report);
+    }
+
+    /// The wire shape is the contract callers build UIs on: a boolean that
+    /// reads as "this key is unused" must never appear in it, however the
+    /// report is populated.
+    #[test]
+    fn the_wire_shape_asserts_nothing_about_absence_of_use() {
+        let mut referenced = KeyImpactReport::configuration_layer("kms-key-1");
+        referenced.push_reference(bucket_reference("sse-bucket"));
+
+        for report in [KeyImpactReport::configuration_layer("kms-key-1"), referenced] {
+            let json = serde_json::to_value(&report).expect("serialization should succeed");
+            let mut fields: Vec<&str> = json
+                .as_object()
+                .expect("report is a JSON object")
+                .keys()
+                .map(String::as_str)
+                .collect();
+            fields.sort_unstable();
+
+            assert_eq!(fields, vec!["completeness", "coverage", "key_id", "references"]);
+            for forbidden in ["in_use", "unused", "unreferenced", "safe_to_delete", "deletable"] {
+                assert!(!json.to_string().contains(forbidden), "report must not carry a `{forbidden}` claim");
+            }
+        }
+    }
+}

crates/kms/src/lib.rs

@@ -73,6 +73,7 @@ pub mod config;
 pub mod deletion_worker;
 mod encryption;
 mod error;
+pub mod key_impact;
 pub mod manager;
 mod policy;
 pub mod probe;
@@ -83,9 +84,9 @@ pub mod types;
 
 // Re-export public API
 pub use api_types::{
-    CacheSummary, ConfigureKmsRequest, ConfigureKmsResponse, ConfigureLocalKmsRequest, ConfigureStaticKmsRequest,
-    ConfigureVaultKmsRequest, ConfigureVaultTransitKmsRequest, KmsConfigSummary, KmsStatusResponse, StartKmsRequest,
-    StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
+    CacheSummary, ConfigureAwsKmsRequest, ConfigureKmsRequest, ConfigureKmsResponse, ConfigureLocalKmsRequest,
+    ConfigureStaticKmsRequest, ConfigureVaultKmsRequest, ConfigureVaultTransitKmsRequest, KmsConfigSummary, KmsStatusResponse,
+    StartKmsRequest, StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
     UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
 pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
@@ -94,6 +95,7 @@ pub use config::*;
 pub use deletion_worker::DeletionReferenceChecker;
 pub use encryption::is_data_key_envelope;
 pub use error::{KmsError, KmsUnavailableError, Result};
+pub use key_impact::{KeyImpactReport, KeyReference, KeyReferenceKind, ReferenceCompleteness, ReferenceCoverage, ReferenceScope};
 pub use manager::KmsManager;
 pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
 pub use service::{DataKey, ObjectEncryptionService};

crates/kms/src/manager.rs

@@ -18,6 +18,7 @@ use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
 use crate::backends::KmsBackend;
 use crate::cache::{KmsCache, KmsCacheStats};
 use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, KmsConfig};
+use crate::deletion_worker::DeletionReferenceChecker;
 use crate::error::{KmsError, Result};
 use crate::types::{
     CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse,
@@ -41,6 +42,7 @@ pub struct KmsManager {
     backend_kind: &'static str,
     audit_sink: Option<Arc<dyn KmsAuditSink>>,
     allow_immediate_deletion: bool,
+    reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
 }
 
 impl KmsManager {
@@ -60,9 +62,21 @@ impl KmsManager {
             backend_kind: config.backend.as_str(),
             audit_sink: None,
             allow_immediate_deletion: config.allow_immediate_deletion,
+            reference_checker: None,
         }
     }
 
+    /// Consult `checker` before immediate deletion destroys key material.
+    ///
+    /// This is the same checker the deletion worker consults before it removes
+    /// an expired key; installing it here extends that gate to the one
+    /// deletion path that never reaches the worker. It can only add a refusal:
+    /// without a checker the manager behaves exactly as before.
+    pub fn with_deletion_reference_checker(mut self, checker: Option<Arc<dyn DeletionReferenceChecker>>) -> Self {
+        self.reference_checker = checker;
+        self
+    }
+
     /// Send an audit record for every management operation to `sink`.
     ///
     /// Without a sink the manager builds no records at all, so a deployment
@@ -262,6 +276,9 @@ impl KmsManager {
     /// defensive assertion for callers that hold a backend handle directly.
     async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
         self.check_deletion_request(&request)?;
+        if request.force_immediate.unwrap_or(false) {
+            self.refuse_referenced_immediate_deletion(&request.key_id).await?;
+        }
 
         let response = self.backend.delete_key(request).await?;
 
@@ -312,6 +329,41 @@ impl KmsManager {
         Ok(())
     }
 
+    /// Refuse an immediate deletion while configuration still points at the
+    /// key.
+    ///
+    /// A scheduled deletion is re-checked against these same references by the
+    /// deletion worker before it destroys anything, and stays cancellable
+    /// until then. Immediate deletion has neither property: it destroys
+    /// material on the spot and never reaches the worker, so the check has to
+    /// happen here or not at all.
+    ///
+    /// Only ever a refusal. An empty reference set is not a clearance — it
+    /// means the sources consulted here raised no objection, while the caller
+    /// still had to pass the server-side opt-in and the key-id confirmation to
+    /// get this far. With no checker installed the manager has no
+    /// configuration source to consult and behaves as it did before, matching
+    /// the deletion worker, which also skips a checker it was not given.
+    async fn refuse_referenced_immediate_deletion(&self, key_id: &str) -> Result<()> {
+        let mut references = Vec::new();
+        if self.default_key_id.as_deref() == Some(key_id) {
+            references.push("kms-service-default-key".to_string());
+        }
+        if let Some(checker) = &self.reference_checker {
+            references.extend(checker.references(key_id).await);
+        }
+        if references.is_empty() {
+            return Ok(());
+        }
+
+        warn!(
+            key_id,
+            ?references,
+            "immediate KMS key deletion refused; configuration still references the key"
+        );
+        Err(KmsError::key_still_referenced(key_id, references))
+    }
+
     /// Cancel key deletion
     ///
     /// Audited as an internal operation; callers serving an authenticated
@@ -1256,6 +1308,191 @@ mod tests {
         assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
     }
 
+    /// Reference checker whose answer is fixed, standing in for the server's
+    /// bucket-configuration gate.
+    struct StaticReferences(Vec<String>);
+
+    #[async_trait]
+    impl DeletionReferenceChecker for StaticReferences {
+        async fn references(&self, _key_id: &str) -> Vec<String> {
+            self.0.clone()
+        }
+    }
+
+    #[tokio::test]
+    async fn immediate_deletion_is_refused_while_configuration_references_the_key() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let manager = deletion_manager(&temp_dir, true)
+            .await
+            .with_deletion_reference_checker(Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))));
+        let key_id = create_named_key(&manager, "referenced-force-delete").await;
+        let probe = data_key_probe(&manager, &key_id).await;
+
+        // Server opt-in granted and the confirmation exact: the reference is
+        // the only thing left to refuse this.
+        let error = manager
+            .delete_key(DeleteKeyRequest {
+                key_id: key_id.clone(),
+                force_immediate: Some(true),
+                confirm_key_id: Some(key_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .expect_err("immediate deletion must be refused while configuration references the key");
+
+        match error {
+            KmsError::KeyStillReferenced {
+                key_id: refused,
+                references,
+            } => {
+                assert_eq!(refused, key_id);
+                assert_eq!(references, vec!["bucket:sse-bucket".to_string()], "the caller must learn what refused it");
+            }
+            other => panic!("expected KeyStillReferenced, got {other:?}"),
+        }
+
+        assert_key_material_intact(&manager, &key_id, &probe).await;
+    }
+
+    #[tokio::test]
+    async fn immediate_deletion_of_the_service_default_key_is_refused() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        config.allow_immediate_deletion = true;
+        let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
+        let key_id = KmsManager::new(backend.clone(), config.clone())
+            .create_key(CreateKeyRequest {
+                key_name: Some("default-key-force-delete".to_string()),
+                key_usage: KeyUsage::EncryptDecrypt,
+                ..Default::default()
+            })
+            .await
+            .expect("Failed to create key")
+            .key_id;
+
+        config.default_key_id = Some(key_id.clone());
+        let manager = KmsManager::new(backend, config);
+        let probe = data_key_probe(&manager, &key_id).await;
+
+        let error = manager
+            .delete_key(DeleteKeyRequest {
+                key_id: key_id.clone(),
+                force_immediate: Some(true),
+                confirm_key_id: Some(key_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .expect_err("the service default key must not be destroyed out from under the deployment");
+        assert!(
+            matches!(error, KmsError::KeyStillReferenced { .. }),
+            "expected KeyStillReferenced, got {error:?}"
+        );
+
+        assert_key_material_intact(&manager, &key_id, &probe).await;
+    }
+
+    /// A checker that reports nothing must not become a shortcut around the
+    /// gates that were already there: it is not a clearance, only the absence
+    /// of one more objection.
+    #[tokio::test]
+    async fn an_empty_reference_set_grants_no_deletion_on_its_own() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let manager = deletion_manager(&temp_dir, false)
+            .await
+            .with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
+        let key_id = create_named_key(&manager, "unreferenced-force-delete").await;
+        let probe = data_key_probe(&manager, &key_id).await;
+
+        // Server opt-in withheld, then confirmation missing: both still refuse.
+        let error = manager
+            .delete_key(DeleteKeyRequest {
+                key_id: key_id.clone(),
+                force_immediate: Some(true),
+                confirm_key_id: Some(key_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .expect_err("an unreferenced key still needs the server-side opt-in");
+        assert!(
+            matches!(error, KmsError::InvalidOperation { .. }),
+            "expected InvalidOperation, got {error:?}"
+        );
+
+        let allowed = deletion_manager(&temp_dir, true)
+            .await
+            .with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
+        let error = allowed
+            .delete_key(DeleteKeyRequest {
+                key_id: key_id.clone(),
+                force_immediate: Some(true),
+                ..Default::default()
+            })
+            .await
+            .expect_err("an unreferenced key still needs the key-id confirmation");
+        assert!(
+            matches!(error, KmsError::InvalidOperation { .. }),
+            "expected InvalidOperation, got {error:?}"
+        );
+
+        assert_key_material_intact(&manager, &key_id, &probe).await;
+    }
+
+    /// The refusal is the only thing the checker adds: a fully authorized
+    /// immediate deletion of a key nothing points at still goes through.
+    #[tokio::test]
+    async fn immediate_deletion_still_succeeds_when_nothing_references_the_key() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let manager = deletion_manager(&temp_dir, true)
+            .await
+            .with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
+        let key_id = create_named_key(&manager, "unreferenced-confirmed-force-delete").await;
+
+        manager
+            .delete_key(DeleteKeyRequest {
+                key_id: key_id.clone(),
+                force_immediate: Some(true),
+                confirm_key_id: Some(key_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .expect("a confirmed immediate deletion must still be allowed when nothing references the key");
+
+        let error = manager
+            .describe_key(DescribeKeyRequest { key_id: key_id.clone() })
+            .await
+            .expect_err("an immediately deleted key must be gone");
+        assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
+    }
+
+    /// Scheduling stays a schedule: it destroys nothing, stays cancellable,
+    /// and is re-checked against the same references by the deletion worker
+    /// before any material goes away. Turning references into an up-front
+    /// refusal here would let one unreadable bucket block routine operations.
+    #[tokio::test]
+    async fn scheduled_deletion_is_unaffected_by_references() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let manager = deletion_manager(&temp_dir, false)
+            .await
+            .with_deletion_reference_checker(Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))));
+        let key_id = create_named_key(&manager, "referenced-schedule").await;
+
+        manager
+            .delete_key(DeleteKeyRequest {
+                key_id: key_id.clone(),
+                ..Default::default()
+            })
+            .await
+            .expect("a scheduled deletion must still be accepted while configuration references the key");
+
+        let state = manager
+            .describe_key(DescribeKeyRequest { key_id: key_id.clone() })
+            .await
+            .expect("a scheduled key must still be describable")
+            .key_metadata
+            .key_state;
+        assert_eq!(state, KeyState::PendingDeletion);
+    }
+
     #[tokio::test]
     async fn pending_window_outside_the_supported_range_is_refused() {
         let temp_dir = tempdir().expect("Failed to create temp dir");

crates/kms/src/service_manager.rs

@@ -634,7 +634,13 @@ impl KmsServiceManager {
         };
 
         // Create KMS manager
-        let mut kms_manager = KmsManager::new(backend, config.clone());
+        //
+        // The deletion reference checker is handed to the manager as well as to
+        // the worker: immediate deletion destroys material without ever
+        // reaching the worker, so that path has to consult the same gate
+        // itself.
+        let mut kms_manager =
+            KmsManager::new(backend, config.clone()).with_deletion_reference_checker(self.deletion_reference_checker());
         if let Some(sink) = self.audit_sink() {
             kms_manager = kms_manager.with_audit_sink(sink);
         }
@@ -749,6 +755,42 @@ mod tests {
         KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
     }
 
+    /// End-to-end wiring check for the AWS backend: an admin configure request
+    /// must select it, build a real client, and pass the startup health check.
+    ///
+    /// `RUSTFS_KMS_AWS_REGION` names the region; the credential chain supplies
+    /// the rest. No key is created, so this test is not billable on its own.
+    #[tokio::test]
+    #[ignore] // Requires real AWS credentials
+    async fn aws_backend_configure_and_start_end_to_end() {
+        let region = std::env::var("RUSTFS_KMS_AWS_REGION").expect("RUSTFS_KMS_AWS_REGION must name the test region");
+        let config = crate::api_types::ConfigureAwsKmsRequest {
+            region,
+            endpoint_url: None,
+            default_key_id: std::env::var("RUSTFS_KMS_DEFAULT_KEY_ID").ok(),
+            timeout_seconds: None,
+            retry_attempts: None,
+            enable_cache: None,
+            max_cached_keys: None,
+            cache_ttl_seconds: None,
+            allow_insecure_dev_defaults: None,
+        }
+        .to_kms_config();
+
+        let manager = KmsServiceManager::new();
+        manager.configure(config).await.expect("configure the AWS backend");
+        manager.start().await.expect("start the AWS backend");
+
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        let capabilities = manager
+            .get_manager()
+            .await
+            .expect("a running service exposes its manager")
+            .backend_capabilities();
+        assert!(capabilities.schedule_deletion);
+        assert!(!capabilities.physical_delete);
+    }
+
     #[tokio::test]
     async fn configure_rejects_insecure_development_defaults_before_state_update() {
         let manager = KmsServiceManager::new();

crates/obs/src/metrics/collectors/bucket_replication.rs

@@ -17,8 +17,10 @@
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::bucket_replication::{
     BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
-    BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
-    BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
+    BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
+    BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
+    BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
+    BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
     BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
     BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD, BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD,
     BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD, BUCKET_REPL_MRF_PENDING_COUNT_MD,
@@ -37,6 +39,7 @@ use std::borrow::Cow;
 
 const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
 const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11;
+const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4;
 
 #[derive(Debug, Clone, Default)]
 pub struct BucketReplicationTargetStats {
@@ -99,6 +102,16 @@ pub(crate) struct BucketReplicationBacklogStats {
     pub(crate) mrf_missed_count: u64,
     pub(crate) mrf_flush_failures: u64,
     pub(crate) mrf_last_flush_duration_millis: u64,
+    pub(crate) target_backlogs: Vec<BucketReplicationTargetBacklogStats>,
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct BucketReplicationTargetBacklogStats {
+    pub(crate) target_arn: String,
+    pub(crate) current_backlog_count: u64,
+    pub(crate) current_backlog_bytes: u64,
+    pub(crate) durable_mrf_backlog_count: u64,
+    pub(crate) durable_mrf_backlog_bytes: u64,
 }
 
 pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
@@ -290,7 +303,14 @@ pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicat
         return Vec::new();
     }
 
-    let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
+    let metric_count = stats
+        .iter()
+        .map(|stat| {
+            BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET
+                + stat.target_backlogs.len() * BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET
+        })
+        .sum();
+    let mut metrics = Vec::with_capacity(metric_count);
     for stat in stats {
         let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
 
@@ -345,6 +365,42 @@ pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicat
             )
             .with_label(BUCKET_L, bucket_label),
         );
+        for target in &stat.target_backlogs {
+            let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
+            let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
+            metrics.push(
+                PrometheusMetric::from_descriptor(
+                    &BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD,
+                    target.current_backlog_count as f64,
+                )
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_label.clone()),
+            );
+            metrics.push(
+                PrometheusMetric::from_descriptor(
+                    &BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
+                    target.current_backlog_bytes as f64,
+                )
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_label.clone()),
+            );
+            metrics.push(
+                PrometheusMetric::from_descriptor(
+                    &BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD,
+                    target.durable_mrf_backlog_count as f64,
+                )
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_label.clone()),
+            );
+            metrics.push(
+                PrometheusMetric::from_descriptor(
+                    &BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
+                    target.durable_mrf_backlog_bytes as f64,
+                )
+                .with_label(BUCKET_L, bucket_label)
+                .with_label(TARGET_ARN_L, target_label),
+            );
+        }
     }
 
     metrics
@@ -469,10 +525,17 @@ mod tests {
             mrf_missed_count: 4,
             mrf_flush_failures: 5,
             mrf_last_flush_duration_millis: 6,
+            target_backlogs: vec![BucketReplicationTargetBacklogStats {
+                target_arn: "arn:rustfs:replication:target-a".to_string(),
+                current_backlog_count: 3,
+                current_backlog_bytes: 4096,
+                durable_mrf_backlog_count: 2,
+                durable_mrf_backlog_bytes: 2048,
+            }],
         }];
 
         let metrics = collect_bucket_replication_backlog_metrics(&stats);
-        assert_eq!(metrics.len(), 11);
+        assert_eq!(metrics.len(), 15);
 
         let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
         assert!(metrics.iter().any(|metric| {
@@ -509,6 +572,50 @@ mod tests {
                 && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
         }));
 
+        let target_count_name = BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == target_count_name
+                && metric.value == 3.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
+        }));
+
+        let target_bytes_name = BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == target_bytes_name
+                && metric.value == 4096.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
+        }));
+
+        let durable_target_count_name = BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_target_count_name
+                && metric.value == 2.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
+        }));
+
+        let durable_target_bytes_name = BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_target_bytes_name
+                && metric.value == 2048.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+                && metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
+        }));
+
         let pending_count_name = BUCKET_REPL_MRF_PENDING_COUNT_MD.get_full_metric_name();
         assert!(metrics.iter().any(|metric| {
             metric.name == pending_count_name
@@ -574,6 +681,7 @@ mod tests {
             mrf_missed_count: 4,
             mrf_flush_failures: 5,
             mrf_last_flush_duration_millis: 6,
+            target_backlogs: Vec::new(),
         }];
 
         let metrics = collect_bucket_replication_backlog_metrics(&stats);

crates/obs/src/metrics/collectors/mod.rs

@@ -42,7 +42,9 @@ pub mod system_process;
 
 pub use audit::{AuditTargetStats, collect_audit_metrics};
 pub use bucket::{BucketStats, collect_bucket_metrics};
-pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
+pub(crate) use bucket_replication::{
+    BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics,
+};
 pub use bucket_replication::{
     BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
     collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,

crates/obs/src/metrics/scheduler.rs

@@ -80,8 +80,10 @@ use crate::metrics::runtime_sources::bucket_monitor_available;
 use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE_LENGTH_MD, AUDIT_TOTAL_MESSAGES_MD};
 use crate::metrics::schema::bucket_replication::{
     BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
-    BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
-    BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD,
+    BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
+    BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
+    BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
+    BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD,
     BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD, BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD,
     BUCKET_REPL_MRF_PENDING_COUNT_MD, TARGET_ARN_L,
 };
@@ -633,6 +635,17 @@ fn repl_backlog_live_keys(stats: &[BucketReplicationBacklogStats]) -> HashSet<Bu
     stats.iter().map(|s| s.bucket.clone()).collect()
 }
 
+fn repl_backlog_target_live_keys(stats: &[BucketReplicationBacklogStats]) -> HashSet<ReplBwKey> {
+    stats
+        .iter()
+        .flat_map(|stat| {
+            stat.target_backlogs
+                .iter()
+                .map(|target| (stat.bucket.clone(), target.target_arn.clone()))
+        })
+        .collect()
+}
+
 fn update_series_zero_tombstones<T: Clone + Eq + std::hash::Hash>(
     has_seen_valid_snapshot: &mut bool,
     prev_live_keys: &mut HashSet<T>,
@@ -1060,6 +1073,40 @@ fn collect_repl_backlog_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketK
     collect_bucket_replication_backlog_metrics(&stats)
 }
 
+fn collect_repl_backlog_target_zero_tombstone_metrics(zero_tombstones: &HashMap<ReplBwKey, u8>) -> Vec<PrometheusMetric> {
+    if zero_tombstones.is_empty() {
+        return Vec::new();
+    }
+
+    let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4);
+    for (bucket, target_arn) in zero_tombstones.keys() {
+        let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone());
+        let target_arn_label: Cow<'static, str> = Cow::Owned(target_arn.clone());
+        zero_metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, 0.0)
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_arn_label.clone()),
+        );
+        zero_metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD, 0.0)
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_arn_label.clone()),
+        );
+        zero_metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, 0.0)
+                .with_label(BUCKET_L, bucket_label.clone())
+                .with_label(TARGET_ARN_L, target_arn_label.clone()),
+        );
+        zero_metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD, 0.0)
+                .with_label(BUCKET_L, bucket_label)
+                .with_label(TARGET_ARN_L, target_arn_label),
+        );
+    }
+
+    zero_metrics
+}
+
 fn retire_repl_bw_metric_series(bucket: &str, target_arn: &str) -> usize {
     let labels = [
         (BUCKET_L, Cow::Owned(bucket.to_string())),
@@ -1089,6 +1136,17 @@ fn retire_repl_backlog_metric_series(bucket: &str) -> usize {
     .sum()
 }
 
+fn retire_repl_backlog_target_metric_series(bucket: &str, target_arn: &str) -> usize {
+    let labels = [
+        (BUCKET_L, Cow::Owned(bucket.to_string())),
+        (TARGET_ARN_L, Cow::Owned(target_arn.to_string())),
+    ];
+    retire_metric_series(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(), &labels)
+        + retire_metric_series(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(), &labels)
+}
+
 fn expire_repl_bw_zero_tombstones(monitor_available: bool, zero_tombstones: &mut HashMap<ReplBwKey, u8>) -> Vec<ReplBwKey> {
     if !monitor_available {
         return Vec::new();
@@ -1105,6 +1163,17 @@ fn expire_repl_backlog_zero_tombstones(monitor_available: bool, zero_tombstones:
     expire_series_zero_tombstones(zero_tombstones)
 }
 
+fn expire_repl_backlog_target_zero_tombstones(
+    target_metrics_available: bool,
+    zero_tombstones: &mut HashMap<ReplBwKey, u8>,
+) -> Vec<ReplBwKey> {
+    if !target_metrics_available {
+        return Vec::new();
+    }
+
+    expire_series_zero_tombstones(zero_tombstones)
+}
+
 /// Initialize all metrics collectors.
 ///
 /// This function spawns background tasks that periodically collect metrics
@@ -1399,6 +1468,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
         let mut prev_backlog_live_keys: HashSet<BucketKey> = HashSet::new();
         let mut backlog_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
         let mut has_seen_valid_backlog_snapshot = false;
+        let mut prev_backlog_target_live_keys: HashSet<ReplBwKey> = HashSet::new();
+        let mut backlog_target_zero_tombstones: HashMap<ReplBwKey, u8> = HashMap::new();
+        let mut has_seen_valid_backlog_target_snapshot = false;
         loop {
             tokio::select! {
                 _ = interval.tick() => {
@@ -1429,6 +1501,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
 
                             let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
+                            let durable_mrf_available = bucket_replication_backlog.iter().any(|stat| stat.durable_mrf_available);
+                            let backlog_target_metrics_available =
+                                durable_mrf_available || monitor_available || !bucket_replication_backlog.is_empty();
                             update_repl_backlog_zero_tombstones(
                                 monitor_available,
                                 &mut has_seen_valid_backlog_snapshot,
@@ -1437,9 +1512,19 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                                 repl_backlog_live_keys(&bucket_replication_backlog),
                                 repl_bw_zero_tombstone_cycles,
                             );
+                            if backlog_target_metrics_available {
+                                update_series_zero_tombstones(
+                                    &mut has_seen_valid_backlog_target_snapshot,
+                                    &mut prev_backlog_target_live_keys,
+                                    &mut backlog_target_zero_tombstones,
+                                    repl_backlog_target_live_keys(&bucket_replication_backlog),
+                                    repl_bw_zero_tombstone_cycles,
+                                );
+                            }
                             metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
                             metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
                             metrics.extend(collect_repl_backlog_zero_tombstone_metrics(&backlog_zero_tombstones));
+                            metrics.extend(collect_repl_backlog_target_zero_tombstone_metrics(&backlog_target_zero_tombstones));
                             let replication = collect_replication_stats().await;
                             metrics.extend(collect_replication_metrics(&replication));
                             report_metrics(&metrics);
@@ -1451,6 +1536,12 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             for bucket in expire_repl_backlog_zero_tombstones(monitor_available, &mut backlog_zero_tombstones) {
                                 let _ = retire_repl_backlog_metric_series(&bucket);
                             }
+                            for (bucket, target_arn) in expire_repl_backlog_target_zero_tombstones(
+                                backlog_target_metrics_available,
+                                &mut backlog_target_zero_tombstones,
+                            ) {
+                                let _ = retire_repl_backlog_target_metric_series(&bucket, &target_arn);
+                            }
                         },
                     ).await;
                 }
@@ -2259,6 +2350,69 @@ mod tests {
         assert_eq!(prev_live_keys, bucket_keys(&["photos"]));
     }
 
+    #[test]
+    fn repl_backlog_target_tombstones_zero_removed_targets_then_expire() {
+        let mut has_seen_valid_snapshot = false;
+        let mut prev_live_keys = HashSet::new();
+        let mut zero_tombstones = HashMap::new();
+        let key = repl_bw_key("photos", "arn:rustfs:replication:target-a");
+
+        update_series_zero_tombstones(
+            &mut has_seen_valid_snapshot,
+            &mut prev_live_keys,
+            &mut zero_tombstones,
+            repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]),
+            2,
+        );
+        assert!(has_seen_valid_snapshot);
+        assert_eq!(prev_live_keys, repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]));
+        assert!(zero_tombstones.is_empty());
+
+        update_series_zero_tombstones(&mut has_seen_valid_snapshot, &mut prev_live_keys, &mut zero_tombstones, HashSet::new(), 2);
+        assert_eq!(zero_tombstones.get(&key), Some(&2));
+
+        let metrics = collect_repl_backlog_target_zero_tombstone_metrics(&zero_tombstones);
+        assert_eq!(metrics.len(), 4);
+        let expected_names = HashSet::from([
+            BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(),
+            BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(),
+        ]);
+        let mut actual_names = HashSet::new();
+        for metric in metrics {
+            actual_names.insert(metric.name.to_string());
+            assert_eq!(metric.value, 0.0);
+            let labels = metric
+                .labels
+                .into_iter()
+                .map(|(key, value)| (key, value.to_string()))
+                .collect::<HashMap<_, _>>();
+            assert_eq!(labels.get(BUCKET_L).map(String::as_str), Some("photos"));
+            assert_eq!(labels.get(TARGET_ARN_L).map(String::as_str), Some("arn:rustfs:replication:target-a"));
+        }
+        assert_eq!(actual_names, expected_names);
+
+        let expired = expire_repl_backlog_target_zero_tombstones(true, &mut zero_tombstones);
+        assert!(expired.is_empty());
+        assert_eq!(zero_tombstones.get(&key), Some(&1));
+
+        let expired = expire_repl_backlog_target_zero_tombstones(true, &mut zero_tombstones);
+        assert_eq!(expired, vec![key]);
+        assert!(zero_tombstones.is_empty());
+    }
+
+    #[test]
+    fn repl_backlog_target_tombstones_do_not_advance_when_target_metrics_unavailable() {
+        let key = repl_bw_key("videos", "arn:rustfs:replication:target-b");
+        let mut zero_tombstones = HashMap::from([(key.clone(), 1)]);
+
+        let expired = expire_repl_backlog_target_zero_tombstones(false, &mut zero_tombstones);
+
+        assert!(expired.is_empty());
+        assert_eq!(zero_tombstones.get(&key), Some(&1));
+    }
+
     #[test]
     fn bucket_tombstones_zero_removed_buckets_then_expire() {
         let mut has_seen_valid_snapshot = false;

crates/obs/src/metrics/schema/bucket_replication.rs

@@ -35,9 +35,13 @@ const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
 const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
 const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
 const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
+const CURRENT_TARGET_BACKLOG_COUNT: &str = "current_target_backlog_count";
+const CURRENT_TARGET_BACKLOG_BYTES: &str = "current_target_backlog_bytes";
 const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
 const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
 const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
+const DURABLE_MRF_TARGET_BACKLOG_COUNT: &str = "durable_mrf_target_backlog_count";
+const DURABLE_MRF_TARGET_BACKLOG_BYTES: &str = "durable_mrf_target_backlog_bytes";
 const MRF_PENDING_COUNT: &str = "mrf_pending_count";
 const MRF_PENDING_BYTES: &str = "mrf_pending_bytes";
 const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
@@ -108,6 +112,24 @@ pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = La
     )
 });
 
+pub static BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(CURRENT_TARGET_BACKLOG_COUNT),
+        "Current number of target-scoped operations admitted to in-memory replication worker queues for a bucket and target ARN",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(CURRENT_TARGET_BACKLOG_BYTES),
+        "Current bytes in target-scoped operations admitted to in-memory replication worker queues for a bucket and target ARN",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::from(DURABLE_MRF_AVAILABLE),
@@ -135,6 +157,24 @@ pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor>
     )
 });
 
+pub static BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_TARGET_BACKLOG_COUNT),
+        "Current number of target-scoped operations in the durable MRF backlog file for a bucket and target ARN on this node",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_TARGET_BACKLOG_BYTES),
+        "Current bytes in target-scoped operations in the durable MRF backlog file for a bucket and target ARN on this node",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_MRF_PENDING_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::from(MRF_PENDING_COUNT),

crates/obs/src/metrics/stats_collector.rs

@@ -21,11 +21,11 @@
 //! and convert them to the Stats structs used by collectors.
 
 use crate::metrics::collectors::{
-    BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
-    BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
-    CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
-    IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
-    ScannerStats,
+    BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetBacklogStats,
+    BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
+    ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats,
+    HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats,
+    ResourceStats, ScannerStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
@@ -212,6 +212,17 @@ async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>,
             mrf_missed_count: stats.mrf_missed_count,
             mrf_flush_failures: stats.mrf_flush_failures,
             mrf_last_flush_duration_millis: stats.mrf_last_flush_duration_millis,
+            target_backlogs: stats
+                .target_backlogs
+                .iter()
+                .map(|target| BucketReplicationTargetBacklogStats {
+                    target_arn: target.target_arn.clone(),
+                    current_backlog_count: target.current_backlog_count,
+                    current_backlog_bytes: target.current_backlog_bytes,
+                    durable_mrf_backlog_count: target.durable_mrf_backlog_count,
+                    durable_mrf_backlog_bytes: target.durable_mrf_backlog_bytes,
+                })
+                .collect(),
         });
         detail_stats.push(bucket_replication_detail_from_snapshot(stats));
     }

crates/obs/src/metrics/storage_api.rs

@@ -18,7 +18,8 @@ use std::time::Duration;
 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
 use rustfs_ecstore::api::bucket::replication::{
-    DurableMrfBucketBacklog, MrfBucketBacklogObservability, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
+    DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog,
+    durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot, get_global_replication_stats,
     mrf_backlog_observability_snapshot,
 };
 pub(crate) use rustfs_ecstore::api::capacity::{
@@ -44,6 +45,15 @@ pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
     pub(crate) latency_ms: f64,
 }
 
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct ObsBucketReplicationTargetBacklogSnapshot {
+    pub(crate) target_arn: String,
+    pub(crate) current_backlog_count: u64,
+    pub(crate) current_backlog_bytes: u64,
+    pub(crate) durable_mrf_backlog_count: u64,
+    pub(crate) durable_mrf_backlog_bytes: u64,
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub(crate) struct ObsBucketReplicationStatsSnapshot {
     pub(crate) bucket: String,
@@ -84,6 +94,7 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
     pub(crate) mrf_flush_failures: u64,
     pub(crate) mrf_last_flush_duration_millis: u64,
     pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
+    pub(crate) target_backlogs: Vec<ObsBucketReplicationTargetBacklogSnapshot>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq)]
@@ -122,6 +133,15 @@ struct ObsBucketReplicationProxySnapshot {
     proxied_delete_tagging_requests_failures: u64,
 }
 
+#[derive(Debug, Clone, Default, PartialEq)]
+struct ObsBucketReplicationBacklogSnapshot {
+    durable_mrf_available: bool,
+    durable_bucket: DurableMrfBucketBacklog,
+    runtime_targets: Vec<RuntimeReplicationTargetBacklog>,
+    durable_targets: Vec<DurableMrfTargetBacklog>,
+    mrf_observability: MrfBucketBacklogObservability,
+}
+
 #[derive(Debug, Clone, Default, PartialEq)]
 pub(crate) struct ObsReplicationSiteStatsSnapshot {
     pub(crate) average_active_workers: f64,
@@ -157,10 +177,40 @@ fn bucket_replication_stats_snapshot_from_parts(
     bucket: String,
     runtime: ObsBucketReplicationRuntimeSnapshot,
     proxy: ObsBucketReplicationProxySnapshot,
-    durable_mrf_available: bool,
-    durable_bucket: DurableMrfBucketBacklog,
-    mrf_observability: MrfBucketBacklogObservability,
+    backlog: ObsBucketReplicationBacklogSnapshot,
 ) -> ObsBucketReplicationStatsSnapshot {
+    let mut target_backlogs = HashMap::with_capacity(backlog.runtime_targets.len().saturating_add(backlog.durable_targets.len()));
+    for target in backlog.runtime_targets {
+        let entry =
+            target_backlogs
+                .entry(target.target_arn.clone())
+                .or_insert_with(|| ObsBucketReplicationTargetBacklogSnapshot {
+                    target_arn: target.target_arn,
+                    current_backlog_count: 0,
+                    current_backlog_bytes: 0,
+                    durable_mrf_backlog_count: 0,
+                    durable_mrf_backlog_bytes: 0,
+                });
+        entry.current_backlog_count = target.count;
+        entry.current_backlog_bytes = target.bytes;
+    }
+    for target in backlog.durable_targets {
+        let entry =
+            target_backlogs
+                .entry(target.target_arn.clone())
+                .or_insert_with(|| ObsBucketReplicationTargetBacklogSnapshot {
+                    target_arn: target.target_arn,
+                    current_backlog_count: 0,
+                    current_backlog_bytes: 0,
+                    durable_mrf_backlog_count: 0,
+                    durable_mrf_backlog_bytes: 0,
+                });
+        entry.durable_mrf_backlog_count = target.count;
+        entry.durable_mrf_backlog_bytes = target.bytes;
+    }
+    let mut target_backlogs = target_backlogs.into_values().collect::<Vec<_>>();
+    target_backlogs.sort_by(|left, right| left.target_arn.cmp(&right.target_arn));
+
     ObsBucketReplicationStatsSnapshot {
         bucket,
         total_failed_bytes: runtime.total_failed_bytes,
@@ -190,16 +240,17 @@ fn bucket_replication_stats_snapshot_from_parts(
         resync_duration_ms: runtime.resync_duration_ms,
         current_backlog_count: runtime.current_backlog_count,
         current_backlog_bytes: runtime.current_backlog_bytes,
-        durable_mrf_available,
-        durable_mrf_backlog_count: durable_bucket.count,
-        durable_mrf_backlog_bytes: durable_bucket.bytes,
-        mrf_pending_count: mrf_observability.pending_count,
-        mrf_pending_bytes: mrf_observability.pending_bytes,
-        mrf_dropped_count: mrf_observability.dropped_count,
-        mrf_missed_count: mrf_observability.missed_count,
-        mrf_flush_failures: mrf_observability.flush_failure_count,
-        mrf_last_flush_duration_millis: mrf_observability.last_flush_duration_millis,
+        durable_mrf_available: backlog.durable_mrf_available,
+        durable_mrf_backlog_count: backlog.durable_bucket.count,
+        durable_mrf_backlog_bytes: backlog.durable_bucket.bytes,
+        mrf_pending_count: backlog.mrf_observability.pending_count,
+        mrf_pending_bytes: backlog.mrf_observability.pending_bytes,
+        mrf_dropped_count: backlog.mrf_observability.dropped_count,
+        mrf_missed_count: backlog.mrf_observability.missed_count,
+        mrf_flush_failures: backlog.mrf_observability.flush_failure_count,
+        mrf_last_flush_duration_millis: backlog.mrf_observability.last_flush_duration_millis,
         targets: runtime.targets,
+        target_backlogs,
     }
 }
 
@@ -222,6 +273,27 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
         .into_iter()
         .map(|bucket| (bucket.bucket.clone(), bucket))
         .collect::<HashMap<String, DurableMrfBucketBacklog>>();
+    let mut durable_targets_by_bucket: HashMap<String, Vec<DurableMrfTargetBacklog>> = HashMap::new();
+    let durable_mrf_targets = if replication_storage_available && durable_mrf_available {
+        durable_mrf_target_backlog_snapshot()
+    } else {
+        Vec::new()
+    };
+    for target in durable_mrf_targets {
+        durable_targets_by_bucket
+            .entry(target.bucket.clone())
+            .or_default()
+            .push(target);
+    }
+    let mut runtime_targets_by_bucket: HashMap<String, Vec<RuntimeReplicationTargetBacklog>> = HashMap::new();
+    if let Some(stats) = &stats {
+        for target in stats.runtime_target_backlog_snapshot() {
+            runtime_targets_by_bucket
+                .entry(target.bucket.clone())
+                .or_default()
+                .push(target);
+        }
+    }
     let mrf_observability = if replication_storage_available {
         mrf_backlog_observability_snapshot()
     } else {
@@ -236,7 +308,8 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
         all_bucket_stats
             .len()
             .saturating_add(durable_buckets.len())
-            .saturating_add(mrf_observability_buckets.len()),
+            .saturating_add(mrf_observability_buckets.len())
+            .saturating_add(runtime_targets_by_bucket.len()),
     );
     bucket_names.extend(all_bucket_stats.keys().cloned());
     bucket_names.extend(
@@ -251,6 +324,16 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
             .filter(|bucket| !all_bucket_stats.contains_key(*bucket) && !durable_buckets.contains_key(*bucket))
             .cloned(),
     );
+    bucket_names.extend(
+        runtime_targets_by_bucket
+            .keys()
+            .filter(|bucket| {
+                !all_bucket_stats.contains_key(*bucket)
+                    && !durable_buckets.contains_key(*bucket)
+                    && !mrf_observability_buckets.contains_key(*bucket)
+            })
+            .cloned(),
+    );
     let mut buckets = Vec::with_capacity(bucket_names.len());
 
     for bucket in bucket_names {
@@ -327,14 +410,20 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
             runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
         }
         let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
+        let runtime_targets = runtime_targets_by_bucket.remove(&bucket).unwrap_or_default();
+        let durable_targets = durable_targets_by_bucket.remove(&bucket).unwrap_or_default();
         let mrf_observability = mrf_observability_buckets.get(&bucket).cloned().unwrap_or_default();
         buckets.push(bucket_replication_stats_snapshot_from_parts(
             bucket,
             runtime,
             proxy,
-            durable_mrf_available,
-            durable_bucket,
-            mrf_observability,
+            ObsBucketReplicationBacklogSnapshot {
+                durable_mrf_available,
+                durable_bucket,
+                runtime_targets,
+                durable_targets,
+                mrf_observability,
+            },
         ));
     }
 
@@ -426,20 +515,34 @@ mod tests {
                 proxied_get_requests_failures: 1,
                 ..Default::default()
             },
-            true,
-            DurableMrfBucketBacklog {
-                bucket: "runtime-bucket".to_string(),
-                count: 5,
-                bytes: 8192,
-            },
-            MrfBucketBacklogObservability {
-                bucket: "runtime-bucket".to_string(),
-                pending_count: 1,
-                pending_bytes: 512,
-                dropped_count: 2,
-                missed_count: 3,
-                flush_failure_count: 4,
-                last_flush_duration_millis: 5,
+            ObsBucketReplicationBacklogSnapshot {
+                durable_mrf_available: true,
+                durable_bucket: DurableMrfBucketBacklog {
+                    bucket: "runtime-bucket".to_string(),
+                    count: 5,
+                    bytes: 8192,
+                },
+                runtime_targets: vec![RuntimeReplicationTargetBacklog {
+                    bucket: "runtime-bucket".to_string(),
+                    target_arn: "arn:rustfs:replication:target-a".to_string(),
+                    count: 3,
+                    bytes: 4096,
+                }],
+                durable_targets: vec![DurableMrfTargetBacklog {
+                    bucket: "runtime-bucket".to_string(),
+                    target_arn: "arn:rustfs:replication:target-a".to_string(),
+                    count: 2,
+                    bytes: 4096,
+                }],
+                mrf_observability: MrfBucketBacklogObservability {
+                    bucket: "runtime-bucket".to_string(),
+                    pending_count: 1,
+                    pending_bytes: 512,
+                    dropped_count: 2,
+                    missed_count: 3,
+                    flush_failure_count: 4,
+                    last_flush_duration_millis: 5,
+                },
             },
         );
 
@@ -449,6 +552,12 @@ mod tests {
         assert!(snapshot.durable_mrf_available);
         assert_eq!(snapshot.durable_mrf_backlog_count, 5);
         assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
+        assert_eq!(snapshot.target_backlogs.len(), 1);
+        assert_eq!(snapshot.target_backlogs[0].target_arn, "arn:rustfs:replication:target-a");
+        assert_eq!(snapshot.target_backlogs[0].current_backlog_count, 3);
+        assert_eq!(snapshot.target_backlogs[0].current_backlog_bytes, 4096);
+        assert_eq!(snapshot.target_backlogs[0].durable_mrf_backlog_count, 2);
+        assert_eq!(snapshot.target_backlogs[0].durable_mrf_backlog_bytes, 4096);
         assert_eq!(snapshot.mrf_pending_count, 1);
         assert_eq!(snapshot.mrf_pending_bytes, 512);
         assert_eq!(snapshot.mrf_dropped_count, 2);
@@ -466,13 +575,15 @@ mod tests {
             "durable-only".to_string(),
             ObsBucketReplicationRuntimeSnapshot::default(),
             ObsBucketReplicationProxySnapshot::default(),
-            true,
-            DurableMrfBucketBacklog {
-                bucket: "durable-only".to_string(),
-                count: 11,
-                bytes: 2048,
+            ObsBucketReplicationBacklogSnapshot {
+                durable_mrf_available: true,
+                durable_bucket: DurableMrfBucketBacklog {
+                    bucket: "durable-only".to_string(),
+                    count: 11,
+                    bytes: 2048,
+                },
+                ..Default::default()
             },
-            MrfBucketBacklogObservability::default(),
         );
 
         assert_eq!(snapshot.bucket, "durable-only");
@@ -488,16 +599,18 @@ mod tests {
             "mrf-observability-only".to_string(),
             ObsBucketReplicationRuntimeSnapshot::default(),
             ObsBucketReplicationProxySnapshot::default(),
-            true,
-            DurableMrfBucketBacklog::default(),
-            MrfBucketBacklogObservability {
-                bucket: "mrf-observability-only".to_string(),
-                pending_count: 13,
-                pending_bytes: 4096,
-                dropped_count: 1,
-                missed_count: 2,
-                flush_failure_count: 3,
-                last_flush_duration_millis: 4,
+            ObsBucketReplicationBacklogSnapshot {
+                durable_mrf_available: true,
+                mrf_observability: MrfBucketBacklogObservability {
+                    bucket: "mrf-observability-only".to_string(),
+                    pending_count: 13,
+                    pending_bytes: 4096,
+                    dropped_count: 1,
+                    missed_count: 2,
+                    flush_failure_count: 3,
+                    last_flush_duration_millis: 4,
+                },
+                ..Default::default()
             },
         );
 
@@ -525,9 +638,7 @@ mod tests {
                 ..Default::default()
             },
             ObsBucketReplicationProxySnapshot::default(),
-            false,
-            DurableMrfBucketBacklog::default(),
-            MrfBucketBacklogObservability::default(),
+            ObsBucketReplicationBacklogSnapshot::default(),
         );
 
         assert_eq!(snapshot.current_backlog_count, 1);

crates/policy/src/policy/policy.rs

@@ -365,8 +365,48 @@ pub mod default {
 
     use super::Policy;
 
+    /// Name of the built-in policy granting KMS key lifecycle management.
+    pub const KMS_KEY_ADMINISTRATOR: &str = "KMSKeyAdministrator";
+    /// Name of the built-in policy granting cryptographic use of KMS keys.
+    pub const KMS_KEY_USER: &str = "KMSKeyUser";
+    /// Name of the built-in policy granting read-only visibility into KMS keys.
+    pub const KMS_AUDITOR: &str = "KMSAuditor";
+
+    /// Every KMS key, in the resource grammar identity policies use.
+    ///
+    /// The built-in KMS policies ship unscoped so they behave like the other canned
+    /// policies; an operator narrows a copy to `arn:aws:kms:::key/<key_id>` per workload.
+    const ALL_KMS_KEYS: &str = "*";
+
+    /// A KMS statement allowing `actions` on every key.
+    fn kms_allow(actions: Vec<Action>) -> Statement {
+        Statement {
+            sid: "".into(),
+            effect: Effect::Allow,
+            actions: ActionSet(actions),
+            not_actions: ActionSet(Default::default()),
+            resources: ResourceSet(vec![Resource::Kms(ALL_KMS_KEYS.into())]),
+            conditions: Functions::default(),
+            ..Default::default()
+        }
+    }
+
+    /// The `sts:AssumeRole` grant every canned policy carries so an STS session may
+    /// assume it.
+    fn assume_role_allow() -> Statement {
+        Statement {
+            sid: "".into(),
+            effect: Effect::Allow,
+            actions: ActionSet(vec![Action::StsAction(StsAction::AssumeRoleAction)]),
+            not_actions: ActionSet(Default::default()),
+            resources: ResourceSet(Default::default()),
+            conditions: Functions::default(),
+            ..Default::default()
+        }
+    }
+
     #[allow(clippy::incompatible_msrv)]
-    pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 5]> = LazyLock::new(|| {
+    pub static DEFAULT_POLICIES: LazyLock<[(&'static str, Policy); 8]> = LazyLock::new(|| {
         [
             (
                 "readwrite",
@@ -534,6 +574,65 @@ pub mod default {
                     ],
                 },
             ),
+            // KMS role templates. They deliberately carry no S3 or admin grants, so an
+            // operator combines one with a data-plane policy ("readwrite,KMSKeyUser").
+            //
+            // None of them grants kms:Configure, kms:ServiceControl, kms:ClearCache,
+            // kms:Backup or kms:Restore: those act on the KMS service or on the material
+            // of every key at once, which is a cluster-administration power rather than a
+            // key-management one, and they stay with consoleAdmin. Key creation currently
+            // shares kms:Configure with backend configuration, so it stays there too.
+            (
+                KMS_KEY_ADMINISTRATOR,
+                Policy {
+                    id: "".into(),
+                    version: DEFAULT_VERSION.into(),
+                    statements: vec![
+                        // Separation of duties: a key administrator governs a key's
+                        // lifecycle but is never able to encrypt or decrypt with it.
+                        kms_allow(vec![
+                            Action::KmsAction(KmsAction::DescribeKeyAction),
+                            Action::KmsAction(KmsAction::ListKeysAction),
+                            Action::KmsAction(KmsAction::EnableKeyAction),
+                            Action::KmsAction(KmsAction::DisableKeyAction),
+                            Action::KmsAction(KmsAction::RotateKeyAction),
+                            Action::KmsAction(KmsAction::DeleteKeyAction),
+                        ]),
+                        assume_role_allow(),
+                    ],
+                },
+            ),
+            (
+                KMS_KEY_USER,
+                Policy {
+                    id: "".into(),
+                    version: DEFAULT_VERSION.into(),
+                    statements: vec![
+                        // The two actions the SSE-KMS data path evaluates, plus the
+                        // metadata read a client needs to tell which key it is using.
+                        kms_allow(vec![
+                            Action::KmsAction(KmsAction::GenerateDataKeyAction),
+                            Action::KmsAction(KmsAction::DecryptAction),
+                            Action::KmsAction(KmsAction::DescribeKeyAction),
+                        ]),
+                        assume_role_allow(),
+                    ],
+                },
+            ),
+            (
+                KMS_AUDITOR,
+                Policy {
+                    id: "".into(),
+                    version: DEFAULT_VERSION.into(),
+                    statements: vec![
+                        kms_allow(vec![
+                            Action::KmsAction(KmsAction::DescribeKeyAction),
+                            Action::KmsAction(KmsAction::ListKeysAction),
+                        ]),
+                        assume_role_allow(),
+                    ],
+                },
+            ),
         ]
     });
 }
@@ -542,6 +641,7 @@ pub mod default {
 mod test {
     use super::*;
     use crate::error::Result;
+    use crate::policy::action::{AdminAction, KmsAction, S3Action};
 
     #[tokio::test]
     async fn test_parse_policy() -> Result<()> {
@@ -709,6 +809,186 @@ mod test {
         }
     }
 
+    // ------------------------------------------------------------------------
+    // Built-in KMS role templates
+    // ------------------------------------------------------------------------
+
+    fn default_policy(name: &str) -> &'static Policy {
+        default::DEFAULT_POLICIES
+            .iter()
+            .find_map(|(candidate, policy)| (*candidate == name).then_some(policy))
+            .unwrap_or_else(|| panic!("built-in policy {name} should exist"))
+    }
+
+    /// Evaluate `policy` for `account` against `action` on `key_id`.
+    ///
+    /// Mirrors the admin and SSE call sites: the requested key identifier travels in
+    /// `object` with `bucket` left empty. An empty `key_id` is the unscoped call.
+    async fn kms_allows(policy: &Policy, account: &str, action: KmsAction, key_id: &str) -> bool {
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        policy
+            .is_allowed(&Args {
+                account,
+                groups: &None,
+                action: Action::KmsAction(action),
+                bucket: "",
+                conditions: &conditions,
+                is_owner: false,
+                object: key_id,
+                claims: &claims,
+                deny_only: false,
+            })
+            .await
+    }
+
+    const KMS_LIFECYCLE_ACTIONS: [KmsAction; 4] = [
+        KmsAction::EnableKeyAction,
+        KmsAction::DisableKeyAction,
+        KmsAction::RotateKeyAction,
+        KmsAction::DeleteKeyAction,
+    ];
+
+    const KMS_CRYPTO_ACTIONS: [KmsAction; 2] = [KmsAction::GenerateDataKeyAction, KmsAction::DecryptAction];
+
+    /// Actions that act on the service or on every key's material at once. No role
+    /// template may confer them.
+    const KMS_CLUSTER_ADMIN_ACTIONS: [KmsAction; 6] = [
+        KmsAction::AllActions,
+        KmsAction::ConfigureAction,
+        KmsAction::ServiceControlAction,
+        KmsAction::ClearCacheAction,
+        KmsAction::BackupAction,
+        KmsAction::RestoreAction,
+    ];
+
+    const KMS_ROLE_TEMPLATES: [&str; 3] = [default::KMS_KEY_ADMINISTRATOR, default::KMS_KEY_USER, default::KMS_AUDITOR];
+
+    #[tokio::test]
+    async fn kms_key_administrator_manages_keys_but_cannot_use_them() {
+        let policy = default_policy(default::KMS_KEY_ADMINISTRATOR);
+
+        for action in KMS_LIFECYCLE_ACTIONS {
+            assert!(
+                kms_allows(policy, "keyadmin", action, "app-key").await,
+                "KMSKeyAdministrator should allow {action:?}"
+            );
+        }
+        assert!(kms_allows(policy, "keyadmin", KmsAction::DescribeKeyAction, "app-key").await);
+        assert!(kms_allows(policy, "keyadmin", KmsAction::ListKeysAction, "").await);
+
+        for action in KMS_CRYPTO_ACTIONS {
+            assert!(
+                !kms_allows(policy, "keyadmin", action, "app-key").await,
+                "KMSKeyAdministrator must not allow {action:?}"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn kms_key_user_uses_keys_but_cannot_manage_them() {
+        let policy = default_policy(default::KMS_KEY_USER);
+
+        for action in KMS_CRYPTO_ACTIONS {
+            assert!(
+                kms_allows(policy, "appuser", action, "app-key").await,
+                "KMSKeyUser should allow {action:?}"
+            );
+        }
+        assert!(kms_allows(policy, "appuser", KmsAction::DescribeKeyAction, "app-key").await);
+
+        for action in KMS_LIFECYCLE_ACTIONS {
+            assert!(
+                !kms_allows(policy, "appuser", action, "app-key").await,
+                "KMSKeyUser must not allow {action:?}"
+            );
+        }
+        assert!(!kms_allows(policy, "appuser", KmsAction::ListKeysAction, "").await);
+    }
+
+    #[tokio::test]
+    async fn kms_auditor_only_reads_key_metadata() {
+        let policy = default_policy(default::KMS_AUDITOR);
+
+        assert!(kms_allows(policy, "auditor", KmsAction::DescribeKeyAction, "app-key").await);
+        assert!(kms_allows(policy, "auditor", KmsAction::ListKeysAction, "").await);
+
+        for action in KMS_LIFECYCLE_ACTIONS.iter().chain(KMS_CRYPTO_ACTIONS.iter()) {
+            assert!(
+                !kms_allows(policy, "auditor", *action, "app-key").await,
+                "KMSAuditor must not allow {action:?}"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn kms_role_templates_withhold_service_and_bundle_actions() {
+        for name in KMS_ROLE_TEMPLATES {
+            let policy = default_policy(name);
+            for action in KMS_CLUSTER_ADMIN_ACTIONS {
+                assert!(
+                    !kms_allows(policy, "someone", action, "app-key").await,
+                    "{name} must not allow {action:?}"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn kms_role_templates_grant_nothing_outside_kms() {
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let foreign_actions = [
+            Action::S3Action(S3Action::GetObjectAction),
+            Action::S3Action(S3Action::PutObjectAction),
+            Action::AdminAction(AdminAction::ServerInfoAdminAction),
+        ];
+
+        for name in KMS_ROLE_TEMPLATES {
+            let policy = default_policy(name);
+            for action in &foreign_actions {
+                let allowed = policy
+                    .is_allowed(&Args {
+                        account: "someone",
+                        groups: &None,
+                        action: *action,
+                        bucket: "any-bucket",
+                        conditions: &conditions,
+                        is_owner: false,
+                        object: "any-object",
+                        claims: &claims,
+                        deny_only: false,
+                    })
+                    .await;
+                assert!(!allowed, "{name} must not allow {action:?}");
+            }
+        }
+    }
+
+    /// The narrowing an operator is told to apply must actually deny the other keys.
+    #[tokio::test]
+    async fn narrowed_kms_role_template_denies_other_keys() -> Result<()> {
+        let narrowed = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:GenerateDataKey", "kms:Decrypt", "kms:DescribeKey"],
+      "Resource": ["arn:aws:kms:::key/reports-*"]
+    }
+  ]
+}"#,
+        )?;
+
+        assert!(kms_allows(&narrowed, "appuser", KmsAction::GenerateDataKeyAction, "reports-2026").await);
+        assert!(kms_allows(&narrowed, "appuser", KmsAction::DecryptAction, "reports-2026").await);
+        assert!(!kms_allows(&narrowed, "appuser", KmsAction::DecryptAction, "payroll-2026").await);
+        assert!(!kms_allows(&narrowed, "appuser", KmsAction::DisableKeyAction, "reports-2026").await);
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_deny_only_checks_only_deny_statements() -> Result<()> {
         let data = r#"

crates/replication/src/delete.rs

@@ -49,6 +49,11 @@ impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
                 .delete_object
                 .delete_marker_mtime
                 .and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
+            target_arns: if self.target_arn.is_empty() {
+                Vec::new()
+            } else {
+                vec![self.target_arn.clone()]
+            },
         }
     }
 
@@ -111,6 +116,7 @@ mod tests {
                 delete_marker_mtime: Some(mtime),
                 ..Default::default()
             },
+            target_arn: "arn:target-a".to_string(),
             ..Default::default()
         };
 
@@ -129,6 +135,7 @@ mod tests {
             Some(mtime.unix_timestamp_nanos() as i64),
             "delete-marker mtime must be persisted in the MRF entry"
         );
+        assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
         assert_eq!(info.get_object(), "object");
     }
 
@@ -148,6 +155,7 @@ mod tests {
         };
 
         assert_eq!(info.to_mrf_entry().delete_marker_mtime, None);
+        assert!(info.to_mrf_entry().target_arns.is_empty());
     }
 
     #[test]

crates/replication/src/filemeta.rs

@@ -593,6 +593,9 @@ pub struct MrfReplicateEntry {
     // to preserve pre-existing behaviour (backlog#867).
     #[serde(rename = "deleteMarkerMtime", skip_serializing_if = "Option::is_none", default)]
     pub delete_marker_mtime: Option<i64>,
+
+    #[serde(rename = "targetARNs", skip_serializing_if = "Vec::is_empty", default)]
+    pub target_arns: Vec<String>,
 }
 
 fn retry_count_to_mrf(retry_count: u32) -> i32 {
@@ -672,6 +675,18 @@ impl ReplicateDecision {
         }
         if result.is_empty() { None } else { Some(result) }
     }
+
+    pub fn replicate_target_arns(&self) -> Vec<String> {
+        let mut arns = self
+            .targets_map
+            .values()
+            .filter(|target| target.replicate && !target.arn.is_empty())
+            .map(|target| target.arn.clone())
+            .collect::<Vec<_>>();
+        arns.sort();
+        arns.dedup();
+        arns
+    }
 }
 
 impl fmt::Display for ReplicateDecision {
@@ -799,6 +814,7 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: self.dsc.replicate_target_arns(),
         }
     }
 
@@ -853,6 +869,7 @@ impl ReplicateObjectInfo {
             delete_marker_version_id: None,
             delete_marker: false,
             delete_marker_mtime: None,
+            target_arns: self.dsc.replicate_target_arns(),
         }
     }
 }
@@ -994,6 +1011,62 @@ impl Default for ResyncDecision {
 mod tests {
     use super::*;
 
+    #[test]
+    fn replicate_decision_returns_sorted_unique_replicating_target_arns() {
+        let mut decision = ReplicateDecision::new();
+        decision.set(ReplicateTargetDecision {
+            arn: "arn:target-b".to_string(),
+            replicate: true,
+            ..Default::default()
+        });
+        decision.set(ReplicateTargetDecision {
+            arn: "arn:target-a".to_string(),
+            replicate: true,
+            ..Default::default()
+        });
+        decision.set(ReplicateTargetDecision {
+            arn: "arn:target-c".to_string(),
+            replicate: false,
+            ..Default::default()
+        });
+        decision.set(ReplicateTargetDecision {
+            arn: String::new(),
+            replicate: true,
+            ..Default::default()
+        });
+
+        assert_eq!(
+            decision.replicate_target_arns(),
+            vec!["arn:target-a".to_string(), "arn:target-b".to_string()]
+        );
+    }
+
+    #[test]
+    fn replicate_object_info_mrf_entry_carries_replicating_targets() {
+        let mut decision = ReplicateDecision::new();
+        decision.set(ReplicateTargetDecision {
+            arn: "arn:target-a".to_string(),
+            replicate: true,
+            ..Default::default()
+        });
+        decision.set(ReplicateTargetDecision {
+            arn: "arn:target-b".to_string(),
+            replicate: false,
+            ..Default::default()
+        });
+        let info = ReplicateObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            size: 42,
+            dsc: decision,
+            ..Default::default()
+        };
+
+        let entry = info.to_mrf_entry();
+
+        assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
+    }
+
     #[test]
     fn target_state_reads_resync_timestamp_from_target_reset_header_key() {
         let arn = "arn:rustfs:replication:us-east-1:target:bucket";

crates/replication/src/mrf.rs

@@ -71,6 +71,7 @@ mod tests {
                 delete_marker_version_id: None,
                 delete_marker: false,
                 delete_marker_mtime: None,
+                target_arns: vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
             },
             MrfReplicateEntry {
                 bucket: "bucket-a".to_string(),
@@ -82,6 +83,7 @@ mod tests {
                 delete_marker_version_id: Some(del_vid),
                 delete_marker: true,
                 delete_marker_mtime: Some(1_705_312_200_123_456_789),
+                target_arns: vec!["arn:target-a".to_string()],
             },
         ];
 
@@ -91,9 +93,11 @@ mod tests {
         assert_eq!(decoded.len(), 2);
         assert_eq!(decoded[0].version_id, Some(obj_vid));
         assert_eq!(decoded[0].op, MrfOpKind::Object);
+        assert_eq!(decoded[0].target_arns, vec!["arn:target-a".to_string(), "arn:target-b".to_string()]);
         assert_eq!(decoded[0].delete_marker_mtime, None);
         assert_eq!(decoded[1].delete_marker_version_id, Some(del_vid));
         assert_eq!(decoded[1].op, MrfOpKind::Delete);
+        assert_eq!(decoded[1].target_arns, vec!["arn:target-a".to_string()]);
         assert!(decoded[1].delete_marker);
         assert_eq!(
             decoded[1].delete_marker_mtime,
@@ -129,6 +133,7 @@ mod tests {
         assert_eq!(decoded[0].retry_count, 2);
         assert_eq!(decoded[0].size, 100);
         assert_eq!(decoded[0].op, MrfOpKind::Object);
+        assert!(decoded[0].target_arns.is_empty());
         // Old files lack the deleteMarkerMtime key; it must default to None so replay keeps the
         // pre-#867 fallback to the current time.
         assert_eq!(decoded[0].delete_marker_mtime, None);

crates/security-governance/src/admin_matrix.rs

@@ -63,6 +63,14 @@ pub enum RouteRiskLevel {
     Normal,
     Sensitive,
     High,
+    /// A single authorized request can destroy stored data beyond every
+    /// recovery path the server offers — no undo, no waiting window, no
+    /// backup taken on the caller's behalf.
+    ///
+    /// This is deliberately narrower than [`Self::High`], which covers routes
+    /// that change state an operator can put back. Reserve it for routes whose
+    /// worst case is permanent loss of user data.
+    Critical,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]

docs/operations/kms-backend-security.md

@@ -2,7 +2,7 @@
 
 RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
 
-For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md).
+For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md).
 
 ## Backend comparison
 
@@ -51,7 +51,7 @@ Notes:
 
 ## Master key rotation: retention, destruction, and upgrade ordering
 
-Rotation support differs per backend. Local and Static reject rotation outright (`InvalidOperation`); their single key material is never overwritten. Vault Transit delegates rotation to the Transit engine's own key versioning (ciphertext is version-prefixed, e.g. `vault:v1:...`). Vault KV2 rotates by retaining every historical version, as described below. Rotation is currently only reachable through the backend-level `KmsClient::rotate_key` API; it is not exposed through the admin or S3 surface.
+Rotation support differs per backend. Local and Static reject rotation outright (`InvalidOperation`); their single key material is never overwritten. Vault Transit delegates rotation to the Transit engine's own key versioning (ciphertext is version-prefixed, e.g. `vault:v1:...`). Vault KV2 rotates by retaining every historical version, as described below. Rotation is reachable through the admin API as `POST /rustfs/admin/v3/kms/keys/rotate`, which the route policy classifies as high risk and gates behind `kms:RotateKey`; it is not exposed through the S3 surface. The upgrade ordering constraint below therefore applies to an operator action, not only to a call from inside the process.
 
 ### Vault KV2 versioned retention model
 
@@ -63,8 +63,19 @@ Decryption loads exactly the version recorded in the envelope and fails closed w
 
 - Every version record that any stored DEK envelope references must remain readable. Until an object rewrap/migration capability exists, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped.
 - Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely.
-- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must echo the key id back as `confirm_key_id`. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key.
+- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must be a `DELETE` with a JSON body that sets `force_immediate` and echoes the key id back as `confirm_key_id` — the query-parameter form (`?force_immediate=true`) is refused outright, whatever the gate is set to. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key.
 - For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext.
+- `force_immediate` is additionally refused, with a `409 Conflict`, while any bucket's default encryption configuration still names the key, or while the key is the KMS service default key. A scheduled deletion is not refused for that reason: it destroys nothing and stays cancellable, and the background sweep re-checks the same references before it destroys the material.
+
+### Reading the `impact` section
+
+`DeleteKey` responses always carry an `impact` section listing the configuration that currently points at the key — the buckets whose default encryption names it, and whether it is the service default key — so the references that will refuse the destruction are visible when the deletion is scheduled rather than only in a server-side log once the window has run out.
+
+`DescribeKey` (`GET /rustfs/admin/v3/kms/keys/{key_id}`) can return the same section, but only when the request asks for it with `impact=true`. It is opt-in there because collecting it lists every bucket and `DescribeKey` is polled; without the parameter the endpoint does exactly the work it did before and returns no `impact` field at all. A value other than `true` or `false` is rejected with `400` rather than treated as `false`, so a typo can never answer a request for the section with a response that merely lacks one. **An absent section means "not collected", never "nothing references this key".**
+
+Read it for what it says and nothing more. `coverage.scanned` names the sources that were read; `coverage.not_scanned` names the ones that were not, which currently includes every object encrypted under the key. `completeness` is `exact` only over the scanned sources, and `unavailable` when a source could not be read at all — an unavailable report is not an empty one, and both an unreadable source and an outstanding reference will stop the sweep from destroying the material.
+
+**An empty `references` list does not mean the key is unused.** No object metadata is consulted, so a key with no configuration references can still protect an arbitrary amount of live data, which stays readable only until the material is gone. There is no field in the response that asserts otherwise, and none should be inferred from one.
 
 ### Upgrade before first rotation (hard constraint)
 
@@ -74,7 +85,7 @@ This is the sharpest instance of a broader class of constraints; the rest are co
 
 ## Mixed-version clusters during a rolling upgrade
 
-During a rolling upgrade the cluster runs two RustFS builds at once. That window matters more for KMS than for most subsystems, because KMS state is shared three ways: **Vault** holds the key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and — for configuration — can disagree for as long as the operator leaves them running.
+During a rolling upgrade the cluster runs two RustFS builds at once. That window matters more for KMS than for most subsystems, because KMS state is shared three ways: **Vault** holds the key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and — for configuration — can disagree for as long as the operator leaves them running, because the reload broadcast that converges configuration is one of the things an older build rejects.
 
 This section states only what is true of the current implementation. It is written for the KV2 and Transit backends; the Local backend is unsupported for multi-node deployments regardless of version (see the [deployment support matrix](#deployment-support-matrix)).
 
@@ -103,23 +114,29 @@ Even with every node on the same build, some state is process-local. These windo
 | What can diverge | Bound | Mechanism |
 | --- | --- | --- |
 | Transit key lifecycle state used by the `encrypt` and `generate_data_key` gates | ≤ 300 s (`METADATA_CACHE_TTL`) | Each node caches Transit metadata in process, TTL- and capacity-bounded, with targeted invalidation when a data-path call reports the key is gone server-side. A disable or schedule-deletion performed on one node is enforced on the others within one TTL at the latest, sooner if they hit that signal. |
-| `describe_key` output | ≤ 300 s | The manager-level key metadata cache. This is a reporting cache; the KV2 state gates do not read it. |
+| `describe_key` output | One metadata cache TTL: 300 s by default, otherwise whatever `cache_ttl_seconds` was configured with, clamped to 24 h | The manager-level key metadata cache, built from the configured cache settings. This is a reporting cache; the KV2 state gates do not read it. |
 | KV2 key lifecycle state | None | The KV2 backend re-reads the key record from Vault for every lifecycle and data-key operation, so a committed disable is effective on every upgraded node immediately. |
-| Active KMS configuration | Until the remaining nodes are restarted | See below. |
+| Active KMS configuration | One best-effort reload broadcast; unbounded for any peer that did not apply it | See below. |
 
 Builds older than rustfs/rustfs#5520 held the Transit metadata cache with no TTL and no capacity bound. On such a node the divergence window is not 300 seconds but "until the process restarts": it can keep encrypting under a key that another node disabled, indefinitely.
 
-### Configuration is persisted cluster-wide but applied per node
+The `describe_key` bound is the only one on that list an operator sets, so compute it rather than assuming the default: the window is the `cache_ttl_seconds` the KMS configure request was given, 300 s when it was omitted, clamped down to 24 h at use if it is larger (clamped rather than rejected, so an oversized setting still starts). Zero is refused outright while caching is enabled. `kms service-status` and the KMS configuration endpoint report the effective, post-clamp value, so the number the admin API shows is the number the cache honours. Note that this is the Transit row's neighbour and not its equal: `METADATA_CACHE_TTL` above is a separate, deliberately non-tunable 300 s, because that cache does gate cryptographic operations.
 
-`POST /rustfs/admin/v3/kms/reconfigure` currently does two things: it switches the KMS service **on the node that handled the request**, and it persists the new configuration to cluster storage at `config/kms_config.json`. It does not notify peers. Every other node keeps running the configuration it started with until it is restarted, at which point it loads the persisted configuration during startup.
+One upgrade caveat: builds older than rustfs/rustfs#5569 ignored `cache_ttl_seconds` and ran a hardcoded 300 s, while their configure converters persisted 3600 s as the default value. A cluster configured through the admin API before that fix therefore widens its `describe_key` staleness window from an effective 300 s to the 3600 s already stored in `config/kms_config.json`, with no configuration change of its own. Read the reported value back after upgrading instead of assuming it stayed at 300 s. No cryptographic or authorization path widens with it — encrypt, decrypt and data-key generation go straight to the backend and never read this cache.
 
-The practical consequences today:
+### Configuration changes converge through a best-effort peer reload
 
-- The configuration-split window has no upper bound other than the operator restarting the remaining nodes. Convergence is a restart, not a timeout.
-- During the window both configurations are live. If the reconfiguration changed backends, or changed the Vault mount or key prefix, different nodes write new key material to different places, and a key created through one node is invisible to the others.
-- `kms status` reflects the node that answered the request, so a single successful status response is not evidence that the cluster is consistent. Query every node.
+`POST /rustfs/admin/v3/kms/configure` and `POST /rustfs/admin/v3/kms/reconfigure` persist the new configuration to cluster storage at `config/kms_config.json`, switch the KMS service **on the node that handled the request**, and then broadcast a reload signal to every peer. A peer that accepts the signal re-reads the persisted configuration and reconfigures itself, so a runtime change normally reaches the whole cluster without any restart. A peer already running that exact configuration treats the signal as a no-op.
 
-Treat `reconfigure` as the first step of a cluster-wide operation, not as the operation itself.
+Convergence is best effort by contract, and the request never fails on account of a peer: the local node has already switched, and KMS configuration has no quorum or authoritative holder to roll back to. What that leaves:
+
+- The broadcast is sent **once**, with no background retry. A peer that is unreachable, that rejects the signal because its build predates the KMS subsystem, or whose reload itself fails keeps serving its previous configuration until a later `reconfigure` reaches it, or until it restarts and loads the persisted configuration during startup. For those peers the split window is still unbounded.
+- The admin response reports success either way, but its message names every peer that did not converge, and the server logs one `kms_peer_config_reload_failed` warning per peer. Read the message: an operation that reports success can still have left the cluster split.
+- For as long as a split lasts, both configurations are live. If the change switched backends, or changed the Vault mount or key prefix, different nodes write new key material to different places, and a key created through one node is invisible to the others.
+
+`GET /rustfs/admin/v3/kms/service-status` makes the split observable from a single request: it returns a `cluster_config` object holding one redacted configuration fingerprint per node plus a `consistent` flag. `consistent` is true only when every node answered with the same fingerprint — an unreachable peer, a peer whose build reports no fingerprint, and a node with no configuration at all each read as divergent rather than as agreement. Secrets are substituted out before a configuration is fingerprinted, so two nodes on the same backend holding different credentials still fingerprint alike; the field detects a configuration split, not a credential split.
+
+Treat a `configure` or `reconfigure` whose response names unconverged peers as an unfinished cluster-wide operation: re-issue it once those peers are reachable, or restart them.
 
 ### Recommended rolling upgrade order
 
@@ -130,7 +147,7 @@ Follow the node-at-a-time procedure in the [multi-node restart runbook](rolling-
 3. **Verify no node is left behind** before unfreezing. A single old node is enough to reintroduce blind writes and to strip `baseline_version` on its next lifecycle write.
 4. **Resume administrative traffic.**
 5. **Only then perform the first rotation of any key.** Once the whole cluster understands `master_key_version`, rotation is safe; before that it is not.
-6. **If the KMS configuration was changed at any point**, restart the nodes that did not handle the request so they reload the persisted configuration, and confirm each one reports the intended backend.
+6. **If the KMS configuration was changed at any point**, confirm `cluster_config.consistent` is true in the `service-status` response, and re-issue the change — or restart the node — for every peer still reporting a different fingerprint. A peer whose build predates the reload signal never converges on its own.
 
 ### Do not do these during a mixed-version window
 
@@ -138,7 +155,7 @@ Follow the node-at-a-time procedure in the [multi-node restart runbook](rolling-
 - **Issue any KV2 lifecycle write to an old node.** Its blind write can clobber a concurrent check-and-set commit and will drop `baseline_version` from the record.
 - **Create the same key ID from two nodes.** The create path is create-only on upgraded builds, but an old node's blind write does not honor that: the later writer's material wins and every DEK already wrapped with the earlier material becomes permanently unwrappable.
 - **Assume a disable or schedule-deletion took effect cluster-wide.** Old Transit nodes cache lifecycle state without expiry; confirm per node, or restart the old nodes, before treating a key as no longer in use.
-- **Reconfigure the KMS backend and consider it done.** The change applies to one node and is persisted; the rest need a restart.
+- **Reconfigure the KMS backend and consider it done.** The reload broadcast is exactly what an old build rejects, so during a mixed-version window the change reaches only the node that served it and the already-upgraded peers. Check the response message and `cluster_config.consistent` before assuming otherwise.
 - **Delete or prune version records** under `{prefix}/{key_id}/versions/*` for any reason. This is never safe, mixed-version or not; see [Retention and destruction preconditions](#retention-and-destruction-preconditions).
 
 ## Choosing between Vault KV2 and Vault Transit
@@ -164,7 +181,9 @@ Two consequences follow from that last row: **SSE-S3 key auto-creation and the s
 
 Key versions are opaque. AWS addresses backing keys internally and picks the right one to decrypt with, so RustFS reports `key_version` as 1 and cannot enumerate versions. Rotation uses `RotateKeyOnDemand`, which retains prior backing keys for decryption; AWS's separate automatic yearly rotation is neither enabled nor reported on by RustFS.
 
-The AWS backend is not configurable through the KMS admin API — use the environment variables above at startup.
+The KMS admin API accepts the AWS backend as `"backend_type": "AWS"` (aliases `aws`, `aws-kms`, `aws_kms`, `AwsKms`) on `/v3/kms/configure` and `/v3/kms/reconfigure`. The body carries `region` (**required**), and optionally `endpoint_url`, `default_key_id`, and the shared timeout/retry/cache settings. It accepts no credential fields at all — unknown fields are rejected — because every node resolves credentials through its own provider chain.
+
+`region` is mandatory on this path even though `RUSTFS_KMS_AWS_REGION` is optional at startup: the admin configuration is persisted once and replayed on every node, so a request that left the region to each node's ambient chain would let nodes address different regions, and therefore different keys, while reporting an identical configuration. `default_key_id` must be an AWS key id or ARN that already exists — this backend never creates keys by name.
 
 ## Local backend durability and deployment support matrix
 

docs/operations/kms-disaster-recovery-drill.md

@@ -0,0 +1,113 @@
+# KMS disaster-recovery drill
+
+A KMS backup that has never been restored is a hypothesis. This runbook turns it into evidence: it rehearses the complete loop — back up, lose the persistence layer, preflight, restore, and read historical objects again — and files a machine-readable evidence bundle for each run. For what each backend's backup actually covers, see [KMS backend security properties](kms-backend-security.md); for the metrics and alerts around KMS operations, see the [KMS observability runbook](kms-observability-runbook.md).
+
+The acceptance criterion of a drill is not that files came back. It is that objects encrypted before the disaster decrypt after the restore. The harness keeps the ciphertext and encryption metadata of every object it sealed before the disaster and, once the restore is complete, decrypts each one through a freshly opened backend and compares against the pre-disaster digest. Anything less proves only that a bundle is well formed.
+
+## Scope
+
+The drill covers the **Local** backend, which is the only backend RustFS produces a full-material bundle for. The responsibility split is deliberate and is described in `crates/kms/src/backup/capability.rs`:
+
+| Backend | What a RustFS bundle carries | What restores it |
+| --- | --- | --- |
+| Local | Key records, all stored versions, the KDF salt, sanitized configuration | The RustFS restore in this runbook |
+| Static | Non-sensitive references only | The operator re-supplies the secret out of band |
+| Vault KV2 + Transit | KV metadata and Transit ciphertext references | Vault's native snapshot restore, then the RustFS orchestration |
+| Vault Transit | Metadata, configuration references, verification data | Vault's native snapshot restore, then the RustFS orchestration |
+
+For the Vault backends there is no RustFS-side export, so there is no loop for a drill to close end to end: the cryptographic root is non-exportable and comes back through Vault's own disaster-recovery flow. What RustFS owns there is the refusal to proceed before that has happened, plus the ordering of everything after it. Rehearse it with the Vault section below.
+
+## What the drill measures
+
+**Recovery point (RPO).** The harness creates one key *after* the export fence closed and seals objects under it. After the restore that key is absent and its objects do not decrypt — as they must not. The recovery point of a KMS backup is therefore the snapshot generation of the last bundle, not the moment of the disaster, and `rpo.rpo_window_millis` in the evidence is the width of that window in the rehearsal. In production the same window is the age of your newest bundle, which is what your backup schedule sets.
+
+**Recovery time (RTO).** `rto_millis` is the sum of the phases an operator waits on: quarantine, preflight, restore, and verification. Seeding, sealing and the disaster itself are drill scaffolding and are excluded. Human decision time is excluded too and belongs to your incident process, not to this number. Per-phase costs are in `timings`.
+
+An RTO figure only transfers to production if the rehearsed deployment is comparable, so size `RUSTFS_KMS_DRILL_KEYS` to the number of master keys the real deployment holds. Both numbers are recorded in the evidence (`dataset`), so a stale measurement is visibly stale.
+
+## Running a drill
+
+```bash
+export RUSTFS_KMS_DRILL_WORKSPACE=/var/lib/rustfs/dr-drill/$(date -u +%Y%m%dT%H%M%SZ)
+export RUSTFS_KMS_DRILL_MASTER_KEY='<throwaway at-rest key for the rehearsal deployment>'
+export RUSTFS_KMS_BACKUP_KEK='<base64 of the 32-byte backup KEK>'
+export RUSTFS_KMS_BACKUP_KEK_ID='<kek identifier>'
+export RUSTFS_KMS_DRILL_KEYS=64
+
+cargo run -q -p rustfs-kms --example kms_dr_drill
+```
+
+The workspace must be an absolute path that does not already hold a rehearsal; give each run its own directory so the evidence and the quarantined state stay side by side. The runner exits 0 only when every check held, so a scheduled drill fails its job on a bad result instead of quietly filing a bad report.
+
+`RUSTFS_KMS_BACKUP_KEK` and `RUSTFS_KMS_BACKUP_KEK_ID` are the same variables the admin backup API reads. Use the KEK real bundles are sealed under: retrieving it is the one part of a recovery no bundle can attest to, and a drill that invents its own KEK does not test it. The rehearsal deployment's at-rest master key is separate and throwaway — the drill creates and destroys that deployment itself and never touches the running KMS.
+
+Optional variables: `RUSTFS_KMS_DRILL_DISASTER` (see below), `RUSTFS_KMS_DRILL_ID`, `RUSTFS_KMS_DRILL_DEPLOYMENT`, `RUSTFS_KMS_DRILL_OBJECTS_PER_KEY`, `RUSTFS_KMS_DRILL_OBJECT_BYTES`, `RUSTFS_KMS_DRILL_FILE_PERMISSIONS`, `RUSTFS_KMS_BACKUP_KEK_VERSION`, and `RUSTFS_KMS_DRILL_EVIDENCE` (evidence path, default `<workspace>/evidence.json`).
+
+## Disaster matrix
+
+Run all three; they exercise different failure surfaces and converge on the same procedure, which is the point — an operator does not have to diagnose the failure mode before acting.
+
+| `RUSTFS_KMS_DRILL_DISASTER` | Simulates |
+| --- | --- |
+| `key-directory-lost` (default) | The whole key directory is gone: lost volume, wiped host |
+| `master-key-salt-lost` | Records survive but the KDF salt is gone, so no material unwraps |
+| `key-record-corrupted` | One key record is truncated in place: torn write, partial media failure |
+
+Bundle-side faults — tampered, truncated, wrong-KEK, incomplete, unknown-version — are not drill scenarios. They are covered exhaustively and deterministically by the unit tests of `crates/kms/src/backup/local_export.rs` and `crates/kms/src/backup/local_restore.rs`, and re-running them as a drill would add fixtures without adding evidence.
+
+## Recovery procedure
+
+The drill executes exactly this sequence; running it by hand against a real deployment is the same procedure with your own paths.
+
+1. **Stop the KMS.** A restore must never race a running backend.
+2. **Quarantine, do not delete.** Move the damaged key directory aside rather than clearing it. The drill records the quarantine path and its contents in the evidence, and nothing damaged is ever destroyed: a restore that turns out to be the wrong call has to stay reversible, and forensics need the original bytes. Restore refuses a non-empty target anyway, including one holding only an orphan salt.
+3. **Preflight.** A dry-run decodes the whole bundle, checks the KDF descriptor against this build, verifies the operator-supplied master key against the bundle's one-way verifier, and enumerates conflicts. It writes nothing; the drill proves that by digesting the target before and after. Read every blocker before going further.
+4. **Restore.** Artifacts are staged inside the target, decryption-probed, then published through a commit marker and an atomic cutover.
+5. **Verify.** Open the restored directory and decrypt historical objects. This is the acceptance criterion, not step 4.
+6. **Observation period.** Keep the quarantined directory and the bundle until you have observed the recovered deployment serving reads. Nothing in the restore path deletes old material for you.
+
+## Reading the evidence
+
+`evidence.json` is `DrillEvidence` (`format_version` 1). It carries identifiers, digests and durations only: no key material, no master key, no plaintext, no ciphertext. Archive it next to the incident or audit record.
+
+| Field | Why it matters |
+| --- | --- |
+| `verdict`, `findings` | The result and every check that did not hold |
+| `envelope_probes[].verified` | Per-object proof that a historical data key still unwraps |
+| `rpo.post_snapshot_objects_recovered` | Must be `0`; anything else means the bundle was not a point-in-time snapshot |
+| `bundle.manifest_digest` vs `manifest_digest_after_recovery`, `bundle.source_unchanged` | The restore treated its bundle as strictly read-only |
+| `recovery.dry_run_zero_write` | The preflight wrote nothing to the target |
+| `recovery.commit_marker_cleared` | The cutover completed rather than leaving the target mid-restore |
+| `recovery.repeat_restore_refused` and `..._left_target_unchanged` | Re-running the procedure cannot damage a healthy deployment |
+| `dataset`, `timings`, `rto_millis`, `rpo.rpo_window_millis` | The measurement, and the deployment size it is valid for |
+
+## Interruptions and re-entry
+
+A restore has exactly one commit point: the durably published `.restore-commit.json` marker. Before it, the target's top level is untouched and re-running starts over. With it published, the backend refuses to start — a key directory mid-cutover must not serve requests — and you have two ways out:
+
+- **Roll forward**: re-run the restore with the same bundle. The marker is bound to the bundle by backup id and manifest digest, so a different bundle is refused rather than merged.
+- **Roll back**: abort the restore, which takes back exactly the files the marker names, then the marker, then the staged state. The target returns to its pre-restore state and a fresh restore can start.
+
+Both paths are exercised by the drill's interrupted-cutover tests in `crates/kms/src/backup/drill.rs`, which crash a restore precisely at its commit point rather than simulating the resulting state. Every interruption converges on the complete old state or the complete new state; there is no half-activated outcome.
+
+## Vault backends
+
+There is no RustFS-side Vault export, so a Vault drill is not the same closed loop. Rehearse it as:
+
+1. Restore the Vault cluster from its own native snapshot, following your Vault runbook. Transit keys are non-exportable and RustFS never attempts to import them.
+2. Run the RustFS preflight against the recovered cluster. It verifies cluster, namespace, mounts, the Transit key's identity and its version window, and the KV generation, and refuses on the first mismatch. A Transit key that was not restored, a `min_decryption_version` raised above what the bundle still needs, or a version history restored below the bundle's snapshot point are each reported as a distinct mismatch.
+3. Only then let the restore publish KV records, in the order material and versions, then metadata, then configuration and cutover.
+
+`crates/kms/src/backup/drill.rs` carries an `#[ignore]`d leg for step 2 that drills the refusal against a real server. It needs a Vault with the transit engine enabled and reads `RUSTFS_KMS_VAULT_ADDR` and `RUSTFS_KMS_VAULT_TOKEN`:
+
+```bash
+cargo test -p rustfs-kms --lib backup::drill -- --ignored --nocapture
+```
+
+## Running the drill matrix as tests
+
+```bash
+cargo test -p rustfs-kms --lib backup::drill
+```
+
+This runs the offline matrix — all three disasters, the evidence contract, and both interrupted-cutover outcomes — with no external dependencies. Use it in CI to keep the harness itself honest; use the operator entry point above to produce evidence for a real deployment size.

docs/operations/kms-observability-runbook.md

@@ -1,10 +1,14 @@
 # KMS observability runbook
 
-This runbook covers the KMS backend operation metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md).
+This runbook covers the KMS metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md).
 
 ## Metric reference
 
-All four metrics are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through. Label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression.
+Across every family below, label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression.
+
+### Backend operation metrics
+
+All four are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through.
 
 | Metric | Type | Labels | Meaning |
 | --- | --- | --- | --- |
@@ -20,15 +24,73 @@ Label values:
 - `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version).
 - `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`.
 
-Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. These metrics do not carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`).
+Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of these four series on a cluster using those backends is expected, not an outage. The families below sit above the backend layer and are emitted regardless.
 
-Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of KMS series on a cluster using those backends is expected, not an outage.
+### Key metadata cache metrics
+
+Emitted by the manager-level key metadata cache (`crates/kms/src/cache.rs`), which every backend shares. Publication is gated by the cache's `enable_metrics` setting, which defaults to on and which no configure-request field sets today, so in practice these are always published. The counters behind the admin status API are maintained either way, so the switch could never blind `kms service-status`.
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `rustfs_kms_metadata_cache_lookups_total` | counter | `result` | Key metadata lookups, by `hit` or `miss` |
+| `rustfs_kms_metadata_cache_evictions_total` | counter | `cause` | Entries dropped from the cache, by removal cause |
+| `rustfs_kms_metadata_cache_entries` | gauge | — | Entries the cache currently holds |
+
+`cause` is `expired` (TTL), `size` (capacity), `explicit` (invalidated by a key lifecycle operation), or `replaced` (overwritten by a newer value). Only `expired` and `size` are true evictions — a sustained `explicit`/`replaced` rate is lifecycle traffic, not cache pressure.
+
+The entry gauge is republished from every write path and from lookups that miss, because TTL expiry drops entries without any write taking place; a cache that goes completely idle can therefore hold a stale value until the next lookup. Note also that this cache only serves key metadata reads such as `describe_key` — encrypt, decrypt and data key generation never consult it, so a low hit ratio is not a data-path problem.
+
+### Key lifecycle metrics
+
+Published by the background deletion worker (`crates/kms/src/deletion_worker.rs`) at the end of each sweep, derived from the pages the sweep already walks, so observing the lifecycle costs no extra backend call. The worker only runs on backends whose capabilities include `schedule_deletion`, so a deployment on a backend without it emits none of these.
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `rustfs_kms_pending_deletion_keys` | gauge | — | Keys scheduled for deletion whose deadline has not passed |
+| `rustfs_kms_deletion_tombstone_keys` | gauge | — | Keys left tombstoned by an interrupted removal, still awaiting the sweep |
+| `rustfs_kms_oldest_key_rotation_age_seconds` | gauge | — | Seconds since the least recently rotated usable key was rotated, counting from creation for keys with no recorded rotation; `0` when there are none |
+| `rustfs_kms_deletion_sweep_keys_total` | counter | `outcome` | Keys the sweep acted on, by outcome |
+
+`outcome` is `removed`, `blocked` (live configuration — the default key, or a reference reported by the injected checker — still points at the key, so the sweep refuses to remove it), `skipped` (pending but not yet due, or the state changed between inspection and removal), or `failed` (the removal attempt failed and is retried next sweep). Every series is emitted at zero from the first sweep on, so a `rate()` over it is defined immediately.
+
+The three gauges are republished only by a sweep that saw the whole key set; a sweep that could not finish listing leaves the previous, complete values standing rather than understating them. Keys already on their way out are excluded from the rotation-age gauge, so it does not stay pinned high by a key that will never be rotated again.
+
+The rotation age comes from whatever the backend reports as the last rotation, and backends only report a rotation they recorded themselves. A key rotated before its backend persisted rotation timestamps therefore ages from creation until its next rotation stamps the record: the gauge overstates that key's age rather than inventing a rotation it cannot vouch for, so an alert on it fires early rather than late. Backends that cannot rotate at all (Local, Static) age every key from creation by construction.
+
+### Vault credential metrics
+
+Published by the Vault credential provider (`crates/kms/src/backends/vault_credentials.rs`), so they exist only on Vault-backed backends. Both are label-less: there is exactly one credential generation to describe, and the Vault address, mount, auth path and token are all off limits as label values.
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `rustfs_kms_vault_token_ttl_seconds` | gauge | — | Seconds left before the active Vault token expires; `0` once it has |
+| `rustfs_kms_vault_credentials_fail_closed` | gauge | — | `1` while the provider refuses to hand out its token because it is inside the fail-closed safety window, `0` otherwise |
+
+The renewal loop republishes both on a 10-second cadence while it waits, generating no extra Vault traffic, so a scrape landing between refresh cycles never reads a TTL frozen at the last refresh. `rustfs_kms_vault_credentials_fail_closed` at `1` is the metric form of the fail-closed window described in the [Vault KMS authentication runbook](vault-kms-authentication.md): while it is set, Vault-backed operations fail rather than run on a credential that may already be invalid.
+
+### Synthetic probe metrics
+
+Published by the background probe worker (`crates/kms/src/probe.rs`), which generates a data key under a reserved probe key, decrypts it, and compares the material. It runs every `RUSTFS_KMS_PROBE_INTERVAL_SECS` seconds (default 60, raised to a floor of 5, `0` disables the probe entirely), and the status it publishes is what KMS readiness reads.
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `rustfs_kms_probe_rounds_total` | counter | `result` | Probe rounds completed, by `success`, `failure` or `unsupported` |
+| `rustfs_kms_probe_failures_total` | counter | `failure_kind` | Failed rounds, by the round-trip stage that failed |
+| `rustfs_kms_probe_duration_seconds` | histogram | `result` | Wall-clock duration of one probe round |
+| `rustfs_kms_probe_last_success_timestamp_seconds` | gauge | — | Unix timestamp of the most recent successful round |
+| `rustfs_kms_probe_consecutive_failures` | gauge | — | Rounds that have failed since the last success |
+
+`failure_kind` is `key_provisioning` (the probe key could not be described or created), `generate`, `decrypt`, or `mismatch` — the last means both calls answered but the material did not survive the round trip, which is as serious as an outage and is reported as loudly.
+
+`unsupported` means the backend cannot host the probe key. It is deliberately counted as its own result and never as a failure, and the worker stops after recording it, so failure-counter alerts stay silent on such deployments; the AWS KMS backend is the case in practice, because it refuses a caller-named create. Note also that `rustfs_kms_probe_last_success_timestamp_seconds` only ever moves forward on a success, so while the probe fails its age keeps growing — alert on that age, not on the presence of a failure counter.
+
+Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. None of these metrics carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`).
 
 ## Dashboard
 
 Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` (Prometheus data source) and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed there.
 
-The dashboard's "Planned Panels (TODO)" text panel lists metric families designed under rustfs/backlog#1584 but not yet landed (key-cache effectiveness, lifecycle gauges, token TTL, synthetic probe). Do not add panels or alerts for those names until the emitting code is merged; keep that panel and the [Coverage gaps](#coverage-gaps-and-planned-metrics) section below in sync as they land.
+The shipped dashboard covers the backend operation metrics only. Its "Planned Panels (TODO)" text panel still describes the cache, lifecycle, Vault credential and probe families as not landed — that panel is stale: the emitting code is merged and the metric names, types and label values are in [Metric reference](#metric-reference) above. Until real panels replace it, query those families ad hoc; nothing in the shipped dashboard or alert rules reads them. See [Coverage gaps](#coverage-gaps).
 
 ## Alert rules
 
@@ -45,7 +107,7 @@ Meaning: attempts are failing with `error_class="fatal"` — failures the policy
 Investigation:
 
 1. Break the rate down by operation: `sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m]))`.
-2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs.
+2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs; `rustfs_kms_vault_credentials_fail_closed` at `1`, or `rustfs_kms_vault_token_ttl_seconds` at or near `0`, confirms that state without reading logs.
 3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the minimal policy in [KMS backend security properties](kms-backend-security.md) (a policy that drifted or was re-scoped produces 403s that classify as fatal), and check the Vault audit log for the corresponding denied requests.
 4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the rotation retention preconditions in [KMS backend security properties](kms-backend-security.md) and verify nobody destroyed version records under the key subtree.
 5. Confirm blast radius with the outcome view: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="fatal"}[5m]))`.
@@ -112,16 +174,15 @@ Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff wa
 
 Every numeric threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).
 
-## Coverage gaps and planned metrics
+## Coverage gaps
 
-The following families are designed under rustfs/backlog#1584 but land in separate changes; they are intentionally absent from the dashboard and alert rules, and referencing their names before the emitting code merges would produce permanently-empty panels and never-firing alerts:
+The four metric families designed under rustfs/backlog#1584 — key-cache effectiveness, key lifecycle, Vault credentials, synthetic probe — have all landed and are documented in [Metric reference](#metric-reference). What is still missing:
 
-- Key-cache effectiveness: hit/miss counters, entry gauge, eviction counter (replacing the former hardcoded-zero miss statistic).
-- Key lifecycle: pending-deletion/tombstone gauges from the deletion-worker sweep and an aggregate rotation-age gauge.
-- Vault credentials: token TTL remaining and fail-closed state gauges (today only the log warnings quoted above exist).
-- Synthetic backend probe: probe outcome and failure-class metrics feeding the readiness cache.
+- **No dashboard panels and no alert rules for those four families.** They are emitted but neither visualized nor alerted on, so they surface only in ad-hoc queries. Building against them is safe now: the names and label values above are what the code emits.
+- **The Local and Static backends emit no operation metrics**, because they do not flow through the operation-policy choke point; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Their cache metrics are emitted normally.
+- **No formal SLO targets**, deliberately, until a production baseline exists — see [Threshold calibration](#threshold-calibration).
 
-When one of these lands, add its panels, extend the alert rules, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel, and update this section.
+When a panel or alert rule for one of the landed families is added, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel and update this section.
 
 ## Related documents
 

docs/operations/kms-per-key-authorization.md

@@ -0,0 +1,107 @@
+# Per-key KMS authorization
+
+RustFS authorizes KMS access with identity policies. A statement may name the keys it applies to, so a grant such as `kms:DisableKey` no longer implies every key in the cluster, and an SSE-KMS request is checked against the key it actually resolves to.
+
+This page covers the resource grammar, the built-in role templates, the two enforcement planes, and the migration path. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md).
+
+## Resource grammar
+
+KMS resources use the same empty-account ARN form as S3 resources:
+
+| Pattern | Matches |
+| --- | --- |
+| `arn:aws:kms:::key/<key_id>` | exactly that key |
+| `arn:aws:kms:::key/app-*` | every key whose id starts with `app-` |
+| `arn:aws:kms:::*` | every key |
+| `arn:aws:kms:::alias/<name>` | reserved; matches nothing until alias resolution ships |
+
+Rules worth knowing before writing a policy:
+
+- A statement mixing `kms:` actions with `s3:` actions is rejected. Put KMS grants in their own statement.
+- A statement carrying a KMS resource with non-KMS actions is rejected.
+- A KMS statement with **no** resource matches every key. This is the legacy form and stays valid, so existing policies keep their current effect.
+- A KMS statement whose resources are S3 ARNs predates KMS resources. It is still loadable, but the S3 patterns never constrained key access, so evaluation ignores them, treats the statement as unscoped, and logs a warning. Rewrite these with real KMS resources.
+- `PutBucketPolicy` rejects any statement carrying a `kms:` action or a KMS resource. KMS grants belong to identities. Bucket policies stored before this check still load; evaluation skips their pure-KMS statements and warns.
+- `Deny` wins, as everywhere else: a `Deny` on `arn:aws:kms:::key/payroll-*` overrides an `Allow` on `arn:aws:kms:::*`.
+- The key identifier matched is the one the request names, before any alias resolution.
+
+## Built-in role templates
+
+Three canned policies ship alongside `readwrite`, `readonly`, `writeonly`, `diagnostics` and `consoleAdmin`:
+
+| Policy | Grants | Intended holder |
+| --- | --- | --- |
+| `KMSKeyAdministrator` | `kms:DescribeKey`, `kms:ListKeys`, `kms:EnableKey`, `kms:DisableKey`, `kms:RotateKey`, `kms:DeleteKey` | the team that governs key lifecycle |
+| `KMSKeyUser` | `kms:GenerateDataKey`, `kms:Decrypt`, `kms:DescribeKey` | workloads that read and write SSE-KMS objects |
+| `KMSAuditor` | `kms:DescribeKey`, `kms:ListKeys` | reviewers who need visibility and nothing else |
+
+They express separation of duties: `KMSKeyAdministrator` can disable or delete a key but can never encrypt or decrypt with it, and `KMSKeyUser` can use a key but can never change its state.
+
+None of the three grants `kms:Configure`, `kms:ServiceControl`, `kms:ClearCache`, `kms:Backup` or `kms:Restore`. Those act on the KMS service or on the material of every key at once, which is a cluster-administration power rather than a key-management one; they stay with `consoleAdmin`. Key creation currently shares `kms:Configure` with backend configuration, so creating keys is a `consoleAdmin` operation too.
+
+The templates carry no S3 or admin grants, so combine one with a data-plane policy. Policy mappings accept a comma-separated list:
+
+```
+PUT /rustfs/admin/v3/set-user-or-group-policy?policyName=readwrite,KMSKeyUser&userOrGroup=app-writer&isGroup=false
+```
+
+They are also unscoped (`arn:aws:kms:::*`) so they behave like the other canned policies. For least privilege, copy one and narrow it per workload:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:*"],
+      "Resource": ["arn:aws:s3:::reports", "arn:aws:s3:::reports/*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["kms:GenerateDataKey", "kms:Decrypt", "kms:DescribeKey"],
+      "Resource": ["arn:aws:kms:::key/reports-*"]
+    }
+  ]
+}
+```
+
+## Enforcement planes
+
+### Admin endpoints
+
+Every `/rustfs/admin/v3/kms/...` endpoint that names a key authorizes against that key. The identifier is taken from the request body's `key_id`, falling back to the `keyId` query parameter. An endpoint that names no key (for example `GET /rustfs/admin/v3/kms/keys`) matches any KMS resource in the policy, as before.
+
+This plane is always on. It needs no migration: a policy that never mentioned a KMS resource keeps matching every key.
+
+### SSE-KMS data path
+
+An SSE-KMS write additionally requires `kms:GenerateDataKey`, and an SSE-KMS read additionally requires `kms:Decrypt`, both evaluated as the requesting identity against the key the request resolves to. This closes the confused-deputy gap where the server called KMS with its own authority on behalf of any caller holding `s3:PutObject`.
+
+Scope and exemptions:
+
+- **SSE-KMS only.** SSE-S3 wraps its data key with a server-owned key the caller never names, and SSE-C never reaches KMS; both are exempt, matching AWS.
+- **The resolved key**, not the header. A bucket default encryption rule naming a KMS key is authorized the same way an explicit `x-amz-server-side-encryption-aws-kms-key-id` header is.
+- **Anonymous requests are exempt.** They have no identity policy to evaluate, and denying them would break public buckets holding SSE-KMS objects. They remain governed by bucket policy.
+- **Internal work is exempt.** Replication, lifecycle transitions, healing and the scanner run as the system principal.
+- **Authorization runs before key state is checked**, so a denial cannot be used to probe whether a key exists, is disabled, or is pending deletion. The response is always `AccessDenied`.
+- **Multipart uploads are authorized at create time**, where the session data key is generated. Part uploads and completion reuse that envelope and are not re-authorized against the destination key.
+- **Copies are checked on both ends**: `kms:Decrypt` on the source object's key and `kms:GenerateDataKey` on the destination key. `UploadPartCopy` authorizes its source read the same way.
+
+## Migration
+
+Data-path enforcement is **off by default in this release** because it changes the outcome of requests that succeed today: an identity holding only `s3:PutObject` can currently encrypt under any key. Turning it on without preparing policies will produce `AccessDenied` on working workloads.
+
+```bash
+RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true
+```
+
+The server logs the configured mode once at startup, and warns while enforcement is off. A later release defaults it to enabled.
+
+Recommended sequence:
+
+1. Leave enforcement off. Inventory which identities read and write SSE-KMS objects and under which keys — the KMS fields on S3 audit entries (`sseType`, `kmsKeyId`, `kmsOutcome`) report the key each request used.
+2. Attach `KMSKeyUser`, or a narrowed copy, to those identities. This is a no-op while enforcement is off, so it can be rolled out ahead of time.
+3. Enable enforcement on one node and confirm the workloads still succeed.
+4. Roll it out cluster-wide.
+
+If a workload starts failing with `AccessDenied` on an SSE-KMS object after step 3, the identity is missing `kms:GenerateDataKey` (writes) or `kms:Decrypt` (reads) on the key named in the audit entry.

rustfs/src/admin/handlers/kms_dynamic.rs

@@ -129,6 +129,9 @@ fn normalize_configure_request_secrets(
         ConfigureKmsRequest::VaultTransit(req) => token_is_blank(&req.auth_method),
         ConfigureKmsRequest::Local(_) => false,
         ConfigureKmsRequest::Static(_) => false,
+        // AWS credentials come from the aws-config chain, so there is nothing
+        // to carry over from the existing configuration.
+        ConfigureKmsRequest::Aws(_) => false,
     };
 
     if !needs_existing_auth {
@@ -144,6 +147,7 @@ fn normalize_configure_request_secrets(
         ConfigureKmsRequest::VaultTransit(req) => req.auth_method = existing_auth,
         ConfigureKmsRequest::Local(_) => {}
         ConfigureKmsRequest::Static(_) => {}
+        ConfigureKmsRequest::Aws(_) => {}
     }
 
     Ok(())
@@ -1181,9 +1185,9 @@ impl Operation for ReconfigureKmsHandler {
 #[cfg(test)]
 mod tests {
     use super::{
-        decode_persisted_kms_config, ensure_kms_config_persistable, kms_config_fingerprint, kms_config_is_unchanged,
-        kms_configure_actions, kms_service_control_actions, local_success_with_peer_report, normalize_configure_request_secrets,
-        redacted_canonical_config,
+        decode_persisted_kms_config, ensure_kms_config_persistable, ensure_kms_request_persistable, kms_config_fingerprint,
+        kms_config_is_unchanged, kms_configure_actions, kms_service_control_actions, local_success_with_peer_report,
+        normalize_configure_request_secrets, redacted_canonical_config,
     };
     use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
     use std::path::PathBuf;
@@ -1420,6 +1424,72 @@ mod tests {
         assert_eq!(backend_error, "Changing from the Local KMS backend is not supported");
     }
 
+    fn aws_configure_request(region: &str) -> rustfs_kms::ConfigureKmsRequest {
+        rustfs_kms::ConfigureKmsRequest::Aws(rustfs_kms::ConfigureAwsKmsRequest {
+            region: region.to_string(),
+            endpoint_url: None,
+            default_key_id: Some("arn:aws:kms:us-east-1:111122223333:key/1234abcd".to_string()),
+            timeout_seconds: None,
+            retry_attempts: None,
+            enable_cache: None,
+            max_cached_keys: None,
+            cache_ttl_seconds: None,
+            allow_insecure_dev_defaults: None,
+        })
+    }
+
+    /// The AWS backend holds no credential material of its own, so its
+    /// configuration is safe to persist cluster-wide and needs nothing carried
+    /// over from a previous configuration.
+    #[test]
+    fn aws_configure_request_is_persistable_and_needs_no_existing_credentials() {
+        let mut request = aws_configure_request("us-east-1");
+        normalize_configure_request_secrets(&mut request, None).expect("aws request needs no existing credentials");
+        assert!(ensure_kms_request_persistable(&request).is_ok());
+
+        let config = request.to_kms_config();
+        assert!(ensure_kms_config_persistable(&config).is_ok());
+
+        let canonical = redacted_canonical_config(&config).expect("aws configuration should serialize");
+        assert!(canonical.contains("us-east-1"), "the pinned region must drive the fingerprint");
+        for credential_field in ["access_key", "secret_access_key", "session_token"] {
+            assert!(
+                !canonical.contains(credential_field),
+                "aws configuration must carry no credential material: {canonical}"
+            );
+        }
+    }
+
+    /// Two nodes cannot be allowed to read the same AWS configuration as
+    /// different regions, so the pinned region has to be part of what a
+    /// fingerprint comparison would flag as a split.
+    #[test]
+    fn aws_config_fingerprint_tracks_the_pinned_region() {
+        let first = kms_config_fingerprint(&aws_configure_request("us-east-1").to_kms_config())
+            .expect("fingerprint should be computable");
+        let same = kms_config_fingerprint(&aws_configure_request("us-east-1").to_kms_config())
+            .expect("fingerprint should be computable");
+        let other = kms_config_fingerprint(&aws_configure_request("eu-central-1").to_kms_config())
+            .expect("fingerprint should be computable");
+
+        assert_eq!(first, same);
+        assert_ne!(first, other);
+    }
+
+    #[test]
+    fn local_backend_cannot_be_switched_to_aws() {
+        let mut existing = rustfs_kms::KmsConfig::local(PathBuf::from("/var/lib/rustfs/kms"));
+        let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
+            panic!("local constructor must create local backend config");
+        };
+        existing_local.master_key = Some("stored-master-key".to_string());
+
+        let mut request = aws_configure_request("us-east-1");
+        let error = normalize_configure_request_secrets(&mut request, Some(&existing))
+            .expect_err("switching away from the local backend must be rejected");
+        assert_eq!(error, "Changing from the Local KMS backend is not supported");
+    }
+
     const VAULT_TOKEN: &str = "hvs-super-secret-token";
 
     fn vault_config(address: &str, token: &str) -> rustfs_kms::KmsConfig {

rustfs/src/admin/handlers/kms_keys.rs

@@ -19,12 +19,13 @@ use crate::admin::auth::{validate_admin_request, validate_admin_request_with_kms
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::{current_kms_runtime_service_manager, current_or_init_kms_runtime_service_manager};
 use crate::auth::{check_key_valid, get_session_token};
+use crate::kms_deletion_gate::current_key_impact;
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
 use base64::Engine;
 use hyper::{HeaderMap, Method, StatusCode};
 use matchit::Params;
 use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
-use rustfs_kms::{KmsAuditOperation, KmsError, types::*};
+use rustfs_kms::{KeyImpactReport, KmsAuditOperation, KmsError, types::*};
 use rustfs_policy::policy::action::{Action, KmsAction};
 use s3s::header::CONTENT_TYPE;
 use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
@@ -371,11 +372,14 @@ impl Operation for DescribeKeyHandler {
 #[cfg(test)]
 mod tests {
     use super::{
-        CancelKmsKeyDeletionRequest, CreateKeyApiRequest, CreateKmsKeyRequest, DeleteKmsKeyRequest, GenerateDataKeyApiRequest,
-        extract_key_id, kms_create_key_actions, kms_delete_key_actions, kms_describe_key_actions, kms_generate_data_key_actions,
-        kms_list_keys_actions, scoped_key_id,
+        CancelKmsKeyDeletionRequest, CreateKeyApiRequest, CreateKmsKeyRequest, DeleteKmsKeyRequest, DeleteKmsKeyResponse,
+        DescribeKmsKeyResponse, GenerateDataKeyApiRequest, delete_key_error_status, delete_request_from_query, extract_key_id,
+        key_impact_if_requested, kms_create_key_actions, kms_delete_key_actions, kms_describe_key_actions,
+        kms_generate_data_key_actions, kms_list_keys_actions, scoped_key_id, wants_key_impact,
     };
     use http::Uri;
+    use hyper::StatusCode;
+    use rustfs_kms::{KeyImpactReport, KeyReference, KeyReferenceKind, KmsError, ReferenceScope};
     use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
     use rustfs_policy::policy::{Args, Policy};
     use std::collections::HashMap;
@@ -463,6 +467,107 @@ mod tests {
         }
     }
 
+    fn delete_query(query: &str) -> Result<DeleteKmsKeyRequest, Box<DeleteKmsKeyResponse>> {
+        let uri: Uri = format!("/rustfs/admin/v3/kms/keys/delete?{query}")
+            .parse()
+            .expect("uri should parse");
+        delete_request_from_query(&uri)
+    }
+
+    /// The query string can no longer ask for immediate deletion in any form
+    /// (rustfs/backlog#1585). Refusing beats downgrading to a scheduled
+    /// deletion: a caller who asked for destruction must not read the answer as
+    /// "destroyed".
+    #[test]
+    fn delete_query_cannot_request_immediate_deletion() {
+        for query in [
+            "keyId=key-a&force_immediate=true",
+            "keyId=key-a&force_immediate=True",
+            "keyId=key-a&force_immediate=1",
+            "keyId=key-a&force_immediate=",
+            "keyId=key-a&confirm_key_id=key-a",
+            "keyId=key-a&force_immediate=false&confirm_key_id=key-a",
+        ] {
+            let Err(refused) = delete_query(query) else {
+                panic!("{query} must be refused");
+            };
+            assert!(!refused.success, "{query} must not report success");
+            assert_eq!(refused.key_id, "key-a");
+            assert!(refused.deletion_date.is_none(), "{query} must not report a deletion date");
+            assert!(
+                refused.message.contains("JSON body"),
+                "{query} must point the caller at the body form: {}",
+                refused.message
+            );
+        }
+    }
+
+    /// The scheduled path keeps its query form, including the explicit
+    /// `force_immediate=false` some clients send.
+    #[test]
+    fn delete_query_still_schedules_a_deletion() {
+        for (query, expected_window) in [
+            ("keyId=key-a", None),
+            ("keyId=key-a&force_immediate=false", None),
+            ("keyId=key-a&pending_window_in_days=7", Some(7)),
+        ] {
+            let request = delete_query(query).expect("a scheduled deletion must still parse from the query");
+            assert_eq!(request.key_id, "key-a");
+            assert_eq!(request.pending_window_in_days, expected_window);
+            assert_eq!(request.force_immediate, None, "{query} must reach the service as scheduled");
+            assert_eq!(request.confirm_key_id, None, "{query} must not carry a confirmation");
+        }
+    }
+
+    #[test]
+    fn delete_query_without_a_key_id_is_refused() {
+        let uri: Uri = "/rustfs/admin/v3/kms/keys/delete".parse().expect("uri should parse");
+        let refused = delete_request_from_query(&uri).expect_err("a delete without a key must be refused");
+        assert!(refused.message.contains("keyId"));
+        assert!(refused.key_id.is_empty());
+    }
+
+    /// The body form still carries both immediate-deletion fields; the service
+    /// gate, not the transport, decides whether they are honoured.
+    #[test]
+    fn delete_body_still_carries_the_confirmation_fields() {
+        let request =
+            serde_json::from_str::<DeleteKmsKeyRequest>(r#"{"key_id":"key-a","force_immediate":true,"confirm_key_id":"key-a"}"#)
+                .expect("delete body should parse");
+
+        assert_eq!(request.force_immediate, Some(true));
+        assert_eq!(request.confirm_key_id.as_deref(), Some("key-a"));
+    }
+
+    /// A refused waiting window and a refused immediate deletion are the
+    /// caller's input to fix, so the endpoint answers 400 rather than blaming
+    /// the backend.
+    #[test]
+    fn refused_deletions_report_a_client_error() {
+        for error in [
+            KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"),
+            KmsError::invalid_operation("immediate deletion of key key-a is not allowed"),
+            KmsError::validation_error("bad input"),
+        ] {
+            assert_eq!(delete_key_error_status(&error), StatusCode::BAD_REQUEST, "{error} must be a 400");
+        }
+
+        assert_eq!(delete_key_error_status(&KmsError::key_not_found("key-a")), StatusCode::NOT_FOUND);
+        assert_eq!(
+            delete_key_error_status(&KmsError::backend_error("vault is down")),
+            StatusCode::INTERNAL_SERVER_ERROR
+        );
+    }
+
+    /// A key the deployment still points at is refused with 409, not 400: the
+    /// request is well formed and the key exists, and what has to change to
+    /// make it succeed is the configuration, not the request.
+    #[test]
+    fn a_still_referenced_key_reports_a_conflict() {
+        let error = KmsError::key_still_referenced("key-a", vec!["bucket:sse-bucket".to_string()]);
+        assert_eq!(delete_key_error_status(&error), StatusCode::CONFLICT);
+    }
+
     #[test]
     fn scoped_key_id_reads_the_body_first_and_falls_back_to_the_query() {
         let uri: Uri = "/rustfs/admin/v3/kms/keys/delete?keyId=query-key"
@@ -701,6 +806,162 @@ mod tests {
         }
     }
 
+    fn referenced_impact() -> KeyImpactReport {
+        let mut impact = KeyImpactReport::configuration_layer("key-a");
+        impact.push_reference(KeyReference {
+            kind: KeyReferenceKind::BucketDefaultEncryption,
+            id: "sse-bucket".to_string(),
+            detail: "bucket default encryption names this key".to_string(),
+        });
+        impact
+    }
+
+    /// Scheduling a deletion for a key that bucket configuration still points
+    /// at succeeds — it destroys nothing and stays cancellable — but the
+    /// caller has to be told, in the same response, what will refuse the
+    /// destruction once the window runs out.
+    #[test]
+    fn a_scheduled_deletion_succeeds_and_still_names_what_will_block_it() {
+        let response = DeleteKmsKeyResponse {
+            success: true,
+            message: "key deleted successfully".to_string(),
+            key_id: "key-a".to_string(),
+            deletion_date: Some("2026-01-01T00:00:00Z".to_string()),
+            impact: Some(referenced_impact()),
+        };
+
+        let json = serde_json::to_value(&response).expect("response should serialize");
+        assert_eq!(
+            json["success"], true,
+            "an outstanding reference must not change the outcome of a schedule"
+        );
+        assert_eq!(json["impact"]["references"][0]["kind"], "bucket-default-encryption");
+        assert_eq!(json["impact"]["references"][0]["id"], "sse-bucket");
+    }
+
+    /// The impact section is exhaustive only over what it read, and says so.
+    /// A caller must be able to tell an empty reference list apart from a
+    /// key that nothing uses, because this report cannot distinguish them.
+    #[test]
+    fn the_impact_section_states_its_coverage_and_claims_no_key_is_unused() {
+        let empty = DescribeKmsKeyResponse {
+            success: true,
+            message: "Key described successfully".to_string(),
+            key_metadata: None,
+            impact: Some(KeyImpactReport::configuration_layer("key-a")),
+        };
+
+        let json = serde_json::to_value(&empty).expect("response should serialize");
+        assert_eq!(json["impact"]["references"].as_array().expect("references is an array").len(), 0);
+        assert_eq!(json["impact"]["completeness"], "exact");
+        assert_eq!(json["impact"]["coverage"]["scanned"][0], "bucket-default-encryption");
+        assert_eq!(
+            json["impact"]["coverage"]["not_scanned"][0], "object-envelopes",
+            "an empty reference list is only honest next to the scopes it did not read"
+        );
+
+        let referenced = serde_json::to_value(DeleteKmsKeyResponse {
+            success: true,
+            message: String::new(),
+            key_id: "key-a".to_string(),
+            deletion_date: None,
+            impact: Some(referenced_impact()),
+        })
+        .expect("response should serialize");
+
+        for body in [json.to_string(), referenced.to_string()] {
+            for forbidden in ["in_use", "unused", "unreferenced", "safe_to_delete", "deletable"] {
+                assert!(!body.contains(forbidden), "the impact section must not carry a `{forbidden}` claim");
+            }
+        }
+    }
+
+    /// The impact section is a report, never an input to a decision here. The
+    /// deletion worker's fail-closed gate stays the only thing that decides
+    /// whether material is destroyed; a handler that branched on this report
+    /// would be treating "nothing found in the configuration layer" as
+    /// "nothing uses this key", which it is not.
+    #[test]
+    fn handlers_report_the_key_impact_without_acting_on_it() {
+        let src = include_str!("kms_keys.rs");
+
+        // Deleting is the request whose consequences the caller cannot
+        // otherwise see, and it is not polled, so it always reports.
+        let delete = operation_block(src, "DeleteKmsKeyHandler");
+        assert!(
+            delete.contains("let impact = current_key_impact("),
+            "the delete path must report the configuration impact unconditionally"
+        );
+
+        // Describing is polled, so the same collection is opt-in there.
+        let describe = operation_block(src, "DescribeKmsKeyHandler");
+        assert!(
+            describe.contains("key_impact_if_requested(wants_impact,"),
+            "the describe path must collect the impact only when the request asked for it"
+        );
+        assert!(
+            !describe.contains("current_key_impact("),
+            "the describe path must not reach the collection except through the opt-in"
+        );
+
+        // Neither handler may read what the report says. Constructing it and
+        // handing it to the response is the whole of their business: a handler
+        // that inspected it would be treating "nothing found in the
+        // configuration layer" as "nothing uses this key", which it is not.
+        for (handler, block) in [("DeleteKmsKeyHandler", delete), ("DescribeKmsKeyHandler", describe)] {
+            for inspection in [
+                "impact.blocks_destruction",
+                "impact.references",
+                "impact.completeness",
+                "impact.coverage",
+                "impact.key_id",
+                "match impact",
+            ] {
+                assert!(!block.contains(inspection), "{handler} must not read the impact report (`{inspection}`)");
+            }
+        }
+    }
+
+    fn describe_uri(query: &str) -> Uri {
+        format!("/rustfs/admin/v3/kms/keys/key-a{query}")
+            .parse()
+            .expect("uri should parse")
+    }
+
+    #[test]
+    fn the_impact_section_is_opt_in_and_refuses_an_unreadable_opt_in() {
+        assert_eq!(
+            wants_key_impact(&describe_uri("")),
+            Ok(false),
+            "the default read path must not ask for it"
+        );
+        assert_eq!(wants_key_impact(&describe_uri("?impact=false")), Ok(false));
+        assert_eq!(wants_key_impact(&describe_uri("?impact=true")), Ok(true));
+
+        // A typo must not be read as "off": the caller asked for the section,
+        // and an absent section would be indistinguishable from an empty one.
+        for query in ["?impact=maybe", "?impact=1", "?impact=", "?impact=TRUE"] {
+            let error = wants_key_impact(&describe_uri(query)).expect_err("a malformed opt-in must be refused");
+            assert!(error.contains("expected 'true' or 'false'"), "unhelpful message for {query}: {error}");
+        }
+    }
+
+    /// A report can only come from the collection, so `None` is proof that the
+    /// default read path never reached the bucket listing behind it.
+    #[tokio::test]
+    async fn the_default_describe_path_never_collects_the_impact() {
+        assert!(key_impact_if_requested(false, "key-a", None).await.is_none());
+
+        let collected = key_impact_if_requested(true, "key-a", None)
+            .await
+            .expect("an opted-in describe must carry the section");
+        assert_eq!(collected.key_id, "key-a");
+        assert_eq!(
+            collected.coverage.not_scanned,
+            vec![ReferenceScope::ObjectEnvelopes, ReferenceScope::InProgressMultipartUploads]
+        );
+    }
+
     fn operation_block<'a>(src: &'a str, handler: &str) -> &'a str {
         let marker = format!("impl Operation for {handler}");
         let block = src.split_once(&marker).expect("handler impl should exist").1;
@@ -1041,6 +1302,90 @@ pub struct DeleteKmsKeyResponse {
     pub message: String,
     pub key_id: String,
     pub deletion_date: Option<String>,
+    /// Configuration that still points at the key when the request was
+    /// handled. A scheduled deletion succeeds regardless — it destroys
+    /// nothing and stays cancellable — but the deletion worker will refuse to
+    /// destroy the material while any of these references remain, so an
+    /// operator has to be able to see them at the moment they schedule it
+    /// rather than only in a server-side log once the window has run out.
+    ///
+    /// `None` when the request never got far enough to identify a key or
+    /// reach the KMS service. See [`KeyImpactReport`] for what an empty
+    /// reference list does and does not mean.
+    pub impact: Option<KeyImpactReport>,
+}
+
+const IMMEDIATE_DELETION_QUERY_RETIRED: &str = "immediate deletion is no longer accepted as a query parameter; send it as a JSON body with force_immediate and confirm_key_id set to the key id";
+
+/// Delete request carried entirely by the query string, which can only ever
+/// schedule a deletion.
+///
+/// Immediate deletion destroys key material outright and takes every object
+/// encrypted under the key with it, so it is reachable only through the JSON
+/// body form (rustfs/backlog#1585): a URL travels through shell history, proxy
+/// logs and browser bars, and asking for it there is far too easy to do by
+/// accident. An immediate-deletion attempt made this way is refused rather
+/// than downgraded to a scheduled deletion, so the caller cannot mistake one
+/// outcome for the other.
+///
+/// The refusal is boxed because it is a full response body, not an error code:
+/// carrying one inline would make every `Ok` of this function as wide as the
+/// widest failure it can describe.
+fn delete_request_from_query(uri: &hyper::Uri) -> Result<DeleteKmsKeyRequest, Box<DeleteKmsKeyResponse>> {
+    let query_params = extract_query_params(uri);
+    let refuse = |key_id: &str, message: &str| {
+        Box::new(DeleteKmsKeyResponse {
+            success: false,
+            message: message.to_string(),
+            key_id: key_id.to_string(),
+            deletion_date: None,
+            // The request never reached the KMS service, so nothing was
+            // collected; this is not a report that found no references.
+            impact: None,
+        })
+    };
+
+    let Some(key_id) = query_params.get("keyId") else {
+        return Err(refuse("", "missing required parameter: 'keyId'"));
+    };
+
+    // `force_immediate=false` is the scheduled path spelled out, so it stays
+    // accepted; anything else in that parameter is an immediate-deletion
+    // attempt, including a value that does not parse as a boolean.
+    if query_params.get("force_immediate").is_some_and(|value| value != "false") || query_params.contains_key("confirm_key_id") {
+        return Err(refuse(key_id, IMMEDIATE_DELETION_QUERY_RETIRED));
+    }
+
+    Ok(DeleteKmsKeyRequest {
+        key_id: key_id.clone(),
+        pending_window_in_days: query_params.get("pending_window_in_days").and_then(|s| s.parse::<u32>().ok()),
+        force_immediate: None,
+        confirm_key_id: None,
+    })
+}
+
+/// Status for a deletion the KMS refused.
+///
+/// A rejected waiting window and a refused immediate deletion both arrive as
+/// [`KmsError::InvalidOperation`], and both are the caller's input to fix, so
+/// they must surface as 400 rather than as a server fault.
+fn delete_key_error_status(error: &KmsError) -> StatusCode {
+    match error {
+        KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND,
+        KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST,
+        // Damaged or missing key material is an integrity fault of an existing
+        // key: it must surface as a server error, never as NOT_FOUND (the key
+        // exists) and never as a retryable backend outage.
+        KmsError::MaterialMissing { .. }
+        | KmsError::MaterialCorrupt { .. }
+        | KmsError::MaterialAuthenticationFailed { .. }
+        | KmsError::UnsupportedFormatVersion { .. } => StatusCode::INTERNAL_SERVER_ERROR,
+        // The request was well formed and the key exists; it is the state of
+        // the deployment around it that refuses, and that state can change
+        // without the caller changing anything about the request.
+        KmsError::KeyStillReferenced { .. } => StatusCode::CONFLICT,
+        _ => StatusCode::INTERNAL_SERVER_ERROR,
+    }
 }
 
 /// Delete a KMS key
@@ -1081,31 +1426,15 @@ impl Operation for DeleteKmsKeyHandler {
         let body = body.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
 
         let request: DeleteKmsKeyRequest = if body.is_empty() {
-            let query_params = extract_query_params(&req.uri);
-            let Some(key_id) = query_params.get("keyId") else {
-                let response = DeleteKmsKeyResponse {
-                    success: false,
-                    message: "missing required parameter: 'keyId'".to_string(),
-                    key_id: "".to_string(),
-                    deletion_date: None,
-                };
-                let data =
-                    serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
-                let mut headers = HeaderMap::new();
-                headers.insert(CONTENT_TYPE, "application/json".parse().expect("operation should succeed"));
-                return Ok(S3Response::with_headers((StatusCode::BAD_REQUEST, Body::from(data)), headers));
-            };
-
-            // Extract pending_window_in_days and force_immediate from query parameters
-            let pending_window_in_days = query_params.get("pending_window_in_days").and_then(|s| s.parse::<u32>().ok());
-            let force_immediate = query_params.get("force_immediate").and_then(|s| s.parse::<bool>().ok());
-            let confirm_key_id = query_params.get("confirm_key_id").map(|s| s.to_string());
-
-            DeleteKmsKeyRequest {
-                key_id: key_id.clone(),
-                pending_window_in_days,
-                force_immediate,
-                confirm_key_id,
+            match delete_request_from_query(&req.uri) {
+                Ok(request) => request,
+                Err(response) => {
+                    let data = serde_json::to_vec(&response)
+                        .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+                    let mut headers = HeaderMap::new();
+                    headers.insert(CONTENT_TYPE, "application/json".parse().expect("operation should succeed"));
+                    return Ok(S3Response::with_headers((StatusCode::BAD_REQUEST, Body::from(data)), headers));
+                }
             }
         } else {
             serde_json::from_slice(&body).map_err(|e| s3_error!(InvalidRequest, "invalid JSON: {}", e))?
@@ -1117,6 +1446,7 @@ impl Operation for DeleteKmsKeyHandler {
                 message: "kms service manager is not initialized".to_string(),
                 key_id: request.key_id,
                 deletion_date: None,
+                impact: None,
             };
             let data =
                 serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
@@ -1131,6 +1461,7 @@ impl Operation for DeleteKmsKeyHandler {
                 message: "kms service is not running".to_string(),
                 key_id: request.key_id,
                 deletion_date: None,
+                impact: None,
             };
             let data =
                 serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
@@ -1146,6 +1477,13 @@ impl Operation for DeleteKmsKeyHandler {
             confirm_key_id: request.confirm_key_id.clone(),
         };
 
+        // Collected before the request is carried out, so the response
+        // describes the deployment the caller is acting on. It is reported,
+        // never acted on here: the deletion worker re-runs this check against
+        // live configuration before it destroys anything, which is the gate
+        // that decides the outcome.
+        let impact = current_key_impact(&request.key_id, manager.get_default_key_id().map(String::as_str)).await;
+
         match manager.delete_key_with_context(kms_request, audit.context()).await {
             Ok(kms_response) => {
                 info!(
@@ -1162,6 +1500,7 @@ impl Operation for DeleteKmsKeyHandler {
                     message: "key deleted successfully".to_string(),
                     key_id: kms_response.key_id,
                     deletion_date: kms_response.deletion_date,
+                    impact: Some(impact),
                 };
 
                 let data =
@@ -1183,23 +1522,13 @@ impl Operation for DeleteKmsKeyHandler {
                     error = %e,
                     "admin kms keys state"
                 );
-                let status = match &e {
-                    KmsError::KeyNotFound { .. } => StatusCode::NOT_FOUND,
-                    KmsError::InvalidOperation { .. } | KmsError::ValidationError { .. } => StatusCode::BAD_REQUEST,
-                    // Damaged or missing key material is an integrity fault of an existing
-                    // key: it must surface as a server error, never as NOT_FOUND (the key
-                    // exists) and never as a retryable backend outage.
-                    KmsError::MaterialMissing { .. }
-                    | KmsError::MaterialCorrupt { .. }
-                    | KmsError::MaterialAuthenticationFailed { .. }
-                    | KmsError::UnsupportedFormatVersion { .. } => StatusCode::INTERNAL_SERVER_ERROR,
-                    _ => StatusCode::INTERNAL_SERVER_ERROR,
-                };
+                let status = delete_key_error_status(&e);
                 let response = DeleteKmsKeyResponse {
                     success: false,
                     message: format!("Failed to delete key: {e}"),
                     key_id: request.key_id,
                     deletion_date: None,
+                    impact: Some(impact),
                 };
 
                 let data =
@@ -1513,6 +1842,42 @@ pub struct DescribeKmsKeyResponse {
     pub success: bool,
     pub message: String,
     pub key_metadata: Option<KeyMetadata>,
+    /// Configuration that currently points at the key, so the blast radius of
+    /// a deletion can be read off without scheduling one first.
+    ///
+    /// Present only when the request asked for it with `impact=true`; `None`
+    /// otherwise, and whenever the key could not be described at all. Absent
+    /// therefore means "not collected", never "nothing references this key" —
+    /// see [`KeyImpactReport`] for what a collected one does and does not say.
+    pub impact: Option<KeyImpactReport>,
+}
+
+/// Whether the caller asked for the configuration impact section.
+///
+/// Off unless asked for. Collecting the section lists every bucket, and this
+/// endpoint is polled, so the default read path must not carry that fan-out;
+/// the delete path reports unconditionally instead, because that is the
+/// request whose consequences the operator cannot otherwise see.
+///
+/// A value that is neither `true` nor `false` is refused rather than read as
+/// "off". Silently downgrading a typo would answer a request for the section
+/// with a response that has none, and an absent section must never be
+/// mistaken for one that found nothing.
+fn wants_key_impact(uri: &hyper::Uri) -> Result<bool, String> {
+    match extract_query_params(uri).get("impact") {
+        None => Ok(false),
+        Some(value) => value
+            .parse::<bool>()
+            .map_err(|_| format!("invalid value for 'impact': expected 'true' or 'false', got '{value}'")),
+    }
+}
+
+/// Configuration impact of the described key, collected only when requested.
+async fn key_impact_if_requested(requested: bool, key_id: &str, default_key_id: Option<&str>) -> Option<KeyImpactReport> {
+    if !requested {
+        return None;
+    }
+    Some(current_key_impact(key_id, default_key_id).await)
 }
 
 /// Describe a KMS key
@@ -1550,6 +1915,7 @@ impl Operation for DescribeKmsKeyHandler {
                 success: false,
                 message: "missing required parameter: 'keyId'".to_string(),
                 key_metadata: None,
+                impact: None,
             };
             let data =
                 serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
@@ -1558,11 +1924,31 @@ impl Operation for DescribeKmsKeyHandler {
             return Ok(S3Response::with_headers((StatusCode::BAD_REQUEST, Body::from(data)), headers));
         };
 
+        // Validated before the key is looked up, so a malformed opt-in fails as
+        // the input error it is instead of riding along on the key's outcome.
+        let wants_impact = match wants_key_impact(&req.uri) {
+            Ok(wants_impact) => wants_impact,
+            Err(message) => {
+                let response = DescribeKmsKeyResponse {
+                    success: false,
+                    message,
+                    key_metadata: None,
+                    impact: None,
+                };
+                let data =
+                    serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+                let mut headers = HeaderMap::new();
+                headers.insert(CONTENT_TYPE, "application/json".parse().expect("operation should succeed"));
+                return Ok(S3Response::with_headers((StatusCode::BAD_REQUEST, Body::from(data)), headers));
+            }
+        };
+
         let Some(service_manager) = kms_service_manager_from_context() else {
             let response = DescribeKmsKeyResponse {
                 success: false,
                 message: "kms service manager is not initialized".to_string(),
                 key_metadata: None,
+                impact: None,
             };
             let data =
                 serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
@@ -1576,6 +1962,7 @@ impl Operation for DescribeKmsKeyHandler {
                 success: false,
                 message: "kms service is not running".to_string(),
                 key_metadata: None,
+                impact: None,
             };
             let data =
                 serde_json::to_vec(&response).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
@@ -1599,10 +1986,16 @@ impl Operation for DescribeKmsKeyHandler {
                     state = "completed",
                     "admin kms keys state"
                 );
+                // Read-only, and opt-in: the same configuration-layer facts the
+                // delete path reports, so the blast radius of a deletion can be
+                // inspected without scheduling one first.
+                let impact =
+                    key_impact_if_requested(wants_impact, key_id, manager.get_default_key_id().map(String::as_str)).await;
                 let response = DescribeKmsKeyResponse {
                     success: true,
                     message: "Key described successfully".to_string(),
                     key_metadata: Some(kms_response.key_metadata),
+                    impact,
                 };
 
                 let data =
@@ -1641,6 +2034,7 @@ impl Operation for DescribeKmsKeyHandler {
                     success: false,
                     message: format!("Failed to describe key: {e}"),
                     key_metadata: None,
+                    impact: None,
                 };
 
                 let data =

rustfs/src/admin/handlers/replication.rs

@@ -835,6 +835,10 @@ struct MrfTargetBacklog {
     failed_count: i64,
     #[serde(rename = "FailedSize")]
     failed_size: i64,
+    #[serde(rename = "DurableCount")]
+    durable_count: i64,
+    #[serde(rename = "DurableSize")]
+    durable_size: i64,
     #[serde(rename = "ObservationScope")]
     observation_scope: &'static str,
 }
@@ -842,7 +846,7 @@ struct MrfTargetBacklog {
 /// Response body for `GET /v3/replication/mrf`.
 ///
 /// Runtime failed/queued totals and the durable MRF recovery backlog are kept
-/// separate because persisted MRF entries do not contain a target ARN.
+/// separate because older persisted MRF entries do not contain a target ARN.
 #[derive(Debug, Serialize)]
 struct MrfResponse {
     #[serde(rename = "Bucket")]
@@ -888,32 +892,60 @@ fn build_mrf_response(
         "partial_cluster"
     };
     let mut targets: Vec<MrfTargetBacklog> = Vec::with_capacity(bucket_stats.replication_stats.stats.len());
+    let mut targets_by_arn: HashMap<String, usize> = HashMap::with_capacity(bucket_stats.replication_stats.stats.len());
     let mut total_failed_count: i64 = 0;
     let mut total_failed_size: i64 = 0;
     for (arn, stat) in &bucket_stats.replication_stats.stats {
         total_failed_count = total_failed_count.saturating_add(stat.failed.count);
         total_failed_size = total_failed_size.saturating_add(stat.failed.size);
+        targets_by_arn.insert(arn.clone(), targets.len());
         targets.push(MrfTargetBacklog {
             arn: arn.clone(),
             failed_count: stat.failed.count,
             failed_size: stat.failed.size,
+            durable_count: 0,
+            durable_size: 0,
             observation_scope,
         });
     }
-    targets.sort_by(|a, b| a.arn.cmp(&b.arn));
 
     let queued = &bucket_stats.replication_stats.q_stat.curr;
-    let (durable_count, durable_size) = if durable.available {
-        durable
-            .entries
-            .iter()
-            .filter(|entry| entry.bucket == bucket)
-            .fold((0i64, 0i64), |(count, size), entry| {
-                (count.saturating_add(1), size.saturating_add(entry.size))
-            })
+    let (durable_count, durable_size, missing_target_arns) = if durable.available {
+        durable.entries.iter().filter(|entry| entry.bucket == bucket).fold(
+            (0i64, 0i64, false),
+            |(count, size, missing_target_arns), entry| {
+                let mut missing_target_arns = missing_target_arns;
+                for target_arn in &entry.target_arns {
+                    if target_arn.is_empty() {
+                        continue;
+                    }
+                    let index = if let Some(index) = targets_by_arn.get(target_arn).copied() {
+                        index
+                    } else {
+                        targets_by_arn.insert(target_arn.clone(), targets.len());
+                        targets.push(MrfTargetBacklog {
+                            arn: target_arn.clone(),
+                            failed_count: 0,
+                            failed_size: 0,
+                            durable_count: 0,
+                            durable_size: 0,
+                            observation_scope,
+                        });
+                        targets.len() - 1
+                    };
+                    targets[index].durable_count = targets[index].durable_count.saturating_add(1);
+                    targets[index].durable_size = targets[index].durable_size.saturating_add(entry.size);
+                }
+                if entry.target_arns.is_empty() {
+                    missing_target_arns = true;
+                }
+                (count.saturating_add(1), size.saturating_add(entry.size), missing_target_arns)
+            },
+        )
     } else {
-        (0, 0)
+        (0, 0, false)
     };
+    targets.sort_by(|a, b| a.arn.cmp(&b.arn));
 
     MrfResponse {
         bucket,
@@ -930,7 +962,7 @@ fn build_mrf_response(
         durable_backlog_available: durable.available,
         durable_count,
         durable_size,
-        per_target_durable_entries_available: false,
+        per_target_durable_entries_available: durable.available && !missing_target_arns,
     }
 }
 
@@ -940,8 +972,9 @@ fn build_mrf_response(
 ///
 /// Compatibility note: MinIO returns a stream of individual MRF entries. RustFS
 /// deliberately returns aggregate runtime and durable counters instead.
-/// `PerObjectEntriesAvailable` and `PerTargetDurableEntriesAvailable` remain
-/// false until an enumerable API and target-bearing durable format exist.
+/// `PerObjectEntriesAvailable` remains false until an enumerable API exists.
+/// `PerTargetDurableEntriesAvailable` is false when the durable backlog includes
+/// older entries that cannot be attributed to a target.
 pub struct ReplicationMrfHandler {}
 
 #[async_trait::async_trait]
@@ -1059,6 +1092,7 @@ mod tests {
                     delete_marker_version_id: None,
                     delete_marker: false,
                     delete_marker_mtime: None,
+                    target_arns: vec!["arn-a".to_string(), "arn-durable-only".to_string()],
                 },
                 MrfReplicateEntry {
                     bucket: "other-bucket".to_string(),
@@ -1070,6 +1104,7 @@ mod tests {
                     delete_marker_version_id: None,
                     delete_marker: false,
                     delete_marker_mtime: None,
+                    target_arns: Vec::new(),
                 },
             ],
         };
@@ -1087,7 +1122,64 @@ mod tests {
         assert_eq!(json["ClusterComplete"], false);
         assert_eq!(json["Targets"][0]["ObservationScope"], "partial_cluster");
         assert_eq!(json["PerObjectEntriesAvailable"], false);
+        assert_eq!(json["PerTargetDurableEntriesAvailable"], true);
+
+        let targets = json["Targets"].as_array().expect("targets should serialize as an array");
+        let target_a = targets
+            .iter()
+            .find(|target| target["ARN"] == "arn-a")
+            .expect("runtime target should remain present");
+        assert_eq!(target_a["FailedCount"], 3);
+        assert_eq!(target_a["FailedSize"], 900);
+        assert_eq!(target_a["DurableCount"], 1);
+        assert_eq!(target_a["DurableSize"], 250);
+        let target_durable_only = targets
+            .iter()
+            .find(|target| target["ARN"] == "arn-durable-only")
+            .expect("durable-only target should be surfaced");
+        assert_eq!(target_durable_only["FailedCount"], 0);
+        assert_eq!(target_durable_only["FailedSize"], 0);
+        assert_eq!(target_durable_only["DurableCount"], 1);
+        assert_eq!(target_durable_only["DurableSize"], 250);
+    }
+
+    #[test]
+    fn mrf_response_keeps_legacy_durable_entries_bucket_only() {
+        let mut stats = BucketStats::default();
+        stats.replication_stats.provider_available = true;
+        stats.replication_stats.cluster_complete = true;
+        stats.replication_stats.observed_node_count = 1;
+        stats.replication_stats.expected_node_count = 1;
+        let durable = DurableMrfBacklog {
+            available: true,
+            entries: vec![MrfReplicateEntry {
+                bucket: "bucket-a".to_string(),
+                object: "legacy-object".to_string(),
+                version_id: None,
+                retry_count: 0,
+                size: 250,
+                op: MrfOpKind::Object,
+                delete_marker_version_id: None,
+                delete_marker: false,
+                delete_marker_mtime: None,
+                target_arns: Vec::new(),
+            }],
+        };
+
+        let response = build_mrf_response("bucket-a".to_string(), &stats, durable);
+        let json = serde_json::to_value(response).expect("MRF response should serialize");
+
+        assert_eq!(json["DurableBacklogAvailable"], true);
+        assert_eq!(json["DurableCount"], 1);
+        assert_eq!(json["DurableSize"], 250);
         assert_eq!(json["PerTargetDurableEntriesAvailable"], false);
+        assert_eq!(
+            json["Targets"]
+                .as_array()
+                .expect("targets should serialize as an array")
+                .len(),
+            0
+        );
     }
 
     #[test]
@@ -1115,6 +1207,7 @@ mod tests {
         assert_eq!(valid_empty_json["DurableBacklogAvailable"], true);
         assert_eq!(valid_empty_json["TotalFailedCount"], 0);
         assert_eq!(valid_empty_json["DurableCount"], 0);
+        assert_eq!(valid_empty_json["PerTargetDurableEntriesAvailable"], true);
     }
 
     #[test]

rustfs/src/admin/route_policy.rs

@@ -766,11 +766,16 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
     ),
     admin(HttpMethod::Post, "/rustfs/admin/v3/kms/reconfigure", KMS_CONFIGURE, RouteRiskLevel::High),
     admin(HttpMethod::Post, "/rustfs/admin/v3/kms/keys", KMS_CONFIGURE, RouteRiskLevel::High),
+    // Critical rather than High: destroying a master key makes every object
+    // encrypted under it permanently unreadable, and nothing on the server can
+    // bring those objects back (rustfs/backlog#1585). The waiting window and
+    // cancel-deletion are the only recovery path, which is why the route below
+    // that reopens it stays merely High.
     admin(
         HttpMethod::Delete,
         "/rustfs/admin/v3/kms/keys/delete",
         KMS_DELETE_KEY,
-        RouteRiskLevel::High,
+        RouteRiskLevel::Critical,
     ),
     admin(
         HttpMethod::Post,
@@ -1880,6 +1885,39 @@ mod tests {
         }
     }
 
+    /// `Critical` marks the routes whose worst case is permanent loss of user
+    /// data, so the inventory is pinned in both directions: the KMS key
+    /// deletion route may not be quietly downgraded, and no other route may be
+    /// promoted without the same review.
+    #[test]
+    fn route_policy_reserves_critical_risk_for_unrecoverable_destruction() {
+        let critical = ADMIN_ROUTE_POLICY_SPECS
+            .iter()
+            .filter(|spec| spec.risk_level() == RouteRiskLevel::Critical)
+            .map(|spec| route_key(spec.method(), spec.path()))
+            .collect::<BTreeSet<_>>();
+
+        assert_eq!(
+            critical,
+            BTreeSet::from([route_key(HttpMethod::Delete, "/rustfs/admin/v3/kms/keys/delete")]),
+            "the Critical set changed; classify the new route deliberately or restore the old one"
+        );
+    }
+
+    /// The routes that recover from, or merely describe, a pending deletion are
+    /// not Critical: refusing them is what leaves an operator without a way
+    /// back.
+    #[test]
+    fn route_policy_keeps_kms_deletion_recovery_below_critical() {
+        for (method, path) in [
+            (HttpMethod::Post, "/rustfs/admin/v3/kms/keys/cancel-deletion"),
+            (HttpMethod::Get, "/rustfs/admin/v3/kms/keys/{key_id}"),
+            (HttpMethod::Get, "/rustfs/admin/v3/kms/keys"),
+        ] {
+            assert_risk_below_critical(method, path);
+        }
+    }
+
     /// Backup and restore expose the whole key inventory at once, so no other
     /// KMS action may reach them: holding `kms:Configure` or a per-key action
     /// must not be enough.
@@ -2039,6 +2077,14 @@ mod tests {
         assert_ne!(spec.access().admin_action(), Some(action));
     }
 
+    fn assert_risk_below_critical(method: HttpMethod, path: &str) {
+        let spec = ADMIN_ROUTE_POLICY_SPECS
+            .iter()
+            .find(|spec| spec.method() == method && spec.path() == path)
+            .expect("expected direct route policy");
+        assert_ne!(spec.risk_level(), RouteRiskLevel::Critical, "{path} must not be Critical");
+    }
+
     fn assert_public(method: HttpMethod, path: &str, kind: PublicRouteKind) {
         let spec = ADMIN_ROUTE_POLICY_SPECS
             .iter()

rustfs/src/config/cli.rs

@@ -314,7 +314,7 @@ pub struct ServerOpts {
     #[arg(long, default_value_t = false, env = "RUSTFS_KMS_ENABLE")]
     pub kms_enable: bool,
 
-    /// KMS backend type: local, vault or vault-kv2 (plain Vault KV v2 storage), vault-transit
+    /// KMS backend type: local, vault or vault-kv2 (plain Vault KV v2 storage), vault-transit, static, aws
     #[arg(long, default_value_t = rustfs_config::DEFAULT_KMS_BACKEND.to_string(), env = "RUSTFS_KMS_BACKEND")]
     pub kms_backend: String,
 

rustfs/src/init.rs

@@ -430,6 +430,37 @@ fn build_static_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::
     Ok(kms_config)
 }
 
+/// Build KMS configuration for the AWS KMS backend
+///
+/// No credential material is read here: AWS credentials are resolved by the
+/// standard `aws-config` provider chain (environment, shared profile,
+/// container/IMDS role), so only the two non-credential settings are taken
+/// from the environment. An unresolvable region fails the backend closed when
+/// the service starts.
+fn build_aws_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::config::KmsConfig> {
+    use rustfs_kms::config::{AwsKmsConfig, ENV_KMS_AWS_ENDPOINT_URL, ENV_KMS_AWS_REGION};
+
+    let kms_config = rustfs_kms::config::KmsConfig {
+        backend: rustfs_kms::config::KmsBackend::Aws,
+        backend_config: rustfs_kms::config::BackendConfig::Aws(Box::new(AwsKmsConfig {
+            region: rustfs_utils::get_env_opt_str(ENV_KMS_AWS_REGION),
+            endpoint_url: rustfs_utils::get_env_opt_str(ENV_KMS_AWS_ENDPOINT_URL),
+        })),
+        allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
+        allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
+        // Keys are never auto-created on this backend: it refuses
+        // caller-named creation because AWS assigns identifiers, so the
+        // default key must already exist in AWS and be named by key id or ARN.
+        default_key_id: cfg.kms_default_key_id.clone(),
+        ..Default::default()
+    };
+
+    kms_config
+        .validate()
+        .map_err(|e| Error::other(format!("AWS KMS configuration validation failed: {e}")))?;
+    Ok(kms_config)
+}
+
 /// Configure and start KMS service
 async fn configure_and_start_kms(
     service_manager: &std::sync::Arc<rustfs_kms::KmsServiceManager>,
@@ -506,6 +537,7 @@ pub async fn init_kms_system(config: &config::Config) -> std::io::Result<()> {
             "vault" | "vault-kv2" | "vault_kv2" => build_vault_kms_config(config)?,
             "vault-transit" | "vault_transit" => build_vault_transit_kms_config(config)?,
             "static" => build_static_kms_config(config)?,
+            "aws" | "aws-kms" | "aws_kms" => build_aws_kms_config(config)?,
             _ => return Err(Error::other(format!("Unsupported KMS backend: {}", config.kms_backend))),
         };
 
@@ -1373,7 +1405,7 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
 
 #[cfg(test)]
 mod tests {
-    use super::{notification_config_to_event_rules, resolve_buffer_profile_config};
+    use super::{build_aws_kms_config, notification_config_to_event_rules, resolve_buffer_profile_config};
     use crate::config::{BufferConfig, WorkloadProfile};
     use rustfs_config::KI_B;
     use rustfs_s3_types::EventName;
@@ -1466,4 +1498,57 @@ mod tests {
 
         assert!(err.to_string().contains("Invalid ARN"), "unexpected error: {err}");
     }
+
+    fn aws_kms_test_config() -> crate::config::Config {
+        let mut config = crate::config::Config::new("127.0.0.1:9000", vec!["/tmp/rustfs-aws-kms".to_string()]);
+        config.kms_enable = true;
+        config.kms_backend = "aws".to_string();
+        config.kms_default_key_id = Some("arn:aws:kms:us-east-1:111122223333:key/1234abcd".to_string());
+        config
+    }
+
+    /// Startup takes only the two non-credential AWS settings from the
+    /// environment; credentials stay with the `aws-config` provider chain.
+    #[test]
+    fn build_aws_kms_config_reads_only_non_credential_settings() {
+        let config = temp_env::with_vars(
+            [
+                ("RUSTFS_KMS_AWS_REGION", Some("eu-central-1")),
+                ("RUSTFS_KMS_AWS_ENDPOINT_URL", None),
+            ],
+            || build_aws_kms_config(&aws_kms_test_config()).expect("aws KMS configuration should build"),
+        );
+
+        assert_eq!(config.backend, rustfs_kms::config::KmsBackend::Aws);
+        let aws = config.aws_kms_config().expect("aws backend config");
+        assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
+        assert_eq!(aws.endpoint_url, None);
+        assert_eq!(config.default_key_id.as_deref(), Some("arn:aws:kms:us-east-1:111122223333:key/1234abcd"));
+    }
+
+    /// A plaintext endpoint override exposes every KMS request, plaintext data
+    /// keys included, so startup refuses it without the development opt-in.
+    #[test]
+    fn build_aws_kms_config_refuses_a_plaintext_endpoint_without_opt_in() {
+        let vars = [
+            ("RUSTFS_KMS_AWS_REGION", Some("us-east-1")),
+            ("RUSTFS_KMS_AWS_ENDPOINT_URL", Some("http://localhost:4566")),
+        ];
+
+        temp_env::with_vars(vars, || {
+            let error =
+                build_aws_kms_config(&aws_kms_test_config()).expect_err("a plaintext AWS endpoint must not start the server");
+            assert!(error.to_string().contains("https"), "unexpected error: {error}");
+        });
+
+        temp_env::with_vars(vars, || {
+            let mut config = aws_kms_test_config();
+            config.kms_allow_insecure_dev_defaults = true;
+            let config = build_aws_kms_config(&config).expect("the development opt-in should accept a plaintext endpoint");
+            assert_eq!(
+                config.aws_kms_config().expect("aws backend config").endpoint_url.as_deref(),
+                Some("http://localhost:4566")
+            );
+        });
+    }
 }

rustfs/src/kms_deletion_gate.rs

@@ -17,18 +17,27 @@
 //! must not be removed. Object-level references (existing envelopes) are out
 //! of scope here; those objects stay decryptable only until the key is gone,
 //! which is why the pending-deletion window exists.
+//!
+//! The same collection also backs the impact section of the admin key
+//! responses, so an operator sees the references that will block a deletion at
+//! the moment they schedule it rather than only in a server-side log once the
+//! window has run out. Both consumers read the same collection: the report
+//! cannot describe a deployment the gate does not enforce.
+//!
+//! Cost is bounded by the number of buckets — one bucket listing plus one
+//! cached metadata lookup each. Nothing here lists objects or versions.
 
 use crate::runtime_sources::current_object_store_handle;
 use crate::storage_api::kms::contract::bucket::{BucketOperations, BucketOptions};
 use crate::storage_api::kms::{ECStore, StorageError, get_bucket_sse_config};
 use async_trait::async_trait;
-use rustfs_kms::DeletionReferenceChecker;
+use rustfs_kms::{DeletionReferenceChecker, KeyImpactReport, KeyReference, KeyReferenceKind};
 use s3s::dto::ServerSideEncryptionConfiguration;
 use std::sync::Arc;
 use tracing::warn;
 
-/// Reference reported when bucket configuration cannot be inspected at all.
-const BUCKET_CONFIG_UNAVAILABLE: &str = "bucket-encryption-config:unavailable";
+/// Source name reported when bucket configuration cannot be inspected at all.
+const BUCKET_ENCRYPTION_SOURCE: &str = "bucket-encryption-config";
 
 /// Blocks deletion of keys referenced by any bucket's SSE configuration
 /// (default bucket KMS key). Registered on the KMS service manager at startup
@@ -38,33 +47,73 @@ pub(crate) struct BucketEncryptionReferenceChecker;
 #[async_trait]
 impl DeletionReferenceChecker for BucketEncryptionReferenceChecker {
     async fn references(&self, key_id: &str) -> Vec<String> {
-        collect_references(key_id, current_object_store_handle()).await
+        // Bucket configuration only, and no service default key: the deletion
+        // worker checks the default key itself before it consults a checker,
+        // so this reports exactly the reference set it always has.
+        collect_key_impact(key_id, None, current_object_store_handle())
+            .await
+            .references
+            .iter()
+            .map(blocking_reference)
+            .collect()
     }
 }
 
-async fn collect_references(key_id: &str, store: Option<Arc<ECStore>>) -> Vec<String> {
+/// Configuration-layer impact of deleting `key_id`.
+///
+/// Exhaustive over what it covers, and explicit about what it does not:
+/// object envelopes written under the key are not enumerated, so an empty
+/// reference list is never a statement that the key is unused.
+pub(crate) async fn current_key_impact(key_id: &str, default_key_id: Option<&str>) -> KeyImpactReport {
+    collect_key_impact(key_id, default_key_id, current_object_store_handle()).await
+}
+
+async fn collect_key_impact(key_id: &str, default_key_id: Option<&str>, store: Option<Arc<ECStore>>) -> KeyImpactReport {
+    let mut report = KeyImpactReport::configuration_layer(key_id);
+
+    if default_key_id == Some(key_id) {
+        report.push_reference(KeyReference {
+            kind: KeyReferenceKind::ServiceDefaultKey,
+            id: key_id.to_string(),
+            detail: "the KMS service is configured to use this key as its default key".to_string(),
+        });
+    }
+
     // Fail closed: destroying key material is irreversible while the deletion
     // worker retries every sweep, so an uninspectable configuration must block
     // the removal rather than wave it through. This also covers the worker
     // racing server startup, before the object store is published.
     let Some(store) = store else {
         warn!(key_id, "KMS deletion reference check: object store not ready; blocking removal");
-        return vec![BUCKET_CONFIG_UNAVAILABLE.to_string()];
+        report.push_reference(unreadable_source(
+            "object store is not ready, so bucket encryption configuration could not be read",
+        ));
+        return report;
     };
     let buckets = match store.list_bucket(&BucketOptions::default()).await {
         Ok(buckets) => buckets,
         Err(error) => {
             warn!(key_id, %error, "KMS deletion reference check: listing buckets failed; blocking removal");
-            return vec![BUCKET_CONFIG_UNAVAILABLE.to_string()];
+            report.push_reference(unreadable_source(format!("buckets could not be listed: {error}")));
+            return report;
         }
     };
 
-    let mut references = Vec::new();
     for bucket in &buckets {
         let lookup = get_bucket_sse_config(&bucket.name).await;
-        references.extend(bucket_reference(&bucket.name, lookup, key_id));
+        if let Some(reference) = bucket_reference(&bucket.name, lookup, key_id) {
+            report.push_reference(reference);
+        }
+    }
+    report
+}
+
+fn unreadable_source(detail: impl Into<String>) -> KeyReference {
+    KeyReference {
+        kind: KeyReferenceKind::UnreadableSource,
+        id: BUCKET_ENCRYPTION_SOURCE.to_string(),
+        detail: detail.into(),
     }
-    references
 }
 
 /// `Some(reference)` when the bucket's encryption configuration references
@@ -74,9 +123,13 @@ fn bucket_reference(
     bucket: &str,
     lookup: Result<ServerSideEncryptionConfiguration, StorageError>,
     key_id: &str,
-) -> Option<String> {
+) -> Option<KeyReference> {
     match lookup {
-        Ok(config) if sse_config_references_key(&config, key_id) => Some(format!("bucket:{bucket}")),
+        Ok(config) if sse_config_references_key(&config, key_id) => Some(KeyReference {
+            kind: KeyReferenceKind::BucketDefaultEncryption,
+            id: bucket.to_string(),
+            detail: "bucket default encryption names this key, so new objects are written under it".to_string(),
+        }),
         Ok(_) => None,
         Err(StorageError::ConfigNotFound) => None,
         Err(error) => {
@@ -86,11 +139,29 @@ fn bucket_reference(
                 %error,
                 "KMS deletion reference check: unreadable bucket encryption config; blocking removal"
             );
-            Some(format!("bucket:{bucket}:encryption-config-unreadable"))
+            Some(KeyReference {
+                kind: KeyReferenceKind::UnreadableResource,
+                id: bucket.to_string(),
+                detail: format!("bucket encryption configuration could not be read: {error}"),
+            })
         }
     }
 }
 
+/// Reference identifier handed to the deletion worker.
+///
+/// The worker only needs an identifier to log and to count as non-empty, so
+/// these strings stay exactly as they were before the structured report
+/// existed; a reference is a reference regardless of how it renders.
+fn blocking_reference(reference: &KeyReference) -> String {
+    match reference.kind {
+        KeyReferenceKind::BucketDefaultEncryption => format!("bucket:{}", reference.id),
+        KeyReferenceKind::UnreadableResource => format!("bucket:{}:encryption-config-unreadable", reference.id),
+        KeyReferenceKind::UnreadableSource => format!("{}:unavailable", reference.id),
+        KeyReferenceKind::ServiceDefaultKey => format!("kms-service-default-key:{}", reference.id),
+    }
+}
+
 fn sse_config_references_key(config: &ServerSideEncryptionConfiguration, key_id: &str) -> bool {
     config.rules.iter().any(|rule| {
         rule.apply_server_side_encryption_by_default
@@ -103,6 +174,7 @@ fn sse_config_references_key(config: &ServerSideEncryptionConfiguration, key_id:
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustfs_kms::ReferenceCompleteness;
     use s3s::dto::{ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionRule};
 
     fn sse_kms_config(key_id: Option<&str>) -> ServerSideEncryptionConfiguration {
@@ -119,10 +191,10 @@ mod tests {
 
     #[test]
     fn referencing_bucket_blocks_deletion() {
-        assert_eq!(
-            bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1"),
-            Some("bucket:sse-bucket".to_string())
-        );
+        let reference = bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1")
+            .expect("a bucket that names the key must be reported");
+        assert_eq!(reference.kind, KeyReferenceKind::BucketDefaultEncryption);
+        assert_eq!(reference.id, "sse-bucket");
     }
 
     #[test]
@@ -135,13 +207,82 @@ mod tests {
 
     #[test]
     fn unreadable_config_blocks_deletion() {
-        let reference = bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1");
-        assert_eq!(reference, Some("bucket:broken:encryption-config-unreadable".to_string()));
+        let reference = bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1")
+            .expect("an unreadable bucket must be reported");
+        assert_eq!(reference.kind, KeyReferenceKind::UnreadableResource);
+        assert_eq!(reference.id, "broken");
+    }
+
+    /// The deletion worker's gate is the only thing standing between an
+    /// expired key and destroyed material; reshaping the collection it reads
+    /// must not change a single identifier it receives. The report is the
+    /// second reading of that same collection, so it must never look clear
+    /// where the gate objects.
+    #[test]
+    fn worker_reference_identifiers_are_unchanged() {
+        let cases = [
+            (
+                bucket_reference("sse-bucket", Ok(sse_kms_config(Some("kms-key-1"))), "kms-key-1"),
+                "bucket:sse-bucket",
+            ),
+            (
+                bucket_reference("broken", Err(StorageError::FaultyDisk), "kms-key-1"),
+                "bucket:broken:encryption-config-unreadable",
+            ),
+            (
+                Some(unreadable_source("object store is not ready")),
+                "bucket-encryption-config:unavailable",
+            ),
+        ];
+
+        for (reference, expected) in cases {
+            let reference = reference.expect("case must produce a reference");
+            assert_eq!(blocking_reference(&reference), expected);
+
+            let mut report = KeyImpactReport::configuration_layer("kms-key-1");
+            report.push_reference(reference);
+            assert!(report.blocks_destruction(), "{expected} must read as blocking in the report too");
+        }
     }
 
     #[tokio::test]
     async fn missing_object_store_blocks_deletion() {
-        let references = collect_references("kms-key-1", None).await;
-        assert_eq!(references, vec![BUCKET_CONFIG_UNAVAILABLE.to_string()]);
+        let references: Vec<String> = collect_key_impact("kms-key-1", None, None)
+            .await
+            .references
+            .iter()
+            .map(blocking_reference)
+            .collect();
+        assert_eq!(references, vec!["bucket-encryption-config:unavailable".to_string()]);
+    }
+
+    /// An unreachable object store must never render as "we looked and found
+    /// nothing": the report says the configuration could not be read.
+    #[tokio::test]
+    async fn missing_object_store_reports_unavailable_completeness() {
+        let report = collect_key_impact("kms-key-1", None, None).await;
+
+        assert_eq!(report.completeness, ReferenceCompleteness::Unavailable);
+        assert!(report.blocks_destruction());
+        assert_eq!(report.references.len(), 1);
+        assert_eq!(report.references[0].kind, KeyReferenceKind::UnreadableSource);
+    }
+
+    /// The service default key is a configuration reference in its own right,
+    /// and it is decidable without touching the object store.
+    #[tokio::test]
+    async fn service_default_key_is_reported_before_any_store_lookup() {
+        let report = collect_key_impact("kms-key-1", Some("kms-key-1"), None).await;
+
+        assert_eq!(report.references[0].kind, KeyReferenceKind::ServiceDefaultKey);
+        assert_eq!(report.references[0].id, "kms-key-1");
+
+        let other = collect_key_impact("kms-key-1", Some("kms-key-2"), None).await;
+        assert!(
+            !other
+                .references
+                .iter()
+                .any(|reference| reference.kind == KeyReferenceKind::ServiceDefaultKey)
+        );
     }
 }

rustfs/src/storage/sse.rs

@@ -3144,12 +3144,14 @@ pub fn is_managed_sse(server_side_encryption: &ServerSideEncryption) -> bool {
 /// Encryption metadata describes the physical source representation and must never be
 /// inherited by a plaintext destination or by a destination using a different key.
 pub fn strip_managed_encryption_metadata(metadata: &mut HashMap<String, String>) {
-    const KEYS: [&str; 19] = [
+    const KEYS: [&str; 21] = [
         "x-amz-server-side-encryption",
         "x-amz-server-side-encryption-aws-kms-key-id",
         "x-amz-server-side-encryption-customer-algorithm",
         "x-amz-server-side-encryption-customer-key-md5",
         SSEC_ORIGINAL_SIZE_HEADER,
+        INTERNAL_ENCRYPTION_KEY_ID_HEADER,
+        INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
         INTERNAL_ENCRYPTION_IV_HEADER,
         "x-rustfs-encryption-tag",
         INTERNAL_ENCRYPTION_KEY_HEADER,
@@ -4549,6 +4551,38 @@ mod tests {
         assert!(metadata.contains_key("content-type"));
     }
 
+    /// A copy destination must not inherit any key that `is_object_encryption_marker`
+    /// accepts as proof the payload is encrypted.
+    ///
+    /// Guards this chain: `ObjectInfo::is_encrypted` is keyed on that predicate, so a
+    /// single leftover marker makes a plaintext destination report itself encrypted.
+    /// `object_api::readers` then takes its encrypted branch and demands read material,
+    /// but the material itself was stripped, so `sse_decryption` reports no encryption
+    /// and the read fails with "encrypted object metadata is incomplete" — a destination
+    /// that CopyObject wrote successfully becomes permanently unreadable.
+    #[test]
+    fn test_strip_managed_encryption_metadata_clears_encryption_markers() {
+        let mut metadata = HashMap::from([
+            ("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
+            ("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "source-key".to_string()),
+            (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "source-key".to_string()),
+            (INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "AES256".to_string()),
+            (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "wrapped-dek".to_string()),
+            (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "base-nonce".to_string()),
+            ("content-type".to_string(), "text/plain".to_string()),
+        ]);
+
+        strip_managed_encryption_metadata(&mut metadata);
+
+        let inherited: Vec<&str> = metadata
+            .keys()
+            .filter(|key| rustfs_utils::http::is_object_encryption_marker(key))
+            .map(String::as_str)
+            .collect();
+        assert!(inherited.is_empty(), "copy destination inherited encryption markers: {inherited:?}");
+        assert!(metadata.contains_key("content-type"));
+    }
+
     #[cfg(feature = "rio-v2")]
     #[test]
     fn test_legacy_managed_metadata_excludes_sealed_keys() {