ci: stop archiving every dependency's unpacked source in each cache (#5566)

The four consolidated ci keys plus the build keys still do not fit the
repository's fixed 10GB Actions cache quota, so LRU keeps evicting them:
measured demand is ci-dev 2331MB + ci-feat-proto 2310MB + ci-feat-rio 1951MB +
ci-uring 1317MB + two build legs at ~1429MB + cargo-deny 844MB, and main pushes
add two more build legs. That is roughly 14.4GB against 10.24GB. The symptom is
misleading: Cache Warm reports every job successful and the restores log "full
match: true", yet ci-feat-rio and ci-uring disappear from the cache list between
runs.

cache-all-crates was the wrong default for this repository. With it set to true,
rust-cache's cleanup returns before pruning ~/.cargo/registry/src and its config
archives the whole registry, so every cache carried the unpacked source tree of
every dependency — not, as the name suggests, just a few extra crates.

Setting it to false is rust-cache's own default and loses no coverage: the
package set comes from `cargo metadata --all-features`, a strict superset of any
single lane's feature closure; -sys crates are explicitly exempt from pruning,
since their source timestamps would otherwise trigger rebuilds; and everything
pruned is re-unpacked from the .crate files still in registry/cache, whose
mtimes crates.io normalises, so cargo fingerprints stay valid.

Applied to the setup composite and to audit.yml's own rust-cache. Cache Warm
now also reports the sizes of registry/src, registry/cache, registry/index,
~/.cargo/git and target/ to the step summary, immediately before rust-cache's
post step archives them, so the size of the effect is measured rather than
assumed.

The new sizes only appear once the cache key next rotates, since rust-cache
skips the save entirely on an exact key hit. Deliberately not forcing that by
bumping prefix-key: it would invalidate every family at once and produce a
repository-wide cold build.

Refs: rustfs/backlog#1598, rustfs/backlog#1600

.config/nextest.toml

@@ -300,9 +300,6 @@ path = "junit.xml"
 # negative-path siblings of each family stay in as regression guards.
 #   * rustfs#4843 — over-limit archive entry paths hard-reject the whole
 #     archive even under ignore-errors semantics.
-#   * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
-#     under parallel load (multi-node in-process clusters; natural home is
-#     ci-7's nightly cluster lane).
 [profile.e2e-full]
 default-filter = """
     package(e2e_test)
@@ -311,7 +308,6 @@ default-filter = """
     & !test(/^replication_extension_test::/)
     & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
     & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
-    & !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
 """
 fail-fast = false
 

.github/actions/setup/action.yml

@@ -111,7 +111,18 @@ runs:
     - name: Setup Rust cache
       uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
       with:
-        cache-all-crates: true
+        # false is rust-cache's own default. With true, cleanup.ts returns
+        # *before* pruning ~/.cargo/registry/src, and config.ts archives the
+        # whole registry — so every cache carried the unpacked source tree of
+        # every dependency, not just "a few extra crates".
+        #
+        # No coverage is lost: getPackages runs `cargo metadata --all-features`,
+        # a strict superset of any single lane's feature closure, and -sys crates
+        # are explicitly exempted from pruning (their src timestamps would
+        # otherwise trigger rebuilds). Anything pruned is re-unpacked from the
+        # .crate files still in registry/cache, whose mtimes crates.io
+        # normalises, so cargo fingerprints stay valid.
+        cache-all-crates: false
         cache-on-failure: true
         shared-key: ${{ inputs.cache-shared-key }}
         save-if: ${{ inputs.cache-save-if }}

.github/workflows/architecture-migration-rules.yml

@@ -49,6 +49,8 @@ jobs:
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install ripgrep
         run: |

.github/workflows/audit.yml

@@ -80,6 +80,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # cargo-deny compiles nothing, so the full setup composite (apt packages,
       # protoc, flatc, nextest, rustfmt/clippy) was pure overhead here. It does
@@ -98,7 +100,9 @@ jobs:
       - name: Setup Rust cache
         uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
         with:
-          cache-all-crates: true
+          # Same reasoning as the setup composite: true archives every
+          # dependency's unpacked source tree.
+          cache-all-crates: false
           cache-on-failure: true
           shared-key: rustfs-cargo-deny
           save-if: ${{ github.ref == 'refs/heads/main' }}
@@ -119,6 +123,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Report unpinned GitHub Actions
         run: ./scripts/security/check_workflow_pins.sh --enforce
@@ -129,6 +135,9 @@ jobs:
       - name: Check every job declares a timeout
         run: ./scripts/security/check_job_timeouts.sh
 
+      - name: Check checkouts clear their credentials
+        run: ./scripts/security/check_persist_credentials.sh
+
       - name: Check preview release workflow policy
         run: ./scripts/security/check_preview_release_workflow.sh
 
@@ -143,6 +152,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Dependency Review
         uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5
@@ -174,6 +185,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/build.yml

@@ -99,6 +99,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Determine build strategy
         id: check
@@ -257,6 +259,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           fetch-depth: 0
 
       - name: Setup Rust environment
@@ -795,6 +798,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           fetch-depth: 0
 
       - name: Create GitHub Release
@@ -852,6 +856,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download all build artifacts
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7

.github/workflows/cache-warm.yml

@@ -65,6 +65,14 @@ on:
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
   workflow_dispatch:
+    inputs:
+      emit_timings:
+        description: >-
+          Also emit cargo --timings for the ci-dev build and upload it. Used to
+          decide whether sccache is worth adopting (rustfs/backlog#1601 gate).
+        required: false
+        default: false
+        type: boolean
 
 permissions:
   contents: read
@@ -99,6 +107,33 @@ jobs:
           cache-save-if: 'true'
           install-build-packaging-tools: 'false'
 
+      # rustfs/backlog#1601 gate. sccache can only cache compilation units whose
+      # --emit includes link, so it covers workspace rlibs and nothing else:
+      # clippy is metadata-only, and the ~100 test binaries, the rustfs bin and
+      # every build script invoke the system linker. Before spending a bucket,
+      # credentials and a supply-chain boundary on it, measure how much of the
+      # build is actually rlib codegen.
+      #
+      # Read from the report: workspace lib codegen as a share of the build, and
+      # s3select-query's own rlib as a share. The plan adopts sccache only above
+      # 50% and 25% respectively; if linking dominates instead, the answer is
+      # mold/lld plus split-debuginfo, which is exactly the part sccache cannot
+      # touch. Off by default — this doubles the ci-dev build.
+      - name: Build ci-dev superset (with --timings)
+        if: inputs.emit_timings
+        env:
+          CARGO_BUILD_JOBS: "2"
+        run: cargo build --workspace --all-targets --timings
+
+      - name: Upload cargo timings report
+        if: inputs.emit_timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cargo-timings-ci-dev
+          path: target/cargo-timings/
+          retention-days: 30
+          if-no-files-found: error
+
       # --all-targets covers the test binaries nextest builds, including
       # e2e_test, which test-and-lint's own run excludes. The second build adds
       # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
@@ -113,6 +148,21 @@ jobs:
           cargo build --workspace --all-targets
           cargo build -p rustfs --bins --features e2e-test-hooks
 
+      # Runs before rust-cache's post step, so these are the sizes it is about
+      # to archive. Reported so the cache-all-crates decision stays evidence-led:
+      # registry/src is what that flag prunes, registry/cache is what the pruned
+      # sources are re-unpacked from. See rustfs/backlog#1600.
+      - name: Report cache input sizes
+        if: always()
+        run: |
+          {
+            echo "### Cache input sizes (ci-dev)"
+            echo '```'
+            du -sh ~/.cargo/registry/src ~/.cargo/registry/cache ~/.cargo/registry/index \
+                   ~/.cargo/git target 2>/dev/null || true
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
   # Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
   warm-ci-feat-rio:
     name: Warm ci-feat-rio

.github/workflows/ci-docs-only.yml

@@ -79,6 +79,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install ripgrep
         run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -125,6 +127,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Docs-only PRs skip the full code CI, but they are exactly where a
       # planning-type document could be slipped in (git add -f bypasses

.github/workflows/ci.yml

@@ -95,6 +95,8 @@ jobs:
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
@@ -112,6 +114,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install ripgrep
         run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -353,6 +357,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -391,6 +397,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -433,6 +441,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -461,6 +471,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -492,6 +504,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -532,6 +546,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -594,6 +610,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Full setup with dependency caching: the smoke-suite step below
       # compiles the e2e_test crate, which pulls in most of the workspace.
@@ -678,6 +696,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -721,6 +741,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Clean up previous test run
         run: |
@@ -775,6 +797,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download debug binary
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -847,6 +871,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download debug binary
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7

.github/workflows/cla.yml

@@ -22,11 +22,18 @@ on:
   issue_comment:
     types: [created, edited]
 
+# Least privilege at the top, widened per job below. This workflow runs on
+# pull_request_target and issue_comment, so it holds full secrets on every fork
+# PR and on any comment anyone writes — the one place in this repository where a
+# compromised action would be handed a repo-write token. It does not check out
+# or execute PR code, so there is no pwn-request path today, but the blast
+# radius should not depend on that staying true.
+#
+# contents: write in particular was never used: the signature records are
+# written to rustfs/cla through the scoped app token created below, and nothing
+# here writes to this repository's contents.
 permissions:
-  contents: write
-  pull-requests: write
-  issues: write
-  checks: write
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
@@ -36,6 +43,9 @@ jobs:
   cancel-closed-pr-runs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
+    # Echoes one line; the run exists only so the concurrency group cancels the
+    # in-flight run of a closed PR.
+    permissions: {}
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -44,6 +54,13 @@ jobs:
 
   cla:
     if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }}
+    # checks: write reports the merge-queue check run; pull-requests and issues
+    # let cla-bot comment and label. contents stays read — see the note above.
+    permissions:
+      contents: read
+      checks: write
+      issues: write
+      pull-requests: write
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:

.github/workflows/coverage.yml

@@ -62,6 +62,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -116,6 +118,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/docker.yml

@@ -98,6 +98,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           # For workflow_run events, checkout the specific commit that triggered the workflow
           ref: ${{ github.event.workflow_run.head_sha || github.sha }}
 
@@ -305,6 +306,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Login to Docker Hub
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3

.github/workflows/e2e-replication-nightly.yml

@@ -64,6 +64,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -124,6 +126,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/e2e-s3tests.yml

@@ -142,6 +142,8 @@ jobs:
       TEST_MODE: ${{ matrix.test-mode }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Provision Python explicitly rather than trusting the runner image to
       # ship a working pip (ci-1: a bare python3 without pip is what broke the
@@ -361,6 +363,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/fuzz.yml

@@ -87,6 +87,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -152,6 +154,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -207,6 +211,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -254,6 +260,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/helm-package.yml

@@ -50,6 +50,8 @@ jobs:
     steps:
       - name: Checkout helm chart repo
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Both inputs reach the shell through env rather than `${{ }}`
       # interpolation. A git ref name may contain `$(...)` — anything without a
@@ -121,6 +123,8 @@ jobs:
       - name: Checkout helm package repo
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          # persist-credentials-exempt: this checkout's token IS the push credential —
+          # the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
           repository: rustfs/helm
           token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
 

.github/workflows/minio-interop.yml

@@ -58,6 +58,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup

.github/workflows/mint.yml

@@ -125,6 +125,8 @@ jobs:
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Enable buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -270,6 +272,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/nix-flake-update.yml

@@ -40,6 +40,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        # persist-credentials-exempt: update-flake-lock pushes the branch and opens
+        # the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
+        # than reusing .git/config, but that is unverified — exempt until a
+        # workflow_dispatch run confirms it (rustfs/backlog#1602).
 
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3

.github/workflows/nix.yml

@@ -71,6 +71,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0

.github/workflows/performance-ab.yml

@@ -99,6 +99,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -148,6 +150,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           fetch-depth: 0 # baseline is built from origin/main
 
       - name: Setup Rust environment
@@ -394,6 +397,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/runner-hygiene.yml

@@ -0,0 +1,81 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Asserts that the self-hosted runners are still ephemeral — one job per pod.
+#
+# This repository is public and its pull_request jobs run on those runners,
+# executing the PR's own build.rs, proc-macros and tests. The only thing keeping
+# that code from reaching a later job is that each ARC pod handles exactly one
+# job and is then destroyed. That guarantee lives in the ARC scale-set
+# configuration, outside this repository, where it can be changed without any PR
+# — so it is asserted here from the outside, against real run data, instead of
+# being assumed.
+#
+# Monthly rather than per-PR: the property changes only when someone
+# reconfigures the scale set, and the check costs a few dozen API calls.
+# See docs/ci/runners.md and rustfs/backlog#1602.
+
+name: Runner Hygiene
+
+on:
+  schedule:
+    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: runner-hygiene
+  cancel-in-progress: false
+
+jobs:
+  check-ephemerality:
+    name: Check runner ephemerality
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      # Exit 2 (inconclusive / broken) is deliberately not a pass: a window
+      # where every sm-* job was still queued would otherwise look identical to
+      # a clean bill of health.
+      - name: Assert one job per self-hosted runner
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./scripts/ci/check_runner_ephemerality.sh 40
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [check-ephemerality]
+    # Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
+    # scheduled runs file a tracking issue, manual dispatch stays quiet so
+    # debugging never produces a spurious alert.
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/schedule-failure-alert-drill.yml

@@ -63,6 +63,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

Cargo.lock

@@ -290,11 +290,11 @@ dependencies = [
 
 [[package]]
 name = "ar_archive_writer"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
+checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
 dependencies = [
- "object 0.37.3",
+ "object 0.39.1",
 ]
 
 [[package]]
@@ -599,6 +599,16 @@ dependencies = [
  "syn 2.0.119",
 ]
 
+[[package]]
+name = "assert-json-diff"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "astral-tokio-tar"
 version = "0.6.4"
@@ -943,6 +953,32 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "aws-sdk-kms"
+version = "1.114.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
+dependencies = [
+ "arc-swap",
+ "aws-credential-types",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-observability",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand",
+ "http 0.2.12",
+ "http 1.5.0",
+ "regex-lite",
+ "tracing",
+]
+
 [[package]]
 name = "aws-sdk-s3"
 version = "1.140.0"
@@ -1158,17 +1194,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
 dependencies = [
  "aws-smithy-async",
+ "aws-smithy-protocol-test",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
+ "bytes",
  "h2",
  "http 1.5.0",
+ "http-body 1.1.0",
  "hyper",
  "hyper-rustls",
  "hyper-util",
+ "indexmap 2.14.0",
  "pin-project-lite",
  "rustls",
  "rustls-native-certs",
  "rustls-pki-types",
+ "serde",
+ "serde_json",
  "tokio",
  "tokio-rustls",
  "tower",
@@ -1195,6 +1237,25 @@ dependencies = [
  "aws-smithy-runtime-api",
 ]
 
+[[package]]
+name = "aws-smithy-protocol-test"
+version = "0.64.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a"
+dependencies = [
+ "assert-json-diff",
+ "aws-smithy-runtime-api",
+ "base64-simd",
+ "cbor-diag",
+ "ciborium",
+ "http 0.2.12",
+ "pretty_assertions",
+ "regex-lite",
+ "roxmltree",
+ "serde_json",
+ "thiserror 2.0.19",
+]
+
 [[package]]
 name = "aws-smithy-query"
 version = "0.62.0"
@@ -1528,7 +1589,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -1547,7 +1608,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -1598,7 +1659,7 @@ version = "3.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
 dependencies = [
- "darling 0.20.11",
+ "darling 0.23.0",
  "ident_case",
  "prettyplease",
  "proc-macro2",
@@ -1679,9 +1740,9 @@ dependencies = [
 
 [[package]]
 name = "bytesize"
-version = "2.4.2"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
+checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493"
 
 [[package]]
 name = "bytestring"
@@ -1758,6 +1819,25 @@ dependencies = [
  "cipher 0.5.2",
 ]
 
+[[package]]
+name = "cbor-diag"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429"
+dependencies = [
+ "bs58",
+ "chrono",
+ "data-encoding",
+ "half",
+ "nom 7.1.3",
+ "num-bigint",
+ "num-rational",
+ "num-traits",
+ "separator",
+ "url",
+ "uuid",
+]
+
 [[package]]
 name = "cc"
 version = "1.4.0"
@@ -1870,7 +1950,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
  "inout 0.1.4",
 ]
 
@@ -1888,9 +1968,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.4"
+version = "4.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
+checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1898,9 +1978,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.2"
+version = "4.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
+checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2328,7 +2408,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "rand_core 0.6.4",
  "subtle",
  "zeroize",
@@ -2353,11 +2433,11 @@ dependencies = [
 
 [[package]]
 name = "crypto-common"
-version = "0.1.7"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "typenum",
 ]
 
@@ -3554,7 +3634,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer 0.10.4",
  "const-oid 0.9.6",
- "crypto-common 0.1.7",
+ "crypto-common 0.1.6",
  "subtle",
 ]
 
@@ -3814,7 +3894,7 @@ dependencies = [
  "crypto-bigint 0.5.5",
  "digest 0.10.7",
  "ff 0.13.1",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "group 0.13.0",
  "hkdf 0.12.4",
  "pem-rfc7468 0.7.0",
@@ -4259,9 +4339,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.7"
+version = "0.14.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
 dependencies = [
  "typenum",
  "version_check",
@@ -4274,7 +4354,7 @@ version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
 dependencies = [
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "rustversion",
  "typenum",
 ]
@@ -5301,7 +5381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
  "block-padding 0.3.3",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
 ]
 
 [[package]]
@@ -5835,8 +5915,7 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.49"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
+source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
 dependencies = [
  "cc",
  "cty",
@@ -6245,8 +6324,7 @@ dependencies = [
 [[package]]
 name = "mimalloc"
 version = "0.1.52"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
+source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
 dependencies = [
  "libmimalloc-sys",
 ]
@@ -6668,6 +6746,17 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-rational"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -6732,7 +6821,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.22.1",
  "chrono",
  "getrandom 0.2.17",
  "http 1.5.0",
@@ -7857,7 +7946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
  "heck",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "log",
  "multimap",
  "once_cell",
@@ -7877,7 +7966,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
  "heck",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "log",
  "multimap",
  "petgraph 0.8.3",
@@ -7898,7 +7987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
 dependencies = [
  "anyhow",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
  "syn 2.0.119",
@@ -7911,7 +8000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
 dependencies = [
  "anyhow",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
  "syn 2.0.119",
@@ -8631,6 +8720,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "roxmltree"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
+dependencies = [
+ "xmlparser",
+]
+
 [[package]]
 name = "rsa"
 version = "0.9.10"
@@ -8724,9 +8822,9 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.62.4"
+version = "0.62.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
+checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
 dependencies = [
  "aes 0.9.2",
  "aws-lc-rs",
@@ -9511,10 +9609,16 @@ dependencies = [
  "arc-swap",
  "argon2",
  "async-trait",
+ "aws-config",
+ "aws-sdk-kms",
+ "aws-smithy-http-client",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
  "base64 0.23.0",
  "chacha20poly1305",
  "hex",
  "hotpath",
+ "http 1.5.0",
  "insta",
  "jiff",
  "md-5 0.11.0",
@@ -9523,6 +9627,7 @@ dependencies = [
  "moka",
  "rand 0.10.2",
  "reqwest",
+ "rustfs-s3-types",
  "rustfs-security-governance",
  "rustfs-utils",
  "rustify",
@@ -9708,6 +9813,7 @@ dependencies = [
  "hotpath",
  "jiff",
  "libc",
+ "log",
  "metrics",
  "num_cpus",
  "nvml-wrapper",
@@ -9743,6 +9849,7 @@ dependencies = [
  "tracing-error",
  "tracing-opentelemetry",
  "tracing-subscriber",
+ "url",
  "zstd",
 ]
 
@@ -9774,6 +9881,7 @@ dependencies = [
  "time",
  "tokio",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -10608,7 +10716,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
  "base16ct 0.2.0",
  "der 0.7.10",
- "generic-array 0.14.7",
+ "generic-array 0.14.9",
  "pkcs8 0.10.2",
  "subtle",
  "zeroize",
@@ -10661,6 +10769,12 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "separator"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5"
+
 [[package]]
 name = "seq-macro"
 version = "0.3.6"
@@ -11598,7 +11712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",

Cargo.toml

@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
 # Serialization and Data Formats
 apache-avro = "0.21.0"
 bytes = { version = "1.12.1" }
-bytesize = "2.4.2"
+bytesize = "2.6.0"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
@@ -227,6 +227,7 @@ atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.10.1" }
 aws-credential-types = { version = "1.3.0" }
+aws-sdk-kms = { default-features = false, version = "1.114.0" }
 aws-sdk-s3 = { default-features = false, version = "1.140.0" }
 aws-sdk-sts = { default-features = false, version = "1.110.0" }
 aws-smithy-http-client = { default-features = false, version = "1.2.0" }
@@ -235,7 +236,7 @@ aws-smithy-types = { version = "1.6.1" }
 base64 = "0.23.0"
 base64-simd = "0.8.0"
 brotli = "8.0.4"
-clap = { version = "4.6.4" }
+clap = { version = "4.6.5" }
 const-str = { version = "1.1.0" }
 convert_case = "0.11.0"
 criterion = { version = "0.8" }
@@ -340,14 +341,15 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.1" }
 rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.62.4" }
+russh = { version = "0.62.5" }
 russh-sftp = "2.3.0"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-mimalloc = "0.1.52"
+mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
+libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
 hotpath = { version = "0.22.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

crates/config/src/constants/app.rs

@@ -230,6 +230,19 @@ pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
 /// Default value: false
 pub const DEFAULT_KMS_ENABLE: bool = false;
 
+/// Environment variable enabling per-key KMS authorization on the SSE-KMS data path.
+///
+/// When enabled, an SSE-KMS write additionally requires `kms:GenerateDataKey` and an
+/// SSE-KMS read additionally requires `kms:Decrypt` on the resolved key, evaluated as
+/// the requesting identity. SSE-S3 and SSE-C are unaffected.
+pub const ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY: &str = "RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY";
+
+/// Default per-key KMS authorization mode for the SSE-KMS data path.
+///
+/// Off for now so deployments whose identity policies only grant s3 actions keep
+/// working; the default flips to on in a later release.
+pub const DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY: bool = false;
+
 /// Environment variable for server KMS backend.
 pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
 

crates/e2e_test/src/admin_iam_crud_test.rs

@@ -26,75 +26,16 @@
 //! Later batches tracked on backlog#1154: config get/set, info, pools status,
 //! group lifecycle, import/export IAM.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::{Client, Config};
-use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use serial_test::serial;
 use std::error::Error;
 use tokio::time::{Duration, sleep};
 
 type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-type BoxError = Box<dyn Error + Send + Sync>;
-
-/// Signs and sends an admin HTTP request with the given credential, returning
-/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
-/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
-async fn admin_request(
-    base_url: &str,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(StatusCode, String), BoxError> {
-    let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut builder = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        builder = builder.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
-    let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request = local_http_client().request(reqwest_method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
-    let status = response.status();
-    let text = response.text().await.unwrap_or_default();
-    Ok((status, text))
-}
-
-/// Root-credential admin request that must succeed; returns the response body.
-async fn admin_ok(
-    env: &RustFSTestEnvironment,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-) -> Result<String, BoxError> {
-    let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
-    if !status.is_success() {
-        return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
-    }
-    Ok(text)
-}
 
 fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
     let config = Config::builder()

crates/e2e_test/src/common.rs

@@ -24,7 +24,12 @@
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use http::header::{CONTENT_TYPE, HOST};
 use reqwest::Client as HttpClient;
+use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::path::{Path, PathBuf};
@@ -75,6 +80,58 @@ pub fn local_http_client() -> HttpClient {
         .expect("failed to build local reqwest client")
 }
 
+/// Signs and sends an admin HTTP request with the given credentials.
+pub(crate) async fn admin_request(
+    base_url: &str,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{base_url}{path_and_query}");
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let mut request = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if body.is_some() {
+        request = request.header(CONTENT_TYPE, "application/json");
+    }
+
+    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
+    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
+
+    let mut request = local_http_client().request(method, &url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    let response = request.send().await?;
+    let status = response.status();
+    let body = response.text().await?;
+    Ok((status, body))
+}
+
+/// Sends a root-credential admin request and returns its successful response body.
+pub(crate) async fn admin_ok(
+    env: &RustFSTestEnvironment,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    let (status, response_body) =
+        admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
+    if !status.is_success() {
+        return Err(format!("{method} {path_and_query} failed: {status} {response_body}").into());
+    }
+    Ok(response_body)
+}
+
 /// Resolve the RustFS binary relative to the workspace.
 pub fn rustfs_binary_path() -> PathBuf {
     rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())

crates/e2e_test/src/reliant/lock.rs

@@ -15,9 +15,7 @@
 
 use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
 use rustfs_lock::client::{LockClient, local::LocalClient};
-use rustfs_lock::{
-    GlobalLockManager, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey,
-};
+use rustfs_lock::{GlobalLockManager, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -35,7 +33,11 @@ struct FailingClient;
 #[async_trait::async_trait]
 impl rustfs_lock::LockClient for FailingClient {
     async fn acquire_lock(&self, _request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
-        Err(LockError::internal("simulated gRPC node failure"))
+        // Match RemoteClient's transport-failure response so the coordinator can count this node toward quorum loss.
+        Ok(LockResponse::failure(
+            "Remote lock RPC failed: simulated gRPC node failure",
+            Duration::ZERO,
+        ))
     }
 
     async fn release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {

crates/e2e_test/src/reliant/tiering.rs

@@ -382,7 +382,6 @@ struct ManualTransitionRunReport {
     skipped_delete_marker: u64,
     skipped_directory: u64,
     skipped_replication: u64,
-    skipped_already_transitioned: u64,
     skipped_already_in_flight: u64,
     skipped_queue_full: u64,
     skipped_queue_closed: u64,
@@ -408,41 +407,6 @@ fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionR
     }
 }
 
-fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) {
-    assert_completed_or_in_flight_partial(state, report, context);
-    if report.skipped_already_in_flight > 0 {
-        assert!(
-            report.scanned <= expected_objects,
-            "{context}: scanned more objects than the conflict scope contains: {report:#?}"
-        );
-        assert!(
-            report.eligible <= expected_objects,
-            "{context}: marked more objects eligible than the conflict scope contains: {report:#?}"
-        );
-        assert_eq!(
-            report.enqueued + report.skipped_already_in_flight,
-            report.eligible,
-            "{context}: partial in-flight accounting must cover every eligible object: {report:#?}"
-        );
-    } else {
-        assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}");
-        assert_eq!(
-            report.eligible + report.skipped_already_transitioned,
-            expected_objects,
-            "{context}: {report:#?}"
-        );
-        assert_eq!(
-            report.enqueued + report.skipped_already_in_flight,
-            expected_objects,
-            "{context}: {report:#?}"
-        );
-    }
-    assert_eq!(
-        report.transition_completed, report.enqueued,
-        "{context}: winner must wait for all queued transitions: {report:#?}"
-    );
-}
-
 #[derive(Debug, Deserialize)]
 struct ManualTransitionQueueSnapshot {
     queue_capacity: u64,
@@ -1276,8 +1240,15 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
-        .await?;
+    hot.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_SCANNER_ENABLED", "false"),
+            ("RUSTFS_SCANNER_CYCLE", "3600"),
+            (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
+        ],
+    )
+    .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1347,20 +1318,21 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     assert_eq!(conflict.cancel_endpoint, status_endpoint);
     assert!(!conflict.scope_key.is_empty());
 
+    manual_transition_job_cancel(&hot, cancel_endpoint).await?;
+
     let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
     assert_eq!(terminal.job_id, job_id);
+    assert_eq!(terminal.status, "cancelled", "terminal conflict winner response: {terminal:#?}");
     assert!(!terminal.report.dry_run);
     assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
     assert_eq!(terminal.report.prefix, accepted.report.prefix);
-    assert_conflict_winner_report(
-        &terminal.status,
-        &terminal.report,
-        MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
-        "terminal conflict winner response",
+    assert!(terminal.report.cancelled, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(terminal.report.scanned, 0, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(terminal.report.enqueued, 0, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(
+        terminal.report.transition_completed, 0,
+        "terminal conflict winner response: {terminal:#?}"
     );
-    assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}");
     let after_remote_count = cold_tier_object_count(&cold_client).await?;
     assert!(after_remote_count >= before_remote_count);
     assert!(after_remote_count <= before_remote_count + MANUAL_ASYNC_CONFLICT_OBJECTS);

crates/e2e_test/src/sts_query_compat_test.rs

@@ -12,22 +12,35 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
 use aws_sdk_sts::config::retry::RetryConfig;
 use aws_sdk_sts::config::{Credentials, Region};
 use aws_sdk_sts::error::ProvideErrorMetadata;
 use aws_sdk_sts::operation::RequestId;
 use aws_sdk_sts::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use http::header::{CONTENT_TYPE, HOST};
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
+use bytes::Bytes;
+use http::header::{AUTHORIZATION, CONTENT_TYPE};
+use http::{Request, Response};
+use http_body_util::{BodyExt, Full};
+use hyper::body::Incoming;
+use hyper::server::conn::http1;
+use hyper::service::service_fn;
+use hyper_util::rt::TokioIo;
+use serde_json::Value;
 use serial_test::serial;
+use std::collections::BTreeSet;
+use std::convert::Infallible;
 use std::error::Error;
+use std::sync::Arc;
+use tokio::net::TcpListener;
+use tokio::sync::{Notify, mpsc};
+use tokio::task::{JoinHandle, JoinSet};
+use tokio::time::{Duration, timeout};
 
 type BoxError = Box<dyn Error + Send + Sync>;
 type TestResult = Result<(), BoxError>;
+const OPA_AUTH_TOKEN: &str = "sts-opa-token";
 
 fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
     let mut config = Config::builder()
@@ -49,32 +62,14 @@ fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Opti
 }
 
 async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
-    let path = "/rustfs/admin/v3/add-service-accounts";
-    let url = format!("{}{path}", env.url);
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
-    let request = http::Request::builder()
-        .method(http::Method::PUT)
-        .uri(uri)
-        .header(HOST, authority)
-        .header(CONTENT_TYPE, "application/json")
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
-        .body(Body::empty())?;
-    let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
-    let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
-    let mut request = local_http_client().put(&url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    let response = request.body(body).send().await?;
-    let status = response.status();
-    let body = response.text().await?;
-    if !status.is_success() {
-        return Err(format!("create service account failed: {status} {body}").into());
-    }
-
-    let response: serde_json::Value = serde_json::from_str(&body)?;
+    let body = admin_ok(
+        env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-service-accounts",
+        Some(serde_json::json!({ "targetUser": env.access_key.clone() }).to_string()),
+    )
+    .await?;
+    let response: Value = serde_json::from_str(&body)?;
     let access_key = response["credentials"]["accessKey"]
         .as_str()
         .ok_or("service account response should contain credentials.accessKey")?
@@ -86,28 +81,237 @@ async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(Str
     Ok((access_key, secret_key))
 }
 
-async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
+async fn create_user_with_policy(
+    env: &RustFSTestEnvironment,
+    user: &str,
+    secret: &str,
+    policy_name: &str,
+    statements: Value,
+) -> TestResult {
+    create_user(env, user, secret).await?;
+    admin_ok(
+        env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(
+            serde_json::json!({
+                "Version": "2012-10-17",
+                "Statement": statements,
+            })
+            .to_string(),
+        ),
+    )
+    .await?;
+    admin_ok(
+        env,
+        http::Method::POST,
+        "/rustfs/admin/v3/idp/builtin/policy/attach",
+        Some(serde_json::json!({ "policies": [policy_name], "user": user }).to_string()),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn create_user(env: &RustFSTestEnvironment, user: &str, secret: &str) -> TestResult {
+    admin_ok(
+        env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+        Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
     let error = client
         .assume_role()
         .role_arn("arn:aws:iam::123456789012:role/test")
         .role_session_name("sts-query-compat-e2e")
         .send()
         .await
-        .expect_err("credential chaining must be denied");
+        .expect_err("AssumeRole must be denied");
     let service_error = error
         .as_service_error()
-        .ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
+        .ok_or_else(|| format!("{context} should deserialize as an STS service error: {error:?}"))?;
 
     assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
     assert_eq!(service_error.code(), Some("AccessDenied"));
     assert_eq!(service_error.message(), Some("Access Denied"));
     assert!(
         error.request_id().is_some_and(|request_id| !request_id.is_empty()),
-        "{credential_kind} denial should include a request ID"
+        "{context} should include a request ID"
     );
     Ok(())
 }
 
+async fn handle_opa_request(
+    request: Request<Incoming>,
+    requests: mpsc::UnboundedSender<Value>,
+    validation_started: mpsc::UnboundedSender<()>,
+    validation_mode: OpaValidationMode,
+    expected_authorization: Option<String>,
+) -> Result<Response<Full<Bytes>>, Infallible> {
+    if let Some(expected_authorization) = expected_authorization
+        && request.headers().get(AUTHORIZATION).and_then(|value| value.to_str().ok()) != Some(expected_authorization.as_str())
+    {
+        return Ok(Response::builder()
+            .status(401)
+            .body(Full::new(Bytes::new()))
+            .expect("static OPA unauthorized response must be valid"));
+    }
+
+    let body = match request.into_body().collect().await {
+        Ok(body) => body.to_bytes(),
+        Err(error) => {
+            return Ok(Response::builder()
+                .status(400)
+                .body(Full::new(Bytes::from(error.to_string())))
+                .expect("static OPA error response must be valid"));
+        }
+    };
+
+    let payload = if body.is_empty() {
+        None
+    } else {
+        match serde_json::from_slice::<Value>(&body) {
+            Ok(payload) => Some(payload),
+            Err(error) => {
+                return Ok(Response::builder()
+                    .status(400)
+                    .body(Full::new(Bytes::from(error.to_string())))
+                    .expect("static OPA error response must be valid"));
+            }
+        }
+    };
+    if payload.is_none() {
+        let _ = validation_started.send(());
+        if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
+            release.notified().await;
+            return Ok(Response::builder()
+                .status(503)
+                .body(Full::new(Bytes::new()))
+                .expect("static OPA unavailable response must be valid"));
+        }
+    }
+    let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
+        Some(Value::String(account)) if account == "opaallow" => payload
+            .as_ref()
+            .and_then(|value| value.pointer("/input/context/deny_only"))
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+        Some(Value::String(account)) if account == "opadeny" => false,
+        None => true,
+        _ => false,
+    };
+    if let Some(payload) = payload {
+        let _ = requests.send(payload);
+    }
+    let body =
+        serde_json::to_vec(&serde_json::json!({ "result": { "allow": allow } })).expect("static OPA response must serialize");
+    Ok(Response::builder()
+        .header(CONTENT_TYPE, "application/json")
+        .body(Full::new(Bytes::from(body)))
+        .expect("static OPA response must be valid"))
+}
+
+#[derive(Clone)]
+enum OpaValidationMode {
+    Ready,
+    DelayedUnavailable(Arc<Notify>),
+}
+
+struct OpaMock {
+    url: String,
+    requests: mpsc::UnboundedReceiver<Value>,
+    validation_started: mpsc::UnboundedReceiver<()>,
+    validation_release: Option<Arc<Notify>>,
+    task: JoinHandle<()>,
+}
+
+impl OpaMock {
+    async fn start() -> Result<Self, BoxError> {
+        Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
+    }
+
+    async fn start_delayed_unavailable() -> Result<Self, BoxError> {
+        let release = Arc::new(Notify::new());
+        Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
+    }
+
+    async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let url = format!("http://{}/v1/data/rustfs/authz/allow", listener.local_addr()?);
+        let (requests_tx, requests) = mpsc::unbounded_channel();
+        let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
+        let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
+        let validation_release = match &validation_mode {
+            OpaValidationMode::Ready => None,
+            OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
+        };
+        let task = tokio::spawn(async move {
+            let mut connections = JoinSet::new();
+            loop {
+                tokio::select! {
+                    accepted = listener.accept() => {
+                        let Ok((stream, _)) = accepted else { break };
+                        let requests = requests_tx.clone();
+                        let validation_started = validation_started_tx.clone();
+                        let validation_mode = validation_mode.clone();
+                        let expected_authorization = expected_authorization.clone();
+                        connections.spawn(async move {
+                            let handler = service_fn(move |request| {
+                                handle_opa_request(
+                                    request,
+                                    requests.clone(),
+                                    validation_started.clone(),
+                                    validation_mode.clone(),
+                                    expected_authorization.clone(),
+                                )
+                            });
+                            let _ = http1::Builder::new()
+                                .serve_connection(TokioIo::new(stream), handler)
+                                .await;
+                        });
+                    }
+                    _ = connections.join_next(), if !connections.is_empty() => {}
+                }
+            }
+        });
+        Ok(Self {
+            url,
+            requests,
+            validation_started,
+            validation_release,
+            task,
+        })
+    }
+
+    async fn next_request(&mut self) -> Result<Value, BoxError> {
+        timeout(Duration::from_secs(5), self.requests.recv())
+            .await?
+            .ok_or_else(|| "OPA request channel closed".into())
+    }
+
+    async fn wait_for_validation(&mut self) -> TestResult {
+        timeout(Duration::from_secs(5), self.validation_started.recv())
+            .await?
+            .ok_or_else(|| "OPA validation channel closed".into())
+    }
+
+    fn release_validation(&self) {
+        if let Some(release) = &self.validation_release {
+            release.notify_one();
+        }
+    }
+}
+
+impl Drop for OpaMock {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+
 #[tokio::test]
 #[serial]
 async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
@@ -155,7 +359,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
         "signature rejection should include a request ID"
     );
 
-    assert_chaining_denied(
+    assert_access_denied(
         &sts_client(
             &env.url,
             temporary.access_key_id(),
@@ -167,7 +371,187 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
     .await?;
 
     let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
-    assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
+    assert_access_denied(
+        &sts_client(&env.url, &service_access_key, &service_secret_key, None),
+        "service-account denial",
+    )
+    .await?;
+
+    let implicit_user = "stsimplicit";
+    let explicit_allow_user = "stsallow";
+    let explicit_deny_user = "stsdeny";
+    let policyless_user = "stspolicyless";
+    let secret = "stsAuthzSecret123";
+    create_user(&env, policyless_user, secret).await?;
+    assert_access_denied(&sts_client(&env.url, policyless_user, secret, None), "policyless user").await?;
+    create_user_with_policy(
+        &env,
+        implicit_user,
+        secret,
+        "sts-implicit-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+    create_user_with_policy(
+        &env,
+        explicit_allow_user,
+        secret,
+        "sts-allow-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["sts:AssumeRole"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+    create_user_with_policy(
+        &env,
+        explicit_deny_user,
+        secret,
+        "sts-deny-policy",
+        serde_json::json!([
+            {
+                "Effect": "Allow",
+                "Action": ["sts:AssumeRole"],
+                "Resource": ["arn:aws:s3:::*"],
+            },
+            {
+                "Effect": "Deny",
+                "Action": ["sts:AssumeRole"],
+                "Resource": ["arn:aws:s3:::*"],
+            }
+        ]),
+    )
+    .await?;
+
+    for user in [implicit_user, explicit_allow_user] {
+        let output = sts_client(&env.url, user, secret, None)
+            .assume_role()
+            .role_arn("arn:aws:iam::123456789012:role/test")
+            .role_session_name("sts-authz-e2e")
+            .send()
+            .await
+            .map_err(|error| format!("{user} should be allowed to call AssumeRole: {error:?}"))?;
+        let credentials = output
+            .credentials()
+            .ok_or_else(|| format!("{user} AssumeRole response should contain credentials"))?;
+        assert!(!credentials.access_key_id().is_empty());
+        assert!(!credentials.secret_access_key().is_empty());
+        assert!(!credentials.session_token().is_empty());
+    }
+    assert_access_denied(&sts_client(&env.url, explicit_deny_user, secret, None), "explicit sts:AssumeRole Deny").await?;
+
+    env.stop_server();
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_assume_role_opa_contract() -> TestResult {
+    init_logging();
+
+    let mut opa = OpaMock::start().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+            ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
+        ],
+    )
+    .await?;
+
+    let secret = "stsOpaSecret123";
+    create_user_with_policy(
+        &env,
+        "opaallow",
+        secret,
+        "sts-opa-local-deny-policy",
+        serde_json::json!([{
+            "Effect": "Deny",
+            "Action": ["sts:AssumeRole"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+    create_user_with_policy(
+        &env,
+        "opadeny",
+        secret,
+        "sts-opa-local-allow-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["sts:AssumeRole"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+
+    sts_client(&env.url, "opaallow", secret, None)
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/test")
+        .role_session_name("sts-opa-contract")
+        .send()
+        .await
+        .map_err(|error| format!("OPA allow should override the local explicit Deny: {error:?}"))?;
+    assert_access_denied(
+        &sts_client(&env.url, "opadeny", secret, None),
+        "OPA denial despite local sts:AssumeRole Allow",
+    )
+    .await?;
+
+    let mut accounts = BTreeSet::new();
+    for _ in 0..2 {
+        let request = opa.next_request().await?;
+        assert_eq!(request.pointer("/input/action").and_then(Value::as_str), Some("sts:AssumeRole"));
+        assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(true));
+        let account = request
+            .pointer("/input/identity/account")
+            .and_then(Value::as_str)
+            .ok_or("OPA input should include identity.account")?;
+        accounts.insert(account.to_owned());
+    }
+    assert_eq!(accounts, BTreeSet::from(["opaallow".to_owned(), "opadeny".to_owned()]));
+
+    env.stop_server();
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
+    init_logging();
+
+    let mut opa = OpaMock::start_delayed_unavailable().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
+        .await?;
+    opa.wait_for_validation().await?;
+
+    let user = "opaunavailable";
+    let secret = "stsOpaUnavailableSecret123";
+    create_user_with_policy(
+        &env,
+        user,
+        secret,
+        "sts-opa-unavailable-local-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+
+    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
+
+    opa.release_validation();
+    tokio::time::sleep(Duration::from_millis(200)).await;
+    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
 
     env.stop_server();
     Ok(())

crates/ecstore/src/api/mod.rs

@@ -172,6 +172,9 @@ pub mod bucket {
     }
 
     pub mod replication {
+        pub use crate::bucket::replication::replication_pool::{
+            DurableMrfBacklogSummary, DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot,
+        };
         pub use crate::bucket::replication::{
             BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
             MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
@@ -419,15 +422,15 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
-        PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
-        ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
-        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
-        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
-        verify_tonic_rpc_signature_with_bootstrap,
+        AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
+        PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
+        ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
+        gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
+        node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
+        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
+        tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
+        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
+        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/bucket/replication/replication_pool.rs

@@ -46,7 +46,11 @@ use super::replication_target_boundary::{ReplicationTargetStore, replication_obj
 use super::replication_versioning_boundary::ReplicationVersioningStore;
 use super::runtime_boundary as runtime_sources;
 use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
+use std::collections::HashMap;
+use std::collections::hash_map::Entry;
 use std::sync::Arc;
+use std::sync::LazyLock;
+use std::sync::RwLock as StdRwLock;
 use std::sync::atomic::AtomicI32;
 use std::sync::atomic::Ordering;
 use time::OffsetDateTime;
@@ -74,6 +78,66 @@ pub struct DurableMrfBacklog {
     pub entries: Vec<MrfReplicateEntry>,
 }
 
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct DurableMrfBucketBacklog {
+    pub bucket: String,
+    pub count: u64,
+    pub bytes: u64,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct DurableMrfBacklogSummary {
+    pub available: bool,
+    pub buckets: Vec<DurableMrfBucketBacklog>,
+}
+
+static DURABLE_MRF_BACKLOG_SUMMARY: LazyLock<StdRwLock<DurableMrfBacklogSummary>> =
+    LazyLock::new(|| StdRwLock::new(DurableMrfBacklogSummary::default()));
+
+fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSummary
+where
+    I: IntoIterator<Item = (String, i64)>,
+{
+    let mut buckets = HashMap::<String, DurableMrfBucketBacklog>::new();
+    for (bucket_name, entry_size) in entries {
+        let Ok(size) = u64::try_from(entry_size) else {
+            return DurableMrfBacklogSummary::default();
+        };
+
+        let bucket = match buckets.entry(bucket_name) {
+            Entry::Occupied(entry) => entry.into_mut(),
+            Entry::Vacant(entry) => {
+                let bucket = entry.key().clone();
+                entry.insert(DurableMrfBucketBacklog {
+                    bucket,
+                    ..Default::default()
+                })
+            }
+        };
+        bucket.count = bucket.count.saturating_add(1);
+        bucket.bytes = bucket.bytes.saturating_add(size);
+    }
+
+    DurableMrfBacklogSummary {
+        available: true,
+        buckets: buckets.into_values().collect(),
+    }
+}
+
+fn set_durable_mrf_backlog_summary(summary: DurableMrfBacklogSummary) {
+    match DURABLE_MRF_BACKLOG_SUMMARY.write() {
+        Ok(mut guard) => *guard = summary,
+        Err(poisoned) => *poisoned.into_inner() = summary,
+    }
+}
+
+pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
+    match DURABLE_MRF_BACKLOG_SUMMARY.read() {
+        Ok(guard) => guard.clone(),
+        Err(poisoned) => poisoned.into_inner().clone(),
+    }
+}
+
 fn durable_mrf_backlog_from_read(result: Result<Vec<u8>, EcstoreError>) -> DurableMrfBacklog {
     match result {
         Ok(data) => match decode_mrf_file(&data) {
@@ -233,6 +297,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
             let active_counter = self.active_lrg_workers.clone();
             let storage = self.storage.clone();
+            let stats = self.stats.clone();
 
             let handle = tokio::spawn(async move {
                 let mut rx = rx;
@@ -241,10 +306,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
                     match operation {
                         ReplicationOperation::Object(obj_info) => {
+                            let bucket = obj_info.bucket.clone();
+                            let size = obj_info.size;
+                            let delete_marker = obj_info.delete_marker;
+                            let op_type = obj_info.op_type;
                             replicate_object(*obj_info, storage.clone()).await;
+                            stats.dec_q(&bucket, size, delete_marker, op_type);
                         }
                         ReplicationOperation::Delete(del_info) => {
+                            let bucket = del_info.bucket.clone();
+                            let op_type = del_info.op_type;
                             replicate_delete(*del_info, storage.clone()).await;
+                            stats.dec_q(&bucket, 0, true, op_type);
                         }
                     }
 
@@ -310,23 +383,22 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
                     match operation {
                         ReplicationOperation::Object(obj_info) => {
-                            stats
-                                .inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                                .await;
-
+                            let bucket = obj_info.bucket.clone();
+                            let size = obj_info.size;
+                            let delete_marker = obj_info.delete_marker;
+                            let op_type = obj_info.op_type;
                             // Perform actual replication (placeholder)
-                            replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
+                            replicate_object(*obj_info, storage.clone()).await;
 
-                            stats
-                                .dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                                .await;
+                            stats.dec_q(&bucket, size, delete_marker, op_type);
                         }
                         ReplicationOperation::Delete(del_info) => {
-                            stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
+                            let bucket = del_info.bucket.clone();
+                            let op_type = del_info.op_type;
                             // Perform actual delete replication (placeholder)
-                            replicate_delete(del_info.as_ref().clone(), storage.clone()).await;
+                            replicate_delete(*del_info, storage.clone()).await;
 
-                            stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
+                            stats.dec_q(&bucket, 0, true, op_type);
                         }
                     }
 
@@ -379,16 +451,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     active_counter.fetch_add(1, Ordering::SeqCst);
                     match operation {
                         ReplicationOperation::Object(obj_info) => {
-                            stats
-                                .inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                                .await;
-                            replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
-                            stats
-                                .dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                                .await;
+                            let bucket = obj_info.bucket.clone();
+                            let size = obj_info.size;
+                            let delete_marker = obj_info.delete_marker;
+                            let op_type = obj_info.op_type;
+                            replicate_object(*obj_info, storage.clone()).await;
+                            stats.dec_q(&bucket, size, delete_marker, op_type);
                         }
                         ReplicationOperation::Delete(del_info) => {
+                            let bucket = del_info.bucket.clone();
+                            let op_type = del_info.op_type;
                             replicate_delete(*del_info, storage.clone()).await;
+                            stats.dec_q(&bucket, 0, true, op_type);
                         }
                     }
                     active_counter.fetch_sub(1, Ordering::SeqCst);
@@ -529,9 +603,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
             if !lrg_workers.is_empty() {
                 let index = (hash as usize) % lrg_workers.len();
 
-                if let Some(worker) = lrg_workers.get(index)
-                    && worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
-                {
+                if let Some(worker) = lrg_workers.get(index) {
+                    self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
+                    if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
+                        return ReplicationQueueAdmission::Queued;
+                    }
+                    self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
+
                     // Try to add more workers if possible
                     let max_l_workers = *self.max_l_workers.read().await;
                     let existing = lrg_workers.len();
@@ -539,17 +617,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     drop(lrg_workers);
 
                     // Queue to MRF if worker is busy.
-                    let admission =
-                        queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "large_object")
-                            .await;
+                    let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "large_object").await;
 
                     if let Some(resize) = resize {
                         self.resize_lrg_workers(resize.new_count, resize.existing_count).await;
                     }
                     return admission;
                 }
-
-                return ReplicationQueueAdmission::Queued;
             }
             return ReplicationQueueAdmission::Missed;
         }
@@ -562,12 +636,14 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
             return ReplicationQueueAdmission::Missed;
         };
 
+        self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
         if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
             return ReplicationQueueAdmission::Queued;
         }
+        self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
 
         // Queue to MRF if all workers are busy.
-        let admission = queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "object").await;
+        let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
 
         // Try to scale up workers based on priority
         self.apply_queue_backpressure("object", true, "Replication queue is backpressured")
@@ -586,18 +662,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
             return ReplicationQueueAdmission::Missed;
         };
 
+        self.stats.inc_q(&doi.bucket, 0, true, doi.op_type);
         if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
             return ReplicationQueueAdmission::Queued;
         }
+        self.stats.dec_q(&doi.bucket, 0, true, doi.op_type);
 
-        let admission = queue_mrf_save_admission(
-            &self.mrf_save_tx,
-            doi.to_mrf_entry(),
-            &doi.bucket,
-            &doi.delete_object.object_name,
-            "delete",
-        )
-        .await;
+        let admission = self.queue_mrf_save_admission(doi.to_mrf_entry(), "delete").await;
 
         self.apply_queue_backpressure("delete", false, "Replication delete queue is backpressured")
             .await;
@@ -607,7 +678,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
     /// Queues an MRF save operation
     async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
-        let _ = queue_mrf_save_admission(&self.mrf_save_tx, entry, "", "", "mrf_worker").await;
+        let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
+    }
+
+    async fn queue_mrf_save_admission(&self, entry: MrfReplicateEntry, queue_type: &'static str) -> ReplicationQueueAdmission {
+        let bucket = entry.bucket.clone();
+        let size = entry.size;
+        let is_delete = matches!(entry.op, MrfOpKind::Delete);
+        let admission = queue_mrf_save_entry(&self.mrf_save_tx, entry, queue_type).await;
+        if admission == ReplicationQueueAdmission::Queued {
+            self.stats.inc_q(&bucket, size, is_delete, ReplicationType::Heal);
+        }
+        admission
     }
 
     /// Starts the MRF processor — one-shot at startup.
@@ -622,7 +704,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
         let handle = tokio::spawn(async move {
             let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
                 Ok(d) => d,
-                Err(EcstoreError::ConfigNotFound) => return, // no file yet — normal on first start
+                Err(EcstoreError::ConfigNotFound) => {
+                    set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
+                        available: true,
+                        buckets: Vec::new(),
+                    });
+                    return;
+                }
                 Err(e) => {
                     warn!(
                         component = LOG_COMPONENT_ECSTORE,
@@ -653,6 +741,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     return;
                 }
             };
+            set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
+                entries.iter().map(|entry| (entry.bucket.clone(), entry.size)),
+            ));
 
             let total = entries.len();
             let mut queued_count = 0usize;
@@ -765,6 +856,11 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     error = %e,
                     "Failed to clear MRF recovery file after replay — entries may be replayed again on next restart"
                 );
+            } else {
+                set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
+                    available: true,
+                    buckets: Vec::new(),
+                });
             }
 
             if queued_count > 0 {
@@ -793,6 +889,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
             return;
         };
         let storage = self.storage.clone();
+        let stats = self.stats.clone();
 
         let handle = tokio::spawn(async move {
             // The on-disk MRF file is a restart-recovery backstop: entries are
@@ -818,6 +915,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                     entry = rx.recv() => match entry {
                         Some(e) => {
                             if pending.len() >= MRF_PENDING_CAP {
+                                dec_mrf_entries(stats.as_ref(), std::slice::from_ref(&e));
                                 if !capped {
                                     capped = true;
                                     warn!(
@@ -836,20 +934,31 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
                             // set, not the absolute length, so a large backlog is
                             // not rewritten on every single add).
                             if pending.len() - flushed_len >= 1000 && flush_mrf_to_disk(&pending, &storage).await {
+                                set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
+                                    pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
+                                ));
+                                dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
                                 flushed_len = pending.len();
                                 dirty = false;
                             }
                         }
                         None => {
                             // Channel closed (pool shutting down) — final flush.
-                            if dirty {
-                                flush_mrf_to_disk(&pending, &storage).await;
+                            if dirty && flush_mrf_to_disk(&pending, &storage).await {
+                                set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
+                                    pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
+                                ));
+                                dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
                             }
                             break;
                         }
                     },
                     _ = interval.tick() => {
                         if dirty && flush_mrf_to_disk(&pending, &storage).await {
+                            set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
+                                pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
+                            ));
+                            dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
                             flushed_len = pending.len();
                             dirty = false;
                         }
@@ -872,24 +981,22 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
             match operation {
                 ReplicationOperation::Object(obj_info) => {
-                    stats
-                        .inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                        .await;
-
+                    let bucket = obj_info.bucket.clone();
+                    let size = obj_info.size;
+                    let delete_marker = obj_info.delete_marker;
+                    let op_type = obj_info.op_type;
                     // Perform actual replication (placeholder)
-                    replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
+                    replicate_object(*obj_info, self.storage.clone()).await;
 
-                    stats
-                        .dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                        .await;
+                    stats.dec_q(&bucket, size, delete_marker, op_type);
                 }
                 ReplicationOperation::Delete(del_info) => {
-                    stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
-
+                    let bucket = del_info.bucket.clone();
+                    let op_type = del_info.op_type;
                     // Perform actual delete replication (placeholder)
-                    replicate_delete(del_info.as_ref().clone(), self.storage.clone()).await;
+                    replicate_delete(*del_info, self.storage.clone()).await;
 
-                    stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
+                    stats.dec_q(&bucket, 0, true, op_type);
                 }
             }
 
@@ -898,16 +1005,30 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     }
 
     /// Worker function for handling large object replication operations
-    async fn add_large_worker(&self, mut rx: Receiver<ReplicationOperation>, active_counter: Arc<AtomicI32>, storage: Arc<S>) {
+    async fn add_large_worker(
+        &self,
+        mut rx: Receiver<ReplicationOperation>,
+        active_counter: Arc<AtomicI32>,
+        stats: Arc<ReplicationStats>,
+        storage: Arc<S>,
+    ) {
         while let Some(operation) = rx.recv().await {
             active_counter.fetch_add(1, Ordering::SeqCst);
 
             match operation {
                 ReplicationOperation::Object(obj_info) => {
+                    let bucket = obj_info.bucket.clone();
+                    let size = obj_info.size;
+                    let delete_marker = obj_info.delete_marker;
+                    let op_type = obj_info.op_type;
                     replicate_object(*obj_info, storage.clone()).await;
+                    stats.dec_q(&bucket, size, delete_marker, op_type);
                 }
                 ReplicationOperation::Delete(del_info) => {
+                    let bucket = del_info.bucket.clone();
+                    let op_type = del_info.op_type;
                     replicate_delete(*del_info, storage.clone()).await;
+                    stats.dec_q(&bucket, 0, true, op_type);
                 }
             }
 
@@ -927,18 +1048,19 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
 
             match operation {
                 ReplicationOperation::Object(obj_info) => {
-                    stats
-                        .inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                        .await;
-
+                    let bucket = obj_info.bucket.clone();
+                    let size = obj_info.size;
+                    let delete_marker = obj_info.delete_marker;
+                    let op_type = obj_info.op_type;
                     replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
 
-                    stats
-                        .dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
-                        .await;
+                    stats.dec_q(&bucket, size, delete_marker, op_type);
                 }
                 ReplicationOperation::Delete(del_info) => {
+                    let bucket = del_info.bucket.clone();
+                    let op_type = del_info.op_type;
                     replicate_delete(*del_info, self.storage.clone()).await;
+                    stats.dec_q(&bucket, 0, true, op_type);
                 }
             }
 
@@ -1273,29 +1395,34 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
     }
 }
 
-async fn queue_mrf_save_admission(
+async fn queue_mrf_save_entry(
     tx: &Sender<MrfReplicateEntry>,
     entry: MrfReplicateEntry,
-    bucket: &str,
-    object: &str,
     queue_type: &'static str,
 ) -> ReplicationQueueAdmission {
-    if tx.send(entry).await.is_ok() {
+    let Err(error) = tx.send(entry).await else {
         return ReplicationQueueAdmission::Queued;
-    }
+    };
+    let entry = error.0;
 
     warn!(
         event = EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE,
         component = LOG_COMPONENT_ECSTORE,
         subsystem = LOG_SUBSYSTEM_REPLICATION,
-        bucket = %bucket,
-        object = %object,
+        bucket = %entry.bucket,
+        object = %entry.object,
         queue_type = queue_type,
         "MRF save channel unavailable — replication failure entry could not be persisted for retry"
     );
     ReplicationQueueAdmission::Missed
 }
 
+fn dec_mrf_entries(stats: &ReplicationStats, entries: &[MrfReplicateEntry]) {
+    for entry in entries {
+        stats.dec_q(&entry.bucket, entry.size, matches!(entry.op, MrfOpKind::Delete), ReplicationType::Heal);
+    }
+}
+
 /// Encodes `entries` and overwrites the MRF persistence file.
 /// Returns `true` on success; on failure logs the error and returns `false`.
 /// Callers must NOT clear their in-memory buffer on `false` so the next tick
@@ -2016,6 +2143,147 @@ mod tests {
         })
     }
 
+    async fn current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str) -> (i64, i64) {
+        let stats = pool.stats.get_latest_replication_stats(bucket).await;
+        (stats.replication_stats.q_stat.curr.count, stats.replication_stats.q_stat.curr.bytes)
+    }
+
+    async fn wait_for_current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, expected: (i64, i64)) {
+        tokio::time::timeout(Duration::from_secs(10), async {
+            loop {
+                if current_queue(pool, bucket).await == expected {
+                    break;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("replication queue should reach the expected state");
+    }
+
+    #[tokio::test]
+    async fn regular_worker_admission_counts_channel_backlog_before_receive() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, _rx) = mpsc::channel(1);
+        pool.workers.write().await.push(tx);
+
+        let admission = pool
+            .queue_replica_task(ReplicateObjectInfo {
+                bucket: "admission-bucket".to_string(),
+                name: "object".to_string(),
+                size: 4096,
+                op_type: ReplicationType::Object,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
+    }
+
+    #[tokio::test]
+    async fn large_worker_admission_counts_channel_backlog_before_receive() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, _rx) = mpsc::channel(1);
+        pool.lrg_workers.write().await.push(tx);
+        let size = 128 * 1024 * 1024;
+
+        let admission = pool
+            .queue_replica_task(ReplicateObjectInfo {
+                bucket: "large-admission-bucket".to_string(),
+                name: "large-object".to_string(),
+                size,
+                op_type: ReplicationType::Object,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        assert_eq!(current_queue(&pool, "large-admission-bucket").await, (1, size));
+    }
+
+    #[tokio::test]
+    async fn delete_admission_counts_channel_backlog_before_receive() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        let (tx, _rx) = mpsc::channel(1);
+        pool.workers.write().await.push(tx);
+
+        let admission = pool
+            .queue_replica_delete_task(DeletedObjectReplicationInfo {
+                bucket: "delete-admission-bucket".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "deleted-object".to_string(),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        assert_eq!(current_queue(&pool, "delete-admission-bucket").await, (1, 0));
+    }
+
+    #[tokio::test]
+    async fn regular_worker_drains_current_backlog_after_processing() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        pool.resize_workers(1, 0).await;
+
+        let admission = pool
+            .queue_replica_task(ReplicateObjectInfo {
+                bucket: "regular-drain-bucket".to_string(),
+                name: "object".to_string(),
+                size: 4096,
+                op_type: ReplicationType::Object,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        wait_for_current_queue(&pool, "regular-drain-bucket", (0, 0)).await;
+    }
+
+    #[tokio::test]
+    async fn large_worker_drains_current_backlog_after_processing() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        pool.resize_lrg_workers(1, 0).await;
+        let size = 128 * 1024 * 1024;
+
+        let admission = pool
+            .queue_replica_task(ReplicateObjectInfo {
+                bucket: "large-drain-bucket".to_string(),
+                name: "large-object".to_string(),
+                size,
+                op_type: ReplicationType::Object,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        wait_for_current_queue(&pool, "large-drain-bucket", (0, 0)).await;
+    }
+
+    #[tokio::test]
+    async fn regular_delete_worker_drains_current_backlog_after_processing() {
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
+        pool.resize_workers(1, 0).await;
+
+        let admission = pool
+            .queue_replica_delete_task(DeletedObjectReplicationInfo {
+                bucket: "delete-drain-bucket".to_string(),
+                delete_object: ReplicationDeletedObject {
+                    object_name: "deleted-object".to_string(),
+                    ..Default::default()
+                },
+                op_type: ReplicationType::Delete,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        wait_for_current_queue(&pool, "delete-drain-bucket", (0, 0)).await;
+    }
+
     fn load_resync_test_metadata() -> Vec<u8> {
         let mut status = BucketReplicationResyncStatus::new();
         status.targets_map.insert(
@@ -2301,6 +2569,37 @@ mod tests {
         assert_eq!(admission, ReplicationQueueAdmission::Missed);
     }
 
+    #[tokio::test]
+    async fn queue_replica_task_counts_mrf_pending_backlog_when_worker_queue_is_full() {
+        let shared = empty_resync_shared_state();
+        let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared))).await;
+        let (tx, _rx) = mpsc::channel(1);
+        tx.try_send(ReplicationOperation::Object(Box::new(ReplicateObjectInfo {
+            bucket: "runtime-backlog".to_string(),
+            name: "already-buffered".to_string(),
+            size: 1,
+            op_type: ReplicationType::Object,
+            ..Default::default()
+        })))
+        .expect("test setup should fill the worker queue");
+        pool.workers.write().await.push(tx);
+
+        let admission = pool
+            .queue_replica_task(ReplicateObjectInfo {
+                bucket: "runtime-backlog".to_string(),
+                name: "fallback-object".to_string(),
+                size: 2048,
+                op_type: ReplicationType::Object,
+                ..Default::default()
+            })
+            .await;
+
+        assert_eq!(admission, ReplicationQueueAdmission::Queued);
+        let queued = pool.stats.get_latest_replication_stats("runtime-backlog").await;
+        assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
+        assert_eq!(queued.replication_stats.q_stat.curr.bytes, 2048);
+    }
+
     #[test]
     fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
         let checksum = bytes::Bytes::from_static(b"ssec-checksum");
@@ -2342,7 +2641,7 @@ mod tests {
 
         tx.try_send(first).expect("first MRF entry should fill the test channel");
 
-        let admission = queue_mrf_save_admission(&tx, second, "bucket", "second", "test");
+        let admission = queue_mrf_save_entry(&tx, second, "test");
         tokio::pin!(admission);
 
         assert!(
@@ -2687,6 +2986,30 @@ mod tests {
         assert!(missing_file.entries.is_empty());
     }
 
+    #[test]
+    fn durable_mrf_summary_aggregates_entries_by_bucket_for_obs() {
+        let summary =
+            durable_mrf_backlog_summary_from_sizes([("b1".to_string(), 1024), ("b1".to_string(), 512), ("b2".to_string(), 0)]);
+
+        assert!(summary.available);
+        let buckets = summary
+            .buckets
+            .into_iter()
+            .map(|bucket| (bucket.bucket.clone(), bucket))
+            .collect::<HashMap<_, _>>();
+        assert_eq!(buckets["b1"].count, 2);
+        assert_eq!(buckets["b1"].bytes, 1536);
+        assert_eq!(buckets["b2"].count, 1);
+        assert_eq!(buckets["b2"].bytes, 0);
+    }
+
+    #[test]
+    fn durable_mrf_summary_marks_invalid_sizes_unavailable() {
+        let invalid = durable_mrf_backlog_summary_from_sizes([("bucket".to_string(), -1)]);
+        assert!(!invalid.available);
+        assert!(invalid.buckets.is_empty());
+    }
+
     #[test]
     fn durable_mrf_snapshot_marks_corrupt_or_invalid_data_unavailable() {
         let corrupt = durable_mrf_backlog_from_read(Ok(vec![0, 1, 2]));

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -23,8 +23,8 @@ use super::replication_stats_boundary::{
 };
 use super::runtime_boundary as runtime_sources;
 use std::collections::HashMap;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
+use std::sync::{Arc, Mutex as StdMutex};
 use std::time::{Duration, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
@@ -143,7 +143,7 @@ pub struct ReplicationStats {
     // Active worker statistics
     pub workers: Arc<Mutex<ActiveWorkerStat>>,
     // Queue statistics cache
-    pub q_cache: Arc<Mutex<QueueCache>>,
+    pub q_cache: Arc<StdMutex<QueueCache>>,
     // Proxy statistics cache
     pub p_cache: Arc<Mutex<ProxyStatsCache>>,
     // MRF backlog statistics (simplified)
@@ -158,7 +158,7 @@ impl ReplicationStats {
         Self {
             sr_stats: Arc::new(SRStats::new()),
             workers: Arc::new(Mutex::new(ActiveWorkerStat::new())),
-            q_cache: Arc::new(Mutex::new(QueueCache::new())),
+            q_cache: Arc::new(StdMutex::new(QueueCache::new())),
             p_cache: Arc::new(Mutex::new(ProxyStatsCache::new())),
             mrf_stats: HashMap::new(),
             cache: Arc::new(RwLock::new(HashMap::new())),
@@ -198,8 +198,9 @@ impl ReplicationStats {
             let mut interval = interval(Duration::from_secs(2));
             loop {
                 interval.tick().await;
-                let mut cache = q_cache_clone.lock().await;
-                cache.update();
+                if let Ok(mut cache) = q_cache_clone.lock() {
+                    cache.update();
+                }
             }
         });
     }
@@ -391,12 +392,13 @@ impl ReplicationStats {
         drop(cache);
 
         {
-            let q_cache = self.q_cache.lock().await;
-            for (bucket, queue_stats) in &q_cache.bucket_stats {
-                let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
-                bucket_stats.q_stat = queue_stats.snapshot();
-                bucket_stats.mark_node_local_provider_available();
-                bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
+            if let Ok(q_cache) = self.q_cache.lock() {
+                for (bucket, queue_stats) in &q_cache.bucket_stats {
+                    let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+                    bucket_stats.q_stat = queue_stats.snapshot();
+                    bucket_stats.mark_node_local_provider_available();
+                    bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
+                }
             }
         }
 
@@ -429,8 +431,11 @@ impl ReplicationStats {
         let boot_time = SystemTime::UNIX_EPOCH; // simplified implementation
         let uptime = SystemTime::now().duration_since(boot_time).unwrap_or_default().as_secs() as i64;
 
-        let q_cache = self.q_cache.lock().await;
-        let queued = q_cache.get_site_stats();
+        let queued = self
+            .q_cache
+            .lock()
+            .map(|q_cache| q_cache.get_site_stats())
+            .unwrap_or_default();
 
         let p_cache = self.p_cache.lock().await;
         let proxied = p_cache.get_site_stats();
@@ -633,8 +638,9 @@ impl ReplicationStats {
         drop(cache);
 
         {
-            let q_cache = self.q_cache.lock().await;
-            if let Some(queue_stats) = q_cache.bucket_stats.get(bucket) {
+            if let Ok(q_cache) = self.q_cache.lock()
+                && let Some(queue_stats) = q_cache.bucket_stats.get(bucket)
+            {
                 replication_stats.q_stat = queue_stats.snapshot();
             }
         }
@@ -665,31 +671,17 @@ impl ReplicationStats {
     }
 
     /// Increase queue statistics
-    pub async fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
-        let mut q_cache = self.q_cache.lock().await;
-        let stats = q_cache
-            .bucket_stats
-            .entry(bucket.to_string())
-            .or_insert_with(InQueueMetric::default);
-        stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
-        stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
-
-        q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
+    pub fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
+        if let Ok(mut q_cache) = self.q_cache.lock() {
+            q_cache.inc(bucket, size);
+        }
     }
 
     /// Decrease queue statistics
-    pub async fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
-        let mut q_cache = self.q_cache.lock().await;
-        let stats = q_cache
-            .bucket_stats
-            .entry(bucket.to_string())
-            .or_insert_with(InQueueMetric::default);
-        stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
-
-        q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
-        q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
+    pub fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
+        if let Ok(mut q_cache) = self.q_cache.lock() {
+            q_cache.dec(bucket, size);
+        }
     }
 
     /// Increase proxy metrics
@@ -801,14 +793,14 @@ mod tests {
     async fn latest_stats_include_queue_until_drained() {
         let stats = ReplicationStats::new();
 
-        stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object).await;
+        stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object);
         let queued = stats.get_latest_replication_stats("queued-bucket").await;
         assert!(queued.replication_stats.provider_available);
         assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
         assert_eq!(queued.replication_stats.q_stat.curr.bytes, 4096);
         assert_eq!(queued.replication_stats.queue_scope, ReplicationMetricScope::NodeLocal);
 
-        stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object).await;
+        stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object);
         let drained = stats.get_latest_replication_stats("queued-bucket").await;
         assert_eq!(drained.replication_stats.q_stat.curr.count, 0);
         assert_eq!(drained.replication_stats.q_stat.curr.bytes, 0);
@@ -911,7 +903,7 @@ mod tests {
         for _ in 0..32 {
             let stats = Arc::clone(&stats);
             tasks.push(tokio::spawn(async move {
-                stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object).await;
+                stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object);
             }));
         }
         for task in tasks {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -44,7 +44,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub(crate) use peer_rest_client::TierConfigReloadOutcome;
 pub use peer_rest_client::{
-    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
     SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -45,9 +45,9 @@ use rustfs_protos::proto_gen::node_service::{
     HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
     LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
-    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
-    StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
-    TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
+    StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
+    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -71,6 +71,12 @@ use uuid::Uuid;
 
 pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
 pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
+/// Dynamic config subsystem for the cluster-persisted KMS configuration.
+///
+/// KMS configuration lives in its own cluster object rather than in the server
+/// config document, so it is not a `ServerConfig` subsystem; it only shares the
+/// reload signal transport.
+pub const KMS_SIGNAL_SUBSYSTEM: &str = "kms";
 const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
 const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
 const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
@@ -99,8 +105,13 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
 }
 
 fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
+    // The version stays pinned to DYNAMIC_CONFIG_PROTOCOL_VERSION rather than
+    // being bumped per subsystem: the comparison is shared, so raising it would
+    // retire peers that already converge scanner and heal config correctly.
+    // Subsystems added after a peer was built are rejected by that peer's own
+    // subsystem allow-list, which surfaces as an explicit failed signal.
     if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
-        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
+        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS | KMS_SIGNAL_SUBSYSTEM)
         && protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
     {
         return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
@@ -1500,6 +1511,22 @@ impl PeerRestClient {
     }
 
     pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
+        self.signal_service_checked(sig, sub_sys, dry_run).await.map(|_| ())
+    }
+
+    /// Report the KMS configuration fingerprint the peer is currently running.
+    ///
+    /// Sent as a dry-run reload signal so the peer answers without swapping its
+    /// own configuration. `None` means the peer has no KMS configuration. The
+    /// fingerprint is advisory and feeds cluster status reporting only, so the
+    /// response is not proof-signed.
+    pub async fn kms_config_fingerprint(&self) -> Result<Option<String>> {
+        self.signal_service_checked(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, true)
+            .await
+            .map(|response| response.config_fingerprint)
+    }
+
+    async fn signal_service_checked(&self, sig: u64, sub_sys: &str, dry_run: bool) -> Result<SignalServiceResponse> {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
@@ -1520,7 +1547,7 @@ impl PeerRestClient {
                     return Err(Error::other(""));
                 }
                 validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
-                Ok(())
+                Ok(response)
             }
             .await,
         )
@@ -2347,6 +2374,19 @@ mod tests {
             .expect("full refresh compatibility is guarded by its scanner preflight");
     }
 
+    #[test]
+    fn dynamic_kms_config_requires_versioned_peer_acknowledgement() {
+        let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, 0)
+            .expect_err("an unversioned peer must not claim KMS config convergence");
+        assert!(err.to_string().contains("does not support dynamic"));
+        validate_signal_service_protocol(
+            SERVICE_SIGNAL_RELOAD_DYNAMIC,
+            KMS_SIGNAL_SUBSYSTEM,
+            rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
+        )
+        .expect("a current peer should support dynamic KMS config");
+    }
+
     #[test]
     fn peer_rest_client_marks_network_like_errors() {
         assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));

crates/ecstore/src/disk/local.rs

@@ -5078,7 +5078,7 @@ impl LocalDisk {
         Ok((buf, mtime))
     }
 
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "LocalDisk")]
     async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
         check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
 
@@ -5121,7 +5121,7 @@ impl LocalDisk {
         Ok((data, modtime))
     }
 
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "LocalDisk")]
     async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
         // TODO: timeout support
         let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;

crates/ecstore/src/erasure/coding/bitrot.rs

@@ -103,7 +103,7 @@ where
     /// or `out` is larger than one shard. On error `out`'s contents are
     /// unspecified but never contain bytes that failed the hash check — the copy
     /// happens only after verification.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "BitrotReader")]
     pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
         let want = out.len();
         self.begin_read(want)?;
@@ -303,7 +303,7 @@ where
 
     /// Write a (hash+data) block. Returns the number of data bytes written.
     /// Returns an error if called after a short write or if data exceeds shard_size.
-    #[hotpath::measure(label = "BitrotWriter::write")]
+    #[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
     pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
         if buf.is_empty() {
             return Ok(0);

crates/ecstore/src/erasure/coding/decode.rs

@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
 where
     R: crate::erasure::coding::ShardSource,
 {
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "ParallelReader")]
     pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
         // On the reconstruction-verifying GET path, read every live shard reader
         // in lockstep so all readers advance one block per stripe and stay
@@ -1505,7 +1505,7 @@ where
 }
 
 impl Erasure {
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub async fn decode<W, R>(
         &self,
         writer: &mut W,

crates/ecstore/src/erasure/coding/encode.rs

@@ -28,8 +28,11 @@ use std::vec;
 use tokio::io::AsyncRead;
 use tokio::runtime::RuntimeFlavor;
 use tokio::sync::mpsc;
+use tokio::task::{JoinError, JoinHandle};
 use tracing::error;
 
+/// Queue-capacity input for encoded blocks awaiting shard writers; it is not a
+/// per-PUT or process-RSS memory limit.
 const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
 const ENV_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: &str = "RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS";
 const ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: &str = "RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST";
@@ -87,6 +90,33 @@ fn use_bytesmut_ingest() -> bool {
         rustfs_utils::get_env_bool(ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST, DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST)
     })
 }
+
+/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
+/// task when its `JoinHandle` is dropped, so the producer must be aborted when
+/// an upload is cancelled before the encode pipeline finishes.
+struct AbortOnDropTask<T>(JoinHandle<T>);
+
+impl<T> AbortOnDropTask<T> {
+    fn new(task: JoinHandle<T>) -> Self {
+        Self(task)
+    }
+
+    async fn abort_and_wait(&mut self) {
+        self.0.abort();
+        let _ = (&mut self.0).await;
+    }
+
+    async fn join(&mut self) -> Result<T, JoinError> {
+        (&mut self.0).await
+    }
+}
+
+impl<T> Drop for AbortOnDropTask<T> {
+    fn drop(&mut self) {
+        self.0.abort();
+    }
+}
+
 /// Read up to `limit` bytes into `buf`'s uninitialized spare capacity, appending after its
 /// current length, and distinguish a clean EOF from a short read.
 ///
@@ -133,22 +163,65 @@ fn queued_block_bytes(block: &[Bytes]) -> usize {
     block.iter().map(Bytes::len).sum()
 }
 
-async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
-    while let Some(block) = rx.recv().await {
-        rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
+/// Owns an encoded queue entry's gauge contribution until its consumer takes it.
+struct QueuedInflightBytes {
+    bytes: usize,
+}
+
+impl QueuedInflightBytes {
+    fn new(bytes: usize) -> Self {
+        rustfs_io_metrics::add_ec_encode_inflight_bytes(bytes);
+        Self { bytes }
     }
+
+    fn settle(&mut self) {
+        let bytes = std::mem::take(&mut self.bytes);
+        if bytes != 0 {
+            rustfs_io_metrics::remove_ec_encode_inflight_bytes(bytes);
+        }
+    }
+}
+
+impl Drop for QueuedInflightBytes {
+    fn drop(&mut self) {
+        self.settle();
+    }
+}
+
+/// Couples an encoded block with its queue gauge contribution. Keeping the
+/// guard in the queue entry also covers Tokio sends that complete through a
+/// permit after the receiver has closed.
+struct InflightEntry<T> {
+    entry: T,
+    accounting: QueuedInflightBytes,
+}
+
+impl<T> InflightEntry<T> {
+    fn new(entry: T, bytes: usize) -> Self {
+        Self {
+            entry,
+            accounting: QueuedInflightBytes::new(bytes),
+        }
+    }
+
+    fn into_inner(mut self) -> T {
+        self.accounting.settle();
+        self.entry
+    }
+}
+
+async fn send_queued<T>(
+    sender: &mpsc::Sender<InflightEntry<T>>,
+    entry: T,
+    bytes: usize,
+) -> Result<(), mpsc::error::SendError<InflightEntry<T>>> {
+    sender.send(InflightEntry::new(entry, bytes)).await
 }
 
 fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
     batch.iter().map(|block| queued_block_bytes(block)).sum()
 }
 
-async fn drain_queued_batched_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Vec<Bytes>>>) {
-    while let Some(batch) = rx.recv().await {
-        rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
-    }
-}
-
 fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
     summary.dominant_error_label
 }
@@ -504,7 +577,7 @@ impl Erasure {
         Ok((reader, total))
     }
 
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub async fn encode<R>(
         self: Arc<Self>,
         reader: R,
@@ -540,13 +613,14 @@ impl Erasure {
             ));
         }
 
-        // Bound queued encoded blocks by memory budget to avoid per-request spikes.
+        // Bound queued encoded blocks by a queue budget; this does not bound
+        // reader, encoder, writer, allocator, or process-RSS memory.
         let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
         let max_inflight_bytes = erasure_encode_max_inflight_bytes();
         let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
-        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
+        let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
 
-        let task = tokio::spawn(async move {
+        let mut task = AbortOnDropTask::new(tokio::spawn(async move {
             let block_size = self.block_size;
             let mut total = 0;
             if use_bytesmut_ingest {
@@ -567,10 +641,8 @@ impl Erasure {
                             let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
                             buf = BytesMut::with_capacity(ingest_capacity);
                             let queued_bytes = queued_block_bytes(&res);
-                            rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                             let send_wait_stage_start = stage_timer_if_enabled();
-                            if let Err(err) = tx.send(res).await {
-                                rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
+                            if let Err(err) = send_queued(&tx, res, queued_bytes).await {
                                 return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                             }
                             record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -598,10 +670,8 @@ impl Erasure {
                             let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
                             buf = returned_buf;
                             let queued_bytes = queued_block_bytes(&res);
-                            rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                             let send_wait_stage_start = stage_timer_if_enabled();
-                            if let Err(err) = tx.send(res).await {
-                                rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
+                            if let Err(err) = send_queued(&tx, res, queued_bytes).await {
                                 return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                             }
                             record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -626,7 +696,7 @@ impl Erasure {
             }
 
             Ok((reader, total))
-        });
+        }));
 
         let mut writers = MultiWriter::new(writers, quorum);
 
@@ -638,11 +708,10 @@ impl Erasure {
                 break;
             };
             record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
+            let block = block.into_inner();
             if block.is_empty() {
                 break;
             }
-            let queued_bytes = queued_block_bytes(&block);
-            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
             let write_stage_start = stage_timer_if_enabled();
             if let Err(err) = writers.write(block).await {
                 write_err = Some(err);
@@ -652,9 +721,8 @@ impl Erasure {
         }
 
         if let Some(err) = write_err {
-            task.abort();
-            let _ = task.await;
-            drain_queued_inflight_bytes(&mut rx).await;
+            task.abort_and_wait().await;
+            drop(rx);
             let shutdown_stage_start = stage_timer_if_enabled();
             if let Err(shutdown_err) = writers.shutdown().await {
                 error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -663,14 +731,14 @@ impl Erasure {
             return Err(err);
         }
 
-        let (reader, total) = task.await??;
+        let (reader, total) = task.join().await??;
         let shutdown_stage_start = stage_timer_if_enabled();
         writers.shutdown().await?;
         record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
         Ok((reader, total))
     }
 
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub async fn encode_batched<R>(
         self: Arc<Self>,
         mut reader: R,
@@ -692,9 +760,9 @@ impl Erasure {
         let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
         let batch_blocks = encode_batch_block_count().min(inflight_blocks);
         let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
-        let (tx, mut rx) = mpsc::channel::<Vec<Vec<Bytes>>>(channel_capacity);
+        let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
 
-        let task = tokio::spawn(async move {
+        let mut task = AbortOnDropTask::new(tokio::spawn(async move {
             let block_size = self.block_size;
             let mut total = 0;
             let mut buf = vec![0u8; block_size];
@@ -713,10 +781,8 @@ impl Erasure {
                         pending_batch.push(res);
 
                         if pending_batch.len() >= batch_blocks {
-                            rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
                             let send_wait_stage_start = stage_timer_if_enabled();
-                            if let Err(err) = tx.send(pending_batch).await {
-                                rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
+                            if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
                                 return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                             }
                             record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
@@ -742,17 +808,15 @@ impl Erasure {
             }
 
             if !pending_batch.is_empty() {
-                rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
                 let send_wait_stage_start = stage_timer_if_enabled();
-                if let Err(err) = tx.send(pending_batch).await {
-                    rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
+                if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
                     return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                 }
                 record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
             }
 
             Ok((reader, total))
-        });
+        }));
 
         let mut writers = MultiWriter::new(writers, quorum);
         let mut write_err = None;
@@ -763,7 +827,7 @@ impl Erasure {
                 break;
             };
             record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
-            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
+            let batch = batch.into_inner();
             let write_stage_start = stage_timer_if_enabled();
             for block in batch {
                 if let Err(err) = writers.write(block).await {
@@ -778,9 +842,8 @@ impl Erasure {
         }
 
         if let Some(err) = write_err {
-            task.abort();
-            let _ = task.await;
-            drain_queued_batched_inflight_bytes(&mut rx).await;
+            task.abort_and_wait().await;
+            drop(rx);
             let shutdown_stage_start = stage_timer_if_enabled();
             if let Err(shutdown_err) = writers.shutdown().await {
                 error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -789,7 +852,7 @@ impl Erasure {
             return Err(err);
         }
 
-        let (reader, total) = task.await??;
+        let (reader, total) = task.join().await??;
         let shutdown_stage_start = stage_timer_if_enabled();
         writers.shutdown().await?;
         record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
@@ -798,7 +861,7 @@ impl Erasure {
 
     /// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
     /// Reads all data, encodes directly, writes shards sequentially.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub async fn encode_inline_small<R>(
         self: Arc<Self>,
         reader: R,
@@ -813,7 +876,7 @@ impl Erasure {
 
     /// Fast path for single-block non-inline objects: avoids the producer/consumer
     /// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub async fn encode_single_block_non_inline<R>(
         self: Arc<Self>,
         reader: R,
@@ -839,7 +902,104 @@ mod tests {
     use std::sync::{Arc, Mutex};
     use std::task::{Context, Poll};
     use std::time::Duration;
-    use tokio::io::{AsyncWrite, AsyncWriteExt};
+    use tokio::io::{AsyncWrite, AsyncWriteExt, ReadBuf};
+    use tokio::sync::oneshot;
+
+    struct PendingReader {
+        entered: Option<oneshot::Sender<()>>,
+        dropped: Option<oneshot::Sender<()>>,
+    }
+
+    impl PendingReader {
+        fn new() -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>) {
+            let (entered_tx, entered_rx) = oneshot::channel();
+            let (dropped_tx, dropped_rx) = oneshot::channel();
+            (
+                Self {
+                    entered: Some(entered_tx),
+                    dropped: Some(dropped_tx),
+                },
+                entered_rx,
+                dropped_rx,
+            )
+        }
+    }
+
+    impl AsyncRead for PendingReader {
+        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+            if let Some(entered) = self.entered.take() {
+                let _ = entered.send(());
+            }
+            Poll::Pending
+        }
+    }
+
+    impl Drop for PendingReader {
+        fn drop(&mut self) {
+            if let Some(dropped) = self.dropped.take() {
+                let _ = dropped.send(());
+            }
+        }
+    }
+
+    struct BlocksThenPendingReader {
+        blocks_remaining: usize,
+        block: Vec<u8>,
+        blocked: Option<oneshot::Sender<()>>,
+        dropped: Option<oneshot::Sender<()>>,
+        final_block: Option<oneshot::Sender<()>>,
+    }
+
+    impl BlocksThenPendingReader {
+        fn new(
+            blocks_remaining: usize,
+            block_size: usize,
+        ) -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>, oneshot::Receiver<()>) {
+            let (blocked_tx, blocked_rx) = oneshot::channel();
+            let (dropped_tx, dropped_rx) = oneshot::channel();
+            let (final_block_tx, final_block_rx) = oneshot::channel();
+            (
+                Self {
+                    blocks_remaining,
+                    block: vec![0x5a; block_size],
+                    blocked: Some(blocked_tx),
+                    dropped: Some(dropped_tx),
+                    final_block: Some(final_block_tx),
+                },
+                blocked_rx,
+                dropped_rx,
+                final_block_rx,
+            )
+        }
+    }
+
+    impl AsyncRead for BlocksThenPendingReader {
+        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
+            if self.blocks_remaining == 0 {
+                if let Some(blocked) = self.blocked.take() {
+                    let _ = blocked.send(());
+                }
+                return Poll::Pending;
+            }
+
+            if self.blocks_remaining == 1
+                && let Some(final_block) = self.final_block.take()
+            {
+                let _ = final_block.send(());
+            }
+            self.blocks_remaining -= 1;
+            buf.put_slice(&self.block);
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    impl Drop for BlocksThenPendingReader {
+        fn drop(&mut self) {
+            if let Some(dropped) = self.dropped.take() {
+                let _ = dropped.send(());
+            }
+        }
+    }
 
     fn erasure_with_zero_block_size() -> Erasure {
         let mut erasure = Erasure::default();
@@ -897,6 +1057,54 @@ mod tests {
         }
     }
 
+    struct FailAfterReaderBlocksWriter {
+        reader_blocked: oneshot::Receiver<()>,
+        writes: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    impl AsyncWrite for FailAfterReaderBlocksWriter {
+        fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            match Pin::new(&mut self.reader_blocked).poll(cx) {
+                Poll::Pending => Poll::Pending,
+                Poll::Ready(_) => {
+                    self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                    Poll::Ready(Err(std::io::Error::other("injected write failure after producer blocks")))
+                }
+            }
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    struct StallOnWriteWithSignal {
+        entered: Option<oneshot::Sender<()>>,
+        writes: Arc<std::sync::atomic::AtomicUsize>,
+    }
+
+    impl AsyncWrite for StallOnWriteWithSignal {
+        fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
+            self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            if let Some(entered) = self.entered.take() {
+                let _ = entered.send(());
+            }
+            Poll::Pending
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
     #[derive(Clone, Default)]
     struct ShortWriteWriter;
 
@@ -1046,6 +1254,202 @@ mod tests {
         BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::None)
     }
 
+    #[derive(Clone, Copy)]
+    enum EncodePipeline {
+        Vec,
+        BytesMut,
+        Batched,
+    }
+
+    async fn aborting_encode_drops_blocked_producer(pipeline: EncodePipeline) {
+        const BLOCK_SIZE: usize = 16;
+
+        let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
+        let committed = Arc::new(Mutex::new(Vec::new()));
+        let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE))];
+        let (reader, entered, dropped) = PendingReader::new();
+        let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
+
+        let encode = match pipeline {
+            EncodePipeline::Vec => {
+                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
+            }
+            EncodePipeline::BytesMut => {
+                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
+            }
+            EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
+        };
+
+        tokio::time::timeout(Duration::from_secs(1), entered)
+            .await
+            .expect("producer should enter the blocked reader before cancellation")
+            .expect("blocked reader should signal entry");
+        encode.abort();
+        assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
+        tokio::time::timeout(Duration::from_secs(1), dropped)
+            .await
+            .expect("cancelling encode should drop the producer reader")
+            .expect("blocked reader should signal producer drop");
+        assert!(
+            committed.lock().expect("committed buffer should be lockable").is_empty(),
+            "cancelling before the first encoded block must not make data visible"
+        );
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            gauge_baseline,
+            "cancelling the encode pipeline must preserve the inflight queue gauge"
+        );
+    }
+
+    async fn writer_error_aborts_blocked_producer(pipeline: EncodePipeline) {
+        const BLOCK_SIZE: usize = 16;
+
+        let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
+        let blocks_before_pending = match pipeline {
+            EncodePipeline::Batched => encode_batch_block_count(),
+            EncodePipeline::Vec | EncodePipeline::BytesMut => 1,
+        };
+        let (reader, reader_blocked, reader_dropped, _final_block) =
+            BlocksThenPendingReader::new(blocks_before_pending, BLOCK_SIZE);
+        let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let mut writers = vec![Some(bitrot_writer(
+            FailAfterReaderBlocksWriter {
+                reader_blocked,
+                writes: writes.clone(),
+            },
+            BLOCK_SIZE,
+        ))];
+        let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
+
+        let result = match pipeline {
+            EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
+            EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
+            EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
+        };
+
+        let err = match result {
+            Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
+            Err(err) => err,
+        };
+        assert!(err.to_string().contains("Failed to write data"));
+        tokio::time::timeout(Duration::from_secs(1), reader_dropped)
+            .await
+            .expect("writer failure should abort the blocked producer")
+            .expect("blocked producer should signal reader drop");
+        assert_eq!(
+            writes.load(std::sync::atomic::Ordering::SeqCst),
+            1,
+            "writer failure must stop the pipeline before any additional shard write"
+        );
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            gauge_baseline,
+            "writer failure must settle all queued and pending encoded bytes"
+        );
+    }
+
+    async fn aborting_full_queue_settles_pending_send() {
+        const BLOCK_SIZE: usize = 16;
+
+        let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
+        let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
+        let inflight_blocks = encode_channel_capacity(
+            erasure.shard_size().saturating_mul(erasure.total_shard_count()),
+            erasure_encode_max_inflight_bytes(),
+        );
+        let (reader, _reader_blocked, reader_dropped, final_block) =
+            BlocksThenPendingReader::new(inflight_blocks + 2, BLOCK_SIZE);
+        let (writer_entered_tx, writer_entered) = oneshot::channel();
+        let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let mut writers = vec![Some(bitrot_writer(
+            StallOnWriteWithSignal {
+                entered: Some(writer_entered_tx),
+                writes: writes.clone(),
+            },
+            BLOCK_SIZE,
+        ))];
+        let erasure_for_task = erasure.clone();
+        let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
+
+        tokio::time::timeout(Duration::from_secs(1), writer_entered)
+            .await
+            .expect("consumer should start the first writer call")
+            .expect("stalling writer should signal entry");
+        tokio::time::timeout(Duration::from_secs(1), final_block)
+            .await
+            .expect("producer should supply the block whose send fills the queue")
+            .expect("reader should signal final block");
+
+        let expected_queued_bytes =
+            u64::try_from((inflight_blocks + 1) * BLOCK_SIZE).expect("queued byte count should fit the gauge");
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while rustfs_io_metrics::current_ec_encode_inflight_bytes() < gauge_baseline + expected_queued_bytes {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("producer should account for the pending send after the queue fills");
+
+        encode.abort();
+        assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
+        tokio::time::timeout(Duration::from_secs(1), reader_dropped)
+            .await
+            .expect("cancelling a full queue should abort its producer")
+            .expect("full-queue producer should signal reader drop");
+        assert_eq!(
+            writes.load(std::sync::atomic::Ordering::SeqCst),
+            1,
+            "cancellation must not resume the stalled writer"
+        );
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            gauge_baseline,
+            "cancelling a full queue must settle queued and pending bytes"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn cancelling_vec_encode_drops_blocked_producer() {
+        aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn cancelling_bytesmut_encode_drops_blocked_producer() {
+        aborting_encode_drops_blocked_producer(EncodePipeline::BytesMut).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn cancelling_batched_encode_drops_blocked_producer() {
+        aborting_encode_drops_blocked_producer(EncodePipeline::Batched).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn vec_writer_error_aborts_blocked_producer() {
+        writer_error_aborts_blocked_producer(EncodePipeline::Vec).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn bytesmut_writer_error_aborts_blocked_producer() {
+        writer_error_aborts_blocked_producer(EncodePipeline::BytesMut).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn batched_writer_error_aborts_blocked_producer() {
+        writer_error_aborts_blocked_producer(EncodePipeline::Batched).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn cancelling_full_queue_settles_pending_send() {
+        aborting_full_queue_settles_pending_send().await;
+    }
+
     #[tokio::test]
     async fn helper_writers_cover_flush_and_shutdown_paths() {
         let mut failing_write = FailingWriteWriter;
@@ -1329,25 +1733,99 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
-        let (tx, mut rx) = mpsc::channel(2);
-        tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
-        drop(tx);
+    #[serial_test::serial]
+    async fn queued_inflight_bytes_are_settled_on_all_queue_exit_paths() {
+        let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
+        let (tx, rx) = mpsc::channel(1);
+        let queued = vec![Bytes::from_static(b"queued")];
+        let queued_bytes = queued_block_bytes(&queued);
 
-        drain_queued_inflight_bytes(&mut rx).await;
+        send_queued(&tx, queued, queued_bytes)
+            .await
+            .expect("first queue entry should fit");
 
-        assert!(rx.recv().await.is_none());
+        let blocked = vec![Bytes::from_static(b"blocked")];
+        let blocked_bytes = queued_block_bytes(&blocked);
+        {
+            let pending_send = send_queued(&tx, blocked, blocked_bytes);
+            tokio::pin!(pending_send);
+            assert!(
+                futures::poll!(pending_send.as_mut()).is_pending(),
+                "full queue must suspend producer send"
+            );
+        }
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            baseline + u64::try_from(queued_bytes).expect("queue bytes fit the gauge"),
+            "dropping a pending producer send must compensate its bytes"
+        );
+
+        drop(rx);
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            baseline,
+            "dropping the receiver must settle every buffered entry"
+        );
+
+        let rejected = vec![Bytes::from_static(b"rejected")];
+        let rejected_bytes = queued_block_bytes(&rejected);
+        assert!(
+            send_queued(&tx, rejected, rejected_bytes).await.is_err(),
+            "closed receiver must reject a new send"
+        );
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            baseline,
+            "failed sends must compensate their bytes"
+        );
     }
 
     #[tokio::test]
-    async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
-        let (tx, mut rx) = mpsc::channel(2);
-        tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
-        drop(tx);
+    #[serial_test::serial]
+    async fn queued_batch_entry_settles_bytes_before_handoff() {
+        let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
+        let (tx, rx) = mpsc::channel(2);
+        let mut rx = rx;
+        let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
+        let batch_bytes = queued_batch_bytes(&batch);
 
-        drain_queued_batched_inflight_bytes(&mut rx).await;
+        send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
+        let batch = rx.recv().await.expect("queued batch should be received").into_inner();
+        assert_eq!(batch_bytes, queued_batch_bytes(&batch));
 
-        assert!(rx.recv().await.is_none());
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            baseline,
+            "receiving a batch must settle all contained block bytes before shard writes"
+        );
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn queued_entry_settles_when_a_closed_receiver_accepts_an_outstanding_permit() {
+        let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
+        let (tx, mut rx) = mpsc::channel(1);
+        let permit = tx
+            .clone()
+            .reserve_owned()
+            .await
+            .expect("open receiver should reserve queue capacity");
+        rx.close();
+        assert!(
+            matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
+            "an outstanding permit must leave the closed queue observably empty"
+        );
+
+        let block = vec![Bytes::from_static(b"late-permit")];
+        let block_bytes = queued_block_bytes(&block);
+        permit.send(InflightEntry::new(block, block_bytes));
+        drop(rx);
+
+        assert_eq!(
+            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+            baseline,
+            "a queue entry sent through an outstanding permit must settle when Tokio drops it"
+        );
     }
 
     #[tokio::test]

crates/ecstore/src/erasure/coding/erasure.rs

@@ -640,7 +640,7 @@ impl Erasure {
     /// # Returns
     /// A vector of encoded shards as `Bytes`.
     #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
@@ -688,7 +688,7 @@ impl Erasure {
 
     /// Encode owned data, avoiding a copy when the caller already has a heap buffer.
     /// Falls back to copying into a new buffer if zero-copy conversion fails.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
@@ -752,7 +752,7 @@ impl Erasure {
     /// block), the `resize(need_total_size)` below stays within capacity for every
     /// `data_len <= block_size` — both shard-size formulas are monotone in
     /// `data_len` — so this function never reallocates the buffer.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
@@ -805,7 +805,7 @@ impl Erasure {
     ///
     /// # Returns
     /// Ok if reconstruction succeeds, error otherwise.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {
@@ -825,7 +825,7 @@ impl Erasure {
     }
 
     /// Decode and reconstruct missing data shards, then regenerate parity shards.
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "Erasure")]
     pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {

crates/ecstore/src/io_support/bitrot.rs

@@ -19,6 +19,8 @@ use crate::diagnostics::get::{
     GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
     GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
 };
+#[cfg(feature = "hotpath")]
+use crate::disk::FileWriter;
 use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
 use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
 use bytes::Bytes;
@@ -36,6 +38,11 @@ use std::time::Instant;
 use tokio::io::{AsyncRead, ReadBuf};
 use tracing::debug;
 
+#[cfg(all(test, feature = "hotpath"))]
+tokio::task_local! {
+    static FORCE_MMAP_COPY_FAILURE_FOR_TEST: ();
+}
+
 /// A shard source for the bitrot reader.
 ///
 /// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
@@ -360,10 +367,21 @@ async fn open_disk_reader(
             mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
             direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
         });
-        match disk
-            .read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
-            .await
-        {
+        let mmap_result = {
+            #[cfg(all(test, feature = "hotpath"))]
+            if FORCE_MMAP_COPY_FAILURE_FOR_TEST.try_with(|_| ()).is_ok() {
+                Err(DiskError::other("forced mmap-copy failure for test"))
+            } else {
+                disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
+                    .await
+            }
+            #[cfg(not(all(test, feature = "hotpath")))]
+            {
+                disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
+                    .await
+            }
+        };
+        match mmap_result {
             Ok(bytes) => {
                 let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
 
@@ -398,7 +416,11 @@ async fn open_disk_reader(
                 }
 
                 return match stream_result {
-                    Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
+                    Ok(reader) => {
+                        #[cfg(feature = "hotpath")]
+                        let reader = instrument_raw_shard_reader(reader, disk.is_local());
+                        Ok(wrap_first_read_metrics(reader, metrics_path))
+                    }
                     Err(_) => Err(err),
                 };
             }
@@ -407,6 +429,8 @@ async fn open_disk_reader(
 
     let stream_start = stage_metrics_enabled.then(Instant::now);
     let reader = disk.read_file_stream(bucket, path, offset, length).await?;
+    #[cfg(feature = "hotpath")]
+    let reader = instrument_raw_shard_reader(reader, disk.is_local());
     if let Some(metrics_path) = metrics_path {
         record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
     }
@@ -427,6 +451,37 @@ fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str
     ShardReader::Stream(reader)
 }
 
+// The labels are deliberately fixed: object keys, disk paths, and remote hosts
+// are all high-cardinality or sensitive and belong nowhere in a profiling report.
+#[cfg(feature = "hotpath")]
+const RAW_SHARD_READ_LOCAL_LABEL: &str = "EC raw shard read local";
+#[cfg(feature = "hotpath")]
+const RAW_SHARD_READ_REMOTE_LABEL: &str = "EC raw shard read remote";
+#[cfg(feature = "hotpath")]
+const RAW_SHARD_WRITE_LOCAL_LABEL: &str = "EC raw shard write local";
+#[cfg(feature = "hotpath")]
+const RAW_SHARD_WRITE_REMOTE_LABEL: &str = "EC raw shard write remote";
+
+#[cfg(feature = "hotpath")]
+fn instrument_raw_shard_reader(reader: FileReader, is_local: bool) -> FileReader {
+    // `io!` aggregates by call site, so the local and remote branches must remain distinct.
+    if is_local {
+        Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_LOCAL_LABEL))
+    } else {
+        Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_REMOTE_LABEL))
+    }
+}
+
+#[cfg(feature = "hotpath")]
+fn instrument_raw_shard_writer(writer: FileWriter, is_local: bool) -> FileWriter {
+    // `io!` aggregates by call site, so the local and remote branches must remain distinct.
+    if is_local {
+        Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_LOCAL_LABEL))
+    } else {
+        Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_REMOTE_LABEL))
+    }
+}
+
 fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
     (
         offset.div_ceil(shard_size) * checksum_algo.size() + offset,
@@ -686,6 +741,8 @@ pub async fn create_bitrot_writer(
         };
 
         let file = disk.create_file("", volume, path, length).await?;
+        #[cfg(feature = "hotpath")]
+        let file = instrument_raw_shard_writer(file, disk.is_local());
         CustomWriter::new_tokio_writer(file)
     } else {
         return Err(DiskError::DiskNotFound);
@@ -698,6 +755,182 @@ pub async fn create_bitrot_writer(
 mod tests {
     use super::*;
 
+    #[cfg(feature = "hotpath")]
+    use crate::cluster::rpc::RemoteDisk;
+    #[cfg(feature = "hotpath")]
+    use crate::cluster::rpc::internode_data_transport::{
+        InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
+    };
+    #[cfg(feature = "hotpath")]
+    use crate::disk::{Disk, DiskOption, error::Result};
+
+    #[cfg(feature = "hotpath")]
+    #[derive(Debug, Clone, Default)]
+    struct TestRemoteDataTransport {
+        bytes: Arc<Mutex<Vec<u8>>>,
+    }
+
+    #[cfg(feature = "hotpath")]
+    impl TestRemoteDataTransport {
+        fn bytes(&self) -> Vec<u8> {
+            self.bytes
+                .lock()
+                .expect("test remote transport bytes lock should not be poisoned")
+                .clone()
+        }
+    }
+
+    #[cfg(feature = "hotpath")]
+    #[derive(Debug)]
+    struct TestRemoteWriter {
+        bytes: Arc<Mutex<Vec<u8>>>,
+    }
+
+    #[cfg(feature = "hotpath")]
+    impl tokio::io::AsyncWrite for TestRemoteWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
+            self.bytes
+                .lock()
+                .expect("test remote transport bytes lock should not be poisoned")
+                .extend_from_slice(buf);
+            Poll::Ready(Ok(buf.len()))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[cfg(feature = "hotpath")]
+    #[async_trait::async_trait]
+    impl InternodeDataTransport for TestRemoteDataTransport {
+        async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
+            Ok(Box::new(Cursor::new(self.bytes())))
+        }
+
+        async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
+            Ok(Box::new(TestRemoteWriter {
+                bytes: Arc::clone(&self.bytes),
+            }))
+        }
+
+        async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
+            panic!("open_walk_dir must not be used by the raw shard I/O test")
+        }
+
+        fn name(&self) -> &'static str {
+            "bitrot-test-remote"
+        }
+
+        fn capabilities(&self) -> InternodeDataTransportCapabilities {
+            InternodeDataTransportCapabilities::tcp_http()
+        }
+    }
+
+    async fn local_test_disk() -> (DiskStore, tempfile::TempDir) {
+        use crate::disk::endpoint::Endpoint;
+        use crate::disk::{DiskOption, new_disk};
+
+        let dir = tempfile::tempdir().expect("tempdir should be created");
+        let mut endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(0);
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("local disk should be created");
+
+        (disk, dir)
+    }
+
+    #[cfg(feature = "hotpath")]
+    async fn remote_test_disk() -> (DiskStore, TestRemoteDataTransport) {
+        use crate::disk::endpoint::Endpoint;
+
+        let endpoint = Endpoint {
+            url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("test remote endpoint should parse"),
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+        let transport = TestRemoteDataTransport::default();
+        let remote = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+            Arc::new(transport.clone()),
+        )
+        .await
+        .expect("test remote disk should be created");
+
+        (Arc::new(Disk::Remote(Box::new(remote))), transport)
+    }
+
+    async fn round_trip_disk_bitrot(disk: &DiskStore, bucket: &str, path: &str, payload: &[u8], shard_size: usize) -> Vec<u8> {
+        disk.make_volume(bucket).await.expect("volume should be created");
+        let mut writer = create_bitrot_writer(
+            false,
+            Some(disk),
+            bucket,
+            path,
+            i64::try_from(payload.len()).expect("test payload length should fit i64"),
+            shard_size,
+            HashAlgorithm::None,
+        )
+        .await
+        .expect("disk bitrot writer should open the raw shard file");
+        for chunk in payload.chunks(shard_size) {
+            writer
+                .write(chunk)
+                .await
+                .expect("disk bitrot writer should preserve each shard block");
+        }
+        writer.shutdown().await.expect("disk bitrot writer should close cleanly");
+
+        let mut reader = create_bitrot_reader(
+            None,
+            Some(disk),
+            bucket,
+            path,
+            0,
+            payload.len(),
+            shard_size,
+            HashAlgorithm::None,
+            false,
+            false,
+        )
+        .await
+        .expect("disk bitrot reader should open the raw shard file")
+        .expect("disk bitrot reader should exist");
+        let mut actual = Vec::with_capacity(payload.len());
+        while actual.len() < payload.len() {
+            let remaining = payload.len() - actual.len();
+            let mut chunk = vec![0; remaining.min(shard_size)];
+            let read = reader
+                .read(&mut chunk)
+                .await
+                .expect("disk bitrot reader should return the complete shard body");
+            assert!(read > 0, "disk bitrot reader must not end before the expected shard body is complete");
+            actual.extend_from_slice(&chunk[..read]);
+        }
+
+        actual
+    }
+
     #[test]
     fn object_mmap_read_enabled_accepts_legacy_zero_copy_alias() {
         temp_env::with_vars(
@@ -724,6 +957,165 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "hotpath")]
+    #[test]
+    fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes() {
+        const CHILD_ENV: &str = "RUSTFS_HOTPATH_RAW_SHARD_IO_TEST_CHILD";
+        if std::env::var_os(CHILD_ENV).is_none() {
+            let status = std::process::Command::new(std::env::current_exe().expect("test executable path should be available"))
+                .arg("--exact")
+                .arg("io_support::bitrot::tests::raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes")
+                .arg("--nocapture")
+                .env(CHILD_ENV, "1")
+                .status()
+                .expect("isolated HotPath I/O test process should start");
+            assert!(status.success(), "isolated HotPath I/O test process should pass");
+            return;
+        }
+
+        tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("test runtime should be created")
+            .block_on(raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process());
+    }
+
+    #[cfg(feature = "hotpath")]
+    async fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process() {
+        use hotpath::{Format, HotpathGuardBuilder, Section};
+        use tokio::io::AsyncReadExt;
+
+        let report_dir = tempfile::tempdir().expect("report tempdir should be created");
+        let report_path = report_dir.path().join("hotpath-io.json");
+        let guard = HotpathGuardBuilder::new("raw_shard_io_test")
+            .format(Format::Json)
+            .output_path(&report_path)
+            .sections(vec![Section::Io])
+            .build();
+
+        let (disk, _dir) = local_test_disk().await;
+        let bucket = "test-bucket";
+        let path = "obj/hotpath-part.1";
+        let payload = b"local shard bytes";
+        let shard_size = 4;
+        let local_read = round_trip_disk_bitrot(&disk, bucket, path, payload, shard_size).await;
+        assert_eq!(local_read, payload, "local raw shard I/O instrumentation must not alter stored bytes");
+
+        let fallback_path = "obj/hotpath-mmap-fallback-part.1";
+        let fallback_payload = b"mmap fallback shard bytes";
+        disk.write_all("test-bucket", fallback_path, Bytes::from_static(fallback_payload))
+            .await
+            .expect("fallback shard file should be written");
+        let mut fallback_reader = FORCE_MMAP_COPY_FAILURE_FOR_TEST
+            .scope((), open_disk_reader(&disk, bucket, fallback_path, 0, fallback_payload.len(), true, None))
+            .await
+            .expect("mmap-copy failure should fall back to a raw shard stream");
+        assert!(
+            matches!(fallback_reader, ShardReader::Stream(_)),
+            "forced mmap-copy failure must use the streaming fallback"
+        );
+        let mut fallback_read = Vec::new();
+        fallback_reader
+            .read_to_end(&mut fallback_read)
+            .await
+            .expect("mmap-copy fallback stream should preserve bytes");
+        assert_eq!(fallback_read, fallback_payload);
+
+        let (remote_disk, remote_transport) = remote_test_disk().await;
+        let remote_payload = b"remote shard bytes";
+        let mut remote_writer = create_bitrot_writer(
+            false,
+            Some(&remote_disk),
+            bucket,
+            "obj/hotpath-remote-part.1",
+            i64::try_from(remote_payload.len()).expect("remote payload length should fit i64"),
+            shard_size,
+            HashAlgorithm::None,
+        )
+        .await
+        .expect("remote bitrot writer should use the production raw writer path");
+        for chunk in remote_payload.chunks(shard_size) {
+            remote_writer
+                .write(chunk)
+                .await
+                .expect("remote bitrot writer should preserve bytes");
+        }
+        remote_writer
+            .shutdown()
+            .await
+            .expect("remote bitrot writer should close cleanly");
+        assert_eq!(remote_transport.bytes(), remote_payload);
+
+        let mut reader = create_bitrot_reader(
+            None,
+            Some(&remote_disk),
+            bucket,
+            "obj/hotpath-remote-part.1",
+            0,
+            remote_payload.len(),
+            shard_size,
+            HashAlgorithm::None,
+            false,
+            true,
+        )
+        .await
+        .expect("remote bitrot reader should use the production raw reader path")
+        .expect("remote bitrot reader should exist");
+        let mut remote_read = Vec::with_capacity(remote_payload.len());
+        while remote_read.len() < remote_payload.len() {
+            let remaining = remote_payload.len() - remote_read.len();
+            let mut chunk = vec![0; remaining.min(shard_size)];
+            let read = reader
+                .read(&mut chunk)
+                .await
+                .expect("remote bitrot reader should preserve bytes");
+            assert!(read > 0, "remote bitrot reader must not end before the expected shard body is complete");
+            remote_read.extend_from_slice(&chunk[..read]);
+        }
+        assert_eq!(remote_read, remote_payload);
+
+        drop(guard);
+        let report = std::fs::read_to_string(&report_path).expect("HotPath I/O report should be written");
+        let report: serde_json::Value = serde_json::from_str(&report).expect("HotPath I/O report should be valid JSON");
+        let entries = report["io"]["data"]
+            .as_array()
+            .expect("HotPath I/O report should include data rows");
+        let io_bytes = |label: &str, direction: &str| {
+            let byte_count: u64 = entries
+                .iter()
+                .filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
+                .filter_map(|entry| entry[direction]["bytes"].as_u64())
+                .sum();
+            assert!(byte_count > 0, "report must include fixed label {label}");
+            byte_count
+        };
+        let payload_len = u64::try_from(payload.len()).expect("test payload length should fit u64");
+        assert_eq!(
+            io_bytes(RAW_SHARD_READ_LOCAL_LABEL, "read"),
+            payload_len + u64::try_from(fallback_payload.len()).expect("fallback payload length should fit u64")
+        );
+        assert_eq!(io_bytes(RAW_SHARD_WRITE_LOCAL_LABEL, "write"), payload_len);
+        assert_eq!(
+            io_bytes(RAW_SHARD_READ_REMOTE_LABEL, "read"),
+            u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
+        );
+        assert_eq!(
+            io_bytes(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
+            u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
+        );
+        for label in [
+            RAW_SHARD_READ_LOCAL_LABEL,
+            RAW_SHARD_READ_REMOTE_LABEL,
+            RAW_SHARD_WRITE_LOCAL_LABEL,
+            RAW_SHARD_WRITE_REMOTE_LABEL,
+        ] {
+            assert!(
+                !label.contains(['/', ':', '?', '@']),
+                "raw shard I/O labels must not carry a path, host, query, or credential delimiter: {label}"
+            );
+        }
+    }
+
     #[test]
     fn object_mmap_read_max_length_defaults_and_env_override() {
         temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
@@ -741,25 +1133,9 @@ mod tests {
     // be materialized in memory by the mmap-copy path; over-cap reads stream.
     #[tokio::test]
     async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
-        use crate::disk::endpoint::Endpoint;
-        use crate::disk::{DiskOption, new_disk};
         use tokio::io::AsyncReadExt;
 
-        let dir = tempfile::tempdir().expect("tempdir should be created");
-        let mut endpoint =
-            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(0);
-        let disk = new_disk(
-            &endpoint,
-            &DiskOption {
-                cleanup: false,
-                health_check: false,
-            },
-        )
-        .await
-        .expect("local disk should be created");
+        let (disk, _dir) = local_test_disk().await;
 
         let payload = vec![7u8; 4096];
         disk.make_volume("test-bucket").await.expect("volume should be created");
@@ -814,6 +1190,17 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    async fn disk_bitrot_reader_and_writer_preserve_full_shard_body() {
+        let (disk, _dir) = local_test_disk().await;
+        let bucket = "test-bucket";
+        let path = "obj/wrapped-part.1";
+        let payload = b"wrapped shard body";
+        let actual = round_trip_disk_bitrot(&disk, bucket, path, payload, 4).await;
+
+        assert_eq!(actual, payload, "raw shard I/O instrumentation must not alter stored bytes");
+    }
+
     #[tokio::test]
     async fn test_create_bitrot_reader_with_inline_data() {
         let test_data = b"hello world test data";

crates/ecstore/src/services/notification_sys.rs

@@ -337,6 +337,15 @@ pub struct NotificationPeerErr {
     pub err: Option<Error>,
 }
 
+/// One peer's answer to a KMS configuration fingerprint probe.
+pub struct PeerKmsConfigFingerprint {
+    pub host: String,
+    /// `None` when the peer has no KMS configuration of its own, or could not
+    /// be asked at all, in which case `err` carries the reason.
+    pub fingerprint: Option<String>,
+    pub err: Option<Error>,
+}
+
 fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
     NotificationPeerErr { host, err: result.err() }
 }
@@ -518,6 +527,47 @@ impl NotificationSys {
         self.signal_dynamic_config(sub_sys, false).await
     }
 
+    /// Ask every peer to re-read the cluster-persisted KMS configuration.
+    ///
+    /// Best-effort by contract: the caller has already switched locally, so a
+    /// peer that fails is reported rather than rolled back. Peers built before
+    /// the KMS subsystem existed reject the signal with an explicit error.
+    pub async fn reload_kms_config(&self) -> Vec<NotificationPeerErr> {
+        self.reload_dynamic_config(crate::cluster::rpc::KMS_SIGNAL_SUBSYSTEM).await
+    }
+
+    /// Collect the KMS configuration fingerprint each peer is running.
+    ///
+    /// A peer whose build predates the KMS subsystem rejects the probe, so it
+    /// is reported as an error rather than silently agreeing with this node.
+    pub async fn kms_config_fingerprints(&self) -> Vec<PeerKmsConfigFingerprint> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                let Some(client) = client else {
+                    return PeerKmsConfigFingerprint {
+                        host: String::new(),
+                        fingerprint: None,
+                        err: Some(Error::other("peer is not reachable")),
+                    };
+                };
+                match client.kms_config_fingerprint().await {
+                    Ok(fingerprint) => PeerKmsConfigFingerprint {
+                        host: client.host.to_string(),
+                        fingerprint,
+                        err: None,
+                    },
+                    Err(e) => PeerKmsConfigFingerprint {
+                        host: client.host.to_string(),
+                        fingerprint: None,
+                        err: Some(e),
+                    },
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
     pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {

crates/ecstore/src/set_disk/read.rs

@@ -199,7 +199,7 @@ impl SetDisks {
         );
     }
 
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "SetDisks")]
     pub async fn read_version_optimized(
         &self,
         bucket: &str,
@@ -238,7 +238,7 @@ impl SetDisks {
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "SetDisks")]
     pub(super) async fn get_object_fileinfo(
         &self,
         bucket: &str,
@@ -410,7 +410,7 @@ impl SetDisks {
         Ok((fi, parts_metadata, op_online_disks))
     }
 
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "SetDisks")]
     pub(super) async fn get_object_info_and_quorum(
         &self,
         bucket: &str,
@@ -605,7 +605,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "SetDisks")]
     pub(super) async fn get_object_with_fileinfo<W>(
         // &self,
         bucket: &str,
@@ -1140,7 +1140,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "SetDisks")]
     pub(super) async fn get_object_decode_reader_with_fileinfo(
         bucket: &str,
         object: &str,
@@ -1296,7 +1296,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "SetDisks")]
     async fn build_codec_streaming_part_reader(
         bucket: &str,
         object: &str,

crates/ecstore/src/store/multipart.rs

@@ -238,7 +238,7 @@ impl ECStore {
     }
 
     #[instrument(skip(self, data))]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "ECStore")]
     pub(super) async fn handle_put_object_part(
         &self,
         bucket: &str,

crates/ecstore/src/store/object.rs

@@ -815,7 +815,7 @@ impl ECStore {
     }
 
     #[instrument(level = "debug", skip(self))]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "ECStore")]
     pub(super) async fn handle_get_object_reader(
         &self,
         bucket: &str,
@@ -849,7 +849,7 @@ impl ECStore {
     }
 
     #[instrument(level = "debug", skip(self, data))]
-    #[hotpath::measure]
+    #[hotpath::measure(impl_type = "ECStore")]
     pub(super) async fn handle_put_object(
         &self,
         bucket: &str,

crates/ecstore/tests/hotpath_cpu_attribution_contract_test.rs

@@ -0,0 +1,95 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const STORE_MULTIPART: &str = include_str!("../src/store/multipart.rs");
+const STORE_OBJECT: &str = include_str!("../src/store/object.rs");
+const ERASURE: &str = include_str!("../src/erasure/coding/erasure.rs");
+const DECODE: &str = include_str!("../src/erasure/coding/decode.rs");
+const ENCODE: &str = include_str!("../src/erasure/coding/encode.rs");
+const LOCAL_DISK: &str = include_str!("../src/disk/local.rs");
+const BITROT: &str = include_str!("../src/erasure/coding/bitrot.rs");
+const SET_DISK_READ: &str = include_str!("../src/set_disk/read.rs");
+
+fn assert_measured_as(source: &str, impl_type: &str, function: &str) {
+    let attribute = format!("#[hotpath::measure(impl_type = \"{impl_type}\")]\n");
+    let function_start = format!("fn {function}");
+    let mut remaining = source;
+
+    while let Some(attribute_offset) = remaining.find(&attribute) {
+        let measured_source = &remaining[attribute_offset + attribute.len()..];
+        if measured_source.find(&function_start).is_some_and(|offset| offset < 256) {
+            return;
+        }
+        remaining = measured_source;
+    }
+
+    panic!("missing HotPath impl_type={impl_type} for {function}");
+}
+
+#[test]
+fn inherent_hotpath_measurements_have_cpu_attribution_types() {
+    assert_measured_as(STORE_MULTIPART, "ECStore", "handle_put_object_part");
+    for function in ["handle_get_object_reader", "handle_put_object"] {
+        assert_measured_as(STORE_OBJECT, "ECStore", function);
+    }
+    for function in [
+        "encode_data",
+        "encode_data_owned",
+        "encode_data_bytes_mut",
+        "decode_data",
+        "decode_data_and_parity",
+    ] {
+        assert_measured_as(ERASURE, "Erasure", function);
+    }
+    assert_measured_as(DECODE, "ParallelReader", "read");
+    assert_measured_as(DECODE, "Erasure", "decode");
+    for function in [
+        "encode",
+        "encode_batched",
+        "encode_inline_small",
+        "encode_single_block_non_inline",
+    ] {
+        assert_measured_as(ENCODE, "Erasure", function);
+    }
+    for function in ["read_metadata_with_dmtime", "read_all_data"] {
+        assert_measured_as(LOCAL_DISK, "LocalDisk", function);
+    }
+    assert_measured_as(BITROT, "BitrotReader", "read");
+    assert!(
+        BITROT.contains("#[hotpath::measure(label = \"BitrotWriter::write\", impl_type = \"BitrotWriter\")]"),
+        "BitrotWriter::write must retain its stable label and CPU impl_type",
+    );
+    for function in [
+        "read_version_optimized",
+        "get_object_fileinfo",
+        "get_object_info_and_quorum",
+        "get_object_with_fileinfo",
+        "get_object_decode_reader_with_fileinfo",
+        "build_codec_streaming_part_reader",
+    ] {
+        assert_measured_as(SET_DISK_READ, "SetDisks", function);
+    }
+}
+
+#[test]
+fn module_hotpath_measurements_remain_untyped() {
+    assert!(
+        BITROT.contains("#[hotpath::measure]\npub async fn bitrot_verify"),
+        "bitrot_verify is a module function and must not claim an impl type",
+    );
+    assert!(
+        SET_DISK_READ.contains("#[hotpath::measure]\nasync fn setup_multipart_part_readers"),
+        "setup_multipart_part_readers is a module function and must not claim an impl type",
+    );
+}

crates/iam/src/sys.rs

@@ -41,7 +41,7 @@ use rustfs_policy::policy::Args;
 use rustfs_policy::policy::opa;
 use rustfs_policy::policy::{Policy, PolicyDoc, iam_policy_claim_name_sa, policy_needs_existing_object_tag_for_args};
 use serde_json::Value;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::sync::OnceLock;
 use time::OffsetDateTime;
@@ -61,10 +61,51 @@ const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-polic
 const FEDERATED_POLICY_BOUNDARY_CLAIM: &str = "x-rustfs-internal-federated-boundary";
 pub(crate) const SITE_REPLICATOR_CLAIM: &str = "site-replicator";
 
-static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
+#[derive(Clone)]
+enum PolicyPluginState {
+    Disabled,
+    Initializing,
+    Ready(opa::AuthZPlugin),
+    Failed,
+}
 
-fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
-    POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
+static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
+
+fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
+    POLICY_PLUGIN_STATE
+        .get_or_init(|| {
+            let configured = opa::is_configured();
+            let state = Arc::new(RwLock::new(if configured {
+                PolicyPluginState::Initializing
+            } else {
+                PolicyPluginState::Disabled
+            }));
+            if configured {
+                let state = Arc::clone(&state);
+                tokio::spawn(async move {
+                    let next_state = match opa::lookup_config().await {
+                        Ok(conf) if conf.enable() => {
+                            info!("OPA plugin enabled");
+                            PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
+                        }
+                        Ok(_) => PolicyPluginState::Failed,
+                        Err(e) => {
+                            error!(
+                                component = "iam",
+                                subsystem = "policy_plugin",
+                                result = "configuration_load_failed",
+                                error_kind = e.kind(),
+                                "OPA plugin configuration load failed"
+                            );
+                            PolicyPluginState::Failed
+                        }
+                    };
+                    *state.write().await = next_state;
+                });
+            }
+            state
+        })
+        .clone()
 }
 
 pub struct IamSys<T> {
@@ -173,19 +214,7 @@ impl<T: Store> IamSys<T> {
     /// # Returns
     /// A new instance of IamSys
     pub fn new(store: Arc<IamCache<T>>) -> Self {
-        tokio::spawn(async move {
-            match opa::lookup_config().await {
-                Ok(conf) => {
-                    if conf.enable() {
-                        Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
-                        info!("OPA plugin enabled");
-                    }
-                }
-                Err(e) => {
-                    error!("Error loading OPA configuration err:{}", e);
-                }
-            };
-        });
+        get_policy_plugin_state();
 
         Self {
             store,
@@ -206,15 +235,22 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
-        let policy_plugin_client = get_policy_plugin_client();
-        let mut guard = policy_plugin_client.write().await;
-        *guard = Some(client);
+        let policy_plugin_state = get_policy_plugin_state();
+        let mut guard = policy_plugin_state.write().await;
+        *guard = PolicyPluginState::Ready(client);
     }
 
     pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
-        let policy_plugin_client = get_policy_plugin_client();
-        let guard = policy_plugin_client.read().await;
-        guard.clone()
+        let policy_plugin_state = get_policy_plugin_state();
+        let guard = policy_plugin_state.read().await;
+        match &*guard {
+            PolicyPluginState::Ready(client) => Some(client.clone()),
+            PolicyPluginState::Disabled | PolicyPluginState::Initializing | PolicyPluginState::Failed => None,
+        }
+    }
+
+    async fn policy_plugin_state() -> PolicyPluginState {
+        get_policy_plugin_state().read().await.clone()
     }
 
     pub async fn load_group(&self, name: &str) -> Result<()> {
@@ -1060,11 +1096,20 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        if Self::get_policy_plugin_client().await.is_some() {
-            return PreparedIamAuth {
-                needs_existing_object_tag: false,
-                mode: PreparedIamMode::Opa,
-            };
+        match Self::policy_plugin_state().await {
+            PolicyPluginState::Ready(_) => {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::Opa,
+                };
+            }
+            PolicyPluginState::Initializing | PolicyPluginState::Failed => {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::Deny,
+                };
+            }
+            PolicyPluginState::Disabled => {}
         }
 
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
@@ -1163,7 +1208,22 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        let combined_policy = self.get_combined_policy(&policies).await;
+        let (resolved_policies, combined_policy) = self.store.merge_policies(&policies.join(",")).await;
+        if args.deny_only {
+            let resolved_policy_names: HashSet<&str> = resolved_policies
+                .split(',')
+                .filter(|policy_name| !policy_name.trim().is_empty())
+                .collect();
+            if policies
+                .iter()
+                .any(|policy_name| !resolved_policy_names.contains(policy_name.as_str()))
+            {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::Deny,
+                };
+            }
+        }
         PreparedIamAuth {
             needs_existing_object_tag: policy_needs_existing_object_tag_for_args(&combined_policy, args).await,
             mode: PreparedIamMode::Regular { combined_policy },
@@ -1703,7 +1763,7 @@ mod tests {
     fn deprecated_list_polices_api_is_available() {
         let _ = IamSys::<StsTestMockStore>::list_polices;
     }
-    use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
+    use rustfs_policy::policy::action::{Action, AdminAction, S3Action, StsAction};
     use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
     use serde_json::Value;
     use std::{
@@ -3309,6 +3369,155 @@ mod tests {
         assert!(!iam_sys.eval_prepared(&prepared, &args).await);
     }
 
+    #[tokio::test]
+    async fn regular_deny_only_rejects_unresolved_mapped_policy() {
+        let iam_sys = test_iam_sys().await;
+        let user = "regular-unresolved-policy";
+        let identity = UserIdentity::from(Credentials {
+            access_key: user.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(user, &identity, now);
+            cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
+        });
+
+        let groups = None;
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: user,
+            groups: &groups,
+            action: Action::StsAction(StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Deny));
+        assert!(
+            !iam_sys.eval_prepared(&prepared, &args).await,
+            "deny-only evaluation must fail closed when a mapped policy document is unresolved"
+        );
+    }
+
+    #[tokio::test]
+    async fn regular_full_evaluation_keeps_resolved_part_of_partial_mapping() {
+        let iam_sys = test_iam_sys().await;
+        let user = "regular-partial-policy";
+        let identity = UserIdentity::from(Credentials {
+            access_key: user.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(user, &identity, now);
+            cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
+        });
+
+        let groups = None;
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: user,
+            groups: &groups,
+            action: Action::S3Action(S3Action::GetObjectAction),
+            bucket: "bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "object",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "full evaluation must preserve the historical partial-resolution behavior"
+        );
+    }
+
+    #[tokio::test]
+    async fn regular_deny_only_honors_group_derived_explicit_deny() {
+        let iam_sys = test_iam_sys().await;
+        let deny_policy =
+            Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["sts:AssumeRole"]}]}"#)
+                .expect("group deny policy should parse");
+        let now = OffsetDateTime::now_utc();
+        iam_sys
+            .store
+            .cache
+            .add_or_update_policy_doc("deny-assume-role", &PolicyDoc::new(deny_policy), now);
+        iam_sys
+            .store
+            .cache
+            .add_or_update_group_policy("testgroup", &MappedPolicy::new("deny-assume-role"), now);
+
+        let groups = Some(vec!["testgroup".to_string()]);
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: "sts-fallback-test-parent",
+            groups: &groups,
+            action: Action::StsAction(StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
+        assert!(
+            !iam_sys.eval_prepared(&prepared, &args).await,
+            "group-derived explicit Deny must remain authoritative"
+        );
+    }
+
+    #[tokio::test]
+    async fn regular_deny_only_rejects_unresolved_group_mapping() {
+        let iam_sys = test_iam_sys().await;
+        iam_sys.store.cache.add_or_update_group_policy(
+            "testgroup",
+            &MappedPolicy::new("readwrite,missing-policy"),
+            OffsetDateTime::now_utc(),
+        );
+
+        let groups = Some(vec!["testgroup".to_string()]);
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: "sts-fallback-test-parent",
+            groups: &groups,
+            action: Action::StsAction(StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Deny));
+        assert!(
+            !iam_sys.eval_prepared(&prepared, &args).await,
+            "deny-only evaluation must fail closed for an unresolved group-derived mapping"
+        );
+    }
+
     #[tokio::test]
     async fn test_sts_claim_policy_ignores_unsafe_and_missing_policy_names() {
         let store = StsTestMockStore::new(true);

crates/io-metrics/src/lib.rs

@@ -2195,18 +2195,22 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
     }
 }
 
-/// Track encoded bytes currently queued between erasure encode and disk writers.
+/// Track encoded bytes in the queue hand-off between erasure encode and disk writers.
+///
+/// This is queue occupancy, not a per-request or process-RSS memory limit. The
+/// erasure encoder settles it on failed/cancelled sends, receiver drop, and
+/// consumer hand-off before shard writes begin.
 #[inline(always)]
 pub fn add_ec_encode_inflight_bytes(bytes: usize) {
     let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
-    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
+    gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
 }
 
 /// Remove encoded bytes from the tracked erasure encode in-flight gauge.
 #[inline(always)]
 pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
     let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
-    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
+    gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
 }
 
 /// Return the current tracked EC encode in-flight bytes.

crates/kms/Cargo.toml

@@ -64,6 +64,10 @@ md-5 = { workspace = true }
 arc-swap = { workspace = true }
 rustfs-utils = { workspace = true }
 rustfs-security-governance = { workspace = true }
+# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
+# so the audit sink can live outside this crate without a second, drifting
+# copy of the event vocabulary.
+rustfs-s3-types = { workspace = true }
 
 # HTTP client for Vault
 reqwest = { workspace = true }
@@ -73,6 +77,16 @@ vaultrs = { workspace = true }
 rustify = { workspace = true }
 tokio-util = { workspace = true }
 
+# AWS KMS backend. Credentials come from the standard aws-config provider chain
+# (environment, shared profile, IMDS/container roles); this crate never handles
+# AWS credential material itself.
+aws-config = { workspace = true }
+aws-sdk-kms = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
+# SdkError variants and raw HTTP status are needed to classify AWS failures for
+# the operation policy's retry decisions.
+aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
+aws-smithy-types = { workspace = true }
+
 [dev-dependencies]
 anyhow = { workspace = true }
 # Debugging recorder for asserting emitted metrics in tests.
@@ -82,6 +96,9 @@ tempfile = { workspace = true }
 temp-env = { workspace = true }
 # "net" backs the scripted loopback Vault used by the policy wiring tests.
 tokio = { workspace = true, features = ["net", "test-util"] }
+# Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline.
+aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
+http = { workspace = true }
 
 [features]
 default = []

crates/kms/examples/kms_local_demo.rs

@@ -227,10 +227,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Step 12: Check cache statistics
     println!("12. Checking cache statistics...");
-    if let Some((hits, misses)) = encryption_service.cache_stats().await {
+    if let Some(stats) = encryption_service.cache_stats().await {
         println!("   ✓ Cache statistics:");
-        println!("     - Cache hits: {}", hits);
-        println!("     - Cache misses: {}\n", misses);
+        println!("     - Cached entries: {}", stats.entries);
+        println!("     - Cache hits: {}", stats.hits);
+        println!("     - Cache misses: {}", stats.misses);
+        println!("     - Entries evicted: {}\n", stats.evictions);
     } else {
         println!("   - Cache is disabled\n");
     }

crates/kms/examples/kms_vault_kv_demo.rs

@@ -263,10 +263,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Step 12: Check cache statistics
     println!("12. Checking cache statistics...");
-    if let Some((hits, misses)) = encryption_service.cache_stats().await {
+    if let Some(stats) = encryption_service.cache_stats().await {
         println!("   ✓ Cache statistics:");
-        println!("     - Cache hits: {}", hits);
-        println!("     - Cache misses: {}\n", misses);
+        println!("     - Cached entries: {}", stats.entries);
+        println!("     - Cache hits: {}", stats.hits);
+        println!("     - Cache misses: {}", stats.misses);
+        println!("     - Entries evicted: {}\n", stats.evictions);
     } else {
         println!("   - Cache is disabled\n");
     }

crates/kms/src/api_types.rs

@@ -418,6 +418,13 @@ pub enum BackendSummary {
         /// Configured key identifier
         key_id: String,
     },
+    /// AWS KMS backend summary
+    Aws {
+        /// Configured region, when pinned instead of resolved by the AWS chain
+        region: Option<String>,
+        /// Endpoint override, when set for an emulator or private endpoint
+        endpoint_url: Option<String>,
+    },
 }
 
 impl From<&KmsConfig> for KmsConfigSummary {
@@ -467,6 +474,10 @@ impl From<&KmsConfig> for KmsConfigSummary {
             BackendConfig::Static(static_config) => BackendSummary::Static {
                 key_id: static_config.key_id.clone(),
             },
+            BackendConfig::Aws(aws_config) => BackendSummary::Aws {
+                region: aws_config.region.clone(),
+                endpoint_url: aws_config.endpoint_url.clone(),
+            },
         };
 
         Self {

crates/kms/src/audit.rs

@@ -0,0 +1,423 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Audit contract for KMS management operations.
+//!
+//! [`KmsManager`](crate::manager::KmsManager) builds a [`KmsAuditRecord`] for
+//! every management operation it serves and hands it to the installed
+//! [`KmsAuditSink`]. No sink is installed by default, so a deployment that
+//! does not consume KMS audit records pays nothing beyond the `Option` check.
+//!
+//! The record is deliberately a KMS-side type rather than an audit-crate
+//! entry: keeping the dependency edge out of this crate lets the server map
+//! records onto its existing audit pipeline (and its delivery semantics)
+//! without KMS having to know that pipeline exists.
+//!
+//! # Redaction
+//!
+//! A record has no field that can hold key material, and every value that
+//! originates from a caller passes through [`redact_encryption_context`]
+//! before it is stored. Adding a field here means re-checking that invariant.
+
+use crate::error::KmsError;
+use crate::types::OperationContext;
+use rustfs_s3_types::EventName;
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, HashMap};
+use std::time::Duration;
+use uuid::Uuid;
+
+/// Encryption-context keys that describe *where* an object lives rather than
+/// anything secret about it. Their values are recorded verbatim; every other
+/// key is reduced to a digest.
+const ENCRYPTION_CONTEXT_ALLOWLIST: [&str; 5] = ["bucket", "object", "object_key", "algorithm", "sse_type"];
+
+/// Prefix marking a value that was replaced by a digest of itself.
+const DIGEST_PREFIX: &str = "sha256:";
+
+/// Hex characters of the digest that are kept. Enough to correlate repeated
+/// values across records without being reversible in practice.
+const DIGEST_LEN: usize = 16;
+
+/// A KMS management operation that produces an audit record.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum KmsAuditOperation {
+    /// Creation of a new master key.
+    CreateKey,
+    /// Metadata lookup for a single key.
+    DescribeKey,
+    /// Enumeration of keys.
+    ListKeys,
+    /// Scheduling a key for deletion after the pending window.
+    ScheduleKeyDeletion,
+    /// Cancelling a previously scheduled deletion.
+    CancelKeyDeletion,
+    /// Returning a disabled key to service.
+    EnableKey,
+    /// Taking a key out of service without destroying it.
+    DisableKey,
+    /// Rotating a key to a new version.
+    RotateKey,
+    /// Irreversible removal of key material once the pending window expired.
+    DeleteKey,
+}
+
+impl KmsAuditOperation {
+    /// Stable operation name for audit consumers.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            KmsAuditOperation::CreateKey => "CreateKey",
+            KmsAuditOperation::DescribeKey => "DescribeKey",
+            KmsAuditOperation::ListKeys => "ListKeys",
+            KmsAuditOperation::ScheduleKeyDeletion => "ScheduleKeyDeletion",
+            KmsAuditOperation::CancelKeyDeletion => "CancelKeyDeletion",
+            KmsAuditOperation::EnableKey => "EnableKey",
+            KmsAuditOperation::DisableKey => "DisableKey",
+            KmsAuditOperation::RotateKey => "RotateKey",
+            KmsAuditOperation::DeleteKey => "DeleteKey",
+        }
+    }
+
+    /// Notification/audit event name carried by records for this operation.
+    pub fn event_name(&self) -> EventName {
+        match self {
+            KmsAuditOperation::CreateKey => EventName::KmsKeyCreated,
+            KmsAuditOperation::DescribeKey | KmsAuditOperation::ListKeys => EventName::KmsKeyAccessed,
+            KmsAuditOperation::ScheduleKeyDeletion => EventName::KmsKeyDeletionScheduled,
+            KmsAuditOperation::CancelKeyDeletion => EventName::KmsKeyDeletionCancelled,
+            KmsAuditOperation::EnableKey => EventName::KmsKeyEnabled,
+            KmsAuditOperation::DisableKey => EventName::KmsKeyDisabled,
+            KmsAuditOperation::RotateKey => EventName::KmsKeyRotated,
+            KmsAuditOperation::DeleteKey => EventName::KmsKeyDeleted,
+        }
+    }
+}
+
+/// Whether the audited operation completed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum KmsAuditOutcome {
+    /// The operation completed and its effect is durable.
+    Success,
+    /// The operation failed; `error_class` carries the reason category.
+    Failure,
+}
+
+impl KmsAuditOutcome {
+    /// Stable outcome name for audit consumers.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            KmsAuditOutcome::Success => "success",
+            KmsAuditOutcome::Failure => "failure",
+        }
+    }
+}
+
+/// Coarse, stable classification of a failed KMS operation.
+///
+/// Audit consumers alert on these, so the strings are a wire contract: add
+/// new classes rather than renaming existing ones.
+pub fn error_class(error: &KmsError) -> &'static str {
+    match error {
+        KmsError::ConfigurationError { .. } => "configuration",
+        KmsError::KeyNotFound { .. } => "key_not_found",
+        KmsError::InvalidKey { .. } => "invalid_key",
+        KmsError::CryptographicError { .. } => "cryptographic",
+        KmsError::BackendError { .. } => "backend",
+        KmsError::AccessDenied { .. } => "access_denied",
+        KmsError::KeyAlreadyExists { .. } => "key_already_exists",
+        KmsError::InvalidOperation { .. } => "invalid_operation",
+        KmsError::InternalError { .. } => "internal",
+        KmsError::SerializationError { .. } => "serialization",
+        KmsError::IoError { .. } => "io",
+        KmsError::CacheError { .. } => "cache",
+        KmsError::ValidationError { .. } => "validation",
+        KmsError::UnsupportedAlgorithm { .. } => "unsupported_algorithm",
+        KmsError::InvalidKeySize { .. } => "invalid_key_size",
+        KmsError::ContextMismatch { .. } => "context_mismatch",
+        KmsError::OperationTimedOut { .. } => "timeout",
+        KmsError::OperationCancelled { .. } => "cancelled",
+        KmsError::MaterialMissing { .. } => "material_missing",
+        KmsError::MaterialCorrupt { .. } => "material_corrupt",
+        KmsError::MaterialAuthenticationFailed { .. } => "material_authentication_failed",
+        KmsError::UnsupportedFormatVersion { .. } => "unsupported_format_version",
+        KmsError::KeyVersionNotFound { .. } => "key_version_not_found",
+        KmsError::Backup(_) => "backup",
+        KmsError::UnsupportedCapability { .. } => "unsupported_capability",
+        KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
+    }
+}
+
+/// One audited KMS management operation.
+///
+/// Everything an audit consumer needs to answer "who did what to which key,
+/// against which backend, and did it work" — and nothing that could carry key
+/// material. See the module docs for the redaction rules.
+///
+/// Tenant attribution is intentionally absent: multi-tenancy is not modelled
+/// yet, and an invented tenant value is worse than a missing one. It will
+/// arrive as an entry in [`Self::context`] once tenancy lands.
+#[derive(Debug, Clone)]
+pub struct KmsAuditRecord {
+    /// Correlates every record emitted for one logical request.
+    pub operation_id: Uuid,
+    /// The audited operation.
+    pub operation: KmsAuditOperation,
+    /// Event name published to audit consumers.
+    pub event: EventName,
+    /// Authenticated identity that requested the operation, or
+    /// [`OperationContext::INTERNAL_PRINCIPAL`] for server-initiated work.
+    pub principal: String,
+    /// Client address, when the caller supplied one.
+    pub source_ip: Option<String>,
+    /// Client user agent, when the caller supplied one.
+    pub user_agent: Option<String>,
+    /// Key the operation acted on. `None` for operations that span keys, such
+    /// as listing.
+    pub key_id: Option<String>,
+    /// Key version the operation resolved to, when the result identifies one.
+    /// Operations on a key as a whole leave this unset.
+    pub key_version: Option<u32>,
+    /// Whether the operation succeeded.
+    pub outcome: KmsAuditOutcome,
+    /// Failure category; `None` on success. See [`error_class`].
+    pub error_class: Option<&'static str>,
+    /// Backend that served the operation, e.g. `local` or `vault-transit`.
+    pub backend: &'static str,
+    /// Wall-clock time spent in the audited operation.
+    pub latency: Duration,
+    /// Retries performed above the audit point. `None` means the number is
+    /// not observable here: backend-internal retries are accounted for by the
+    /// `rustfs_kms_backend_operation_attempts` metric instead of being
+    /// duplicated (and possibly contradicted) in the audit trail.
+    pub retry_count: Option<u32>,
+    /// Server-supplied correlation values carried by the operation context.
+    /// Also the reserved slot for tenant attribution.
+    pub context: BTreeMap<String, String>,
+    /// Caller-supplied encryption context, after [`redact_encryption_context`].
+    pub encryption_context: BTreeMap<String, String>,
+}
+
+impl KmsAuditRecord {
+    /// Start a record for `operation` performed under `context`.
+    ///
+    /// The outcome defaults to failure so that a record which somehow escapes
+    /// without [`Self::with_result`] under-reports success rather than
+    /// inventing it.
+    pub fn new(operation: KmsAuditOperation, context: &OperationContext, backend: &'static str) -> Self {
+        Self {
+            operation_id: context.operation_id,
+            operation,
+            event: operation.event_name(),
+            principal: context.principal.clone(),
+            source_ip: context.source_ip.clone(),
+            user_agent: context.user_agent.clone(),
+            key_id: None,
+            key_version: None,
+            outcome: KmsAuditOutcome::Failure,
+            error_class: None,
+            backend,
+            latency: Duration::ZERO,
+            retry_count: None,
+            context: context
+                .additional_context
+                .iter()
+                .map(|(k, v)| (k.clone(), v.clone()))
+                .collect(),
+            encryption_context: BTreeMap::new(),
+        }
+    }
+
+    /// Attach the key the operation acted on.
+    pub fn with_key_id(mut self, key_id: Option<impl Into<String>>) -> Self {
+        self.key_id = key_id.map(Into::into);
+        self
+    }
+
+    /// Attach the key version the operation resolved to.
+    pub fn with_key_version(mut self, key_version: Option<u32>) -> Self {
+        self.key_version = key_version;
+        self
+    }
+
+    /// Attach the measured duration of the operation.
+    pub fn with_latency(mut self, latency: Duration) -> Self {
+        self.latency = latency;
+        self
+    }
+
+    /// Derive outcome and error class from the operation result.
+    pub fn with_result<T>(mut self, result: &crate::error::Result<T>) -> Self {
+        match result {
+            Ok(_) => {
+                self.outcome = KmsAuditOutcome::Success;
+                self.error_class = None;
+            }
+            Err(error) => {
+                self.outcome = KmsAuditOutcome::Failure;
+                self.error_class = Some(error_class(error));
+            }
+        }
+        self
+    }
+
+    /// Attach the caller-supplied encryption context, redacted.
+    ///
+    /// Redaction happens here rather than at the call site so no caller can
+    /// place a raw context value into a record by accident.
+    pub fn with_encryption_context(mut self, encryption_context: &HashMap<String, String>) -> Self {
+        self.encryption_context = redact_encryption_context(encryption_context);
+        self
+    }
+}
+
+/// Reduce a caller-supplied encryption context to something safe to persist.
+///
+/// Keys naming the object's location are kept verbatim because they are the
+/// reason the context is audited at all. Every other value is replaced by a
+/// truncated digest, which still correlates repeated values across records
+/// but does not reproduce whatever the caller put there.
+pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -> BTreeMap<String, String> {
+    encryption_context
+        .iter()
+        .map(|(key, value)| {
+            let recorded = if ENCRYPTION_CONTEXT_ALLOWLIST.contains(&key.as_str()) {
+                value.clone()
+            } else {
+                digest_value(value)
+            };
+            (key.clone(), recorded)
+        })
+        .collect()
+}
+
+fn digest_value(value: &str) -> String {
+    let digest = hex::encode(Sha256::digest(value.as_bytes()));
+    format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
+}
+
+/// Receives KMS audit records.
+///
+/// Implementations must not block: they are called on the task that served
+/// the KMS operation. Delivery failures are the sink's problem — the KMS
+/// operation has already completed by the time a record is emitted, and its
+/// result is never changed by what the sink does.
+pub trait KmsAuditSink: Send + Sync {
+    /// Handle one audit record.
+    fn emit(&self, record: KmsAuditRecord);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn every_operation_maps_to_a_kms_event() {
+        let operations = [
+            KmsAuditOperation::CreateKey,
+            KmsAuditOperation::DescribeKey,
+            KmsAuditOperation::ListKeys,
+            KmsAuditOperation::ScheduleKeyDeletion,
+            KmsAuditOperation::CancelKeyDeletion,
+            KmsAuditOperation::EnableKey,
+            KmsAuditOperation::DisableKey,
+            KmsAuditOperation::RotateKey,
+            KmsAuditOperation::DeleteKey,
+        ];
+
+        for operation in operations {
+            let event = operation.event_name();
+            assert!(event.is_kms(), "{} must map to a KMS event, got {event}", operation.as_str());
+        }
+    }
+
+    #[test]
+    fn record_defaults_to_failure_until_a_result_is_attached() {
+        let context = OperationContext::new("tester".to_string());
+        let record = KmsAuditRecord::new(KmsAuditOperation::CreateKey, &context, "local");
+
+        assert_eq!(record.outcome, KmsAuditOutcome::Failure);
+        assert!(record.error_class.is_none());
+
+        let ok: crate::error::Result<()> = Ok(());
+        assert_eq!(record.clone().with_result(&ok).outcome, KmsAuditOutcome::Success);
+
+        let denied: crate::error::Result<()> = Err(KmsError::access_denied("nope"));
+        let failed = record.with_result(&denied);
+        assert_eq!(failed.outcome, KmsAuditOutcome::Failure);
+        assert_eq!(failed.error_class, Some("access_denied"));
+    }
+
+    #[test]
+    fn record_carries_the_full_operation_context() {
+        let context = OperationContext::new("arn:aws:iam::user/alice".to_string())
+            .with_source_ip("10.0.0.7".to_string())
+            .with_user_agent("aws-cli/2".to_string())
+            .with_context("requestID".to_string(), "req-1".to_string());
+
+        let record = KmsAuditRecord::new(KmsAuditOperation::RotateKey, &context, "vault-transit")
+            .with_key_id(Some("key-1"))
+            .with_key_version(Some(3))
+            .with_latency(Duration::from_millis(12));
+
+        assert_eq!(record.operation_id, context.operation_id);
+        assert_eq!(record.principal, "arn:aws:iam::user/alice");
+        assert_eq!(record.source_ip.as_deref(), Some("10.0.0.7"));
+        assert_eq!(record.user_agent.as_deref(), Some("aws-cli/2"));
+        assert_eq!(record.key_id.as_deref(), Some("key-1"));
+        assert_eq!(record.key_version, Some(3));
+        assert_eq!(record.backend, "vault-transit");
+        assert_eq!(record.latency, Duration::from_millis(12));
+        assert_eq!(record.event, EventName::KmsKeyRotated);
+        assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-1"));
+    }
+
+    #[test]
+    fn encryption_context_keeps_only_allowlisted_values_verbatim() {
+        let secret = "eyJhbGciOiJIUzI1NiJ9.super-secret-grant-token";
+        let context = HashMap::from([
+            ("bucket".to_string(), "photos".to_string()),
+            ("object_key".to_string(), "2026/cat.png".to_string()),
+            ("algorithm".to_string(), "AES256".to_string()),
+            ("grant_token".to_string(), secret.to_string()),
+            ("customer_reference".to_string(), "acct-4711".to_string()),
+        ]);
+
+        let redacted = redact_encryption_context(&context);
+
+        assert_eq!(redacted.get("bucket").map(String::as_str), Some("photos"));
+        assert_eq!(redacted.get("object_key").map(String::as_str), Some("2026/cat.png"));
+        assert_eq!(redacted.get("algorithm").map(String::as_str), Some("AES256"));
+
+        for key in ["grant_token", "customer_reference"] {
+            let value = redacted.get(key).expect("non-allowlisted key should be kept as a digest");
+            assert!(value.starts_with(DIGEST_PREFIX), "{key} should be digested, got {value}");
+        }
+
+        let rendered = format!("{redacted:?}");
+        assert!(!rendered.contains(secret), "redacted context must not reproduce the raw value");
+        assert!(!rendered.contains("acct-4711"), "redacted context must not reproduce the raw value");
+    }
+
+    #[test]
+    fn identical_values_digest_identically_across_records() {
+        // Correlation is the reason a digest is preferable to a fixed
+        // placeholder; assert it actually holds.
+        let first = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
+        let second = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
+        let other = redact_encryption_context(&HashMap::from([("tenant".to_string(), "beta".to_string())]));
+
+        assert_eq!(first.get("tenant"), second.get("tenant"));
+        assert_ne!(first.get("tenant"), other.get("tenant"));
+    }
+}

crates/kms/src/backends/local.rs

@@ -14,7 +14,10 @@
 
 //! Local file-based KMS backend implementation
 
-use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits};
+use crate::backends::{
+    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
+    ensure_tag_keys_are_mutable,
+};
 use crate::config::KmsConfig;
 use crate::config::LocalConfig;
 use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -1294,6 +1297,32 @@ impl LocalKmsClient {
         Ok(())
     }
 
+    /// Read-modify-write of a key's mutable metadata under the per-key write
+    /// lock, carrying the existing key material over untouched.
+    ///
+    /// `mutate` reports whether it changed anything; an unchanged record is
+    /// never rewritten, so a repeated no-op update neither rewrites the key
+    /// file nor risks a failed commit on an unrelated call.
+    async fn update_key_metadata<F>(&self, key_id: &str, mutate: F) -> Result<()>
+    where
+        F: FnOnce(&mut MasterKeyInfo) -> Result<bool>,
+    {
+        let _write_guard = self.lock_key_for_write(key_id).await;
+        let mut master_key = self.load_master_key(key_id).await?;
+        if !mutate(&mut master_key)? {
+            return Ok(());
+        }
+
+        // Preserve the existing key material (see enable_key): a metadata edit
+        // must never regenerate the master key, or every DEK wrapped by it
+        // becomes undecryptable.
+        let key_material = self.get_key_material(key_id).await?;
+        self.save_master_key(&master_key, &key_material).await?;
+
+        debug!(key_id, "Local KMS key metadata updated");
+        Ok(())
+    }
+
     /// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
     #[cfg(test)]
     pub(crate) async fn schedule_key_deletion(
@@ -1386,7 +1415,8 @@ impl LocalKmsBackend {
             crate::config::BackendConfig::Local(local_config) => local_config.clone(),
             crate::config::BackendConfig::VaultKv2(_)
             | crate::config::BackendConfig::VaultTransit(_)
-            | crate::config::BackendConfig::Static(_) => {
+            | crate::config::BackendConfig::Static(_)
+            | crate::config::BackendConfig::Aws(_) => {
                 return Err(KmsError::configuration_error("Expected Local backend configuration"));
             }
         };
@@ -1411,12 +1441,15 @@ impl KmsBackend for LocalKmsBackend {
             // Generate key material
             let key_material = generate_key_material(algorithm)?;
 
-            let master_key = MasterKeyInfo::new_with_description(
+            let mut master_key = MasterKeyInfo::new_with_description(
                 key_id.clone(),
                 algorithm.to_string(),
                 Some("local-kms".to_string()),
                 request.description.clone(),
             );
+            // Persist the caller's tags: the response below reports them, and
+            // describe_key reads them back out of this record.
+            master_key.metadata = request.tags.clone();
 
             // Save to disk
             self.client.save_new_master_key(&master_key, &key_material).await?;
@@ -1684,6 +1717,44 @@ impl KmsBackend for LocalKmsBackend {
         self.client.disable_key(key_id, None).await
     }
 
+    async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
+        self.client
+            .update_key_metadata(key_id, |master_key| {
+                if master_key.description.as_deref() == description {
+                    return Ok(false);
+                }
+                master_key.description = description.map(str::to_string);
+                Ok(true)
+            })
+            .await
+    }
+
+    async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
+        ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
+        self.client
+            .update_key_metadata(key_id, |master_key| {
+                let mut changed = false;
+                for (tag_key, value) in tags {
+                    changed |= master_key.metadata.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
+                }
+                Ok(changed)
+            })
+            .await
+    }
+
+    async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
+        ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
+        self.client
+            .update_key_metadata(key_id, |master_key| {
+                let mut changed = false;
+                for tag_key in tag_keys {
+                    changed |= master_key.metadata.remove(tag_key).is_some();
+                }
+                Ok(changed)
+            })
+            .await
+    }
+
     async fn health_check(&self) -> Result<bool> {
         self.client.health_check().await.map(|_| true)
     }
@@ -1696,6 +1767,7 @@ impl KmsBackend for LocalKmsBackend {
             .with_enable_disable(true)
             .with_schedule_deletion(true)
             .with_physical_delete(true)
+            .with_update_key_metadata(true)
     }
 
     async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {

crates/kms/src/backends/mod.rs

@@ -19,7 +19,9 @@ use crate::types::*;
 use async_trait::async_trait;
 use jiff::Zoned;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 
+pub mod aws;
 #[cfg(test)]
 mod contract_tests;
 pub mod local;
@@ -98,6 +100,32 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat
     ensure_key_state_permits(key_id, &state, operation)
 }
 
+/// Tag key that carries the key's identity rather than user metadata.
+///
+/// The key-creation path lifts `name` out of the caller's tag map and uses it
+/// as the key id, so a key's `name` tag and its id are the same string.
+/// Rewriting or dropping it after creation would leave the key addressable
+/// under an id its own metadata no longer states. Metadata updates therefore
+/// reject it; only creation may set it.
+pub const RESERVED_KEY_NAME_TAG: &str = "name";
+
+/// Reject a metadata update that would rewrite or remove
+/// [`RESERVED_KEY_NAME_TAG`].
+///
+/// Enforced by every backend that implements tag updates, so no call path —
+/// including a direct [`KmsBackend`] user that bypasses the manager — can
+/// detach a key from its identity.
+pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item = &'a str>) -> Result<()> {
+    for tag_key in tag_keys {
+        if tag_key == RESERVED_KEY_NAME_TAG {
+            return Err(KmsError::invalid_parameter(format!(
+                "Tag '{RESERVED_KEY_NAME_TAG}' identifies the key and cannot be updated or removed"
+            )));
+        }
+    }
+    Ok(())
+}
+
 /// Simplified KMS backend interface for manager
 #[async_trait]
 pub trait KmsBackend: Send + Sync {
@@ -153,6 +181,38 @@ pub trait KmsBackend: Send + Sync {
         Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
     }
 
+    /// Replace a key's free-form description; `None` clears it.
+    ///
+    /// Backends that advertise [`BackendCapabilities::update_key_metadata`]
+    /// must override this method; the default rejects the operation.
+    async fn update_key_description(&self, _key_id: &str, _description: Option<&str>) -> Result<()> {
+        Err(KmsError::unsupported_capability(
+            "backend without key metadata updates",
+            "update_key_description",
+        ))
+    }
+
+    /// Add or overwrite the given tags, leaving every other tag untouched.
+    ///
+    /// Implementations must run [`ensure_tag_keys_are_mutable`] before
+    /// persisting anything. Backends that advertise
+    /// [`BackendCapabilities::update_key_metadata`] must override this method;
+    /// the default rejects the operation.
+    async fn tag_key(&self, _key_id: &str, _tags: &HashMap<String, String>) -> Result<()> {
+        Err(KmsError::unsupported_capability("backend without key metadata updates", "tag_key"))
+    }
+
+    /// Remove the given tags.
+    ///
+    /// Tags that are not set are ignored, so repeating the call is a no-op
+    /// rather than an error. Implementations must run
+    /// [`ensure_tag_keys_are_mutable`] before persisting anything. Backends
+    /// that advertise [`BackendCapabilities::update_key_metadata`] must
+    /// override this method; the default rejects the operation.
+    async fn untag_key(&self, _key_id: &str, _tag_keys: &[String]) -> Result<()> {
+        Err(KmsError::unsupported_capability("backend without key metadata updates", "untag_key"))
+    }
+
     /// Health check
     async fn health_check(&self) -> Result<bool>;
 
@@ -222,6 +282,8 @@ pub struct BackendCapabilities {
     pub versioning: bool,
     /// Irreversible physical deletion of key material
     pub physical_delete: bool,
+    /// Updating a key's description and tags after creation
+    pub update_key_metadata: bool,
 }
 
 impl BackendCapabilities {
@@ -238,6 +300,7 @@ impl BackendCapabilities {
             schedule_deletion: false,
             versioning: false,
             physical_delete: false,
+            update_key_metadata: false,
         }
     }
 
@@ -288,6 +351,12 @@ impl BackendCapabilities {
         self.physical_delete = physical_delete;
         self
     }
+
+    /// Set whether description and tag updates are supported
+    pub const fn with_update_key_metadata(mut self, update_key_metadata: bool) -> Self {
+        self.update_key_metadata = update_key_metadata;
+        self
+    }
 }
 
 impl Default for BackendCapabilities {
@@ -366,6 +435,7 @@ mod tests {
         assert!(!capabilities.schedule_deletion);
         assert!(!capabilities.versioning);
         assert!(!capabilities.physical_delete);
+        assert!(!capabilities.update_key_metadata);
     }
 
     #[tokio::test]
@@ -374,6 +444,12 @@ mod tests {
             ("enable_key", MinimalBackend.enable_key("any-key").await),
             ("disable_key", MinimalBackend.disable_key("any-key").await),
             ("rotate_key", MinimalBackend.rotate_key("any-key").await),
+            (
+                "update_key_description",
+                MinimalBackend.update_key_description("any-key", Some("new")).await,
+            ),
+            ("tag_key", MinimalBackend.tag_key("any-key", &HashMap::new()).await),
+            ("untag_key", MinimalBackend.untag_key("any-key", &[]).await),
         ] {
             let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
             assert!(
@@ -395,6 +471,20 @@ mod tests {
         );
     }
 
+    #[test]
+    fn identity_tag_is_rejected_by_metadata_updates() {
+        let error = ensure_tag_keys_are_mutable([RESERVED_KEY_NAME_TAG])
+            .expect_err("the identity tag must not be writable through a metadata update");
+        assert!(
+            matches!(&error, KmsError::InvalidOperation { message } if message.contains(RESERVED_KEY_NAME_TAG)),
+            "expected a typed rejection naming the tag, got {error:?}"
+        );
+
+        // Ordinary tags — including ones that merely contain the reserved name
+        // — stay writable.
+        ensure_tag_keys_are_mutable(["team", "nickname", "Name"]).expect("ordinary tags must remain writable");
+    }
+
     #[tokio::test]
     async fn local_backend_capabilities_golden() {
         let temp_dir = tempfile::tempdir().expect("temp dir should be created");

crates/kms/src/backends/snapshots/rustfs_kms__backends__aws__tests__aws_backend_capabilities.snap

@@ -0,0 +1,15 @@
+---
+source: crates/kms/src/backends/aws.rs
+expression: capabilities_snapshot(backend.capabilities())
+---
+{
+  "decrypt": true,
+  "enable_disable": true,
+  "encrypt": true,
+  "generate_data_key": true,
+  "physical_delete": false,
+  "rotate": true,
+  "schedule_deletion": true,
+  "update_key_metadata": false,
+  "versioning": true
+}

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap

@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": true,
   "rotate": false,
   "schedule_deletion": true,
+  "update_key_metadata": true,
   "versioning": false
 }

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap

@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": false,
   "rotate": false,
   "schedule_deletion": false,
+  "update_key_metadata": false,
   "versioning": false
 }

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap

@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": true,
   "rotate": true,
   "schedule_deletion": true,
+  "update_key_metadata": true,
   "versioning": true
 }

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap

@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": true,
   "rotate": true,
   "schedule_deletion": true,
+  "update_key_metadata": true,
   "versioning": true
 }

crates/kms/src/backends/static_kms.rs

@@ -431,6 +431,32 @@ mod tests {
         (backend, key_id, key)
     }
 
+    #[tokio::test]
+    async fn metadata_updates_report_the_capability_gap() {
+        let (backend, key_id, _key) = create_test_backend().await;
+        assert!(
+            !backend.capabilities().update_key_metadata,
+            "Static KMS owns no mutable key record, so it must not advertise metadata updates"
+        );
+
+        for (operation, result) in [
+            ("update_key_description", backend.update_key_description(&key_id, Some("new")).await),
+            (
+                "tag_key",
+                backend
+                    .tag_key(&key_id, &HashMap::from([("team".to_string(), "storage".to_string())]))
+                    .await,
+            ),
+            ("untag_key", backend.untag_key(&key_id, &["team".to_string()]).await),
+        ] {
+            let error = result.expect_err("Static KMS is read-only: metadata updates must be rejected");
+            assert!(
+                matches!(error, KmsError::UnsupportedCapability { .. }),
+                "expected UnsupportedCapability for {operation}, got {error:?}"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_generate_and_decrypt_data_key() {
         let (backend, key_id, _key) = create_test_backend().await;

crates/kms/src/backends/vault.rs

@@ -20,6 +20,7 @@ use crate::backends::vault_credentials::{
 };
 use crate::backends::{
     BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
+    ensure_tag_keys_are_mutable,
 };
 use crate::config::{KmsConfig, VaultConfig};
 use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -1020,6 +1021,67 @@ impl VaultKmsClient {
         Ok(())
     }
 
+    /// Replace the key's description; `None` clears it.
+    ///
+    /// The write goes through the check-and-set read-modify-write loop, so a
+    /// rotation or state transition landing in between is carried over instead
+    /// of clobbered. A description that already matches is not rewritten.
+    pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
+        self.update_key_data_with_cas(key_id, |key_data| {
+            if key_data.description.as_deref() == description {
+                return Ok(CasMutation::Skip(()));
+            }
+            key_data.description = description.map(str::to_string);
+            Ok(CasMutation::Write(()))
+        })
+        .await?;
+
+        debug!(key_id, "Vault KMS key description updated");
+        Ok(())
+    }
+
+    /// Add or overwrite tags, leaving every other tag untouched.
+    pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
+        ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
+
+        self.update_key_data_with_cas(key_id, |key_data| {
+            let mut changed = false;
+            for (tag_key, value) in tags {
+                changed |= key_data.tags.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
+            }
+            Ok(if changed {
+                CasMutation::Write(())
+            } else {
+                CasMutation::Skip(())
+            })
+        })
+        .await?;
+
+        debug!(key_id, "Vault KMS key tags updated");
+        Ok(())
+    }
+
+    /// Remove tags; tags that are not set are ignored.
+    pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
+        ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
+
+        self.update_key_data_with_cas(key_id, |key_data| {
+            let mut changed = false;
+            for tag_key in tag_keys {
+                changed |= key_data.tags.remove(tag_key).is_some();
+            }
+            Ok(if changed {
+                CasMutation::Write(())
+            } else {
+                CasMutation::Skip(())
+            })
+        })
+        .await?;
+
+        debug!(key_id, "Vault KMS key tags removed");
+        Ok(())
+    }
+
     /// Rotate the master key while keeping every historical version decryptable.
     ///
     /// Commit protocol (all writes check-and-set, in this order):
@@ -1163,7 +1225,8 @@ impl VaultKmsBackend {
             crate::config::BackendConfig::VaultKv2(vault_config) => (**vault_config).clone(),
             crate::config::BackendConfig::Local(_)
             | crate::config::BackendConfig::VaultTransit(_)
-            | crate::config::BackendConfig::Static(_) => {
+            | crate::config::BackendConfig::Static(_)
+            | crate::config::BackendConfig::Aws(_) => {
                 return Err(KmsError::configuration_error("Expected Vault KV2 backend configuration"));
             }
         };
@@ -1446,6 +1509,18 @@ impl KmsBackend for VaultKmsBackend {
         self.client.rotate_key(key_id, None).await.map(|_| ())
     }
 
+    async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
+        self.client.update_key_description(key_id, description).await
+    }
+
+    async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
+        self.client.tag_key(key_id, tags).await
+    }
+
+    async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
+        self.client.untag_key(key_id, tag_keys).await
+    }
+
     async fn health_check(&self) -> Result<bool> {
         self.client.health_check().await.map(|_| true)
     }
@@ -1461,6 +1536,7 @@ impl KmsBackend for VaultKmsBackend {
             .with_schedule_deletion(true)
             .with_versioning(true)
             .with_physical_delete(true)
+            .with_update_key_metadata(true)
     }
 
     async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -2844,6 +2920,63 @@ mod tests {
         );
     }
 
+    /// Tag updates are check-and-set read-modify-writes over the live record,
+    /// never blind overwrites: they preserve the material and the tags they did
+    /// not address.
+    #[tokio::test]
+    async fn wired_tag_key_writeback_is_check_and_set() {
+        let mut key_data = healthy_key_data();
+        key_data.tags = HashMap::from([("name".to_string(), "wired-key".to_string())]);
+        let (vault, client) = scripted_client(vec![
+            ScriptedResponse::ok(kv2_metadata_read_data(1)),
+            ScriptedResponse::ok(kv2_read_data(&key_data)),
+            ScriptedResponse::ok(kv2_write_ack()),
+        ])
+        .await;
+
+        client
+            .tag_key("wired-key", &HashMap::from([("team".to_string(), "storage".to_string())]))
+            .await
+            .expect("tagging must succeed");
+
+        let bodies = vault.request_bodies();
+        let writeback = parse_write_body(&bodies[2]);
+        assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}");
+        assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}");
+        assert_eq!(
+            writeback["data"]["tags"]["name"],
+            serde_json::json!("wired-key"),
+            "a tag update must not drop tags it did not address: {writeback}"
+        );
+        assert_eq!(
+            writeback["data"]["encrypted_key_material"],
+            serde_json::json!(healthy_key_data().encrypted_key_material),
+            "the write-back must preserve the material of the freshly read record: {writeback}"
+        );
+    }
+
+    /// Rejecting the identity tag happens before any Vault call, so a rejected
+    /// request cannot leave a partial write behind.
+    #[tokio::test]
+    async fn wired_identity_tag_update_is_rejected_before_any_vault_call() {
+        let (vault, client) = scripted_client(Vec::new()).await;
+
+        for result in [
+            client
+                .tag_key("wired-key", &HashMap::from([("name".to_string(), "other".to_string())]))
+                .await,
+            client.untag_key("wired-key", &["name".to_string()]).await,
+        ] {
+            let error = result.expect_err("the identity tag must not be writable");
+            assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}");
+        }
+        assert!(
+            vault.request_bodies().is_empty(),
+            "a rejected metadata update must not reach Vault: {:?}",
+            vault.request_bodies()
+        );
+    }
+
     /// A version record above the current pointer means the top-level record
     /// regressed (a lost update rolled back a committed rotation). Resolving
     /// material through such a record must fail closed instead of quietly

crates/kms/src/backends/vault_credentials.rs

@@ -57,6 +57,42 @@ const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5);
 /// Default seconds between token file re-reads for [`TokenFileSource`].
 const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30;
 
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// Both gauges describe the one credential generation currently installed, so
+// they carry no labels: the Vault address, mount, auth path and token are all
+// off limits as label values, and there is exactly one generation to describe.
+// The renewal loop republishes them on a bounded cadence while it waits, so a
+// scrape landing between refresh cycles never reads a TTL frozen at the last
+// refresh, or a fail-closed state that flipped after it.
+// ---------------------------------------------------------------------------
+
+/// Gauge: seconds left before the active Vault token expires; `0` once it has.
+const METRIC_TOKEN_TTL_SECONDS: &str = "rustfs_kms_vault_token_ttl_seconds";
+/// Gauge: `1` while [`VaultCredentialProvider::current`] refuses to hand out
+/// the token because it is inside the fail-closed safety window, `0` otherwise.
+const METRIC_CREDENTIALS_FAIL_CLOSED: &str = "rustfs_kms_vault_credentials_fail_closed";
+
+/// How often the renewal loop republishes the credential gauges while waiting.
+/// Bounds how stale a scrape can be, without any additional Vault traffic.
+const CREDENTIAL_GAUGE_INTERVAL: Duration = Duration::from_secs(10);
+
+/// Register metric descriptions once per process.
+fn describe_credential_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_gauge!(
+            METRIC_TOKEN_TTL_SECONDS,
+            "Seconds remaining before the Vault token backing the KMS backend expires"
+        );
+        metrics::describe_gauge!(
+            METRIC_CREDENTIALS_FAIL_CLOSED,
+            "1 while the Vault credential provider refuses to serve its token because the token is inside the fail-closed safety window"
+        );
+    });
+}
+
 /// A crate-owned secret value, zeroized on drop and redacted in Debug output.
 #[derive(Clone, Zeroize, ZeroizeOnDrop)]
 pub(crate) struct SecretString(String);
@@ -627,6 +663,45 @@ impl VaultCredentialProvider {
         Ok(handle)
     }
 
+    /// Publish the credential gauges for the generation currently installed.
+    ///
+    /// The fail-closed gauge re-evaluates the very gate
+    /// [`VaultCredentialProvider::current`] applies, so what operators see and
+    /// what the request path does cannot drift apart.
+    fn record_credential_gauges(&self) {
+        let handle = self.current.load();
+        let now = Instant::now();
+        let fail_closed = match handle.expires_at() {
+            Some(expires_at) => {
+                metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
+                now + self.policy.safety_window >= expires_at
+            }
+            // A generation without an expiry has no remaining TTL to report
+            // and can never lapse, so it can never fail closed either.
+            None => false,
+        };
+        metrics::gauge!(METRIC_CREDENTIALS_FAIL_CLOSED).set(if fail_closed { 1.0 } else { 0.0 });
+    }
+
+    /// Wait until `deadline`, republishing the credential gauges on the
+    /// observation cadence. Reports `false` when cancellation cut the wait
+    /// short.
+    async fn wait_publishing_gauges(&self, deadline: Instant, cancel: &CancellationToken) -> bool {
+        loop {
+            self.record_credential_gauges();
+            let now = Instant::now();
+            if now >= deadline {
+                return true;
+            }
+            let slice = (deadline - now).min(CREDENTIAL_GAUGE_INTERVAL);
+            tokio::select! {
+                biased;
+                _ = cancel.cancelled() => return false,
+                _ = tokio::time::sleep(slice) => {}
+            }
+        }
+    }
+
     /// Refresh the credentials if generation `observed` is still current.
     ///
     /// Single-flight: concurrent callers serialize on the refresh lock, and a
@@ -711,17 +786,22 @@ impl fmt::Debug for VaultCredentialProvider {
 /// again immediately (the renewal point is already in the past), so the
 /// provider keeps trying to recover even after the fail-closed window has
 /// been reached.
+///
+/// Both waits run through [`VaultCredentialProvider::wait_publishing_gauges`],
+/// which is the only place the credential gauges are published: the loop is
+/// already the component that tracks token expiry, and doing it here keeps the
+/// request path free of any metric work.
 async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: CancellationToken) {
+    describe_credential_metrics();
     loop {
         let handle = provider.snapshot();
         let Some(renew_at) = handle.renew_at() else {
             // The current generation never expires; nothing left to schedule.
+            provider.record_credential_gauges();
             return;
         };
-        tokio::select! {
-            biased;
-            _ = cancel.cancelled() => return,
-            _ = tokio::time::sleep_until(renew_at) => {}
+        if !provider.wait_publishing_gauges(renew_at, &cancel).await {
+            return;
         }
 
         match provider.refresh(handle.generation, &cancel).await {
@@ -733,10 +813,9 @@ async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: Cancellati
                     error = %error,
                     "Vault credential refresh failed; retrying until the credentials recover"
                 );
-                tokio::select! {
-                    biased;
-                    _ = cancel.cancelled() => return,
-                    _ = tokio::time::sleep(provider.policy.retry_interval) => {}
+                let retry_at = Instant::now() + provider.policy.retry_interval;
+                if !provider.wait_publishing_gauges(retry_at, &cancel).await {
+                    return;
                 }
             }
         }
@@ -1354,4 +1433,125 @@ mod tests {
         assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}");
         assert!(rendered.contains("vault-token"), "the file path is not a secret");
     }
+
+    // -- Metric emission ----------------------------------------------------
+    //
+    // Each test installs a thread-local debugging recorder and drives a
+    // paused-clock current-thread runtime inside it, so the renewal loop's
+    // gauge publications land on exact virtual timestamps.
+
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Run `test` on a paused current-thread runtime under a debugging
+    /// recorder and return one snapshot of everything it emitted.
+    ///
+    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
+    /// the recorded state, so taking it per assertion would only show the
+    /// first assertion any data.
+    fn record_metrics<Out>(
+        test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
+    ) -> (Vec<MetricEntry>, Out) {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let out = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_time()
+                .start_paused(true)
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (snapshotter.snapshot().into_vec(), out)
+    }
+
+    /// Last value of a gauge, plus the labels it was published with.
+    fn gauge(snapshot: &[MetricEntry], name: &str) -> Option<(f64, Vec<String>)> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(value)) => Some((
+                    value.into_inner(),
+                    composite.key().labels().map(|label| label.key().to_string()).collect(),
+                )),
+                _ => None,
+            }
+        })
+    }
+
+    #[test]
+    fn renewal_loop_republishes_token_ttl_while_it_waits() {
+        let (snapshot, ()) = record_metrics(|| {
+            Box::pin(async {
+                let (provider, _state) = scripted_provider(
+                    Duration::from_secs(60),
+                    true,
+                    test_policy(Duration::from_secs(10), Duration::from_secs(5)),
+                )
+                .await;
+                let task = provider.spawn_renewal_task().expect("lease-bound tokens need renewal");
+
+                // Well inside the first half of the lease: nothing has been
+                // renewed yet, so only the observation cadence can have moved
+                // the gauge off its initial 60s.
+                tokio::time::sleep(Duration::from_secs(25)).await;
+                task.shutdown().await;
+            })
+        });
+
+        let (ttl, labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
+        assert!(
+            (ttl - 40.0).abs() < 1.0,
+            "expected ~40s left of a 60s lease at the last observation, got {ttl}"
+        );
+        assert!(labels.is_empty(), "credential gauges must carry no labels, got {labels:?}");
+        assert_eq!(
+            gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
+            Some(0.0),
+            "a token outside its safety window must not report fail-closed"
+        );
+    }
+
+    #[test]
+    fn fail_closed_gauge_tracks_the_gate_the_request_path_applies() {
+        let (snapshot, refused) = record_metrics(|| {
+            Box::pin(async {
+                let (provider, state) = scripted_provider(
+                    Duration::from_secs(60),
+                    true,
+                    test_policy(Duration::from_secs(10), Duration::from_secs(5)),
+                )
+                .await;
+                state.fail_renew.store(true, Ordering::SeqCst);
+                state.fail_login.store(true, Ordering::SeqCst);
+                let task = provider.spawn_renewal_task().expect("renewal task");
+
+                // 60s lease minus the 10s safety window: by 51s the provider
+                // is refusing the token, and the gauge must already say so.
+                tokio::time::sleep(Duration::from_secs(51)).await;
+                let refused = provider.current().is_err();
+                task.shutdown().await;
+                refused
+            })
+        });
+
+        assert!(refused, "a token inside the safety window must be refused");
+        assert_eq!(
+            gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
+            Some(1.0),
+            "the gauge must report the same fail-closed state the request path enforces"
+        );
+        let (ttl, _labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
+        assert!(
+            (ttl - 10.0).abs() < 1.0,
+            "expected the TTL gauge to track the lease down into its safety window, got {ttl}"
+        );
+    }
 }

crates/kms/src/backends/vault_transit.rs

@@ -18,7 +18,10 @@ use crate::backends::vault_credentials::{
     CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider,
     token_source_for,
 };
-use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits};
+use crate::backends::{
+    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
+    ensure_tag_keys_are_mutable,
+};
 use crate::config::{KmsConfig, VaultTransitConfig};
 use crate::encryption::{DataKeyEnvelope, generate_key_material};
 use crate::error::{KmsError, Result};
@@ -908,6 +911,46 @@ impl VaultTransitKmsClient {
         .map(|_| ())
     }
 
+    /// Replace the key's description; `None` clears it.
+    ///
+    /// Metadata edits carry no state gate: they neither use nor invalidate key
+    /// material, so they stay available for whatever lifecycle state the key
+    /// is in.
+    pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
+        self.mutate_key_metadata(key_id, |metadata| {
+            metadata.description = description.map(str::to_string);
+            Ok(())
+        })
+        .await
+        .map(|_| ())
+    }
+
+    /// Add or overwrite tags, leaving every other tag untouched.
+    pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
+        ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
+        self.mutate_key_metadata(key_id, |metadata| {
+            metadata
+                .tags
+                .extend(tags.iter().map(|(key, value)| (key.clone(), value.clone())));
+            Ok(())
+        })
+        .await
+        .map(|_| ())
+    }
+
+    /// Remove tags; tags that are not set are ignored.
+    pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
+        ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
+        self.mutate_key_metadata(key_id, |metadata| {
+            for tag_key in tag_keys {
+                metadata.tags.remove(tag_key);
+            }
+            Ok(())
+        })
+        .await
+        .map(|_| ())
+    }
+
     /// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
     #[cfg(test)]
     pub(crate) async fn schedule_key_deletion(
@@ -1012,7 +1055,9 @@ impl VaultTransitKmsBackend {
                 metadata_key_prefix: vault_config.key_path_prefix.clone(),
                 tls: vault_config.tls.clone(),
             },
-            crate::config::BackendConfig::Local(_) | crate::config::BackendConfig::Static(_) => {
+            crate::config::BackendConfig::Local(_)
+            | crate::config::BackendConfig::Static(_)
+            | crate::config::BackendConfig::Aws(_) => {
                 return Err(KmsError::configuration_error("Expected Vault Transit backend configuration"));
             }
         };
@@ -1235,6 +1280,18 @@ impl KmsBackend for VaultTransitKmsBackend {
         self.client.rotate_key(key_id, None).await.map(|_| ())
     }
 
+    async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
+        self.client.update_key_description(key_id, description).await
+    }
+
+    async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
+        self.client.tag_key(key_id, tags).await
+    }
+
+    async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
+        self.client.untag_key(key_id, tag_keys).await
+    }
+
     async fn health_check(&self) -> Result<bool> {
         self.client.health_check().await.map(|_| true)
     }
@@ -1249,6 +1306,7 @@ impl KmsBackend for VaultTransitKmsBackend {
             .with_schedule_deletion(true)
             .with_versioning(true)
             .with_physical_delete(true)
+            .with_update_key_metadata(true)
     }
 
     async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {

crates/kms/src/cache.rs

@@ -16,11 +16,84 @@
 
 use crate::types::KeyMetadata;
 use moka::future::Cache;
+use moka::notification::RemovalCause;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::Duration;
 
+/// Default lifetime of a cached key metadata entry.
+const DEFAULT_METADATA_TTL: Duration = Duration::from_secs(300);
+
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// Lookup outcomes and removals are counted here because moka exposes neither
+// hit nor miss counts; the numbers below are the cache's own, not derived.
+// Label values are exclusively static strings (lookup result, removal cause) —
+// key identifiers and key metadata must never reach a metric label.
+// ---------------------------------------------------------------------------
+
+/// Counter: key metadata lookups, by `result` (`hit` or `miss`).
+const METRIC_CACHE_LOOKUPS_TOTAL: &str = "rustfs_kms_metadata_cache_lookups_total";
+/// Counter: entries dropped from the cache, by `cause` (`expired`, `size`,
+/// `explicit`, `replaced`); only `expired` and `size` are true evictions, the
+/// rest are invalidations driven by key lifecycle operations.
+const METRIC_CACHE_EVICTIONS_TOTAL: &str = "rustfs_kms_metadata_cache_evictions_total";
+/// Gauge: entries currently held by the cache. Republished whenever the entry
+/// set can have changed — every write path, plus the lookups that miss, since
+/// TTL expiry drops entries without any write taking place.
+const METRIC_CACHE_ENTRIES: &str = "rustfs_kms_metadata_cache_entries";
+
+/// Register metric descriptions once per process.
+fn describe_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_counter!(METRIC_CACHE_LOOKUPS_TOTAL, "Total KMS key metadata cache lookups, by hit or miss");
+        metrics::describe_counter!(
+            METRIC_CACHE_EVICTIONS_TOTAL,
+            "Total entries dropped from the KMS key metadata cache, by removal cause"
+        );
+        metrics::describe_gauge!(METRIC_CACHE_ENTRIES, "Entries currently held by the KMS key metadata cache");
+    });
+}
+
+fn removal_cause_label(cause: RemovalCause) -> &'static str {
+    match cause {
+        RemovalCause::Expired => "expired",
+        RemovalCause::Explicit => "explicit",
+        RemovalCause::Replaced => "replaced",
+        RemovalCause::Size => "size",
+    }
+}
+
+/// Cumulative cache counters. Shared with moka's eviction listener, which runs
+/// outside any `&self` borrow, hence the `Arc` and the atomics.
+#[derive(Debug, Default)]
+struct CacheCounters {
+    hits: AtomicU64,
+    misses: AtomicU64,
+    evictions: AtomicU64,
+}
+
+/// Snapshot of the KMS key metadata cache counters.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct KmsCacheStats {
+    /// Entries currently held. Eventually consistent: moka applies pending
+    /// maintenance lazily.
+    pub entries: u64,
+    /// Lookups served from the cache since process start.
+    pub hits: u64,
+    /// Lookups that fell through to the backend since process start.
+    pub misses: u64,
+    /// Entries dropped since process start, whatever the cause (expiry,
+    /// capacity, invalidation, replacement).
+    pub evictions: u64,
+}
+
 /// KMS cache for storing frequently accessed keys and metadata
 pub struct KmsCache {
     key_metadata_cache: Cache<String, KeyMetadata>,
+    counters: Arc<CacheCounters>,
 }
 
 impl KmsCache {
@@ -33,11 +106,26 @@ impl KmsCache {
     /// A new instance of `KmsCache`
     ///
     pub fn new(capacity: u64) -> Self {
+        Self::with_ttl(capacity, DEFAULT_METADATA_TTL)
+    }
+
+    /// Create a new KMS cache with an explicit metadata time-to-live
+    fn with_ttl(capacity: u64, metadata_ttl: Duration) -> Self {
+        describe_metrics();
+
+        let counters = Arc::new(CacheCounters::default());
+        let eviction_counters = Arc::clone(&counters);
+
         Self {
             key_metadata_cache: Cache::builder()
                 .max_capacity(capacity)
-                .time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
+                .time_to_live(metadata_ttl)
+                .eviction_listener(move |_key: Arc<String>, _metadata: KeyMetadata, cause: RemovalCause| {
+                    eviction_counters.evictions.fetch_add(1, Ordering::Relaxed);
+                    metrics::counter!(METRIC_CACHE_EVICTIONS_TOTAL, "cause" => removal_cause_label(cause)).increment(1);
+                })
                 .build(),
+            counters,
         }
     }
 
@@ -50,7 +138,26 @@ impl KmsCache {
     /// An `Option` containing the `KeyMetadata` if found, or `None` if not found
     ///
     pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
-        self.key_metadata_cache.get(key_id).await
+        let cached = self.key_metadata_cache.get(key_id).await;
+        self.record_lookup(cached.is_some());
+
+        if cached.is_none() {
+            // TTL expiry is the one way the entry set shrinks without a write,
+            // and a miss is where it surfaces. moka expires lazily: the miss is
+            // reported at once, but the removal that decrements `entry_count`
+            // and reaches the eviction listener lands in the maintenance pass
+            // moka runs opportunistically on reads, on its own interval. So
+            // this converges the gauge within a housekeeping interval rather
+            // than on the first miss — enough to stop a cache that only ever
+            // expires, with no put, remove or clear to follow, from reporting a
+            // population that is gone. Forcing `run_pending_tasks` would
+            // tighten that at the cost of taking moka's maintenance lock on
+            // every miss, for a gauge `entry_count` only approximates anyway.
+            // Hits, the hot path, are left alone.
+            self.record_entry_count();
+        }
+
+        cached
     }
 
     /// Put key metadata into cache
@@ -62,6 +169,7 @@ impl KmsCache {
     pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
         self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
         self.key_metadata_cache.run_pending_tasks().await;
+        self.record_entry_count();
     }
 
     /// Remove key metadata from cache
@@ -71,6 +179,11 @@ impl KmsCache {
     ///
     pub async fn remove_key_metadata(&mut self, key_id: &str) {
         self.key_metadata_cache.remove(key_id).await;
+
+        // Flush maintenance so the removal notification and the entry gauge
+        // describe the cache as it is once this call returns.
+        self.key_metadata_cache.run_pending_tasks().await;
+        self.record_entry_count();
     }
 
     /// Clear all cached entries
@@ -79,18 +192,35 @@ impl KmsCache {
 
         // Wait for invalidation to complete
         self.key_metadata_cache.run_pending_tasks().await;
+        self.record_entry_count();
     }
 
-    /// Get cache statistics (hit count, miss count)
+    /// Get cache statistics
     ///
     /// # Returns
-    /// A tuple containing total entries and total misses
+    /// A [`KmsCacheStats`] snapshot of entry count, hits, misses and evictions
     ///
-    pub fn stats(&self) -> (u64, u64) {
-        (
-            self.key_metadata_cache.entry_count(),
-            0u64, // moka doesn't provide miss count directly
-        )
+    pub fn stats(&self) -> KmsCacheStats {
+        KmsCacheStats {
+            entries: self.key_metadata_cache.entry_count(),
+            hits: self.counters.hits.load(Ordering::Relaxed),
+            misses: self.counters.misses.load(Ordering::Relaxed),
+            evictions: self.counters.evictions.load(Ordering::Relaxed),
+        }
+    }
+
+    fn record_lookup(&self, hit: bool) {
+        let (counter, result) = if hit {
+            (&self.counters.hits, "hit")
+        } else {
+            (&self.counters.misses, "miss")
+        };
+        counter.fetch_add(1, Ordering::Relaxed);
+        metrics::counter!(METRIC_CACHE_LOOKUPS_TOTAL, "result" => result).increment(1);
+    }
+
+    fn record_entry_count(&self) {
+        metrics::gauge!(METRIC_CACHE_ENTRIES).set(self.key_metadata_cache.entry_count() as f64);
     }
 }
 
@@ -99,37 +229,90 @@ mod tests {
     use super::*;
     use crate::types::{KeyState, KeyUsage};
     use jiff::Zoned;
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
     use std::time::Duration;
 
-    #[derive(Debug, Clone)]
-    struct CacheInfo {
-        key_metadata_count: u64,
-    }
-
-    impl CacheInfo {
-        fn total_entries(&self) -> u64 {
-            self.key_metadata_count
-        }
-    }
-
     impl KmsCache {
-        fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
-            Self {
-                key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
-            }
-        }
-
-        fn info_for_tests(&self) -> CacheInfo {
-            CacheInfo {
-                key_metadata_count: self.key_metadata_cache.entry_count(),
-            }
-        }
-
         fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
             self.key_metadata_cache.contains_key(key_id)
         }
     }
 
+    fn test_metadata(key_id: &str) -> KeyMetadata {
+        KeyMetadata {
+            key_id: key_id.to_string(),
+            key_state: KeyState::Enabled,
+            key_usage: KeyUsage::EncryptDecrypt,
+            description: None,
+            creation_date: Zoned::now(),
+            deletion_date: None,
+            origin: "KMS".to_string(),
+            key_manager: "CUSTOMER".to_string(),
+            tags: std::collections::HashMap::new(),
+        }
+    }
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Drive `test` on a current-thread runtime under a thread-local debugging
+    /// recorder and return its output plus one snapshot of everything emitted.
+    ///
+    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
+    /// the recorded state, so taking it per assertion would leave every
+    /// assertion after the first with nothing to look at.
+    fn record_metrics<F, Fut, T>(test: F) -> (T, Vec<MetricEntry>)
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = T>,
+    {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let output = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (output, snapshotter.snapshot().into_vec())
+    }
+
+    /// Sum of counters with `name` whose labels include every `(key, value)` pair.
+    fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let key = composite.key();
+                let matches = composite.kind() == MetricKind::Counter
+                    && key.name() == name
+                    && labels
+                        .iter()
+                        .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected));
+                match (matches, value) {
+                    (true, DebugValue::Counter(count)) => Some(*count),
+                    _ => None,
+                }
+            })
+            .sum()
+    }
+
+    /// Last value recorded for the unlabelled gauge `name`.
+    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(gauge)) => Some(**gauge),
+                _ => None,
+            }
+        })
+    }
+
     #[tokio::test]
     async fn test_cache_operations() {
         let mut cache = KmsCache::new(100);
@@ -154,19 +337,16 @@ mod tests {
         assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
 
         // Test cache info
-        let info = cache.info_for_tests();
-        assert_eq!(info.key_metadata_count, 1);
-        assert_eq!(info.total_entries(), 1);
+        assert_eq!(cache.stats().entries, 1);
 
         // Test cache clearing
         cache.clear().await;
-        let info_after_clear = cache.info_for_tests();
-        assert_eq!(info_after_clear.total_entries(), 0);
+        assert_eq!(cache.stats().entries, 0);
     }
 
     #[tokio::test]
     async fn test_cache_with_custom_ttl() {
-        let mut cache = KmsCache::with_ttl_for_tests(
+        let mut cache = KmsCache::with_ttl(
             100,
             Duration::from_millis(100), // Short TTL for testing
         );
@@ -217,4 +397,92 @@ mod tests {
 
         assert!(cache.contains_key_metadata_for_tests("contains-test"));
     }
+
+    #[test]
+    fn lookups_report_real_hit_and_miss_counts() {
+        let (stats, snapshot) = record_metrics(|| async {
+            let mut cache = KmsCache::new(100);
+
+            assert!(cache.get_key_metadata("absent").await.is_none());
+            cache.put_key_metadata("present", &test_metadata("present")).await;
+            assert!(cache.get_key_metadata("present").await.is_some());
+            assert!(cache.get_key_metadata("absent").await.is_none());
+
+            cache.stats()
+        });
+
+        assert_eq!(stats.hits, 1);
+        assert_eq!(stats.misses, 2);
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "hit")]), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "miss")]), 2);
+    }
+
+    #[test]
+    fn removals_report_their_cause_and_the_resulting_entry_count() {
+        let (stats, snapshot) = record_metrics(|| async {
+            let mut cache = KmsCache::new(100);
+
+            cache.put_key_metadata("key", &test_metadata("key")).await;
+            cache.put_key_metadata("key", &test_metadata("key")).await;
+            cache.remove_key_metadata("key").await;
+
+            cache.stats()
+        });
+
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "replaced")]), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "explicit")]), 1);
+        assert_eq!(stats.evictions, 2);
+        assert_eq!(stats.entries, 0);
+        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
+    }
+
+    #[test]
+    fn capacity_pressure_reports_size_evictions() {
+        let (stats, snapshot) = record_metrics(|| async {
+            let mut cache = KmsCache::with_ttl(1, DEFAULT_METADATA_TTL);
+
+            cache.put_key_metadata("first", &test_metadata("first")).await;
+            cache.put_key_metadata("second", &test_metadata("second")).await;
+
+            cache.stats()
+        });
+
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "size")]), 1);
+        assert_eq!(stats.evictions, 1);
+        assert_eq!(stats.entries, 1);
+        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(1.0));
+    }
+
+    /// Entries that expire are dropped outside every write path, so the gauge
+    /// has to be republished from the lookup that observes the expiry — nothing
+    /// else runs afterwards to correct it.
+    ///
+    /// moka's clock is internal and cannot be driven by tokio's paused time,
+    /// hence the one short real sleep. The explicit `run_pending_tasks` stands
+    /// in for the maintenance moka schedules by itself on reads, so the test
+    /// does not depend on moka's internal 300ms housekeeping interval.
+    #[test]
+    fn expiry_without_further_writes_converges_the_entry_gauge() {
+        let ttl = Duration::from_millis(100);
+
+        let (stats, snapshot) = record_metrics(move || async move {
+            let mut cache = KmsCache::with_ttl(100, ttl);
+
+            cache.put_key_metadata("expiring", &test_metadata("expiring")).await;
+            assert_eq!(cache.stats().entries, 1);
+
+            tokio::time::sleep(Duration::from_millis(150)).await;
+            cache.key_metadata_cache.run_pending_tasks().await;
+
+            // The lookup that observes the expiry is the last thing to touch
+            // the cache: no put, remove or clear follows it.
+            assert!(cache.get_key_metadata("expiring").await.is_none());
+
+            cache.stats()
+        });
+
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "expired")]), 1);
+        assert_eq!(stats.entries, 0);
+        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
+    }
 }

crates/kms/src/config.rs

@@ -34,6 +34,8 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
 pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
 pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
 pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
+pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
+pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
 pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
 pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
 pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
@@ -128,6 +130,26 @@ pub enum KmsBackend {
     /// Static single-key backend that derives DEKs from a pre-configured key
     #[serde(rename = "Static")]
     Static,
+    /// AWS KMS backend: AWS is the cryptographic source of truth and owns key
+    /// state, versioning, and the deletion window.
+    #[serde(rename = "AWS", alias = "AwsKms")]
+    Aws,
+}
+
+impl KmsBackend {
+    /// Stable identifier for logs, metrics, and audit records.
+    ///
+    /// External consumers key off these values, so treat them as a wire
+    /// contract rather than a rendering detail.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            KmsBackend::VaultKv2 => "vault-kv2",
+            KmsBackend::VaultTransit => "vault-transit",
+            KmsBackend::Local => "local",
+            KmsBackend::Static => "static",
+            KmsBackend::Aws => "aws",
+        }
+    }
 }
 
 /// Main KMS configuration
@@ -185,6 +207,9 @@ pub enum BackendConfig {
     VaultTransit(Box<VaultTransitConfig>),
     /// Static single-key backend configuration
     Static(StaticConfig),
+    /// AWS KMS backend configuration
+    #[serde(rename = "AWS", alias = "AwsKms")]
+    Aws(Box<AwsKmsConfig>),
 }
 
 impl Default for BackendConfig {
@@ -200,6 +225,7 @@ impl fmt::Debug for BackendConfig {
             Self::VaultKv2(config) => f.debug_tuple("VaultKv2").field(config).finish(),
             Self::VaultTransit(config) => f.debug_tuple("VaultTransit").field(config).finish(),
             Self::Static(config) => f.debug_tuple("Static").field(config).finish(),
+            Self::Aws(config) => f.debug_tuple("Aws").field(config).finish(),
         }
     }
 }
@@ -525,6 +551,25 @@ impl Default for CacheConfig {
     }
 }
 
+/// AWS KMS backend configuration.
+///
+/// Deliberately holds no credential material: the backend resolves credentials
+/// through the standard `aws-config` provider chain (environment, shared
+/// profile, container/IMDS role), so RustFS never stores, persists, or redacts
+/// AWS secrets of its own.
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+pub struct AwsKmsConfig {
+    /// AWS region hosting the KMS keys. When unset, the region is resolved by
+    /// the standard chain (`AWS_REGION`, profile, IMDS).
+    #[serde(default)]
+    pub region: Option<String>,
+    /// Override for the KMS endpoint, for local emulators and private
+    /// endpoints. Unset in production, where the SDK derives the regional
+    /// endpoint.
+    #[serde(default)]
+    pub endpoint_url: Option<String>,
+}
+
 impl KmsConfig {
     /// Create a new KMS configuration for local backend (for development and testing only)
     pub fn local(key_dir: PathBuf) -> Self {
@@ -634,6 +679,29 @@ impl KmsConfig {
         }
     }
 
+    /// Create a new KMS configuration for the AWS KMS backend.
+    ///
+    /// Credentials are resolved by the standard `aws-config` provider chain;
+    /// only the region is configured here.
+    pub fn aws(region: Option<String>) -> Self {
+        Self {
+            backend: KmsBackend::Aws,
+            backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
+                region,
+                endpoint_url: None,
+            })),
+            ..Default::default()
+        }
+    }
+
+    /// Get the AWS configuration if backend is AWS KMS
+    pub fn aws_kms_config(&self) -> Option<&AwsKmsConfig> {
+        match &self.backend_config {
+            BackendConfig::Aws(config) => Some(config),
+            _ => None,
+        }
+    }
+
     /// Set default key ID
     pub fn with_default_key(mut self, key_id: String) -> Self {
         self.default_key_id = Some(key_id);
@@ -790,6 +858,25 @@ impl KmsConfig {
                 // Validate that the key can be decoded (right length, valid base64)
                 config.decode_key()?;
             }
+            BackendConfig::Aws(config) => {
+                if let Some(region) = &config.region
+                    && region.is_empty()
+                {
+                    return Err(KmsError::configuration_error("AWS KMS region cannot be empty when set"));
+                }
+
+                if let Some(endpoint) = &config.endpoint_url {
+                    if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
+                        return Err(KmsError::configuration_error("AWS KMS endpoint URL must use http or https scheme"));
+                    }
+                    // A plaintext endpoint override exposes every KMS request,
+                    // including plaintext data keys, so it stays gated on the
+                    // explicit development opt-in.
+                    if endpoint.starts_with("http://") && !self.allow_insecure_dev_defaults {
+                        return Err(development_default_error("AWS KMS endpoint URL must use https"));
+                    }
+                }
+            }
         }
 
         // Validate cache configuration
@@ -811,6 +898,7 @@ impl KmsConfig {
                 "vault" | "vault-kv2" | "vault_kv2" => KmsBackend::VaultKv2,
                 "vault-transit" | "vault_transit" => KmsBackend::VaultTransit,
                 "static" => KmsBackend::Static,
+                "aws" | "aws-kms" | "aws_kms" => KmsBackend::Aws,
                 _ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {backend_type}"))),
             };
         }
@@ -939,6 +1027,14 @@ impl KmsConfig {
                 });
                 config.default_key_id = Some(key_id);
             }
+            KmsBackend::Aws => {
+                // Only non-credential settings are read here; access keys,
+                // profiles, and role assumption stay with the aws-config chain.
+                config.backend_config = BackendConfig::Aws(Box::new(AwsKmsConfig {
+                    region: get_env_opt_str(ENV_KMS_AWS_REGION),
+                    endpoint_url: get_env_opt_str(ENV_KMS_AWS_ENDPOINT_URL),
+                }));
+            }
         }
 
         config.validate()?;
@@ -1540,6 +1636,43 @@ mod tests {
         );
     }
 
+    /// The AWS backend reads only non-credential settings from the
+    /// environment; access keys, profiles, and role assumption stay with the
+    /// aws-config provider chain.
+    #[test]
+    fn test_from_env_selects_aws_backend_without_credentials() {
+        with_vars(
+            vec![
+                ("RUSTFS_KMS_BACKEND", Some("aws")),
+                (ENV_KMS_AWS_REGION, Some("eu-central-1")),
+                (ENV_KMS_AWS_ENDPOINT_URL, None::<&str>),
+            ],
+            || {
+                let config = KmsConfig::from_env().expect("kms config should load from env");
+                assert_eq!(config.backend, KmsBackend::Aws);
+                let aws = config.aws_kms_config().expect("aws backend config");
+                assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
+                assert_eq!(aws.endpoint_url, None);
+            },
+        );
+    }
+
+    /// A plaintext endpoint override exposes every KMS request, including the
+    /// plaintext data keys, so it stays behind the explicit development opt-in.
+    #[test]
+    fn test_from_env_rejects_plaintext_aws_endpoint() {
+        with_vars(
+            vec![
+                ("RUSTFS_KMS_BACKEND", Some("aws")),
+                (ENV_KMS_AWS_REGION, Some("us-east-1")),
+                (ENV_KMS_AWS_ENDPOINT_URL, Some("http://localhost:4566")),
+            ],
+            || {
+                KmsConfig::from_env().expect_err("a plaintext AWS endpoint must be rejected by default");
+            },
+        );
+    }
+
     #[test]
     fn test_from_env_approle_requires_secret_id_or_file() {
         with_vars(

crates/kms/src/deletion_worker.rs

@@ -22,18 +22,123 @@
 //! every node of a deployment concurrently — a key is only ever removed while
 //! its (re-read) record is an expired pending deletion or a tombstone.
 
+use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
 use crate::backends::{ExpiredKeyRemoval, KmsBackend};
-use crate::types::{KeyStatus, ListKeysRequest};
+use crate::error::Result;
+use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
 use async_trait::async_trait;
 use jiff::Zoned;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};
 
 /// How often the worker looks for expired pending deletions.
 pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
 
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// The sweep already pages through the whole key set, so everything below is
+// derived from the pages it has in hand: observing the key lifecycle costs no
+// extra backend call. Each metric is an aggregate — the gauges carry no labels
+// at all and the counter only a static outcome — because a per-key label would
+// carry key identifiers into the metric stream and grow the series count with
+// the key set. "This key is overdue for rotation" is a threshold on the
+// aggregate age and belongs in an alerting rule, not in a label.
+// ---------------------------------------------------------------------------
+
+/// Gauge: keys scheduled for deletion whose deadline has not passed, as of the
+/// end of the last sweep that saw the whole key set.
+const METRIC_PENDING_DELETION_KEYS: &str = "rustfs_kms_pending_deletion_keys";
+/// Gauge: keys left tombstoned by an interrupted removal and still awaiting
+/// the sweep, as of the end of the last sweep that saw the whole key set.
+const METRIC_TOMBSTONE_KEYS: &str = "rustfs_kms_deletion_tombstone_keys";
+/// Gauge: seconds since the least recently rotated usable key was rotated
+/// (its creation time when it was never rotated); `0` when there are none.
+const METRIC_OLDEST_ROTATION_AGE_SECONDS: &str = "rustfs_kms_oldest_key_rotation_age_seconds";
+/// Counter: keys the sweep acted on, by `outcome` (`removed`, `blocked`,
+/// `skipped`, `failed`).
+const METRIC_SWEEP_KEYS_TOTAL: &str = "rustfs_kms_deletion_sweep_keys_total";
+
+/// Register metric descriptions once per process.
+fn describe_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_gauge!(
+            METRIC_PENDING_DELETION_KEYS,
+            "KMS keys scheduled for deletion whose deadline has not passed yet"
+        );
+        metrics::describe_gauge!(
+            METRIC_TOMBSTONE_KEYS,
+            "KMS keys left tombstoned by an interrupted removal, still awaiting the deletion sweep"
+        );
+        metrics::describe_gauge!(
+            METRIC_OLDEST_ROTATION_AGE_SECONDS,
+            "Seconds since the least recently rotated usable KMS key was last rotated, counting from creation for keys that were never rotated"
+        );
+        metrics::describe_counter!(METRIC_SWEEP_KEYS_TOTAL, "Total keys acted on by the KMS deletion sweep, by outcome");
+    });
+}
+
+/// Aggregate view of the key set, accumulated while the sweep pages through it.
+///
+/// Counts and one maximum age only: these feed label-less gauges, so no key
+/// identifier can reach the metric stream through them.
+#[derive(Debug, Default, Clone, Copy, PartialEq)]
+struct KeyCensus {
+    pending_deletion: usize,
+    tombstones: usize,
+    /// Longest time since last rotation across keys that are still usable,
+    /// counting from creation for keys that were never rotated. Keys on their
+    /// way out are excluded: they will never be rotated again, and would
+    /// otherwise pin the gauge high until the sweep finishes removing them.
+    oldest_rotation_age_seconds: f64,
+}
+
+impl KeyCensus {
+    fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
+        match key.status {
+            KeyStatus::PendingDeletion => self.pending_deletion += 1,
+            KeyStatus::Deleted => self.tombstones += 1,
+            KeyStatus::Active | KeyStatus::Disabled => {
+                let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
+                self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
+            }
+        }
+    }
+}
+
+/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
+/// timestamp persisted by a node running ahead) cannot produce a negative age.
+fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
+    (now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
+}
+
+/// Publish one sweep's counters, plus the lifecycle gauges when `census` covers
+/// the whole key set.
+fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
+    describe_metrics();
+    for (outcome, count) in [
+        ("removed", report.removed.len()),
+        ("blocked", report.blocked.len()),
+        ("skipped", report.skipped),
+        ("failed", report.failed),
+    ] {
+        // Emitted even at zero so every outcome series exists from the first
+        // sweep on and a rate over it is defined.
+        metrics::counter!(METRIC_SWEEP_KEYS_TOTAL, "outcome" => outcome).increment(count as u64);
+    }
+
+    // A sweep that could not finish listing saw only part of the key set;
+    // publishing its counts would understate every gauge, so the previous
+    // (complete) values are left standing instead.
+    let Some(census) = census else { return };
+    metrics::gauge!(METRIC_PENDING_DELETION_KEYS).set(census.pending_deletion as f64);
+    metrics::gauge!(METRIC_TOMBSTONE_KEYS).set(census.tombstones as f64);
+    metrics::gauge!(METRIC_OLDEST_ROTATION_AGE_SECONDS).set(census.oldest_rotation_age_seconds);
+}
+
 /// Reports configuration that still references a KMS key.
 ///
 /// Consulted before any material is destroyed; a non-empty result blocks the
@@ -68,6 +173,8 @@ pub(crate) struct DeletionWorker {
     default_key_id: Option<String>,
     reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
     interval: Duration,
+    backend_kind: &'static str,
+    audit_sink: Option<Arc<dyn KmsAuditSink>>,
 }
 
 impl DeletionWorker {
@@ -75,15 +182,24 @@ impl DeletionWorker {
         backend: Arc<dyn KmsBackend>,
         default_key_id: Option<String>,
         reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
+        backend_kind: &'static str,
     ) -> Self {
         Self {
             backend,
             default_key_id,
             reference_checker,
+            backend_kind,
+            audit_sink: None,
             interval: DEFAULT_SWEEP_INTERVAL,
         }
     }
 
+    /// Audit each irreversible key removal to `sink`.
+    pub(crate) fn with_audit_sink(mut self, sink: Option<Arc<dyn KmsAuditSink>>) -> Self {
+        self.audit_sink = sink;
+        self
+    }
+
     pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
         tokio::spawn(async move { self.run(cancel).await })
     }
@@ -114,10 +230,15 @@ impl DeletionWorker {
 
     /// Run one sweep at the given time. Exposed separately so tests can drive
     /// the expiry logic deterministically.
+    ///
+    /// Also feeds the lifecycle gauges: every key it lists is counted into a
+    /// [`KeyCensus`] on the way past, so the observability comes out of the
+    /// pages the sweep already had to fetch.
     pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
         let mut report = SweepReport::default();
+        let mut census = KeyCensus::default();
         let mut marker: Option<String> = None;
-        loop {
+        let listed_everything = loop {
             let request = ListKeysRequest {
                 limit: Some(100),
                 marker: marker.clone(),
@@ -129,54 +250,99 @@ impl DeletionWorker {
                 Err(error) => {
                     warn!(%error, "KMS deletion sweep could not list keys");
                     report.failed += 1;
-                    return report;
+                    break false;
                 }
             };
             for key in &response.keys {
-                if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
-                    self.process_key(&key.key_id, now, &mut report).await;
+                // Keys this sweep destroys are left out of the census: the
+                // gauges describe the key set as it stands once the sweep is
+                // done, not as it was when the page was listed.
+                let removed = if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
+                    self.process_key(&key.key_id, now, &mut report).await
+                } else {
+                    false
+                };
+                if !removed {
+                    census.observe(key, now);
                 }
             }
             if !response.truncated {
-                break;
+                break true;
             }
             match response.next_marker {
                 Some(next_marker) => marker = Some(next_marker),
-                None => break,
+                // Truncated without a marker: the rest of the key set is out
+                // of reach, so the census is incomplete.
+                None => break false,
             }
-        }
+        };
+        record_sweep(&report, listed_everything.then_some(census));
         report
     }
 
-    async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) {
+    /// Handle one key that is on its way out; reports whether it was removed.
+    async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
         // Never remove a key that live configuration still points at. The
         // default key check is built in; broader references (bucket
         // encryption settings, ...) come from the injected checker.
         if self.default_key_id.as_deref() == Some(key_id) {
             warn!(key_id, "expired KMS key is still the default key; refusing removal");
             report.blocked.push(key_id.to_string());
-            return;
+            return false;
         }
         if let Some(checker) = &self.reference_checker {
             let references = checker.references(key_id).await;
             if !references.is_empty() {
                 warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal");
                 report.blocked.push(key_id.to_string());
-                return;
+                return false;
             }
         }
 
         // The backend re-checks state and deadline under its own write
         // synchronization, so a cancellation racing this sweep wins there.
-        match self.backend.remove_expired_key(key_id, now).await {
-            Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()),
-            Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1,
+        let started = Instant::now();
+        let outcome = self.backend.remove_expired_key(key_id, now).await;
+        match &outcome {
+            Ok(ExpiredKeyRemoval::Removed) => {
+                report.removed.push(key_id.to_string());
+                self.audit_removal(key_id, started, &outcome);
+                true
+            }
+            Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => {
+                report.skipped += 1;
+                false
+            }
             Err(error) => {
                 warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep");
                 report.failed += 1;
+                self.audit_removal(key_id, started, &outcome);
+                false
             }
         }
     }
+
+    /// Record an attempted destruction of key material.
+    ///
+    /// Only attempts that reached the backend are audited: a key the sweep
+    /// declined to touch (not yet due, still referenced, state changed under
+    /// us) was never at risk, and recording it as a deletion event would
+    /// drown the real ones.
+    fn audit_removal(&self, key_id: &str, started: Instant, outcome: &Result<ExpiredKeyRemoval>) {
+        let Some(sink) = self.audit_sink.as_ref() else {
+            return;
+        };
+
+        // Removal runs on a background sweep, so there is no request
+        // principal to attribute it to.
+        let context = OperationContext::internal();
+        sink.emit(
+            KmsAuditRecord::new(KmsAuditOperation::DeleteKey, &context, self.backend_kind)
+                .with_key_id(Some(key_id))
+                .with_latency(started.elapsed())
+                .with_result(outcome),
+        );
+    }
 }
 
 #[cfg(test)]
@@ -216,7 +382,7 @@ mod tests {
     }
 
     fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker {
-        DeletionWorker::new(backend, None, None)
+        DeletionWorker::new(backend, None, None, "local")
     }
 
     fn after_window() -> Zoned {
@@ -307,7 +473,7 @@ mod tests {
         schedule(&backend, &key_id).await;
 
         // Blocked while it is the configured default key.
-        let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None);
+        let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None, "local");
         let report = as_default.sweep(&after_window()).await;
         assert_eq!(report.blocked, vec![key_id.clone()]);
         assert!(report.removed.is_empty());
@@ -317,6 +483,7 @@ mod tests {
             backend.clone(),
             None,
             Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))),
+            "local",
         );
         let report = with_references.sweep(&after_window()).await;
         assert_eq!(report.blocked, vec![key_id.clone()]);
@@ -327,11 +494,53 @@ mod tests {
             .expect("blocked key must still exist");
 
         // Removed once nothing references it anymore.
-        let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))));
+        let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))), "local");
         let report = unreferenced.sweep(&after_window()).await;
         assert_eq!(report.removed, vec![key_id.clone()]);
     }
 
+    #[tokio::test]
+    async fn only_attempted_removals_are_audited() {
+        #[derive(Default)]
+        struct CapturingSink {
+            records: std::sync::Mutex<Vec<KmsAuditRecord>>,
+        }
+
+        impl KmsAuditSink for CapturingSink {
+            fn emit(&self, record: KmsAuditRecord) {
+                self.records.lock().expect("sink lock should not be poisoned").push(record);
+            }
+        }
+
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let backend = local_backend(&temp_dir).await;
+        let key_id = create_key(&backend, "audited-removal").await;
+        schedule(&backend, &key_id).await;
+
+        let sink = Arc::new(CapturingSink::default());
+        let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
+
+        // A key that is not yet due was never at risk, so it produces no record.
+        worker.sweep(&Zoned::now()).await;
+        assert!(
+            sink.records.lock().expect("sink lock").is_empty(),
+            "a key the sweep declined to touch must not be audited as a deletion"
+        );
+
+        worker.sweep(&after_window()).await;
+
+        let records = sink.records.lock().expect("sink lock");
+        assert_eq!(records.len(), 1, "the removal should be audited exactly once");
+        let record = &records[0];
+        assert_eq!(record.operation, crate::audit::KmsAuditOperation::DeleteKey);
+        assert_eq!(record.event, rustfs_s3_types::EventName::KmsKeyDeleted);
+        assert_eq!(record.outcome, crate::audit::KmsAuditOutcome::Success);
+        assert_eq!(record.key_id.as_deref(), Some(key_id.as_str()));
+        assert_eq!(record.backend, "local");
+        // Background work has no request principal to attribute.
+        assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
+    }
+
     #[tokio::test]
     async fn deadline_survives_backend_restart_and_sweep_completes_it() {
         let temp_dir = tempfile::tempdir().expect("temp dir");
@@ -395,4 +604,156 @@ mod tests {
         cancel.cancel();
         task.await.expect("worker task must stop after cancellation");
     }
+
+    // -- Metric emission ----------------------------------------------------
+
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Run `test` on a current-thread runtime under a debugging recorder and
+    /// return one snapshot of everything it emitted.
+    ///
+    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
+    /// the recorded state, so taking it per assertion would only show the
+    /// first assertion any data.
+    fn record_metrics<Out>(
+        test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
+    ) -> (Vec<MetricEntry>, Out) {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let out = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (snapshotter.snapshot().into_vec(), out)
+    }
+
+    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(value)) => Some(value.into_inner()),
+                _ => None,
+            }
+        })
+    }
+
+    fn counter_value(snapshot: &[MetricEntry], name: &str, outcome: &str) -> u64 {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let matches = composite.kind() == MetricKind::Counter
+                    && composite.key().name() == name
+                    && composite
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "outcome" && label.value() == outcome);
+                match (matches, value) {
+                    (true, DebugValue::Counter(count)) => Some(*count),
+                    _ => None,
+                }
+            })
+            .sum()
+    }
+
+    fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
+        KeyInfo {
+            key_id: key_id.to_string(),
+            description: None,
+            algorithm: "AES-256".to_string(),
+            usage: KeyUsage::EncryptDecrypt,
+            status,
+            version: 1,
+            metadata: std::collections::HashMap::new(),
+            tags: std::collections::HashMap::new(),
+            created_at,
+            rotated_at,
+            created_by: None,
+        }
+    }
+
+    #[test]
+    fn census_ages_from_rotation_and_ignores_departing_keys() {
+        let now = Zoned::now();
+        let day = Duration::from_secs(86400);
+        let mut census = KeyCensus::default();
+
+        // Rotation beats creation as the age baseline...
+        census.observe(
+            &key_info("rotated", KeyStatus::Active, now.clone() - 30 * day, Some(now.clone() - 2 * day)),
+            &now,
+        );
+        // ...and a key that was never rotated ages from its creation.
+        census.observe(&key_info("never-rotated", KeyStatus::Disabled, now.clone() - 5 * day, None), &now);
+        // Keys on their way out only ever move the counts: they will not be
+        // rotated again, so their age must not drive the rotation gauge.
+        census.observe(&key_info("pending", KeyStatus::PendingDeletion, now.clone() - 400 * day, None), &now);
+        census.observe(&key_info("tombstone", KeyStatus::Deleted, now.clone() - 400 * day, None), &now);
+
+        assert_eq!(census.pending_deletion, 1);
+        assert_eq!(census.tombstones, 1);
+        let expected = 5.0 * 86400.0;
+        assert!(
+            (census.oldest_rotation_age_seconds - expected).abs() < 1.0,
+            "expected the never-rotated key to set the age, got {}",
+            census.oldest_rotation_age_seconds
+        );
+    }
+
+    #[test]
+    fn sweep_publishes_lifecycle_gauges_without_key_labels() {
+        let (snapshot, key_ids) = record_metrics(|| {
+            Box::pin(async {
+                let temp_dir = tempfile::tempdir().expect("temp dir");
+                let backend = local_backend(&temp_dir).await;
+                let live = create_key(&backend, "live-key").await;
+                let doomed = create_key(&backend, "doomed-key").await;
+                schedule(&backend, &doomed).await;
+
+                // Three days in, still inside the seven-day window: the sweep
+                // observes the scheduled key instead of removing it.
+                let report = worker(backend.clone())
+                    .sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
+                    .await;
+                assert_eq!(report.skipped, 1);
+                assert!(report.removed.is_empty());
+                vec![live, doomed]
+            })
+        });
+
+        assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(1.0));
+        assert_eq!(gauge_value(&snapshot, METRIC_TOMBSTONE_KEYS), Some(0.0));
+        let age = gauge_value(&snapshot, METRIC_OLDEST_ROTATION_AGE_SECONDS).expect("rotation age gauge must be published");
+        // Both keys were just created, and only the usable one counts, so the
+        // reported age is the sweep's own offset into the future.
+        assert!(
+            (age - 3.0 * 86400.0).abs() < 60.0,
+            "expected the sweep offset as the rotation age, got {age}"
+        );
+        assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "skipped"), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "removed"), 0);
+
+        for (composite, ..) in &snapshot {
+            for label in composite.key().labels() {
+                for key_id in &key_ids {
+                    assert!(
+                        !label.value().contains(key_id.as_str()),
+                        "metric {} leaked a key identifier through label {}",
+                        composite.key().name(),
+                        label.key()
+                    );
+                }
+            }
+        }
+    }
 }

crates/kms/src/lib.rs

@@ -65,6 +65,7 @@
 
 // Core modules
 pub mod api_types;
+pub mod audit;
 pub mod backends;
 pub mod backup;
 mod cache;
@@ -74,6 +75,7 @@ mod encryption;
 mod error;
 pub mod manager;
 mod policy;
+pub mod probe;
 pub mod service;
 pub mod service_manager;
 mod time_serde;
@@ -86,11 +88,14 @@ pub use api_types::{
     StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
     UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
+pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
+pub use cache::KmsCacheStats;
 pub use config::*;
 pub use deletion_worker::DeletionReferenceChecker;
 pub use encryption::is_data_key_envelope;
 pub use error::{KmsError, KmsUnavailableError, Result};
 pub use manager::KmsManager;
+pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
     KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,

crates/kms/src/manager.rs

@@ -14,16 +14,19 @@
 
 //! KMS manager for handling key operations and backend coordination
 
+use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
 use crate::backends::KmsBackend;
-use crate::cache::KmsCache;
+use crate::cache::{KmsCache, KmsCacheStats};
 use crate::config::KmsConfig;
 use crate::error::Result;
 use crate::types::{
     CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
     DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
-    GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse,
+    GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
 };
+use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Instant;
 use tokio::sync::RwLock;
 
 /// KMS Manager coordinates operations between backends and caching
@@ -33,6 +36,8 @@ pub struct KmsManager {
     cache: Arc<RwLock<KmsCache>>,
     default_key_id: Option<String>,
     enable_cache: bool,
+    backend_kind: &'static str,
+    audit_sink: Option<Arc<dyn KmsAuditSink>>,
 }
 
 impl KmsManager {
@@ -44,16 +49,72 @@ impl KmsManager {
             cache,
             default_key_id: config.default_key_id,
             enable_cache: config.enable_cache,
+            backend_kind: config.backend.as_str(),
+            audit_sink: None,
         }
     }
 
+    /// Send an audit record for every management operation to `sink`.
+    ///
+    /// Without a sink the manager builds no records at all, so a deployment
+    /// that does not consume KMS audit records is unaffected.
+    pub fn with_audit_sink(mut self, sink: Arc<dyn KmsAuditSink>) -> Self {
+        self.audit_sink = Some(sink);
+        self
+    }
+
     /// Get the default key ID if configured
     pub fn get_default_key_id(&self) -> Option<&String> {
         self.default_key_id.as_ref()
     }
 
+    /// Emit an audit record for a completed management operation.
+    ///
+    /// Called after the operation resolved, so nothing here can change its
+    /// result; the record is built only when a sink is installed.
+    fn audit<T>(
+        &self,
+        operation: KmsAuditOperation,
+        context: &OperationContext,
+        key_id: Option<&str>,
+        started: Instant,
+        result: &Result<T>,
+    ) {
+        let Some(sink) = self.audit_sink.as_ref() else {
+            return;
+        };
+
+        sink.emit(
+            KmsAuditRecord::new(operation, context, self.backend_kind)
+                .with_key_id(key_id)
+                .with_latency(started.elapsed())
+                .with_result(result),
+        );
+    }
+
     /// Create a new master key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::create_key_with_context`].
     pub async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
+        self.create_key_with_context(request, &OperationContext::internal()).await
+    }
+
+    /// Create a new master key on behalf of `context`'s principal
+    pub async fn create_key_with_context(
+        &self,
+        request: CreateKeyRequest,
+        context: &OperationContext,
+    ) -> Result<CreateKeyResponse> {
+        let started = Instant::now();
+        let key_name = request.key_name.clone();
+        let result = self.create_key_inner(request).await;
+        let key_id = result.as_ref().ok().map(|r| r.key_id.as_str()).or(key_name.as_deref());
+        self.audit(KmsAuditOperation::CreateKey, context, key_id, started, &result);
+        result
+    }
+
+    async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
         let response = self.backend.create_key(request).await?;
 
         // Cache the key metadata if enabled
@@ -84,7 +145,27 @@ impl KmsManager {
     }
 
     /// Describe a key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::describe_key_with_context`].
     pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
+        self.describe_key_with_context(request, &OperationContext::internal()).await
+    }
+
+    /// Describe a key on behalf of `context`'s principal
+    pub async fn describe_key_with_context(
+        &self,
+        request: DescribeKeyRequest,
+        context: &OperationContext,
+    ) -> Result<DescribeKeyResponse> {
+        let started = Instant::now();
+        let key_id = request.key_id.clone();
+        let result = self.describe_key_inner(request).await;
+        self.audit(KmsAuditOperation::DescribeKey, context, Some(&key_id), started, &result);
+        result
+    }
+
+    async fn describe_key_inner(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
         // Check cache first if enabled
         if self.enable_cache {
             let cache = self.cache.read().await;
@@ -109,12 +190,24 @@ impl KmsManager {
     }
 
     /// List keys
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::list_keys_with_context`].
     pub async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
-        self.backend.list_keys(request).await
+        self.list_keys_with_context(request, &OperationContext::internal()).await
     }
 
-    /// Get cache statistics
-    pub async fn cache_stats(&self) -> Option<(u64, u64)> {
+    /// List keys on behalf of `context`'s principal
+    pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
+        let started = Instant::now();
+        let result = self.backend.list_keys(request).await;
+        // Listing spans keys, so the record carries no key id.
+        self.audit(KmsAuditOperation::ListKeys, context, None, started, &result);
+        result
+    }
+
+    /// Get cache statistics, or `None` when caching is disabled
+    pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
         if self.enable_cache {
             let cache = self.cache.read().await;
             Some(cache.stats())
@@ -133,7 +226,27 @@ impl KmsManager {
     }
 
     /// Delete a key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::delete_key_with_context`].
     pub async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
+        self.delete_key_with_context(request, &OperationContext::internal()).await
+    }
+
+    /// Delete a key on behalf of `context`'s principal
+    pub async fn delete_key_with_context(
+        &self,
+        request: DeleteKeyRequest,
+        context: &OperationContext,
+    ) -> Result<DeleteKeyResponse> {
+        let started = Instant::now();
+        let key_id = request.key_id.clone();
+        let result = self.delete_key_inner(request).await;
+        self.audit(KmsAuditOperation::ScheduleKeyDeletion, context, Some(&key_id), started, &result);
+        result
+    }
+
+    async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
         let response = self.backend.delete_key(request).await?;
 
         // Remove from cache if enabled and key is being deleted
@@ -146,7 +259,28 @@ impl KmsManager {
     }
 
     /// Cancel key deletion
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::cancel_key_deletion_with_context`].
     pub async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
+        self.cancel_key_deletion_with_context(request, &OperationContext::internal())
+            .await
+    }
+
+    /// Cancel key deletion on behalf of `context`'s principal
+    pub async fn cancel_key_deletion_with_context(
+        &self,
+        request: CancelKeyDeletionRequest,
+        context: &OperationContext,
+    ) -> Result<CancelKeyDeletionResponse> {
+        let started = Instant::now();
+        let key_id = request.key_id.clone();
+        let result = self.cancel_key_deletion_inner(request).await;
+        self.audit(KmsAuditOperation::CancelKeyDeletion, context, Some(&key_id), started, &result);
+        result
+    }
+
+    async fn cancel_key_deletion_inner(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
         let response = self.backend.cancel_key_deletion(request).await?;
 
         // Update cache if enabled
@@ -159,22 +293,79 @@ impl KmsManager {
     }
 
     /// Enable a disabled key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::enable_key_with_context`].
     pub async fn enable_key(&self, key_id: &str) -> Result<()> {
-        self.backend.enable_key(key_id).await?;
-        self.invalidate_cached_metadata(key_id).await;
-        Ok(())
+        self.enable_key_with_context(key_id, &OperationContext::internal()).await
+    }
+
+    /// Enable a disabled key on behalf of `context`'s principal
+    pub async fn enable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
+        let started = Instant::now();
+        let result = self.backend.enable_key(key_id).await;
+        if result.is_ok() {
+            self.invalidate_cached_metadata(key_id).await;
+        }
+        self.audit(KmsAuditOperation::EnableKey, context, Some(key_id), started, &result);
+        result
     }
 
     /// Disable a key; existing data remains decryptable
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::disable_key_with_context`].
     pub async fn disable_key(&self, key_id: &str) -> Result<()> {
-        self.backend.disable_key(key_id).await?;
+        self.disable_key_with_context(key_id, &OperationContext::internal()).await
+    }
+
+    /// Disable a key on behalf of `context`'s principal
+    pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
+        let started = Instant::now();
+        let result = self.backend.disable_key(key_id).await;
+        if result.is_ok() {
+            self.invalidate_cached_metadata(key_id).await;
+        }
+        self.audit(KmsAuditOperation::DisableKey, context, Some(key_id), started, &result);
+        result
+    }
+
+    /// Rotate a key to a new version
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::rotate_key_with_context`].
+    pub async fn rotate_key(&self, key_id: &str) -> Result<()> {
+        self.rotate_key_with_context(key_id, &OperationContext::internal()).await
+    }
+
+    /// Rotate a key on behalf of `context`'s principal
+    pub async fn rotate_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
+        let started = Instant::now();
+        let result = self.backend.rotate_key(key_id).await;
+        if result.is_ok() {
+            self.invalidate_cached_metadata(key_id).await;
+        }
+        self.audit(KmsAuditOperation::RotateKey, context, Some(key_id), started, &result);
+        result
+    }
+
+    /// Replace a key's description; `None` clears it
+    pub async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
+        self.backend.update_key_description(key_id, description).await?;
         self.invalidate_cached_metadata(key_id).await;
         Ok(())
     }
 
-    /// Rotate a key to a new version
-    pub async fn rotate_key(&self, key_id: &str) -> Result<()> {
-        self.backend.rotate_key(key_id).await?;
+    /// Add or overwrite key tags, leaving every other tag untouched
+    pub async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
+        self.backend.tag_key(key_id, tags).await?;
+        self.invalidate_cached_metadata(key_id).await;
+        Ok(())
+    }
+
+    /// Remove key tags; tags that are not set are ignored
+    pub async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
+        self.backend.untag_key(key_id, tag_keys).await?;
         self.invalidate_cached_metadata(key_id).await;
         Ok(())
     }
@@ -208,11 +399,470 @@ impl KmsManager {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::audit::KmsAuditOutcome;
     use crate::backends::local::LocalKmsBackend;
-    use crate::types::{KeySpec, KeyState, KeyUsage};
+    use crate::error::KmsError;
+    use crate::types::{KeyMetadata, KeySpec, KeyState, KeyUsage};
+    use async_trait::async_trait;
+    use base64::Engine as _;
+    use jiff::Zoned;
     use std::collections::HashMap;
+    use std::sync::Mutex;
     use tempfile::tempdir;
 
+    /// Sink that keeps every record so tests can assert on the audit trail.
+    #[derive(Default)]
+    struct CapturingSink {
+        records: Mutex<Vec<KmsAuditRecord>>,
+    }
+
+    impl KmsAuditSink for CapturingSink {
+        fn emit(&self, record: KmsAuditRecord) {
+            self.records
+                .lock()
+                .expect("audit records lock should not be poisoned")
+                .push(record);
+        }
+    }
+
+    impl CapturingSink {
+        fn records(&self) -> Vec<KmsAuditRecord> {
+            self.records
+                .lock()
+                .expect("audit records lock should not be poisoned")
+                .clone()
+        }
+
+        fn take_one(&self) -> KmsAuditRecord {
+            let mut records = self.records.lock().expect("audit records lock should not be poisoned");
+            assert_eq!(records.len(), 1, "expected exactly one audit record, got {records:?}");
+            records.remove(0)
+        }
+    }
+
+    /// Backend whose management operations all succeed or all fail, so a
+    /// single test can drive every audited operation down both paths —
+    /// including operations no real backend supports on both.
+    struct ScriptedBackend {
+        failure: Option<KmsError>,
+    }
+
+    impl ScriptedBackend {
+        fn succeeding() -> Self {
+            Self { failure: None }
+        }
+
+        fn failing(failure: KmsError) -> Self {
+            Self { failure: Some(failure) }
+        }
+
+        fn check(&self) -> Result<()> {
+            match &self.failure {
+                Some(failure) => Err(failure.clone()),
+                None => Ok(()),
+            }
+        }
+
+        fn metadata(key_id: &str) -> KeyMetadata {
+            KeyMetadata {
+                key_id: key_id.to_string(),
+                key_state: KeyState::Enabled,
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: None,
+                creation_date: Zoned::now(),
+                deletion_date: None,
+                origin: "RUSTFS_KMS".to_string(),
+                key_manager: "RUSTFS".to_string(),
+                tags: HashMap::new(),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl KmsBackend for ScriptedBackend {
+        async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
+            self.check()?;
+            let key_id = request.key_name.unwrap_or_else(|| "scripted-key".to_string());
+            Ok(CreateKeyResponse {
+                key_metadata: Self::metadata(&key_id),
+                key_id,
+            })
+        }
+
+        async fn encrypt(&self, _request: EncryptRequest) -> Result<EncryptResponse> {
+            unimplemented!("data plane is not audited by the manager")
+        }
+
+        async fn decrypt(&self, _request: DecryptRequest) -> Result<DecryptResponse> {
+            unimplemented!("data plane is not audited by the manager")
+        }
+
+        async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
+            unimplemented!("data plane is not audited by the manager")
+        }
+
+        async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
+            self.check()?;
+            Ok(DescribeKeyResponse {
+                key_metadata: Self::metadata(&request.key_id),
+            })
+        }
+
+        async fn list_keys(&self, _request: ListKeysRequest) -> Result<ListKeysResponse> {
+            self.check()?;
+            Ok(ListKeysResponse {
+                keys: Vec::new(),
+                next_marker: None,
+                truncated: false,
+            })
+        }
+
+        async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
+            self.check()?;
+            Ok(DeleteKeyResponse {
+                key_metadata: Self::metadata(&request.key_id),
+                key_id: request.key_id,
+                deletion_date: None,
+            })
+        }
+
+        async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
+            self.check()?;
+            Ok(CancelKeyDeletionResponse {
+                key_metadata: Self::metadata(&request.key_id),
+                key_id: request.key_id,
+            })
+        }
+
+        async fn enable_key(&self, _key_id: &str) -> Result<()> {
+            self.check()
+        }
+
+        async fn disable_key(&self, _key_id: &str) -> Result<()> {
+            self.check()
+        }
+
+        async fn rotate_key(&self, _key_id: &str) -> Result<()> {
+            self.check()
+        }
+
+        async fn health_check(&self) -> Result<bool> {
+            Ok(true)
+        }
+    }
+
+    const AUDITED_KEY_ID: &str = "audited-key";
+
+    /// Prefix length used when checking that a record embedded no *part* of a
+    /// secret. Long enough that a collision with unrelated text is not a real
+    /// concern.
+    const FRAGMENT_LEN: usize = 24;
+
+    fn scripted_manager(backend: ScriptedBackend) -> (KmsManager, Arc<CapturingSink>) {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let sink = Arc::new(CapturingSink::default());
+        let manager = KmsManager::new(Arc::new(backend), config).with_audit_sink(sink.clone());
+        (manager, sink)
+    }
+
+    fn request_context() -> OperationContext {
+        OperationContext::new("arn:aws:iam::user/alice".to_string())
+            .with_source_ip("192.0.2.10".to_string())
+            .with_user_agent("rustfs-admin/1".to_string())
+            .with_context("requestID".to_string(), "req-42".to_string())
+    }
+
+    /// Drive one management operation and return the single record it emitted.
+    async fn run_operation(manager: &KmsManager, operation: KmsAuditOperation, context: &OperationContext) -> Result<()> {
+        match operation {
+            KmsAuditOperation::CreateKey => manager
+                .create_key_with_context(
+                    CreateKeyRequest {
+                        key_name: Some(AUDITED_KEY_ID.to_string()),
+                        ..Default::default()
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::DescribeKey => manager
+                .describe_key_with_context(
+                    DescribeKeyRequest {
+                        key_id: AUDITED_KEY_ID.to_string(),
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::ListKeys => manager
+                .list_keys_with_context(ListKeysRequest::default(), context)
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::ScheduleKeyDeletion => manager
+                .delete_key_with_context(
+                    DeleteKeyRequest {
+                        key_id: AUDITED_KEY_ID.to_string(),
+                        pending_window_in_days: None,
+                        force_immediate: None,
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::CancelKeyDeletion => manager
+                .cancel_key_deletion_with_context(
+                    CancelKeyDeletionRequest {
+                        key_id: AUDITED_KEY_ID.to_string(),
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::EnableKey => manager.enable_key_with_context(AUDITED_KEY_ID, context).await,
+            KmsAuditOperation::DisableKey => manager.disable_key_with_context(AUDITED_KEY_ID, context).await,
+            KmsAuditOperation::RotateKey => manager.rotate_key_with_context(AUDITED_KEY_ID, context).await,
+            // Physical removal happens on the background sweep, not here.
+            KmsAuditOperation::DeleteKey => unreachable!("removal is audited by the deletion worker"),
+        }
+    }
+
+    /// Every management operation the manager serves, in audit terms.
+    const AUDITED_OPERATIONS: [KmsAuditOperation; 8] = [
+        KmsAuditOperation::CreateKey,
+        KmsAuditOperation::DescribeKey,
+        KmsAuditOperation::ListKeys,
+        KmsAuditOperation::ScheduleKeyDeletion,
+        KmsAuditOperation::CancelKeyDeletion,
+        KmsAuditOperation::EnableKey,
+        KmsAuditOperation::DisableKey,
+        KmsAuditOperation::RotateKey,
+    ];
+
+    #[tokio::test]
+    async fn every_management_operation_emits_a_complete_success_record() {
+        for operation in AUDITED_OPERATIONS {
+            let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
+            let context = request_context();
+
+            let outer = Instant::now();
+            run_operation(&manager, operation, &context)
+                .await
+                .unwrap_or_else(|error| panic!("{} should succeed: {error}", operation.as_str()));
+            let outer_elapsed = outer.elapsed();
+
+            let record = sink.take_one();
+            assert_eq!(record.operation, operation);
+            assert_eq!(record.event, operation.event_name());
+            assert_eq!(record.outcome, KmsAuditOutcome::Success);
+            assert_eq!(record.error_class, None);
+            assert_eq!(record.operation_id, context.operation_id);
+            assert_eq!(record.principal, "arn:aws:iam::user/alice");
+            assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
+            assert_eq!(record.user_agent.as_deref(), Some("rustfs-admin/1"));
+            assert_eq!(record.backend, "local");
+            assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-42"));
+            assert!(
+                record.latency <= outer_elapsed,
+                "{} reported a latency larger than the call it measured",
+                operation.as_str()
+            );
+
+            // Listing spans keys; every other operation names the key it touched.
+            if operation == KmsAuditOperation::ListKeys {
+                assert_eq!(record.key_id, None);
+            } else {
+                assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn every_management_operation_emits_a_failure_record() {
+        for operation in AUDITED_OPERATIONS {
+            let (manager, sink) = scripted_manager(ScriptedBackend::failing(KmsError::access_denied("denied by policy")));
+            let context = request_context();
+
+            let error = run_operation(&manager, operation, &context)
+                .await
+                .expect_err("scripted backend should reject the operation");
+            assert!(matches!(error, KmsError::AccessDenied { .. }));
+
+            let record = sink.take_one();
+            assert_eq!(record.operation, operation);
+            assert_eq!(record.event, operation.event_name());
+            assert_eq!(record.outcome, KmsAuditOutcome::Failure);
+            assert_eq!(record.error_class, Some("access_denied"));
+            assert_eq!(record.operation_id, context.operation_id);
+            assert_eq!(record.principal, "arn:aws:iam::user/alice");
+            assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
+
+            // A denied create still has to name the key the caller asked for,
+            // otherwise the record cannot answer "what were they after".
+            if operation != KmsAuditOperation::ListKeys {
+                assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn operations_are_unaffected_when_no_sink_is_installed() {
+        // The audit trail is optional; without a sink the manager must behave
+        // exactly as it did before records existed.
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let manager = KmsManager::new(Arc::new(ScriptedBackend::succeeding()), config);
+
+        for operation in AUDITED_OPERATIONS {
+            run_operation(&manager, operation, &request_context())
+                .await
+                .unwrap_or_else(|error| panic!("{} should succeed without a sink: {error}", operation.as_str()));
+        }
+    }
+
+    #[tokio::test]
+    async fn context_free_calls_are_attributed_to_the_internal_principal() {
+        // Callers that have no authenticated identity must still be
+        // distinguishable from an identity we failed to record.
+        let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
+
+        manager
+            .describe_key(DescribeKeyRequest {
+                key_id: AUDITED_KEY_ID.to_string(),
+            })
+            .await
+            .expect("describe should succeed");
+
+        let record = sink.take_one();
+        assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
+        assert_eq!(record.source_ip, None);
+        assert_eq!(record.user_agent, None);
+    }
+
+    #[tokio::test]
+    async fn unsupported_lifecycle_operations_are_audited_with_their_own_class() {
+        // The local backend has no version history, so rotation is a capability
+        // gap rather than a policy denial; the audit trail must say so.
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
+        let sink = Arc::new(CapturingSink::default());
+        let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
+
+        let key_id = manager
+            .create_key_with_context(
+                CreateKeyRequest {
+                    key_name: Some("rotate-me".to_string()),
+                    ..Default::default()
+                },
+                &request_context(),
+            )
+            .await
+            .expect("create should succeed")
+            .key_id;
+
+        manager
+            .rotate_key_with_context(&key_id, &request_context())
+            .await
+            .expect_err("local rotation must be rejected");
+
+        let records = sink.records();
+        let rotate = records.last().expect("rotation should be audited");
+        assert_eq!(rotate.operation, KmsAuditOperation::RotateKey);
+        assert_eq!(rotate.outcome, KmsAuditOutcome::Failure);
+        assert_eq!(rotate.error_class, Some("unsupported_capability"));
+        assert_eq!(rotate.key_id.as_deref(), Some(key_id.as_str()));
+    }
+
+    /// Negative assertion: no audit record may reproduce key material. Driven
+    /// against the real local backend so the assertion covers whatever the
+    /// backend actually hands back, not a hand-written stand-in.
+    #[tokio::test]
+    async fn audit_records_never_reproduce_key_material() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
+        let sink = Arc::new(CapturingSink::default());
+        let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
+
+        let grant_token = "grant-token-cec4d4b5a1";
+        let context = request_context();
+
+        let key_id = manager
+            .create_key_with_context(
+                CreateKeyRequest {
+                    key_name: Some("material-key".to_string()),
+                    ..Default::default()
+                },
+                &context,
+            )
+            .await
+            .expect("create should succeed")
+            .key_id;
+
+        // Produce real key material, then keep driving the management plane so
+        // any record built afterwards is covered by the assertions below.
+        let data_key = manager
+            .generate_data_key(GenerateDataKeyRequest {
+                key_id: key_id.clone(),
+                key_spec: KeySpec::Aes256,
+                encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
+            })
+            .await
+            .expect("data key generation should succeed");
+        let decrypted = manager
+            .decrypt(DecryptRequest {
+                ciphertext: data_key.ciphertext_blob.clone(),
+                encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
+                grant_tokens: vec![grant_token.to_string()],
+            })
+            .await
+            .expect("decrypt should succeed");
+
+        manager
+            .describe_key_with_context(DescribeKeyRequest { key_id: key_id.clone() }, &context)
+            .await
+            .expect("describe should succeed");
+        manager
+            .list_keys_with_context(ListKeysRequest::default(), &context)
+            .await
+            .expect("list should succeed");
+        manager
+            .disable_key_with_context(&key_id, &context)
+            .await
+            .expect("disable should succeed");
+        manager
+            .enable_key_with_context(&key_id, &context)
+            .await
+            .expect("enable should succeed");
+
+        let base64 = base64::engine::general_purpose::STANDARD;
+        let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
+        let mut forbidden = vec![grant_token.to_string()];
+        forbidden.extend(encodings(&data_key.plaintext_key));
+        forbidden.extend(encodings(&decrypted.plaintext));
+        forbidden.extend(encodings(&data_key.ciphertext_blob));
+        // Fragments catch a record that embedded only part of a blob.
+        let fragments: Vec<String> = forbidden
+            .iter()
+            .filter(|secret| secret.len() > FRAGMENT_LEN)
+            .map(|secret| secret[..FRAGMENT_LEN].to_string())
+            .collect();
+        forbidden.extend(fragments);
+
+        let records = sink.records();
+        assert!(!records.is_empty(), "management operations should have been audited");
+        for record in &records {
+            let rendered = format!("{record:?}");
+            for secret in &forbidden {
+                assert!(
+                    !rendered.contains(secret.as_str()),
+                    "audit record leaked key material or a grant token: {rendered}"
+                );
+            }
+        }
+    }
+
     #[tokio::test]
     async fn test_manager_operations() {
         let temp_dir = tempdir().expect("Failed to create temp dir");
@@ -254,9 +904,12 @@ mod tests {
         let describe_response = manager.describe_key(describe_request).await.expect("Failed to describe key");
         assert_eq!(describe_response.key_metadata.key_id, create_response.key_id);
 
-        // Test cache stats
-        let stats = manager.cache_stats().await;
-        assert!(stats.is_some());
+        // Creating the key populated the cache, so the describe above was
+        // served from it rather than from the backend.
+        let stats = manager.cache_stats().await.expect("cache is enabled");
+        assert_eq!(stats.entries, 1);
+        assert_eq!(stats.hits, 1);
+        assert_eq!(stats.misses, 0);
 
         // Test health check
         let health = manager.health_check().await.expect("Health check failed");

crates/kms/src/policy.rs

@@ -104,7 +104,9 @@ pub(crate) fn classify_vaultrs(error: &vaultrs::error::ClientError) -> ErrorClas
     }
 }
 
-fn classify_status(code: u16) -> ErrorClass {
+/// Retry classification of an HTTP status, shared by every backend that talks
+/// to an external KMS over HTTP.
+pub(crate) fn classify_status(code: u16) -> ErrorClass {
     match code {
         429 | 500 | 502 | 503 | 504 => ErrorClass::RetryableStatus,
         _ => ErrorClass::Fatal,

crates/kms/src/probe.rs

@@ -0,0 +1,894 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Background synthetic probe for the configured KMS backend.
+//!
+//! A health check that only reports "the backend answered" cannot distinguish a
+//! usable KMS from one that is reachable but no longer returns material that
+//! round-trips. Each probe round therefore generates a data key under a
+//! dedicated key, decrypts the returned ciphertext, and compares the plaintext,
+//! so a backend that answers without being usable becomes visible before object
+//! traffic depends on it.
+//!
+//! The probe key uses a fixed reserved id ([`PROBE_KEY_ID`]) and is created only
+//! when missing, so every node of a deployment converges on the same key and a
+//! create that loses the race against another node is a success, not an error.
+//!
+//! Backends that cannot host such a key — single-key read-only backends, or
+//! backends without data key generation — are reported as
+//! [`ProbeResult::Unsupported`] exactly once and then left alone: an
+//! unsupported backend is a configuration fact, not an outage, and must never
+//! raise failure metrics or alerts.
+//!
+//! Results are published two ways: as a lock-free [`ProbeStatus`] snapshot that
+//! readiness reporting can read without touching the backend, and as
+//! `rustfs_kms_probe_*` metrics. As everywhere else in this crate, metric labels
+//! carry only static outcome classes — never key identifiers, key material, or
+//! ciphertext.
+
+use crate::backends::KmsBackend;
+use crate::error::KmsError;
+use crate::types::{CreateKeyRequest, DecryptRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyUsage};
+use arc_swap::ArcSwap;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use subtle::ConstantTimeEq;
+use tokio::time::Instant;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, info, warn};
+use zeroize::Zeroize;
+
+/// Reserved id of the key every probe round runs through.
+///
+/// Fixed rather than per-node or per-process: all nodes must converge on one
+/// key instead of littering the backend with probe keys, and the reserved
+/// `rustfs-internal-` prefix marks it as RustFS-owned for operators reviewing
+/// the key list.
+pub const PROBE_KEY_ID: &str = "rustfs-internal-kms-probe";
+
+/// Description stored on the probe key when this node creates it.
+const PROBE_KEY_DESCRIPTION: &str = "RustFS internal KMS health probe key; created and used only by the synthetic probe";
+
+/// Default interval between probe rounds.
+pub const DEFAULT_PROBE_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Floor for the configured interval. Probing faster buys no additional signal
+/// and multiplies backend load by the size of the deployment.
+pub const MIN_PROBE_INTERVAL: Duration = Duration::from_secs(5);
+
+/// Probe interval in whole seconds; `0` disables the probe entirely.
+pub const ENV_KMS_PROBE_INTERVAL_SECS: &str = "RUSTFS_KMS_PROBE_INTERVAL_SECS";
+
+/// Result of the most recent probe round.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProbeResult {
+    /// No round has completed yet for this service version.
+    Pending,
+    /// The full generate/decrypt/compare cycle succeeded.
+    Success,
+    /// The backend cannot host the probe key; no further rounds run and no
+    /// failures are recorded.
+    Unsupported,
+    /// The round failed with the given classification.
+    Failure(ProbeFailureKind),
+}
+
+impl ProbeResult {
+    /// Static metric label for this result.
+    fn as_label(self) -> &'static str {
+        match self {
+            ProbeResult::Pending => "pending",
+            ProbeResult::Success => "success",
+            ProbeResult::Unsupported => "unsupported",
+            ProbeResult::Failure(_) => "failure",
+        }
+    }
+}
+
+/// Stage of the round trip that failed, used as a static metric label.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProbeFailureKind {
+    /// The dedicated probe key could not be described or created.
+    KeyProvisioning,
+    /// Data key generation failed.
+    Generate,
+    /// The freshly generated ciphertext could not be decrypted.
+    Decrypt,
+    /// Decryption succeeded but returned material that differs from what was
+    /// generated — the backend answers without round-tripping.
+    Mismatch,
+}
+
+impl ProbeFailureKind {
+    /// Static metric label for this failure class.
+    fn as_label(self) -> &'static str {
+        match self {
+            ProbeFailureKind::KeyProvisioning => "key_provisioning",
+            ProbeFailureKind::Generate => "generate",
+            ProbeFailureKind::Decrypt => "decrypt",
+            ProbeFailureKind::Mismatch => "mismatch",
+        }
+    }
+}
+
+/// Snapshot of what the probe has observed so far.
+///
+/// Consumed by readiness reporting, which owns the staleness and consecutive
+/// failure thresholds; the probe itself only records what happened.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ProbeStatus {
+    /// Result of the most recent completed round.
+    pub result: ProbeResult,
+    /// When the most recent round completed.
+    pub last_round_at: Option<Instant>,
+    /// When the most recent successful round completed.
+    pub last_success_at: Option<Instant>,
+    /// Wall-clock timestamp of the most recent success, in Unix seconds, for
+    /// display and for the exported gauge. Use [`ProbeStatus::last_success_age`]
+    /// for staleness decisions: it is monotonic and immune to clock steps.
+    pub last_success_unix_secs: Option<i64>,
+    /// Failed rounds since the last success; reset by any non-failure result.
+    pub consecutive_failures: u32,
+}
+
+impl ProbeStatus {
+    /// Initial snapshot published before the first round completes.
+    fn pending() -> Self {
+        Self {
+            result: ProbeResult::Pending,
+            last_round_at: None,
+            last_success_at: None,
+            last_success_unix_secs: None,
+            consecutive_failures: 0,
+        }
+    }
+
+    /// How long ago the most recent round completed.
+    pub fn last_round_age(&self) -> Option<Duration> {
+        self.last_round_at.map(|at| at.elapsed())
+    }
+
+    /// How long ago the most recent successful round completed.
+    pub fn last_success_age(&self) -> Option<Duration> {
+        self.last_success_at.map(|at| at.elapsed())
+    }
+}
+
+/// Lock-free publication slot for [`ProbeStatus`].
+///
+/// Written by the single worker task and read by anyone; readers never block
+/// the probe and the probe never blocks a readiness handler.
+struct ProbeState {
+    status: ArcSwap<ProbeStatus>,
+}
+
+impl ProbeState {
+    fn new() -> Self {
+        Self {
+            status: ArcSwap::from_pointee(ProbeStatus::pending()),
+        }
+    }
+
+    fn status(&self) -> Arc<ProbeStatus> {
+        self.status.load_full()
+    }
+
+    /// Fold one round result into the published snapshot.
+    ///
+    /// Safe as a plain load-then-store because the owning worker task is the
+    /// only writer.
+    fn publish(&self, result: ProbeResult) {
+        let previous = self.status.load();
+        let now = Instant::now();
+        let (last_success_at, last_success_unix_secs) = match result {
+            ProbeResult::Success => (Some(now), Some(jiff::Timestamp::now().as_second())),
+            _ => (previous.last_success_at, previous.last_success_unix_secs),
+        };
+        let consecutive_failures = match result {
+            ProbeResult::Failure(_) => previous.consecutive_failures.saturating_add(1),
+            _ => 0,
+        };
+        self.status.store(Arc::new(ProbeStatus {
+            result,
+            last_round_at: Some(now),
+            last_success_at,
+            last_success_unix_secs,
+            consecutive_failures,
+        }));
+        record_state(result, last_success_unix_secs, consecutive_failures);
+    }
+}
+
+/// Owner of one service version's probe worker.
+///
+/// Mirrors the deletion worker's ownership model: cancelling stops the loop,
+/// and dropping the handle — a service version replaced without an explicit
+/// stop — cancels as a safety net.
+pub struct ProbeHandle {
+    state: Arc<ProbeState>,
+    cancel: CancellationToken,
+    task: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
+}
+
+impl ProbeHandle {
+    /// Latest published snapshot.
+    pub fn status(&self) -> Arc<ProbeStatus> {
+        self.state.status()
+    }
+
+    /// Stop the worker. The task observes the cancelled token on its next poll.
+    pub(crate) fn shutdown(&self) {
+        self.cancel.cancel();
+        if let Ok(mut task) = self.task.lock() {
+            drop(task.take());
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn is_cancelled(&self) -> bool {
+        self.cancel.is_cancelled()
+    }
+}
+
+impl Drop for ProbeHandle {
+    fn drop(&mut self) {
+        self.cancel.cancel();
+    }
+}
+
+/// Start the probe for a service version.
+///
+/// Returns `None` when the backend advertises no data key round trip, or when
+/// the operator disabled the probe; both cases leave no background task behind.
+pub(crate) fn spawn_probe_worker(backend: Arc<dyn KmsBackend>) -> Option<ProbeHandle> {
+    let capabilities = backend.capabilities();
+    if !capabilities.generate_data_key || !capabilities.decrypt {
+        return None;
+    }
+    let interval = configured_interval()?;
+    let state = Arc::new(ProbeState::new());
+    let cancel = CancellationToken::new();
+    let task = ProbeWorker::new(backend, state.clone(), interval).spawn(cancel.clone());
+    Some(ProbeHandle {
+        state,
+        cancel,
+        task: std::sync::Mutex::new(Some(task)),
+    })
+}
+
+/// Probe interval from the environment, or `None` when the probe is disabled.
+fn configured_interval() -> Option<Duration> {
+    parse_interval(std::env::var(ENV_KMS_PROBE_INTERVAL_SECS).ok().as_deref())
+}
+
+/// Interpret the configured interval: unset or unparsable falls back to the
+/// default, `0` disables the probe, and anything below [`MIN_PROBE_INTERVAL`]
+/// is raised to it so a misconfigured value cannot turn the probe into a load
+/// generator against the backend.
+fn parse_interval(value: Option<&str>) -> Option<Duration> {
+    let Some(value) = value else {
+        return Some(DEFAULT_PROBE_INTERVAL);
+    };
+    let Ok(seconds) = value.trim().parse::<u64>() else {
+        warn!(
+            variable = ENV_KMS_PROBE_INTERVAL_SECS,
+            "ignoring unparsable KMS probe interval; falling back to the default"
+        );
+        return Some(DEFAULT_PROBE_INTERVAL);
+    };
+    if seconds == 0 {
+        return None;
+    }
+    Some(Duration::from_secs(seconds).max(MIN_PROBE_INTERVAL))
+}
+
+/// Encryption context bound to every probe data key.
+///
+/// Static and free of per-node values, so material generated by one node stays
+/// decryptable by every other node running the same probe.
+fn probe_encryption_context() -> HashMap<String, String> {
+    HashMap::from([("rustfs-purpose".to_string(), "kms-probe".to_string())])
+}
+
+/// Outcome of making sure the dedicated probe key exists.
+enum KeyProvisioning {
+    Ready,
+    Unsupported,
+    Failed,
+}
+
+pub(crate) struct ProbeWorker {
+    backend: Arc<dyn KmsBackend>,
+    state: Arc<ProbeState>,
+    interval: Duration,
+}
+
+impl ProbeWorker {
+    fn new(backend: Arc<dyn KmsBackend>, state: Arc<ProbeState>, interval: Duration) -> Self {
+        Self {
+            backend,
+            state,
+            interval,
+        }
+    }
+
+    fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
+        tokio::spawn(async move { self.run(cancel).await })
+    }
+
+    async fn run(self, cancel: CancellationToken) {
+        describe_metrics();
+        let mut ticker = tokio::time::interval(self.interval);
+        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+        loop {
+            tokio::select! {
+                _ = cancel.cancelled() => {
+                    debug!("KMS probe worker stopped");
+                    return;
+                }
+                _ = ticker.tick() => {}
+            }
+            let started = Instant::now();
+            let result = self.round().await;
+            record_round(result, started.elapsed());
+            self.state.publish(result);
+            if result == ProbeResult::Unsupported {
+                // A capability gap does not change while a service version
+                // lives, so retrying it forever would only produce noise.
+                info!("KMS backend cannot host the synthetic probe key; probe disabled for this service version");
+                return;
+            }
+        }
+    }
+
+    /// Run one round. Exposed separately so tests can drive the round trip
+    /// deterministically without the interval loop.
+    pub(crate) async fn round(&self) -> ProbeResult {
+        let capabilities = self.backend.capabilities();
+        if !capabilities.generate_data_key || !capabilities.decrypt {
+            return ProbeResult::Unsupported;
+        }
+        match self.ensure_probe_key().await {
+            KeyProvisioning::Ready => {}
+            KeyProvisioning::Unsupported => return ProbeResult::Unsupported,
+            KeyProvisioning::Failed => return ProbeResult::Failure(ProbeFailureKind::KeyProvisioning),
+        }
+
+        let mut generated = match self
+            .backend
+            .generate_data_key(GenerateDataKeyRequest {
+                key_id: PROBE_KEY_ID.to_string(),
+                key_spec: KeySpec::Aes256,
+                encryption_context: probe_encryption_context(),
+            })
+            .await
+        {
+            Ok(generated) => generated,
+            Err(error) => {
+                warn!(%error, "KMS probe could not generate a data key");
+                return ProbeResult::Failure(ProbeFailureKind::Generate);
+            }
+        };
+
+        let decrypted = self
+            .backend
+            .decrypt(DecryptRequest {
+                ciphertext: generated.ciphertext_blob.clone(),
+                encryption_context: probe_encryption_context(),
+                grant_tokens: Vec::new(),
+            })
+            .await;
+
+        let result = match decrypted {
+            Ok(mut response) => {
+                let round_trips = bool::from(response.plaintext.ct_eq(&generated.plaintext_key));
+                response.plaintext.zeroize();
+                if round_trips {
+                    debug!("KMS probe round trip succeeded");
+                    ProbeResult::Success
+                } else {
+                    // The backend answered both calls but the material does not
+                    // survive the round trip: report it as loudly as an outage.
+                    warn!("KMS probe decrypted material does not match the generated data key");
+                    ProbeResult::Failure(ProbeFailureKind::Mismatch)
+                }
+            }
+            Err(error) => {
+                warn!(%error, "KMS probe could not decrypt its generated data key");
+                ProbeResult::Failure(ProbeFailureKind::Decrypt)
+            }
+        };
+        generated.plaintext_key.zeroize();
+        result
+    }
+
+    /// Create the probe key if it is missing.
+    ///
+    /// Idempotent by construction: an existing key is used as is, and a create
+    /// that loses the race against another node reports the key as ready
+    /// instead of failing the round.
+    async fn ensure_probe_key(&self) -> KeyProvisioning {
+        match self
+            .backend
+            .describe_key(DescribeKeyRequest {
+                key_id: PROBE_KEY_ID.to_string(),
+            })
+            .await
+        {
+            Ok(_) => return KeyProvisioning::Ready,
+            Err(KmsError::KeyNotFound { .. }) => {}
+            Err(error) if is_capability_gap(&error) => return KeyProvisioning::Unsupported,
+            Err(error) => {
+                warn!(%error, "KMS probe could not describe its probe key");
+                return KeyProvisioning::Failed;
+            }
+        }
+
+        match self
+            .backend
+            .create_key(CreateKeyRequest {
+                key_name: Some(PROBE_KEY_ID.to_string()),
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: Some(PROBE_KEY_DESCRIPTION.to_string()),
+                ..Default::default()
+            })
+            .await
+        {
+            Ok(_) => {
+                info!(key_id = PROBE_KEY_ID, "created the KMS synthetic probe key");
+                KeyProvisioning::Ready
+            }
+            // Another node created the key between the describe and the create.
+            Err(KmsError::KeyAlreadyExists { .. }) => KeyProvisioning::Ready,
+            Err(error) if is_capability_gap(&error) => KeyProvisioning::Unsupported,
+            Err(error) => {
+                warn!(%error, "KMS probe could not create its probe key");
+                KeyProvisioning::Failed
+            }
+        }
+    }
+}
+
+/// Whether an error means "this backend will never host a probe key".
+///
+/// Read-only single-key backends reject key creation outright, and backends
+/// without the operation report an unsupported capability. Both are properties
+/// of the configuration rather than symptoms of an outage, so they stop the
+/// probe instead of counting as failures.
+fn is_capability_gap(error: &KmsError) -> bool {
+    matches!(error, KmsError::UnsupportedCapability { .. } | KmsError::InvalidOperation { .. })
+}
+
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// Label values are exclusively static outcome classes; the probe key id is a
+// compile-time constant and still never becomes a label, so the metric shape
+// stays independent of how many keys or nodes exist.
+// ---------------------------------------------------------------------------
+
+/// Counter: probe rounds completed, by `result`.
+const METRIC_PROBE_ROUNDS_TOTAL: &str = "rustfs_kms_probe_rounds_total";
+/// Counter: failed probe rounds, by `failure_kind`.
+const METRIC_PROBE_FAILURES_TOTAL: &str = "rustfs_kms_probe_failures_total";
+/// Histogram: wall-clock duration of one probe round, in seconds, by `result`.
+const METRIC_PROBE_DURATION_SECONDS: &str = "rustfs_kms_probe_duration_seconds";
+/// Gauge: Unix timestamp of the most recent successful round.
+const METRIC_PROBE_LAST_SUCCESS_TIMESTAMP: &str = "rustfs_kms_probe_last_success_timestamp_seconds";
+/// Gauge: failed rounds since the last success.
+const METRIC_PROBE_CONSECUTIVE_FAILURES: &str = "rustfs_kms_probe_consecutive_failures";
+
+/// Register metric descriptions once per process.
+fn describe_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_counter!(METRIC_PROBE_ROUNDS_TOTAL, "Total synthetic KMS probe rounds completed, by result");
+        metrics::describe_counter!(
+            METRIC_PROBE_FAILURES_TOTAL,
+            "Total failed synthetic KMS probe rounds, by the round-trip stage that failed"
+        );
+        metrics::describe_histogram!(
+            METRIC_PROBE_DURATION_SECONDS,
+            "Wall-clock duration of one synthetic KMS probe round, in seconds"
+        );
+        metrics::describe_gauge!(
+            METRIC_PROBE_LAST_SUCCESS_TIMESTAMP,
+            "Unix timestamp of the most recent successful synthetic KMS probe round"
+        );
+        metrics::describe_gauge!(
+            METRIC_PROBE_CONSECUTIVE_FAILURES,
+            "Synthetic KMS probe rounds that have failed since the last success"
+        );
+    });
+}
+
+/// Record one completed round.
+///
+/// An unsupported backend is deliberately counted as its own result and never
+/// as a failure, so alerts on the failure counter stay silent for deployments
+/// whose backend cannot host the probe.
+fn record_round(result: ProbeResult, elapsed: Duration) {
+    metrics::counter!(METRIC_PROBE_ROUNDS_TOTAL, "result" => result.as_label()).increment(1);
+    if let ProbeResult::Failure(kind) = result {
+        metrics::counter!(METRIC_PROBE_FAILURES_TOTAL, "failure_kind" => kind.as_label()).increment(1);
+    }
+    metrics::histogram!(METRIC_PROBE_DURATION_SECONDS, "result" => result.as_label()).record(elapsed.as_secs_f64());
+}
+
+/// Mirror the published snapshot into the gauges.
+fn record_state(result: ProbeResult, last_success_unix_secs: Option<i64>, consecutive_failures: u32) {
+    if result == ProbeResult::Success
+        && let Some(timestamp) = last_success_unix_secs
+    {
+        // Only ever moved forward by a success, so a failing probe leaves the
+        // gauge at the last known good time and its age keeps growing.
+        metrics::gauge!(METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).set(timestamp as f64);
+    }
+    metrics::gauge!(METRIC_PROBE_CONSECUTIVE_FAILURES).set(f64::from(consecutive_failures));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::backends::local::LocalKmsBackend;
+    use crate::backends::static_kms::StaticKmsBackend;
+    use crate::config::KmsConfig;
+    use crate::types::{
+        CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyResponse, DecryptResponse, DeleteKeyRequest,
+        DeleteKeyResponse, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyResponse, KeyMetadata, KeyState,
+        ListKeysRequest, ListKeysResponse,
+    };
+    use async_trait::async_trait;
+    use base64::Engine as _;
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+    use std::future::Future;
+
+    async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<dyn KmsBackend> {
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build"))
+    }
+
+    async fn static_backend() -> Arc<dyn KmsBackend> {
+        let config =
+            KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
+        Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
+    }
+
+    fn worker(backend: Arc<dyn KmsBackend>) -> ProbeWorker {
+        ProbeWorker::new(backend, Arc::new(ProbeState::new()), DEFAULT_PROBE_INTERVAL)
+    }
+
+    /// Backend that answers every probe call but corrupts the round trip in a
+    /// configurable way, so each failure classification can be provoked without
+    /// a real backend outage.
+    struct BrokenBackend {
+        fault: Fault,
+    }
+
+    #[derive(Clone, Copy)]
+    enum Fault {
+        Generate,
+        Decrypt,
+        /// Decrypt succeeds but returns material other than the generated key.
+        Mismatch,
+    }
+
+    impl BrokenBackend {
+        fn arc(fault: Fault) -> Arc<dyn KmsBackend> {
+            Arc::new(Self { fault })
+        }
+
+        fn probe_key_metadata() -> KeyMetadata {
+            KeyMetadata {
+                key_id: PROBE_KEY_ID.to_string(),
+                key_state: KeyState::Enabled,
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: None,
+                creation_date: jiff::Zoned::now(),
+                deletion_date: None,
+                origin: "test".to_string(),
+                key_manager: "test".to_string(),
+                tags: HashMap::new(),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl KmsBackend for BrokenBackend {
+        async fn create_key(&self, _request: CreateKeyRequest) -> crate::error::Result<CreateKeyResponse> {
+            unimplemented!("the probe key always exists on this backend")
+        }
+
+        async fn encrypt(&self, _request: EncryptRequest) -> crate::error::Result<EncryptResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn decrypt(&self, _request: DecryptRequest) -> crate::error::Result<DecryptResponse> {
+            match self.fault {
+                Fault::Decrypt => Err(KmsError::backend_error("decrypt unavailable")),
+                Fault::Mismatch => Ok(DecryptResponse {
+                    plaintext: vec![0xffu8; 32],
+                    key_id: PROBE_KEY_ID.to_string(),
+                    encryption_algorithm: None,
+                }),
+                Fault::Generate => unreachable!("generation fails before decrypt is reached"),
+            }
+        }
+
+        async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> crate::error::Result<GenerateDataKeyResponse> {
+            match self.fault {
+                Fault::Generate => Err(KmsError::backend_error("data key generation unavailable")),
+                _ => Ok(GenerateDataKeyResponse {
+                    key_id: PROBE_KEY_ID.to_string(),
+                    plaintext_key: vec![0x11u8; 32],
+                    ciphertext_blob: vec![0x22u8; 48],
+                }),
+            }
+        }
+
+        async fn describe_key(&self, _request: DescribeKeyRequest) -> crate::error::Result<DescribeKeyResponse> {
+            Ok(DescribeKeyResponse {
+                key_metadata: Self::probe_key_metadata(),
+            })
+        }
+
+        async fn list_keys(&self, _request: ListKeysRequest) -> crate::error::Result<ListKeysResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn delete_key(&self, _request: DeleteKeyRequest) -> crate::error::Result<DeleteKeyResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn cancel_key_deletion(
+            &self,
+            _request: CancelKeyDeletionRequest,
+        ) -> crate::error::Result<CancelKeyDeletionResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn health_check(&self) -> crate::error::Result<bool> {
+            Ok(true)
+        }
+    }
+
+    #[tokio::test]
+    async fn round_creates_the_probe_key_once_and_round_trips_it() {
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let backend = local_backend(&temp_dir).await;
+        let worker = worker(backend.clone());
+
+        assert_eq!(worker.round().await, ProbeResult::Success);
+        let created = backend
+            .describe_key(DescribeKeyRequest {
+                key_id: PROBE_KEY_ID.to_string(),
+            })
+            .await
+            .expect("the first round must create the probe key");
+        assert_eq!(created.key_metadata.key_id, PROBE_KEY_ID);
+
+        // A second round reuses the key instead of failing on the existing one,
+        // which is also what a second node in the deployment would do.
+        assert_eq!(worker.round().await, ProbeResult::Success);
+        assert_eq!(
+            backend
+                .list_keys(ListKeysRequest {
+                    limit: Some(100),
+                    marker: None,
+                    usage_filter: None,
+                    status_filter: None,
+                })
+                .await
+                .expect("list keys")
+                .keys
+                .iter()
+                .filter(|key| key.key_id == PROBE_KEY_ID)
+                .count(),
+            1,
+            "probe key creation must be idempotent"
+        );
+    }
+
+    #[test]
+    fn read_only_backend_is_reported_unsupported_without_failures() {
+        let (snapshot, result) = record_metrics(|| {
+            Box::pin(async {
+                let worker = worker(static_backend().await);
+                let result = worker.round().await;
+                record_round(result, Duration::from_millis(1));
+                result
+            })
+        });
+
+        assert_eq!(result, ProbeResult::Unsupported);
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "unsupported")]), 1);
+        assert_eq!(
+            counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[]),
+            0,
+            "an unsupported backend must never raise probe failures"
+        );
+    }
+
+    #[tokio::test]
+    async fn each_round_trip_stage_gets_its_own_failure_classification() {
+        for (fault, expected) in [
+            (Fault::Generate, ProbeFailureKind::Generate),
+            (Fault::Decrypt, ProbeFailureKind::Decrypt),
+            (Fault::Mismatch, ProbeFailureKind::Mismatch),
+        ] {
+            assert_eq!(worker(BrokenBackend::arc(fault)).round().await, ProbeResult::Failure(expected));
+        }
+    }
+
+    #[test]
+    fn metrics_and_status_record_success_then_failure() {
+        let (snapshot, status) = record_metrics(|| {
+            Box::pin(async {
+                let temp_dir = tempfile::tempdir().expect("temp dir");
+                let state = Arc::new(ProbeState::new());
+                let healthy = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL);
+                let round = healthy.round().await;
+                record_round(round, Duration::from_millis(10));
+                state.publish(round);
+
+                let broken = ProbeWorker::new(BrokenBackend::arc(Fault::Mismatch), state.clone(), DEFAULT_PROBE_INTERVAL);
+                for _ in 0..2 {
+                    let round = broken.round().await;
+                    record_round(round, Duration::from_millis(10));
+                    state.publish(round);
+                }
+                state.status()
+            })
+        });
+
+        assert_eq!(status.result, ProbeResult::Failure(ProbeFailureKind::Mismatch));
+        assert_eq!(status.consecutive_failures, 2);
+        assert!(
+            status.last_success_at.is_some() && status.last_success_unix_secs.is_some(),
+            "a failing probe must keep reporting when it last succeeded"
+        );
+
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "success")]), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "failure")]), 2);
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[("failure_kind", "mismatch")]), 2);
+        assert_eq!(histogram_values(&snapshot, METRIC_PROBE_DURATION_SECONDS, &[]).len(), 3);
+        assert!(
+            gauge_value(&snapshot, METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).is_some_and(|timestamp| timestamp > 0.0),
+            "the last success gauge must carry a wall-clock timestamp"
+        );
+        assert_eq!(gauge_value(&snapshot, METRIC_PROBE_CONSECUTIVE_FAILURES), Some(2.0));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn worker_publishes_status_and_stops_on_cancel() {
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let state = Arc::new(ProbeState::new());
+        assert_eq!(state.status().result, ProbeResult::Pending);
+
+        let cancel = CancellationToken::new();
+        let task = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
+
+        // The paused clock auto-advances through the worker's interval ticks.
+        let mut observed = None;
+        for _ in 0..100 {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            if state.status().result != ProbeResult::Pending {
+                observed = Some(state.status());
+                break;
+            }
+        }
+        let observed = observed.expect("the worker loop must publish a round result");
+        assert_eq!(observed.result, ProbeResult::Success);
+        assert_eq!(observed.consecutive_failures, 0);
+        assert!(observed.last_success_at.is_some());
+
+        cancel.cancel();
+        task.await.expect("worker task must stop after cancellation");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unsupported_backend_stops_the_worker_without_cancellation() {
+        let state = Arc::new(ProbeState::new());
+        let cancel = CancellationToken::new();
+        let task = ProbeWorker::new(static_backend().await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
+
+        task.await.expect("worker must exit on its own for an unsupported backend");
+        assert_eq!(state.status().result, ProbeResult::Unsupported);
+        assert!(!cancel.is_cancelled());
+    }
+
+    #[test]
+    fn interval_configuration_defaults_clamps_and_disables() {
+        assert_eq!(parse_interval(None), Some(DEFAULT_PROBE_INTERVAL));
+        assert_eq!(parse_interval(Some("not-a-number")), Some(DEFAULT_PROBE_INTERVAL));
+        assert_eq!(parse_interval(Some(" 120 ")), Some(Duration::from_secs(120)));
+        assert_eq!(parse_interval(Some("1")), Some(MIN_PROBE_INTERVAL));
+        assert_eq!(parse_interval(Some("0")), None, "zero must disable the probe");
+    }
+
+    // -- Metric assertion helpers, mirroring the policy engine's tests --------
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Run `test` on a paused current-thread runtime under a debugging recorder
+    /// and return one snapshot of everything it emitted.
+    fn record_metrics<Out>(test: impl FnOnce() -> std::pin::Pin<Box<dyn Future<Output = Out>>>) -> (Vec<MetricEntry>, Out) {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let out = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_time()
+                .start_paused(true)
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (snapshotter.snapshot().into_vec(), out)
+    }
+
+    fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
+        labels
+            .iter()
+            .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected))
+    }
+
+    fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let matches = composite.kind() == MetricKind::Counter
+                    && composite.key().name() == name
+                    && labels_match(composite.key(), labels);
+                match (matches, value) {
+                    (true, DebugValue::Counter(count)) => Some(*count),
+                    _ => None,
+                }
+            })
+            .sum()
+    }
+
+    fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let matches = composite.kind() == MetricKind::Histogram
+                    && composite.key().name() == name
+                    && labels_match(composite.key(), labels);
+                match (matches, value) {
+                    (true, DebugValue::Histogram(values)) => Some(values),
+                    _ => None,
+                }
+            })
+            .flatten()
+            .map(|value| value.into_inner())
+            .collect()
+    }
+
+    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(gauge)) => Some(gauge.into_inner()),
+                _ => None,
+            }
+        })
+    }
+}

crates/kms/src/service.rs

@@ -14,6 +14,10 @@
 
 //! Object encryption service for S3-compatible encryption
 
+use crate::api_types::{
+    TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
+};
+use crate::cache::KmsCacheStats;
 use crate::encryption::ciphers::{create_cipher, generate_iv};
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
@@ -112,6 +116,23 @@ impl ObjectEncryptionService {
         self.kms_manager.create_key(request).await
     }
 
+    /// Create a new master key on behalf of `context`'s principal
+    ///
+    /// # Arguments
+    /// * `request` - CreateKeyRequest with key parameters
+    /// * `context` - Identity and correlation data recorded in the audit trail
+    ///
+    /// # Returns
+    /// CreateKeyResponse with created key details
+    ///
+    pub async fn create_key_with_context(
+        &self,
+        request: CreateKeyRequest,
+        context: &OperationContext,
+    ) -> Result<CreateKeyResponse> {
+        self.kms_manager.create_key_with_context(request, context).await
+    }
+
     /// Describe a master key (delegates to KMS manager)
     ///
     /// # Arguments
@@ -124,6 +145,23 @@ impl ObjectEncryptionService {
         self.kms_manager.describe_key(request).await
     }
 
+    /// Describe a master key on behalf of `context`'s principal
+    ///
+    /// # Arguments
+    /// * `request` - DescribeKeyRequest with key ID
+    /// * `context` - Identity and correlation data recorded in the audit trail
+    ///
+    /// # Returns
+    /// DescribeKeyResponse with key metadata
+    ///
+    pub async fn describe_key_with_context(
+        &self,
+        request: DescribeKeyRequest,
+        context: &OperationContext,
+    ) -> Result<DescribeKeyResponse> {
+        self.kms_manager.describe_key_with_context(request, context).await
+    }
+
     /// List master keys (delegates to KMS manager)
     ///
     /// # Arguments
@@ -136,6 +174,77 @@ impl ObjectEncryptionService {
         self.kms_manager.list_keys(request).await
     }
 
+    /// List master keys on behalf of `context`'s principal
+    ///
+    /// # Arguments
+    /// * `request` - ListKeysRequest with listing parameters
+    /// * `context` - Identity and correlation data recorded in the audit trail
+    ///
+    /// # Returns
+    /// ListKeysResponse with list of keys
+    ///
+    pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
+        self.kms_manager.list_keys_with_context(request, context).await
+    }
+
+    /// Replace a master key's description (delegates to KMS manager)
+    ///
+    /// # Arguments
+    /// * `request` - UpdateKeyDescriptionRequest with key ID and the new
+    ///   description; an empty description clears the stored value
+    ///
+    /// # Returns
+    /// UpdateKeyDescriptionResponse acknowledging the update
+    ///
+    pub async fn update_key_description(&self, request: UpdateKeyDescriptionRequest) -> Result<UpdateKeyDescriptionResponse> {
+        let description = (!request.description.is_empty()).then_some(request.description.as_str());
+        self.kms_manager.update_key_description(&request.key_id, description).await?;
+
+        Ok(UpdateKeyDescriptionResponse {
+            success: true,
+            message: "key description updated".to_string(),
+            key_id: request.key_id,
+        })
+    }
+
+    /// Add or overwrite master key tags (delegates to KMS manager)
+    ///
+    /// # Arguments
+    /// * `request` - TagKeyRequest with key ID and the tags to set; tags
+    ///   outside the request are left untouched
+    ///
+    /// # Returns
+    /// TagKeyResponse acknowledging the update
+    ///
+    pub async fn tag_key(&self, request: TagKeyRequest) -> Result<TagKeyResponse> {
+        self.kms_manager.tag_key(&request.key_id, &request.tags).await?;
+
+        Ok(TagKeyResponse {
+            success: true,
+            message: "key tags updated".to_string(),
+            key_id: request.key_id,
+        })
+    }
+
+    /// Remove master key tags (delegates to KMS manager)
+    ///
+    /// # Arguments
+    /// * `request` - UntagKeyRequest with key ID and the tag keys to remove;
+    ///   tag keys that are not set are ignored, so the call is idempotent
+    ///
+    /// # Returns
+    /// UntagKeyResponse acknowledging the removal
+    ///
+    pub async fn untag_key(&self, request: UntagKeyRequest) -> Result<UntagKeyResponse> {
+        self.kms_manager.untag_key(&request.key_id, &request.tag_keys).await?;
+
+        Ok(UntagKeyResponse {
+            success: true,
+            message: "key tags removed".to_string(),
+            key_id: request.key_id,
+        })
+    }
+
     /// Generate a data encryption key (delegates to KMS manager)
     ///
     /// # Arguments
@@ -160,9 +269,9 @@ impl ObjectEncryptionService {
     /// Get cache statistics
     ///
     /// # Returns
-    /// Option with (hits, misses) if caching is enabled
+    /// A [`KmsCacheStats`] snapshot if caching is enabled, `None` otherwise
     ///
-    pub async fn cache_stats(&self) -> Option<(u64, u64)> {
+    pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
         self.kms_manager.cache_stats().await
     }
 
@@ -918,6 +1027,128 @@ mod tests {
         );
     }
 
+    async fn describe(service: &ObjectEncryptionService, key_id: &str) -> KeyMetadata {
+        service
+            .describe_key(DescribeKeyRequest {
+                key_id: key_id.to_string(),
+            })
+            .await
+            .expect("describe should succeed")
+            .key_metadata
+    }
+
+    async fn create_metadata_test_key(service: &ObjectEncryptionService, key_id: &str) {
+        service
+            .create_key(CreateKeyRequest {
+                key_name: Some(key_id.to_string()),
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: Some("original".to_string()),
+                policy: None,
+                tags: HashMap::from([
+                    ("name".to_string(), key_id.to_string()),
+                    ("team".to_string(), "storage".to_string()),
+                ]),
+                origin: None,
+            })
+            .await
+            .expect("test key should be created");
+    }
+
+    #[tokio::test]
+    async fn key_metadata_updates_are_visible_to_describe() {
+        let (service, _temp_dir) = create_test_service().await;
+        create_metadata_test_key(&service, "metadata-key").await;
+
+        service
+            .update_key_description(UpdateKeyDescriptionRequest {
+                key_id: "metadata-key".to_string(),
+                description: "updated".to_string(),
+            })
+            .await
+            .expect("description update should succeed");
+        service
+            .tag_key(TagKeyRequest {
+                key_id: "metadata-key".to_string(),
+                tags: HashMap::from([
+                    ("team".to_string(), "platform".to_string()),
+                    ("env".to_string(), "prod".to_string()),
+                ]),
+            })
+            .await
+            .expect("tagging should succeed");
+
+        // Reading through the manager's metadata cache must observe the
+        // updates, not the snapshot cached when the key was created.
+        let metadata = describe(&service, "metadata-key").await;
+        assert_eq!(metadata.description.as_deref(), Some("updated"));
+        assert_eq!(metadata.tags.get("team").map(String::as_str), Some("platform"));
+        assert_eq!(metadata.tags.get("env").map(String::as_str), Some("prod"));
+        assert_eq!(
+            metadata.tags.get("name").map(String::as_str),
+            Some("metadata-key"),
+            "tags set at creation must survive a later tag update"
+        );
+
+        // Untagging is idempotent: removing an absent tag is a no-op, not an
+        // error, so a retried request stays safe.
+        for attempt in 1..=2 {
+            service
+                .untag_key(UntagKeyRequest {
+                    key_id: "metadata-key".to_string(),
+                    tag_keys: vec!["env".to_string()],
+                })
+                .await
+                .unwrap_or_else(|error| panic!("untag attempt {attempt} should succeed: {error:?}"));
+        }
+        let metadata = describe(&service, "metadata-key").await;
+        assert!(!metadata.tags.contains_key("env"), "untagged tag must be gone: {:?}", metadata.tags);
+        assert_eq!(
+            metadata.tags.get("team").map(String::as_str),
+            Some("platform"),
+            "untagging must not touch other tags"
+        );
+
+        // An empty description clears the stored value.
+        service
+            .update_key_description(UpdateKeyDescriptionRequest {
+                key_id: "metadata-key".to_string(),
+                description: String::new(),
+            })
+            .await
+            .expect("clearing the description should succeed");
+        assert_eq!(describe(&service, "metadata-key").await.description, None);
+    }
+
+    #[tokio::test]
+    async fn identity_tag_is_not_writable_through_metadata_updates() {
+        let (service, _temp_dir) = create_test_service().await;
+        create_metadata_test_key(&service, "identity-key").await;
+
+        let rewrite = service
+            .tag_key(TagKeyRequest {
+                key_id: "identity-key".to_string(),
+                tags: HashMap::from([("name".to_string(), "other-key".to_string())]),
+            })
+            .await
+            .expect_err("rewriting the identity tag must be rejected");
+        assert!(matches!(rewrite, KmsError::InvalidOperation { .. }), "got {rewrite:?}");
+
+        let removal = service
+            .untag_key(UntagKeyRequest {
+                key_id: "identity-key".to_string(),
+                tag_keys: vec!["team".to_string(), "name".to_string()],
+            })
+            .await
+            .expect_err("removing the identity tag must be rejected");
+        assert!(matches!(removal, KmsError::InvalidOperation { .. }), "got {removal:?}");
+
+        // Both rejections are pre-write: the record is untouched, including
+        // the ordinary tag that shared the rejected untag request.
+        let metadata = describe(&service, "identity-key").await;
+        assert_eq!(metadata.tags.get("name").map(String::as_str), Some("identity-key"));
+        assert_eq!(metadata.tags.get("team").map(String::as_str), Some("storage"));
+    }
+
     #[tokio::test]
     async fn test_decrypt_data_key_uses_object_encryption_context() {
         let (service, _temp_dir) = create_test_service().await;

crates/kms/src/service_manager.rs

@@ -14,12 +14,14 @@
 
 //! KMS service manager for dynamic configuration and runtime management
 
+use crate::audit::KmsAuditSink;
 use crate::backends::vault_credentials::CredentialTaskHandle;
 use crate::backends::{KmsBackend, local::LocalKmsBackend};
 use crate::config::{BackendConfig, KmsConfig};
 use crate::deletion_worker::{DeletionReferenceChecker, DeletionWorker};
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
+use crate::probe::{ProbeHandle, ProbeStatus, spawn_probe_worker};
 use crate::service::ObjectEncryptionService;
 use arc_swap::ArcSwap;
 use sha2::{Digest, Sha256};
@@ -120,13 +122,23 @@ struct ServiceVersion {
     /// Background deletion worker owned by this service version, if the
     /// backend supports deletion scheduling
     deletion_worker: Option<Arc<DeletionWorkerHandle>>,
+    /// Background synthetic probe owned by this service version, if the
+    /// backend can round-trip a data key and the probe is enabled
+    probe_worker: Option<Arc<ProbeHandle>>,
 }
 
 impl ServiceVersion {
-    fn shutdown_deletion_worker(&self) {
+    /// Stop the background workers this version owns.
+    ///
+    /// The credential renewal task is recycled separately because its shutdown
+    /// is asynchronous.
+    fn shutdown_background_workers(&self) {
         if let Some(worker) = &self.deletion_worker {
             worker.shutdown();
         }
+        if let Some(probe) = &self.probe_worker {
+            probe.shutdown();
+        }
     }
 }
 
@@ -171,6 +183,8 @@ pub struct KmsServiceManager {
     lifecycle_mutex: Arc<Mutex<()>>,
     /// External reference checker consulted before expired keys are removed
     deletion_reference_checker: std::sync::RwLock<Option<Arc<dyn DeletionReferenceChecker>>>,
+    /// External sink receiving audit records for management operations
+    audit_sink: std::sync::RwLock<Option<Arc<dyn KmsAuditSink>>>,
 }
 
 impl KmsServiceManager {
@@ -185,9 +199,26 @@ impl KmsServiceManager {
             version_counter: Arc::new(AtomicU64::new(0)),
             lifecycle_mutex: Arc::new(Mutex::new(())),
             deletion_reference_checker: std::sync::RwLock::new(None),
+            audit_sink: std::sync::RwLock::new(None),
         }
     }
 
+    /// Install the sink that receives an audit record for every KMS
+    /// management operation. Takes effect for services built by the next
+    /// start or reconfigure.
+    ///
+    /// Without a sink no records are built, so KMS operations are unaffected
+    /// when auditing is not configured.
+    pub fn set_audit_sink(&self, sink: Arc<dyn KmsAuditSink>) {
+        if let Ok(mut slot) = self.audit_sink.write() {
+            *slot = Some(sink);
+        }
+    }
+
+    fn audit_sink(&self) -> Option<Arc<dyn KmsAuditSink>> {
+        self.audit_sink.read().ok().and_then(|slot| slot.clone())
+    }
+
     /// Install the reference checker consulted before the deletion worker
     /// removes an expired key. Takes effect for workers spawned by the next
     /// start or reconfigure.
@@ -379,7 +410,7 @@ impl KmsServiceManager {
         // Note: Existing Arc references will keep the service alive until operations complete
         let state = self.state.load_full();
         if let Some(current) = state.current_service.as_ref() {
-            current.shutdown_deletion_worker();
+            current.shutdown_background_workers();
         }
         self.state.store(Arc::new(RuntimeState {
             config: state.config.clone(),
@@ -514,6 +545,19 @@ impl KmsServiceManager {
         self.state.load().current_service.as_ref().map(|sv| sv.version)
     }
 
+    /// Latest synthetic probe snapshot of the running service version.
+    ///
+    /// `None` when KMS is not running, when the backend cannot round-trip a
+    /// data key, or when the probe is disabled. The read is lock-free and
+    /// performs no backend I/O, so a health endpoint can consult it without
+    /// turning an external KMS hiccup into probe pressure of its own; the
+    /// staleness and consecutive-failure thresholds belong to the caller.
+    pub fn probe_status(&self) -> Option<Arc<ProbeStatus>> {
+        let state = self.state.load();
+        let service_version = state.current_service.as_ref()?;
+        Some(service_version.probe_worker.as_ref()?.status())
+    }
+
     /// Health check for the KMS service
     pub async fn health_check(&self) -> Result<bool> {
         let checked_state = self.state.load_full();
@@ -582,10 +626,19 @@ impl KmsServiceManager {
                 let backend = crate::backends::static_kms::StaticKmsBackend::new(config.clone()).await?;
                 Arc::new(backend) as Arc<dyn KmsBackend>
             }
+            BackendConfig::Aws(_) => {
+                info!("Creating AWS KMS backend for version {}", version);
+                let backend = crate::backends::aws::AwsKmsBackend::new(config.clone()).await?;
+                Arc::new(backend) as Arc<dyn KmsBackend>
+            }
         };
 
         // Create KMS manager
-        let kms_manager = Arc::new(KmsManager::new(backend, config.clone()));
+        let mut kms_manager = KmsManager::new(backend, config.clone());
+        if let Some(sink) = self.audit_sink() {
+            kms_manager = kms_manager.with_audit_sink(sink);
+        }
+        let kms_manager = Arc::new(kms_manager);
 
         // Create encryption service
         let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
@@ -596,6 +649,7 @@ impl KmsServiceManager {
             manager: kms_manager,
             credential_task,
             deletion_worker: None,
+            probe_worker: None,
         })
     }
 
@@ -609,9 +663,13 @@ impl KmsServiceManager {
 
     fn publish_running(&self, config: KmsConfig, mut service_version: ServiceVersion) {
         if let Some(previous) = self.state.load().current_service.as_ref() {
-            previous.shutdown_deletion_worker();
+            previous.shutdown_background_workers();
         }
         service_version.deletion_worker = self.spawn_deletion_worker(&config, &service_version);
+        // Started at publish time for the same reason as the deletion worker:
+        // a start or reconfigure candidate that never becomes current must not
+        // leak a running task or publish probe results nobody asked for.
+        service_version.probe_worker = spawn_probe_worker(service_version.manager.backend()).map(Arc::new);
         self.state.store(Arc::new(RuntimeState {
             config: Some(config),
             status: KmsServiceStatus::Running,
@@ -629,7 +687,13 @@ impl KmsServiceManager {
             return None;
         }
         let cancel = CancellationToken::new();
-        let worker = DeletionWorker::new(backend, config.default_key_id.clone(), self.deletion_reference_checker());
+        let worker = DeletionWorker::new(
+            backend,
+            config.default_key_id.clone(),
+            self.deletion_reference_checker(),
+            config.backend.as_str(),
+        )
+        .with_audit_sink(self.audit_sink());
         let task = worker.spawn(cancel.clone());
         Some(Arc::new(DeletionWorkerHandle {
             cancel,
@@ -980,6 +1044,49 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn probe_worker_follows_the_service_lifecycle() {
+        use tempfile::TempDir;
+
+        let key_dir = TempDir::new().expect("create local KMS directory");
+        let mut config = KmsConfig::local(key_dir.path().to_path_buf());
+        config.allow_insecure_dev_defaults = true;
+        let manager = KmsServiceManager::new();
+        manager.configure(config).await.expect("configure local KMS");
+        manager.start().await.expect("start local KMS");
+
+        let first_probe = manager
+            .state
+            .load()
+            .current_service
+            .as_ref()
+            .expect("running service")
+            .probe_worker
+            .clone()
+            .expect("a backend with a data key round trip must run a probe");
+        assert!(!first_probe.is_cancelled());
+        assert!(manager.probe_status().is_some(), "a running service must publish probe status");
+
+        // Replacing the service version replaces (and cancels) its probe.
+        manager.restart().await.expect("restart");
+        assert!(first_probe.is_cancelled(), "replaced version's probe must be cancelled");
+        let second_probe = manager
+            .state
+            .load()
+            .current_service
+            .as_ref()
+            .expect("running service")
+            .probe_worker
+            .clone()
+            .expect("restarted service must run a fresh probe");
+        assert!(!second_probe.is_cancelled());
+
+        // Stopping the service stops its probe and withdraws the snapshot.
+        manager.stop().await.expect("stop");
+        assert!(second_probe.is_cancelled(), "stop must cancel the probe worker");
+        assert!(manager.probe_status().is_none(), "a stopped service must publish no status");
+    }
+
     #[tokio::test]
     async fn reconfigure_allows_safe_local_runtime_settings_only() {
         use tempfile::TempDir;

crates/kms/src/types.rs

@@ -484,6 +484,20 @@ impl OperationContext {
         }
     }
 
+    /// Principal recorded for work the server starts on its own behalf.
+    ///
+    /// Namespaced so it can never collide with an access key or an IAM ARN,
+    /// which keeps "no human did this" distinguishable from "we lost track of
+    /// who did this" in the audit trail.
+    pub const INTERNAL_PRINCIPAL: &'static str = "rustfs:internal";
+
+    /// Context for an operation with no authenticated caller, such as
+    /// background maintenance or a call made while serving a data-plane
+    /// request that carries its own audit entry.
+    pub fn internal() -> Self {
+        Self::new(Self::INTERNAL_PRINCIPAL.to_string())
+    }
+
     /// Add additional context
     ///
     /// # Arguments

crates/notify/src/rules/mod.rs

@@ -22,6 +22,8 @@ mod pattern_rules_test;
 #[cfg(test)]
 mod pattern_test;
 mod rules_map;
+#[cfg(test)]
+mod rules_map_test;
 mod subscriber_index;
 mod subscriber_snapshot;
 mod target_id_set;

crates/notify/src/rules/rules_map_test.rs

@@ -0,0 +1,104 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regressions for rule matching against the KMS event namespace.
+//!
+//! KMS events (backlog#1583) are appended to `EventName` for the audit sink,
+//! but they must stay invisible to bucket notification rules: a subscriber
+//! asking for `s3:ObjectCreated:*` — or for everything — must never receive
+//! key management activity.
+
+use super::RulesMap;
+use rustfs_s3_types::EventName;
+use rustfs_targets::arn::TargetID;
+
+const KMS_EVENTS: &[EventName] = &[
+    EventName::KmsKeyCreated,
+    EventName::KmsKeyRotated,
+    EventName::KmsKeyEnabled,
+    EventName::KmsKeyDisabled,
+    EventName::KmsKeyDeletionScheduled,
+    EventName::KmsKeyDeletionCancelled,
+    EventName::KmsKeyDeleted,
+    EventName::KmsKeyAccessed,
+];
+
+fn test_target() -> TargetID {
+    TargetID::new("primary".to_string(), "webhook".to_string())
+}
+
+fn rules_for(events: &[EventName]) -> RulesMap {
+    let mut rules = RulesMap::new();
+    rules.add_rule_config(events, String::new(), test_target());
+    rules
+}
+
+#[test]
+fn s3_wildcard_subscriptions_do_not_match_kms_events() {
+    let rules = rules_for(&[
+        EventName::ObjectCreatedAll,
+        EventName::ObjectAccessedAll,
+        EventName::ObjectRemovedAll,
+        EventName::ObjectTaggingAll,
+        EventName::ObjectReplicationAll,
+        EventName::ObjectRestoreAll,
+        EventName::ObjectTransitionAll,
+        EventName::LifecycleExpirationAll,
+        EventName::ObjectScannerAll,
+    ]);
+
+    for event in KMS_EVENTS {
+        assert!(!rules.has_subscriber(event), "{event} must not have an S3 wildcard subscriber");
+        assert!(
+            rules.match_rules(*event, "any/object").is_empty(),
+            "{event} must not match any S3 wildcard rule"
+        );
+    }
+}
+
+#[test]
+fn everything_subscription_does_not_match_kms_events() {
+    let rules = rules_for(&[EventName::Everything]);
+
+    // Sanity: the catch-all really is subscribed to the S3 surface.
+    assert!(rules.has_subscriber(&EventName::ObjectCreatedPut));
+
+    for event in KMS_EVENTS {
+        assert!(!rules.has_subscriber(event), "{event} must stay outside the s3 catch-all");
+        assert!(
+            rules.match_rules(*event, "any/object").is_empty(),
+            "{event} must not match the s3 catch-all rule"
+        );
+    }
+}
+
+#[test]
+fn kms_subscription_does_not_leak_into_s3_matching() {
+    // The inverse direction: even an explicit KMS subscription must not widen
+    // the mask so that S3 events start matching a KMS-only rule.
+    let rules = rules_for(KMS_EVENTS);
+
+    for event in [
+        EventName::ObjectCreatedPut,
+        EventName::ObjectRemovedDelete,
+        EventName::ObjectAccessedGet,
+        EventName::BucketCreated,
+    ] {
+        assert!(!rules.has_subscriber(&event), "{event} must not match a KMS-only rule");
+        assert!(
+            rules.match_rules(event, "any/object").is_empty(),
+            "{event} must not match a KMS-only rule"
+        );
+    }
+}

crates/obs/Cargo.toml

@@ -146,6 +146,7 @@ tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "tracing-log", "local-time", "json"] }
 tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
 tokio-util = { workspace = true, features = ["io", "compat"] }
+url.workspace = true
 dial9-tokio-telemetry = { workspace = true, optional = true }
 thiserror = { workspace = true }
 zstd = { workspace = true, features = ["zstdmt"] }
@@ -162,3 +163,4 @@ libc = { workspace = true }
 [dev-dependencies]
 tempfile = { workspace = true }
 temp-env = { workspace = true }
+log = "0.4"

crates/obs/src/metrics/collectors/bucket_replication.rs

@@ -16,22 +16,25 @@
 
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::bucket_replication::{
-    BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
-    BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
-    BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD,
-    BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD,
-    BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
-    BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
+    BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
+    BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
+    BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
+    BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
+    BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
+    BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
+    BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
+    TARGET_ARN_L,
 };
 use std::borrow::Cow;
 
 const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
+const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 5;
 
 #[derive(Debug, Clone, Default)]
 pub struct BucketReplicationTargetStats {
@@ -80,6 +83,16 @@ pub struct BucketReplicationStats {
     pub targets: Vec<BucketReplicationTargetStats>,
 }
 
+#[derive(Debug, Clone, Default)]
+pub(crate) struct BucketReplicationBacklogStats {
+    pub(crate) bucket: String,
+    pub(crate) current_backlog_count: u64,
+    pub(crate) current_backlog_bytes: u64,
+    pub(crate) durable_mrf_available: bool,
+    pub(crate) durable_mrf_backlog_count: u64,
+    pub(crate) durable_mrf_backlog_bytes: u64,
+}
+
 pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
     if stats.is_empty() {
         return Vec::new();
@@ -249,7 +262,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
             PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, stat.resync_duration_ms as f64)
                 .with_label(BUCKET_L, bucket_label.clone()),
         );
-
         for target in &stat.targets {
             let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
             metrics.push(
@@ -265,6 +277,43 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
     metrics
 }
 
+pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
+    for stat in stats {
+        let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, stat.current_backlog_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD, stat.current_backlog_bytes as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD,
+                if stat.durable_mrf_available { 1.0 } else { 0.0 },
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, stat.durable_mrf_backlog_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD, stat.durable_mrf_backlog_bytes as f64)
+                .with_label(BUCKET_L, bucket_label),
+        );
+    }
+
+    metrics
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -369,6 +418,56 @@ mod tests {
         }));
     }
 
+    #[test]
+    fn test_collect_bucket_replication_backlog_metrics() {
+        let stats = vec![BucketReplicationBacklogStats {
+            bucket: "b1".to_string(),
+            current_backlog_count: 3,
+            current_backlog_bytes: 4096,
+            durable_mrf_available: true,
+            durable_mrf_backlog_count: 2,
+            durable_mrf_backlog_bytes: 2048,
+        }];
+
+        let metrics = collect_bucket_replication_backlog_metrics(&stats);
+        assert_eq!(metrics.len(), 5);
+
+        let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == backlog_count_name
+                && metric.value == 3.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let backlog_bytes_name = BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == backlog_bytes_name
+                && metric.value == 4096.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let durable_available_name = BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_available_name
+                && metric.value == 1.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let durable_count_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_count_name
+                && metric.value == 2.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let durable_bytes_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_bytes_name
+                && metric.value == 2048.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+    }
+
     #[test]
     fn test_collect_bucket_replication_metrics_empty() {
         let stats: Vec<BucketReplicationStats> = Vec::new();
@@ -376,6 +475,41 @@ mod tests {
         assert!(metrics.is_empty());
     }
 
+    #[test]
+    fn backlog_metrics_are_bucket_scoped_without_target_labels() {
+        let stats = vec![BucketReplicationBacklogStats {
+            bucket: "scope-bucket".to_string(),
+            current_backlog_count: 5,
+            current_backlog_bytes: 8192,
+            durable_mrf_available: true,
+            durable_mrf_backlog_count: 2,
+            durable_mrf_backlog_bytes: 4096,
+        }];
+
+        let metrics = collect_bucket_replication_backlog_metrics(&stats);
+        let backlog_names = [
+            BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name(),
+            BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name(),
+        ];
+
+        for name in backlog_names {
+            let metric = metrics
+                .iter()
+                .find(|metric| metric.name == name)
+                .expect("backlog metric should be emitted");
+            assert!(
+                metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == BUCKET_L && value == "scope-bucket")
+            );
+            assert!(!metric.labels.iter().any(|(key, _)| *key == TARGET_ARN_L));
+        }
+    }
+
     #[test]
     fn test_collect_bucket_replication_bandwidth_metrics() {
         let stats = vec![BucketReplicationBandwidthStats {

crates/obs/src/metrics/collectors/mod.rs

@@ -42,6 +42,7 @@ pub mod system_process;
 
 pub use audit::{AuditTargetStats, collect_audit_metrics};
 pub use bucket::{BucketStats, collect_bucket_metrics};
+pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
 pub use bucket_replication::{
     BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
     collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,

crates/obs/src/metrics/mod.rs

@@ -31,9 +31,9 @@ pub use scheduler::{
     metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
 };
 pub(crate) use storage_api::metrics::{
-    BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, StorageAdminApi,
-    obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
-    obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
+    BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
+    StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor,
+    obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
     obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
     obs_resolve_object_store_handle, obs_transition_state_handle,
 };

crates/obs/src/metrics/scheduler.rs

@@ -35,6 +35,7 @@ use crate::metrics::collectors::{
     ProcessMemoryStats,
     collect_audit_metrics,
     collect_bucket_metrics,
+    collect_bucket_replication_backlog_metrics,
     collect_bucket_replication_bandwidth_metrics,
     collect_bucket_replication_metrics,
     collect_bucket_usage_metrics,
@@ -94,7 +95,7 @@ use crate::metrics::schema::notification_target::{
 };
 use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
 use crate::metrics::stats_collector::{
-    ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
+    ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_stats_bundle,
     collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
     collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats,
     collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats,
@@ -1348,8 +1349,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                             // Phase-1 action: force zero for removed keys during tombstone cycles.
                             metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
 
-                            let bucket_replication = collect_bucket_replication_detail_stats().await;
+                            let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
                             metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
+                            metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
                             let replication = collect_replication_stats().await;
                             metrics.extend(collect_replication_metrics(&replication));
                             report_metrics(&metrics);

crates/obs/src/metrics/schema/bucket_replication.rs

@@ -33,6 +33,11 @@ const RESYNC_COMPLETED_TOTAL: &str = "resync_completed_total";
 const RESYNC_FAILED_TOTAL: &str = "resync_failed_total";
 const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
 const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
+const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
+const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
+const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
+const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
+const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
 
 pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
@@ -79,6 +84,51 @@ pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new
     )
 });
 
+pub static BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(CURRENT_BACKLOG_BYTES),
+        "Current number of bytes admitted to the in-memory replication worker queues for a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(CURRENT_BACKLOG_COUNT),
+        "Current number of objects admitted to the in-memory replication worker queues for a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_AVAILABLE),
+        "Whether the durable MRF backlog snapshot is available for this bucket on this node",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_BACKLOG_COUNT),
+        "Current number of objects in the durable MRF backlog file for a bucket on this node",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_BACKLOG_BYTES),
+        "Current bytes in the durable MRF backlog file for a bucket on this node",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::ProxiedDeleteTaggingRequestsTotal,

crates/obs/src/metrics/schema/replication.rs

@@ -128,7 +128,7 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
 pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::ReplicationRecentBacklogCount,
-        "Total number of objects currently in replication backlog (failed plus queued)",
+        "Legacy replication backlog indicator: failed target objects plus objects currently queued on this node",
         &[],
         subsystems::REPLICATION,
     )

crates/obs/src/metrics/stats_collector.rs

@@ -21,17 +21,18 @@
 //! and convert them to the Stats structs used by collectors.
 
 use crate::metrics::collectors::{
-    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats,
-    ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats,
-    DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats,
-    ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
+    BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
+    BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
+    CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
+    IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
+    ScannerStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
-    BucketOperations, BucketOptions, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot,
-    obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
-    obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
-    obs_resolve_object_store_handle,
+    BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi,
+    obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity,
+    obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
+    obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
 };
 use crate::node_identity::current_local_node_identity;
 use chrono::Utc;
@@ -192,49 +193,65 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
     ilm_runtime_snapshot().await
 }
 
-async fn obs_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
-    obs_bucket_replication_stats_snapshot()
-        .await
-        .into_iter()
-        .map(|stats| BucketReplicationStats {
-            bucket: stats.bucket,
-            total_failed_bytes: stats.total_failed_bytes,
-            total_failed_count: stats.total_failed_count,
-            last_min_failed_bytes: stats.last_min_failed_bytes,
-            last_min_failed_count: stats.last_min_failed_count,
-            last_hour_failed_bytes: stats.last_hour_failed_bytes,
-            last_hour_failed_count: stats.last_hour_failed_count,
-            sent_bytes: stats.sent_bytes,
-            sent_count: stats.sent_count,
-            proxied_get_requests_total: stats.proxied_get_requests_total,
-            proxied_get_requests_failures: stats.proxied_get_requests_failures,
-            proxied_head_requests_total: stats.proxied_head_requests_total,
-            proxied_head_requests_failures: stats.proxied_head_requests_failures,
-            proxied_put_requests_total: stats.proxied_put_requests_total,
-            proxied_put_requests_failures: stats.proxied_put_requests_failures,
-            proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
-            proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
-            proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
-            proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
-            proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
-            proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
-            resync_started_count: stats.resync_started_count,
-            resync_completed_count: stats.resync_completed_count,
-            resync_failed_count: stats.resync_failed_count,
-            resync_canceled_count: stats.resync_canceled_count,
-            resync_duration_ms: stats.resync_duration_ms,
-            targets: stats
-                .targets
-                .into_iter()
-                .map(|target| BucketReplicationTargetStats {
-                    target_arn: target.target_arn,
-                    bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
-                    current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
-                    latency_ms: target.latency_ms,
-                })
-                .collect(),
-        })
-        .collect()
+async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
+    let snapshots = obs_bucket_replication_stats_snapshot().await;
+    let mut detail_stats = Vec::with_capacity(snapshots.len());
+    let mut backlog_stats = Vec::with_capacity(snapshots.len());
+
+    for stats in snapshots {
+        backlog_stats.push(BucketReplicationBacklogStats {
+            bucket: stats.bucket.clone(),
+            current_backlog_count: stats.current_backlog_count,
+            current_backlog_bytes: stats.current_backlog_bytes,
+            durable_mrf_available: stats.durable_mrf_available,
+            durable_mrf_backlog_count: stats.durable_mrf_backlog_count,
+            durable_mrf_backlog_bytes: stats.durable_mrf_backlog_bytes,
+        });
+        detail_stats.push(bucket_replication_detail_from_snapshot(stats));
+    }
+
+    (detail_stats, backlog_stats)
+}
+
+fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
+    BucketReplicationStats {
+        bucket: stats.bucket,
+        total_failed_bytes: stats.total_failed_bytes,
+        total_failed_count: stats.total_failed_count,
+        last_min_failed_bytes: stats.last_min_failed_bytes,
+        last_min_failed_count: stats.last_min_failed_count,
+        last_hour_failed_bytes: stats.last_hour_failed_bytes,
+        last_hour_failed_count: stats.last_hour_failed_count,
+        sent_bytes: stats.sent_bytes,
+        sent_count: stats.sent_count,
+        proxied_get_requests_total: stats.proxied_get_requests_total,
+        proxied_get_requests_failures: stats.proxied_get_requests_failures,
+        proxied_head_requests_total: stats.proxied_head_requests_total,
+        proxied_head_requests_failures: stats.proxied_head_requests_failures,
+        proxied_put_requests_total: stats.proxied_put_requests_total,
+        proxied_put_requests_failures: stats.proxied_put_requests_failures,
+        proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
+        proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
+        proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
+        proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
+        proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
+        proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
+        resync_started_count: stats.resync_started_count,
+        resync_completed_count: stats.resync_completed_count,
+        resync_failed_count: stats.resync_failed_count,
+        resync_canceled_count: stats.resync_canceled_count,
+        resync_duration_ms: stats.resync_duration_ms,
+        targets: stats
+            .targets
+            .into_iter()
+            .map(|target| BucketReplicationTargetStats {
+                target_arn: target.target_arn,
+                bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
+                current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
+                latency_ms: target.latency_ms,
+            })
+            .collect(),
+    }
 }
 
 async fn obs_site_replication_stats() -> ReplicationStats {
@@ -526,7 +543,16 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
 
 /// Collect bucket and target level replication stats from the global replication runtime.
 pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
-    obs_bucket_replication_detail_stats().await
+    obs_bucket_replication_stats_snapshot()
+        .await
+        .into_iter()
+        .map(bucket_replication_detail_from_snapshot)
+        .collect()
+}
+
+pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
+{
+    obs_bucket_replication_stats_bundle().await
 }
 
 /// Collect site-level replication stats from the global replication runtime.

crates/obs/src/metrics/storage_api.rs

@@ -12,11 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::time::Duration;
 
 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
-use rustfs_ecstore::api::bucket::replication::get_global_replication_stats;
+use rustfs_ecstore::api::bucket::replication::{
+    DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
+};
 pub(crate) use rustfs_ecstore::api::capacity::{
     get_total_usable_capacity as obs_get_total_usable_capacity,
     get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
@@ -68,9 +71,50 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
     pub(crate) resync_failed_count: u64,
     pub(crate) resync_canceled_count: u64,
     pub(crate) resync_duration_ms: u64,
+    pub(crate) current_backlog_count: u64,
+    pub(crate) current_backlog_bytes: u64,
+    pub(crate) durable_mrf_available: bool,
+    pub(crate) durable_mrf_backlog_count: u64,
+    pub(crate) durable_mrf_backlog_bytes: u64,
     pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
 }
 
+#[derive(Debug, Clone, Default, PartialEq)]
+struct ObsBucketReplicationRuntimeSnapshot {
+    total_failed_bytes: u64,
+    total_failed_count: u64,
+    last_min_failed_bytes: u64,
+    last_min_failed_count: u64,
+    last_hour_failed_bytes: u64,
+    last_hour_failed_count: u64,
+    sent_bytes: u64,
+    sent_count: u64,
+    resync_started_count: u64,
+    resync_completed_count: u64,
+    resync_failed_count: u64,
+    resync_canceled_count: u64,
+    resync_duration_ms: u64,
+    current_backlog_count: u64,
+    current_backlog_bytes: u64,
+    targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq)]
+struct ObsBucketReplicationProxySnapshot {
+    proxied_get_requests_total: u64,
+    proxied_get_requests_failures: u64,
+    proxied_head_requests_total: u64,
+    proxied_head_requests_failures: u64,
+    proxied_put_requests_total: u64,
+    proxied_put_requests_failures: u64,
+    proxied_put_tagging_requests_total: u64,
+    proxied_put_tagging_requests_failures: u64,
+    proxied_get_tagging_requests_total: u64,
+    proxied_get_tagging_requests_failures: u64,
+    proxied_delete_tagging_requests_total: u64,
+    proxied_delete_tagging_requests_failures: u64,
+}
+
 #[derive(Debug, Clone, Default, PartialEq)]
 pub(crate) struct ObsReplicationSiteStatsSnapshot {
     pub(crate) average_active_workers: f64,
@@ -102,59 +146,85 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
     failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
 }
 
+fn bucket_replication_stats_snapshot_from_parts(
+    bucket: String,
+    runtime: ObsBucketReplicationRuntimeSnapshot,
+    proxy: ObsBucketReplicationProxySnapshot,
+    durable_mrf_available: bool,
+    durable_bucket: DurableMrfBucketBacklog,
+) -> ObsBucketReplicationStatsSnapshot {
+    ObsBucketReplicationStatsSnapshot {
+        bucket,
+        total_failed_bytes: runtime.total_failed_bytes,
+        total_failed_count: runtime.total_failed_count,
+        last_min_failed_bytes: runtime.last_min_failed_bytes,
+        last_min_failed_count: runtime.last_min_failed_count,
+        last_hour_failed_bytes: runtime.last_hour_failed_bytes,
+        last_hour_failed_count: runtime.last_hour_failed_count,
+        sent_bytes: runtime.sent_bytes,
+        sent_count: runtime.sent_count,
+        proxied_get_requests_total: proxy.proxied_get_requests_total,
+        proxied_get_requests_failures: proxy.proxied_get_requests_failures,
+        proxied_head_requests_total: proxy.proxied_head_requests_total,
+        proxied_head_requests_failures: proxy.proxied_head_requests_failures,
+        proxied_put_requests_total: proxy.proxied_put_requests_total,
+        proxied_put_requests_failures: proxy.proxied_put_requests_failures,
+        proxied_put_tagging_requests_total: proxy.proxied_put_tagging_requests_total,
+        proxied_put_tagging_requests_failures: proxy.proxied_put_tagging_requests_failures,
+        proxied_get_tagging_requests_total: proxy.proxied_get_tagging_requests_total,
+        proxied_get_tagging_requests_failures: proxy.proxied_get_tagging_requests_failures,
+        proxied_delete_tagging_requests_total: proxy.proxied_delete_tagging_requests_total,
+        proxied_delete_tagging_requests_failures: proxy.proxied_delete_tagging_requests_failures,
+        resync_started_count: runtime.resync_started_count,
+        resync_completed_count: runtime.resync_completed_count,
+        resync_failed_count: runtime.resync_failed_count,
+        resync_canceled_count: runtime.resync_canceled_count,
+        resync_duration_ms: runtime.resync_duration_ms,
+        current_backlog_count: runtime.current_backlog_count,
+        current_backlog_bytes: runtime.current_backlog_bytes,
+        durable_mrf_available,
+        durable_mrf_backlog_count: durable_bucket.count,
+        durable_mrf_backlog_bytes: durable_bucket.bytes,
+        targets: runtime.targets,
+    }
+}
+
 pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketReplicationStatsSnapshot> {
-    let Some(stats) = get_global_replication_stats() else {
-        return Vec::new();
+    let stats = get_global_replication_stats();
+    let all_bucket_stats = if let Some(stats) = &stats {
+        stats.get_all().await
+    } else {
+        HashMap::new()
     };
+    let durable_mrf_summary = if obs_resolve_object_store_handle().is_some() {
+        durable_mrf_backlog_summary_snapshot()
+    } else {
+        Default::default()
+    };
+    let durable_mrf_available = durable_mrf_summary.available;
+    let durable_buckets = durable_mrf_summary
+        .buckets
+        .into_iter()
+        .map(|bucket| (bucket.bucket.clone(), bucket))
+        .collect::<HashMap<String, DurableMrfBucketBacklog>>();
+    let mut bucket_names = Vec::with_capacity(all_bucket_stats.len().saturating_add(durable_buckets.len()));
+    bucket_names.extend(all_bucket_stats.keys().cloned());
+    bucket_names.extend(
+        durable_buckets
+            .keys()
+            .filter(|bucket| !all_bucket_stats.contains_key(*bucket))
+            .cloned(),
+    );
+    let mut buckets = Vec::with_capacity(bucket_names.len());
 
-    let all_bucket_stats = stats.get_all().await;
-    let mut buckets = Vec::with_capacity(all_bucket_stats.len());
-
-    for (bucket, bucket_stats) in all_bucket_stats {
-        let proxy = stats.get_proxy_stats(&bucket).await;
-        let mut total_failed_bytes = 0u64;
-        let mut total_failed_count = 0u64;
-        let mut last_min_failed_bytes = 0u64;
-        let mut last_min_failed_count = 0u64;
-        let mut last_hour_failed_bytes = 0u64;
-        let mut last_hour_failed_count = 0u64;
-        let mut sent_bytes = 0u64;
-        let mut sent_count = 0u64;
-        let mut targets = Vec::with_capacity(bucket_stats.stats.len());
-
-        for (target_arn, target_stats) in bucket_stats.stats {
-            total_failed_bytes = total_failed_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
-            total_failed_count = total_failed_count.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
-
-            let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
-            last_min_failed_bytes = last_min_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_min.size));
-            last_min_failed_count = last_min_failed_count.saturating_add(i64_to_u64_floor_zero(last_min.count));
-
-            let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
-            last_hour_failed_bytes = last_hour_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_hour.size));
-            last_hour_failed_count = last_hour_failed_count.saturating_add(i64_to_u64_floor_zero(last_hour.count));
-
-            sent_bytes = sent_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
-            sent_count = sent_count.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
-
-            targets.push(ObsBucketReplicationTargetStatsSnapshot {
-                target_arn,
-                bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
-                current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
-                latency_ms: target_stats.latency.curr,
-            });
-        }
-
-        buckets.push(ObsBucketReplicationStatsSnapshot {
-            bucket,
-            total_failed_bytes,
-            total_failed_count,
-            last_min_failed_bytes,
-            last_min_failed_count,
-            last_hour_failed_bytes,
-            last_hour_failed_count,
-            sent_bytes,
-            sent_count,
+    for bucket in bucket_names {
+        let bucket_stats = all_bucket_stats.get(&bucket);
+        let proxy = if let Some(stats) = &stats {
+            stats.get_proxy_stats(&bucket).await
+        } else {
+            Default::default()
+        };
+        let proxy = ObsBucketReplicationProxySnapshot {
             proxied_get_requests_total: i64_to_u64_floor_zero(proxy.get_total),
             proxied_get_requests_failures: i64_to_u64_floor_zero(proxy.get_failed),
             proxied_head_requests_total: i64_to_u64_floor_zero(proxy.head_total),
@@ -167,13 +237,67 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
             proxied_get_tagging_requests_failures: i64_to_u64_floor_zero(proxy.get_tag_failed),
             proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
             proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
-            resync_started_count: i64_to_u64_floor_zero(bucket_stats.resync_started_count),
-            resync_completed_count: i64_to_u64_floor_zero(bucket_stats.resync_completed_count),
-            resync_failed_count: i64_to_u64_floor_zero(bucket_stats.resync_failed_count),
-            resync_canceled_count: i64_to_u64_floor_zero(bucket_stats.resync_canceled_count),
-            resync_duration_ms: i64_to_u64_floor_zero(bucket_stats.resync_duration_ms),
-            targets,
-        });
+        };
+        let mut runtime = ObsBucketReplicationRuntimeSnapshot {
+            targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
+            ..Default::default()
+        };
+
+        if let Some(bucket_stats) = bucket_stats {
+            for (target_arn, target_stats) in &bucket_stats.stats {
+                runtime.total_failed_bytes = runtime
+                    .total_failed_bytes
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
+                runtime.total_failed_count = runtime
+                    .total_failed_count
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
+
+                let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
+                runtime.last_min_failed_bytes = runtime
+                    .last_min_failed_bytes
+                    .saturating_add(i64_to_u64_floor_zero(last_min.size));
+                runtime.last_min_failed_count = runtime
+                    .last_min_failed_count
+                    .saturating_add(i64_to_u64_floor_zero(last_min.count));
+
+                let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
+                runtime.last_hour_failed_bytes = runtime
+                    .last_hour_failed_bytes
+                    .saturating_add(i64_to_u64_floor_zero(last_hour.size));
+                runtime.last_hour_failed_count = runtime
+                    .last_hour_failed_count
+                    .saturating_add(i64_to_u64_floor_zero(last_hour.count));
+
+                runtime.sent_bytes = runtime
+                    .sent_bytes
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
+                runtime.sent_count = runtime
+                    .sent_count
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
+
+                runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
+                    target_arn: target_arn.clone(),
+                    bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
+                    current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
+                    latency_ms: target_stats.latency.curr,
+                });
+            }
+            runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
+            runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
+            runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
+            runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
+            runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
+            runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
+            runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
+        }
+        let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
+        buckets.push(bucket_replication_stats_snapshot_from_parts(
+            bucket,
+            runtime,
+            proxy,
+            durable_mrf_available,
+            durable_bucket,
+        ));
     }
 
     buckets
@@ -243,15 +367,97 @@ mod tests {
     fn replication_backlog_count_floors_negative_failed_and_queue_values() {
         assert_eq!(replication_backlog_count([-3, 4].into_iter(), -2), 4);
     }
+
+    #[test]
+    fn replication_backlog_count_keeps_legacy_failed_backlog_semantics() {
+        assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
+    }
+
+    #[test]
+    fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
+        let snapshot = bucket_replication_stats_snapshot_from_parts(
+            "runtime-bucket".to_string(),
+            ObsBucketReplicationRuntimeSnapshot {
+                current_backlog_count: 3,
+                current_backlog_bytes: 4096,
+                resync_failed_count: 2,
+                ..Default::default()
+            },
+            ObsBucketReplicationProxySnapshot {
+                proxied_get_requests_total: 7,
+                proxied_get_requests_failures: 1,
+                ..Default::default()
+            },
+            true,
+            DurableMrfBucketBacklog {
+                bucket: "runtime-bucket".to_string(),
+                count: 5,
+                bytes: 8192,
+            },
+        );
+
+        assert_eq!(snapshot.bucket, "runtime-bucket");
+        assert_eq!(snapshot.current_backlog_count, 3);
+        assert_eq!(snapshot.current_backlog_bytes, 4096);
+        assert!(snapshot.durable_mrf_available);
+        assert_eq!(snapshot.durable_mrf_backlog_count, 5);
+        assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
+        assert_eq!(snapshot.resync_failed_count, 2);
+        assert_eq!(snapshot.proxied_get_requests_total, 7);
+        assert_eq!(snapshot.proxied_get_requests_failures, 1);
+    }
+
+    #[test]
+    fn bucket_replication_snapshot_reports_durable_only_bucket() {
+        let snapshot = bucket_replication_stats_snapshot_from_parts(
+            "durable-only".to_string(),
+            ObsBucketReplicationRuntimeSnapshot::default(),
+            ObsBucketReplicationProxySnapshot::default(),
+            true,
+            DurableMrfBucketBacklog {
+                bucket: "durable-only".to_string(),
+                count: 11,
+                bytes: 2048,
+            },
+        );
+
+        assert_eq!(snapshot.bucket, "durable-only");
+        assert_eq!(snapshot.current_backlog_count, 0);
+        assert!(snapshot.durable_mrf_available);
+        assert_eq!(snapshot.durable_mrf_backlog_count, 11);
+        assert_eq!(snapshot.durable_mrf_backlog_bytes, 2048);
+    }
+
+    #[test]
+    fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
+        let snapshot = bucket_replication_stats_snapshot_from_parts(
+            "runtime-only".to_string(),
+            ObsBucketReplicationRuntimeSnapshot {
+                current_backlog_count: 1,
+                current_backlog_bytes: 512,
+                ..Default::default()
+            },
+            ObsBucketReplicationProxySnapshot::default(),
+            false,
+            DurableMrfBucketBacklog::default(),
+        );
+
+        assert_eq!(snapshot.current_backlog_count, 1);
+        assert_eq!(snapshot.current_backlog_bytes, 512);
+        assert!(!snapshot.durable_mrf_available);
+        assert_eq!(snapshot.durable_mrf_backlog_count, 0);
+        assert_eq!(snapshot.durable_mrf_backlog_bytes, 0);
+    }
 }
 
 pub(crate) mod metrics {
     pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi};
 
     pub(crate) use super::{
-        ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle,
-        obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
-        obs_is_disk_compression_enabled, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
-        obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
+        ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
+        obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
+        obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
+        obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
+        obs_resolve_object_store_handle, obs_transition_state_handle,
     };
 }

crates/obs/src/telemetry/filter.rs

@@ -18,7 +18,26 @@
 //! used across different logging backends (stdout, file, OpenTelemetry).
 
 use std::env;
-use tracing_subscriber::{EnvFilter, filter::LevelFilter};
+use tracing::Metadata;
+use tracing_subscriber::{
+    EnvFilter,
+    filter::{FilterFn, LevelFilter, filter_fn},
+};
+
+/// Pyroscope emits raw reqwest errors from its background session manager.
+/// Those errors can include the configured endpoint, so they must never reach
+/// a RustFS logging sink even when an operator enables verbose dependency logs.
+pub(super) fn is_pyroscope_dependency_target(target: &str) -> bool {
+    target == "pyroscope" || target.starts_with("pyroscope::") || target.starts_with("Pyroscope::")
+}
+
+fn allows_non_pyroscope_dependency_event(metadata: &Metadata<'_>) -> bool {
+    !is_pyroscope_dependency_target(metadata.target())
+}
+
+pub(super) fn pyroscope_log_filter() -> FilterFn {
+    filter_fn(allows_non_pyroscope_dependency_event)
+}
 
 /// Build an `EnvFilter` from the given log level string.
 ///

crates/obs/src/telemetry/local.rs

@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
 use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
 use crate::config::OtelConfig;
 use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
-use crate::telemetry::filter::build_env_filter;
+use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
 use crate::telemetry::rolling::{RollingAppender, Rotation};
 use metrics::counter;
 use rustfs_config::observability::{
@@ -319,6 +319,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
 
     tracing_subscriber::registry()
         .with(env_filter)
+        .with(pyroscope_log_filter())
         .with(ErrorLayer::default())
         .with(fmt_layer)
         .try_init()
@@ -431,6 +432,7 @@ fn init_file_logging_internal(
 
     tracing_subscriber::registry()
         .with(env_filter)
+        .with(pyroscope_log_filter())
         .with(ErrorLayer::default())
         .with(file_layer)
         .with(stdout_layer)

crates/obs/src/telemetry/otel.rs

@@ -39,7 +39,7 @@
 use crate::cleaner::types::FileMatchMode;
 use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
-use crate::telemetry::filter::build_env_filter;
+use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
 use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
 use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
 use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -68,7 +68,7 @@ use std::{fs, io::IsTerminal, time::Duration};
 use tracing::{info, warn};
 use tracing_error::ErrorLayer;
 use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
-use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
+use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
 
 const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[
     "rustfs_io_get_object_request_duration_seconds",
@@ -83,6 +83,26 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
     0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
 ];
 
+#[cfg(all(
+    feature = "pyroscope",
+    any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+))]
+const REDACTED_PROFILING_ENDPOINT: &str = "[redacted]";
+
+#[cfg(all(
+    feature = "pyroscope",
+    any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+))]
+fn log_profiler_failure(result: &'static str, error_kind: &'static str) {
+    warn!(
+        backend = "pyroscope",
+        endpoint = REDACTED_PROFILING_ENDPOINT,
+        result,
+        error_kind,
+        "Profiling export agent initialization failed"
+    );
+}
+
 /// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
 ///
 /// This function is invoked when at least one OTLP endpoint has been
@@ -158,9 +178,7 @@ pub(super) fn init_observability_http(
         logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
 
         // Build bridge to capture `tracing` events.
-        otel_bridge = logger_provider
-            .as_ref()
-            .map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None)));
+        otel_bridge = logger_provider.as_ref().map(OpenTelemetryTracingBridge::new);
 
         // No separate formatting layer is added here; when OTLP logging is
         // active, the OpenTelemetry bridge is the authoritative sink for
@@ -256,6 +274,7 @@ pub(super) fn init_observability_http(
     let filter = build_env_filter(logger_level, None);
     tracing_subscriber::registry()
         .with(filter)
+        .with(pyroscope_log_filter())
         .with(ErrorLayer::default())
         .with(file_layer_opt)
         .with(stdout_layer_opt)
@@ -531,6 +550,11 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
         return None;
     };
 
+    if url::Url::parse(endpoint).is_err() {
+        log_profiler_failure("profiling_endpoint_invalid", "invalid_endpoint");
+        return None;
+    }
+
     // Configure Pyroscope Agent
     let backend = pprof_backend(PprofConfig::default(), BackendConfig::default());
     let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
@@ -542,28 +566,16 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
         .build()
     {
         Ok(agent) => agent,
-        Err(err) => {
-            warn!(
-                backend = "pyroscope",
-                endpoint,
-                result = "profiling_agent_build_failed",
-                error = %err,
-                "Profiling export agent initialization failed"
-            );
+        Err(_) => {
+            log_profiler_failure("profiling_agent_build_failed", "build");
             return None;
         }
     };
 
     match agent.start() {
         Ok(agent) => Some(agent),
-        Err(err) => {
-            warn!(
-                backend = "pyroscope",
-                endpoint,
-                result = "profiling_agent_start_failed",
-                error = ?err,
-                "Profiling export agent failed to start"
-            );
+        Err(_) => {
+            log_profiler_failure("profiling_agent_start_failed", "start");
             None
         }
     }
@@ -650,6 +662,34 @@ fn resolve_signal_timeout(common_timeout_millis: Option<u64>, signal_timeout_mil
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::env;
+    use std::io::{self, Write};
+    use std::process::Command;
+    use std::sync::{Arc, Mutex};
+
+    const LOG_TRACER_CHILD_ENV: &str = "RUSTFS_OBS_LOG_TRACER_CHILD";
+
+    #[derive(Clone)]
+    struct TestWriter(Arc<Mutex<Vec<u8>>>);
+
+    impl Write for TestWriter {
+        fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+            self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
+            Ok(bytes.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
+        type Writer = Self;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            self.clone()
+        }
+    }
 
     #[test]
     /// Valid ratios should produce trace-id-ratio sampling.
@@ -766,4 +806,107 @@ mod tests {
         assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.first(), Some(&0.0001));
         assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.last(), Some(&10.0));
     }
+
+    #[cfg(all(
+        feature = "pyroscope",
+        any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+    ))]
+    #[test]
+    fn test_init_profiler_invalid_endpoint_redacts_sensitive_components() {
+        let endpoint = "https://profile-user:profile-token@10.24.0.5:invalid/private/profiles?access_token=query-secret";
+        let buffer = Arc::new(Mutex::new(Vec::new()));
+        let writer = TestWriter(Arc::clone(&buffer));
+        let subscriber = tracing_subscriber::fmt()
+            .with_ansi(false)
+            .without_time()
+            .with_writer(writer)
+            .finish();
+
+        let config = OtelConfig {
+            profiling_export_enabled: Some(true),
+            profiling_endpoint: Some(endpoint.to_string()),
+            ..OtelConfig::default()
+        };
+
+        tracing::subscriber::with_default(subscriber, || assert!(init_profiler(&config).is_none()));
+
+        let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
+        assert!(rendered.contains(REDACTED_PROFILING_ENDPOINT));
+        assert!(rendered.contains("error_kind=\"invalid_endpoint\""));
+        for leaked in [
+            "profile-user",
+            "profile-token",
+            "10.24.0.5",
+            "private/profiles",
+            "query-secret",
+        ] {
+            assert!(!rendered.contains(leaked), "profiling log leaked {leaked}: {rendered}");
+        }
+    }
+
+    #[test]
+    fn test_pyroscope_upload_failure_targets_stay_filtered_when_env_filter_enables_them() {
+        let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
+        let buffer = Arc::new(Mutex::new(Vec::new()));
+        let writer = TestWriter(Arc::clone(&buffer));
+        let env_filter = tracing_subscriber::EnvFilter::new("pyroscope=trace")
+            .add_directive("trace".parse().expect("parse trace filter directive"))
+            .add_directive("Pyroscope::Session=trace".parse().expect("parse Pyroscope filter directive"));
+        let subscriber = tracing_subscriber::registry()
+            .with(env_filter)
+            .with(pyroscope_log_filter())
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_ansi(false)
+                    .without_time()
+                    .with_writer(writer),
+            );
+
+        tracing::subscriber::with_default(subscriber, || {
+            tracing::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
+            tracing::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
+        });
+
+        let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
+        assert!(rendered.is_empty(), "filtered Pyroscope upload failure reached a log sink: {rendered}");
+    }
+
+    #[test]
+    fn test_pyroscope_log_facade_upload_failure_is_filtered() {
+        let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
+        if env::var_os(LOG_TRACER_CHILD_ENV).is_some() {
+            tracing_subscriber::registry()
+                .with(build_env_filter("info", None))
+                .with(pyroscope_log_filter())
+                .with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
+                .try_init()
+                .expect("install isolated LogTracer subscriber");
+            log::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
+            log::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
+            return;
+        }
+
+        let output = Command::new(env::current_exe().expect("resolve test executable"))
+            .args([
+                "--exact",
+                "telemetry::otel::tests::test_pyroscope_log_facade_upload_failure_is_filtered",
+                "--nocapture",
+            ])
+            .env(LOG_TRACER_CHILD_ENV, "1")
+            .env("RUST_LOG", "trace,pyroscope=trace,Pyroscope::Session=trace")
+            .output()
+            .expect("run isolated LogTracer test process");
+        assert!(output.status.success(), "isolated LogTracer test failed: {output:?}");
+
+        let rendered = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
+        for leaked in [
+            "profile-user",
+            "profile-token",
+            "10.24.0.5",
+            "private/profiles",
+            "query-secret",
+        ] {
+            assert!(!rendered.contains(leaked), "LogTracer path leaked {leaked}: {rendered}");
+        }
+    }
 }

crates/policy/Cargo.toml

@@ -32,7 +32,6 @@ workspace = true
 default = []
 hotpath = [
     "hotpath/hotpath",
-    "hotpath/reqwest-0-13",
     "rustfs-config/hotpath",
     "rustfs-credentials/hotpath",
     "rustfs-crypto/hotpath",
@@ -80,6 +79,7 @@ pollster.workspace = true
 proptest = "1"
 test-case.workspace = true
 temp-env = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["fmt"] }
 
 [lib]
 doctest = false

crates/policy/src/policy/opa.rs

@@ -30,16 +30,15 @@ impl Args {
     }
 }
 
-#[derive(Debug, Clone)]
-pub struct AuthZPlugin {
-    client: OpaHttpClient,
-    args: Args,
+pub fn is_configured() -> bool {
+    env::var_os(ENV_POLICY_PLUGIN_OPA_URL).is_some_and(|url| !url.is_empty())
 }
 
-#[cfg(feature = "hotpath")]
-type OpaHttpClient = hotpath::wrap::reqwest::Client;
-#[cfg(not(feature = "hotpath"))]
-type OpaHttpClient = reqwest::Client;
+#[derive(Debug, Clone)]
+pub struct AuthZPlugin {
+    client: reqwest::Client,
+    args: Args,
+}
 
 #[derive(Debug, thiserror::Error)]
 pub enum OpaConfigError {
@@ -59,6 +58,44 @@ pub enum OpaConfigError {
     Connection(reqwest::Error),
 }
 
+impl OpaConfigError {
+    pub fn kind(&self) -> &'static str {
+        match self {
+            Self::MissingRequiredEnv(_) => "missing_required_env",
+            Self::InvalidEnvVars(_) => "invalid_env_vars",
+            Self::EnvRead { .. } => "env_read_failed",
+            Self::InvalidStatus(_) => "invalid_status",
+            Self::Connection(_) => "connection_failed",
+        }
+    }
+}
+
+fn redact_opa_connection_error(error: reqwest::Error) -> OpaConfigError {
+    OpaConfigError::Connection(error.without_url())
+}
+
+fn opa_request_error_kind(error: &reqwest::Error) -> &'static str {
+    if error.is_timeout() {
+        "timeout"
+    } else if error.is_connect() {
+        "connect"
+    } else if error.is_request() {
+        "request"
+    } else if error.is_body() {
+        "body"
+    } else if error.is_decode() {
+        "decode"
+    } else if error.is_status() {
+        "status"
+    } else if error.is_builder() {
+        "builder"
+    } else if error.is_redirect() {
+        "redirect"
+    } else {
+        "other"
+    }
+}
+
 fn check() -> Result<(), OpaConfigError> {
     let env_list = env::vars();
     let mut candidate = HashMap::new();
@@ -86,9 +123,14 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
         .timeout(Duration::from_secs(5))
         .connect_timeout(Duration::from_secs(1))
         .build()
-        .map_err(OpaConfigError::Connection)?;
+        .map_err(redact_opa_connection_error)?;
 
-    match client.post(&config.url).send().await {
+    let mut request = client.post(&config.url);
+    if !config.auth_token.is_empty() {
+        request = request.header("Authorization", format!("Bearer {}", config.auth_token));
+    }
+
+    match request.send().await {
         Ok(resp) => {
             match resp.status() {
                 reqwest::StatusCode::OK => {
@@ -100,7 +142,7 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
             };
         }
         Err(err) => {
-            return Err(OpaConfigError::Connection(err));
+            return Err(redact_opa_connection_error(err));
         }
     };
     Ok(())
@@ -146,9 +188,6 @@ impl AuthZPlugin {
                 reqwest::Client::new()
             });
 
-        #[cfg(feature = "hotpath")]
-        let client = hotpath::http!(client, label = "Policy::OPA");
-
         Self { client, args: config }
     }
 
@@ -164,7 +203,13 @@ impl AuthZPlugin {
             Ok(resp) => {
                 let status = resp.status();
                 if !status.is_success() {
-                    error!("OPA returned non-success status: {}", status);
+                    error!(
+                        component = "policy",
+                        subsystem = "opa",
+                        result = "request_rejected",
+                        status = %status,
+                        "OPA request rejected"
+                    );
                     return false;
                 }
 
@@ -174,13 +219,25 @@ impl AuthZPlugin {
                         OpaResponseEnum::AllowResult(response) => response.result.allow,
                     },
                     Err(err) => {
-                        error!("Error parsing OPA response: {:?}", err);
+                        error!(
+                            component = "policy",
+                            subsystem = "opa",
+                            result = "response_decode_failed",
+                            error_kind = opa_request_error_kind(&err),
+                            "OPA response decoding failed"
+                        );
                         false
                     }
                 }
             }
             Err(err) => {
-                error!("Error sending request to OPA: {:?}", err);
+                error!(
+                    component = "policy",
+                    subsystem = "opa",
+                    result = "request_failed",
+                    error_kind = opa_request_error_kind(&err),
+                    "OPA request failed"
+                );
                 false
             }
         }
@@ -249,8 +306,35 @@ enum OpaResponseEnum {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::io::{Read, Write};
+    use std::net::TcpListener;
+    use std::sync::{Arc, Mutex};
     use temp_env;
 
+    #[derive(Clone)]
+    struct TestWriter(Arc<Mutex<Vec<u8>>>);
+
+    impl Write for TestWriter {
+        fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+            self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
+            Ok(bytes.len())
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
+        type Writer = Self;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            self.clone()
+        }
+    }
+
+    fn assert_reqwest_client(_: &reqwest::Client) {}
+
     #[test]
     fn test_check_valid_config() {
         // Use temp_env to temporarily set environment variables
@@ -336,4 +420,130 @@ mod tests {
         };
         assert!(!args_disabled.enable());
     }
+
+    #[test]
+    fn test_sensitive_opa_endpoints_never_use_hotpath_http_wrapper() {
+        let endpoints = [
+            "https://profile-user:profile-token@10.24.0.5/private/opa?access_token=query-secret",
+            "https://service.internal.example/private/path?token=another-secret",
+        ];
+
+        for endpoint in endpoints {
+            let plugin = AuthZPlugin::new(Args {
+                url: endpoint.to_string(),
+                auth_token: "opa-auth-token".to_string(),
+            });
+
+            assert_reqwest_client(&plugin.client);
+            assert_eq!(plugin.args.url, endpoint);
+        }
+    }
+
+    #[test]
+    fn test_opa_connection_error_removes_sensitive_endpoint() {
+        let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
+        let port = listener.local_addr().expect("read local test listener address").port();
+        let server = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().expect("accept local test request");
+            let mut request = [0_u8; 1024];
+            let read = stream.read(&mut request).expect("read local test request");
+            assert!(read > 0, "local test client must send an HTTP request");
+            stream
+                .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .expect("write local test response");
+        });
+        let endpoint = format!("http://profile-user:profile-token@127.0.0.1:{port}/private/opa?access_token=query-secret");
+        let runtime = tokio::runtime::Runtime::new().expect("build tokio test runtime");
+        let error = runtime.block_on(async {
+            reqwest::Client::builder()
+                .no_proxy()
+                .build()
+                .expect("build local test client")
+                .get(&endpoint)
+                .send()
+                .await
+                .expect("send local test request")
+                .error_for_status()
+                .expect_err("test response should be an HTTP error")
+        });
+        server.join().expect("join local test server");
+        assert!(error.url().is_some(), "fixture must carry the endpoint before redaction");
+
+        let OpaConfigError::Connection(error) = redact_opa_connection_error(error) else {
+            panic!("expected a redacted OPA connection error");
+        };
+        let rendered = error.to_string();
+        assert!(error.url().is_none());
+        for leaked in ["profile-user", "profile-token", "127.0.0.1", "private/opa", "query-secret"] {
+            assert!(!rendered.contains(leaked), "redacted error leaked {leaked}: {rendered}");
+        }
+    }
+
+    #[test]
+    fn test_is_allowed_rejection_log_redacts_sensitive_endpoint() {
+        let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
+        let port = listener.local_addr().expect("read local test listener address").port();
+        let server = std::thread::spawn(move || {
+            let (mut stream, _) = listener.accept().expect("accept local test request");
+            let mut request = [0_u8; 1024];
+            let read = stream.read(&mut request).expect("read local test request");
+            assert!(read > 0, "local test client must send an HTTP request");
+            stream
+                .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .expect("write local test response");
+        });
+        let endpoint = format!("http://opa-user:opa-token@127.0.0.1:{port}/private/allow?access_token=query-secret");
+        let plugin = AuthZPlugin::new(Args {
+            url: endpoint,
+            auth_token: "authorization-secret".to_string(),
+        });
+        let groups = None;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let args = PArgs {
+            account: "account",
+            groups: &groups,
+            action: crate::policy::action::Action::None,
+            bucket: "bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "object",
+            claims: &claims,
+            deny_only: false,
+        };
+        let buffer = Arc::new(Mutex::new(Vec::new()));
+        let writer = TestWriter(Arc::clone(&buffer));
+        let subscriber = tracing_subscriber::fmt()
+            .with_ansi(false)
+            .without_time()
+            .with_writer(writer)
+            .finish();
+
+        tracing::subscriber::with_default(subscriber, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("build tokio test runtime");
+            assert!(!runtime.block_on(plugin.is_allowed(&args)));
+        });
+        server.join().expect("join local test server");
+
+        let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
+        assert!(!rendered.is_empty(), "OPA request failure did not reach the test log sink");
+        assert!(
+            rendered.contains("request_rejected"),
+            "OPA request failure did not retain its stable result category"
+        );
+        assert!(rendered.contains("500 Internal Server Error"));
+        for leaked in [
+            "opa-user",
+            "opa-token",
+            "127.0.0.1",
+            "private/allow",
+            "query-secret",
+            "authorization-secret",
+        ] {
+            assert!(!rendered.contains(leaked), "OPA request log leaked {leaked}: {rendered}");
+        }
+    }
 }

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1171,6 +1171,12 @@ pub struct SignalServiceResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
     #[prost(uint32, tag = "3")]
     pub protocol_version: u32,
+    /// Redacted fingerprint of the KMS configuration this node is running.
+    /// Only set for the "kms" dynamic config subsystem, and left unset when the
+    /// responder has no KMS configuration. Peers built before that subsystem
+    /// reject the signal outright rather than answering without this field.
+    #[prost(string, optional, tag = "4")]
+    pub config_fingerprint: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ScannerActivityRequest {

crates/protos/src/node.proto

@@ -817,6 +817,11 @@ message SignalServiceResponse {
   bool success = 1;
   optional string error_info = 2;
   uint32 protocol_version = 3;
+  // Redacted fingerprint of the KMS configuration this node is running.
+  // Only set for the "kms" dynamic config subsystem, and left unset when the
+  // responder has no KMS configuration. Peers built before that subsystem
+  // reject the signal outright rather than answering without this field.
+  optional string config_fingerprint = 4;
 }
 
 message ScannerActivityRequest {

crates/replication/src/stats.rs

@@ -241,6 +241,16 @@ impl InQueueStats {
         Self::default()
     }
 
+    pub fn add_current(&self, bytes: i64, count: i64) {
+        self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
+        self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
+    }
+
+    pub fn subtract_current(&self, bytes: i64, count: i64) {
+        saturating_atomic_sub(&self.now_bytes, bytes.max(0));
+        saturating_atomic_sub(&self.now_count, count.max(0));
+    }
+
     pub fn get_current_bytes(&self) -> i64 {
         self.now_bytes.load(Ordering::Relaxed)
     }
@@ -250,6 +260,14 @@ impl InQueueStats {
     }
 }
 
+fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
+    if delta == 0 {
+        return;
+    }
+
+    let _ = value.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
+}
+
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct InQueueMetric {
     pub curr: InQueueStats,
@@ -359,6 +377,18 @@ impl QueueCache {
         Self::default()
     }
 
+    pub fn inc(&mut self, bucket: &str, size: i64) {
+        let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
+        stats.curr.add_current(size, 1);
+        self.sr_queue_stats.curr.add_current(size, 1);
+    }
+
+    pub fn dec(&mut self, bucket: &str, size: i64) {
+        let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
+        stats.curr.subtract_current(size, 1);
+        self.sr_queue_stats.curr.subtract_current(size, 1);
+    }
+
     pub fn update(&mut self) {
         let observed_at = Instant::now();
         self.sr_queue_stats.observe(observed_at);
@@ -808,12 +838,10 @@ mod tests {
     #[test]
     fn in_queue_metric_observe_updates_rolling_stats() {
         let mut metric = InQueueMetric::default();
-        metric.curr.now_bytes.store(128, Ordering::Relaxed);
-        metric.curr.now_count.store(4, Ordering::Relaxed);
+        metric.curr.add_current(128, 4);
         metric.observe(Instant::now());
 
-        metric.curr.now_bytes.store(256, Ordering::Relaxed);
-        metric.curr.now_count.store(6, Ordering::Relaxed);
+        metric.curr.add_current(128, 2);
         metric.observe(Instant::now());
 
         assert_eq!(metric.curr.bytes, 256);
@@ -824,6 +852,21 @@ mod tests {
         assert_eq!(metric.last_minute.count, 5);
     }
 
+    #[test]
+    fn queue_cache_decrement_saturates_at_zero() {
+        let mut cache = QueueCache::new();
+        cache.inc("bucket", 128);
+        cache.dec("bucket", 512);
+        cache.dec("bucket", 1);
+
+        let bucket = cache.get_bucket_stats("bucket");
+        let site = cache.get_site_stats();
+        assert_eq!(bucket.curr.count, 0);
+        assert_eq!(bucket.curr.bytes, 0);
+        assert_eq!(site.curr.count, 0);
+        assert_eq!(site.curr.bytes, 0);
+    }
+
     #[test]
     fn fail_stats_recent_since_tracks_windows() {
         let mut stats = FailStats::default();

crates/s3-types/src/event_name.rs

@@ -86,6 +86,20 @@ pub enum EventName {
     ObjectRemovedAbortMultipartUpload,
     ObjectCreatedCreateMultipartUpload,
     ObjectRemovedDeleteObjects,
+
+    // KMS management-plane events. They travel to the audit sink only and are
+    // never produced by the bucket notification path, so no compound `s3:`
+    // event expands to them. New variants must keep being appended here: the
+    // discriminant of every preceding variant is a `mask()` bit position, and
+    // inserting in the middle would silently renumber existing bits.
+    KmsKeyCreated,
+    KmsKeyRotated,
+    KmsKeyEnabled,
+    KmsKeyDisabled,
+    KmsKeyDeletionScheduled,
+    KmsKeyDeletionCancelled,
+    KmsKeyDeleted,
+    KmsKeyAccessed,
 }
 
 // Single event type sequential array for Everything.expand()
@@ -186,6 +200,17 @@ impl EventName {
             "s3:Scanner:LargeVersions" => Ok(EventName::ScannerLargeVersions),
             "s3:Scanner:BigPrefix" => Ok(EventName::ScannerBigPrefix),
             "s3:Scanner:*" => Ok(EventName::ObjectScannerAll),
+            // KMS events use their own namespace so a `s3:` wildcard in a bucket
+            // notification config can never select them. They still round-trip
+            // because audit entries are persisted and replayed by the store targets.
+            "kms:Key:Created" => Ok(EventName::KmsKeyCreated),
+            "kms:Key:Rotated" => Ok(EventName::KmsKeyRotated),
+            "kms:Key:Enabled" => Ok(EventName::KmsKeyEnabled),
+            "kms:Key:Disabled" => Ok(EventName::KmsKeyDisabled),
+            "kms:Key:DeletionScheduled" => Ok(EventName::KmsKeyDeletionScheduled),
+            "kms:Key:DeletionCancelled" => Ok(EventName::KmsKeyDeletionCancelled),
+            "kms:Key:Deleted" => Ok(EventName::KmsKeyDeleted),
+            "kms:Key:Accessed" => Ok(EventName::KmsKeyAccessed),
             // `Everything` has no string representation (`as_str` yields ""), so it
             // cannot be parsed back from a string. Every other variant round-trips.
             _ => Err(ParseEventNameError(s.to_string())),
@@ -251,6 +276,14 @@ impl EventName {
             EventName::ObjectRemovedAbortMultipartUpload => "s3:ObjectRemoved:AbortMultipartUpload",
             EventName::ObjectCreatedCreateMultipartUpload => "s3:ObjectCreated:CreateMultipartUpload",
             EventName::ObjectRemovedDeleteObjects => "s3:ObjectRemoved:DeleteObjects",
+            EventName::KmsKeyCreated => "kms:Key:Created",
+            EventName::KmsKeyRotated => "kms:Key:Rotated",
+            EventName::KmsKeyEnabled => "kms:Key:Enabled",
+            EventName::KmsKeyDisabled => "kms:Key:Disabled",
+            EventName::KmsKeyDeletionScheduled => "kms:Key:DeletionScheduled",
+            EventName::KmsKeyDeletionCancelled => "kms:Key:DeletionCancelled",
+            EventName::KmsKeyDeleted => "kms:Key:Deleted",
+            EventName::KmsKeyAccessed => "kms:Key:Accessed",
         }
     }
 
@@ -357,6 +390,25 @@ impl EventName {
                 | EventName::ObjectRemovedDeleteObjects
         )
     }
+
+    /// Returns `true` for KMS management-plane events.
+    ///
+    /// These are audit-only: they are never emitted through the bucket
+    /// notification pipeline, and no `s3:` event selector expands to them.
+    #[inline]
+    pub fn is_kms(&self) -> bool {
+        matches!(
+            self,
+            EventName::KmsKeyCreated
+                | EventName::KmsKeyRotated
+                | EventName::KmsKeyEnabled
+                | EventName::KmsKeyDisabled
+                | EventName::KmsKeyDeletionScheduled
+                | EventName::KmsKeyDeletionCancelled
+                | EventName::KmsKeyDeleted
+                | EventName::KmsKeyAccessed
+        )
+    }
 }
 
 /// Returns the S3 notification event schema version for a given event.
@@ -570,6 +622,26 @@ mod tests {
         EventName::ObjectRemovedAbortMultipartUpload,
         EventName::ObjectCreatedCreateMultipartUpload,
         EventName::ObjectRemovedDeleteObjects,
+        EventName::KmsKeyCreated,
+        EventName::KmsKeyRotated,
+        EventName::KmsKeyEnabled,
+        EventName::KmsKeyDisabled,
+        EventName::KmsKeyDeletionScheduled,
+        EventName::KmsKeyDeletionCancelled,
+        EventName::KmsKeyDeleted,
+        EventName::KmsKeyAccessed,
+    ];
+
+    /// Every KMS management-plane event.
+    const KMS_EVENT_NAMES: &[EventName] = &[
+        EventName::KmsKeyCreated,
+        EventName::KmsKeyRotated,
+        EventName::KmsKeyEnabled,
+        EventName::KmsKeyDisabled,
+        EventName::KmsKeyDeletionScheduled,
+        EventName::KmsKeyDeletionCancelled,
+        EventName::KmsKeyDeleted,
+        EventName::KmsKeyAccessed,
     ];
 
     /// Regression for backlog#965: `mask()` used to recurse forever for the
@@ -682,6 +754,54 @@ mod tests {
         }
     }
 
+    /// KMS events are audit-plane only. No `s3:` selector — including the
+    /// catch-all `Everything` and every compound "All" type — may share a bit
+    /// with them, otherwise a bucket notification rule would silently start
+    /// matching KMS activity.
+    #[test]
+    fn test_kms_event_masks_are_disjoint_from_every_s3_selector() {
+        let s3_selectors: Vec<EventName> = ALL_EVENT_NAMES.iter().copied().filter(|ev| !ev.is_kms()).collect();
+
+        let mut seen = 0u64;
+        for kms in KMS_EVENT_NAMES {
+            let mask = kms.mask();
+            assert_ne!(mask, 0, "KMS event {kms} must have a non-zero mask");
+            assert_eq!(seen & mask, 0, "KMS event {kms} mask overlaps another KMS event");
+            seen |= mask;
+
+            for s3 in &s3_selectors {
+                assert_eq!(s3.mask() & mask, 0, "KMS event {kms} mask collides with S3 selector {s3}");
+            }
+        }
+    }
+
+    /// KMS event names must live in their own namespace so that neither a
+    /// `s3:` prefix filter nor an `s3:...:*` wildcard can select them.
+    #[test]
+    fn test_kms_event_names_are_outside_the_s3_namespace() {
+        for ev in KMS_EVENT_NAMES {
+            assert!(ev.is_kms(), "{ev} should be classified as a KMS event");
+            assert!(ev.as_str().starts_with("kms:"), "unexpected KMS event name {:?}", ev.as_str());
+            assert_eq!(EventName::parse(ev.as_str()).as_ref(), Ok(ev), "KMS event {ev} must round-trip");
+            assert_eq!(ev.expand(), vec![*ev], "KMS event {ev} must expand to itself only");
+        }
+
+        for ev in ALL_EVENT_NAMES.iter().filter(|ev| !ev.is_kms()) {
+            assert!(!ev.as_str().starts_with("kms:"), "{ev} must not claim the KMS namespace");
+        }
+    }
+
+    /// `mask()` shifts by `discriminant - 1`, so the enum may hold at most 64
+    /// mask-bearing variants. Appending past that overflows the shift instead
+    /// of failing loudly, so keep the budget check next to the variants.
+    #[test]
+    fn test_mask_bit_budget_is_not_exhausted() {
+        for ev in ALL_EVENT_NAMES {
+            let value = *ev as u32;
+            assert!(value <= 64, "{ev} has discriminant {value}; mask() only has 64 bits to hand out");
+        }
+    }
+
     /// `Everything` must cover every sequential single-type bit.
     #[test]
     fn test_everything_mask_covers_all_single_types() {

crates/scanner/src/scanner_folder.rs

@@ -75,7 +75,7 @@ const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500;
 const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
 const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
 const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
-const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60);
+const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
 const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
 const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
 const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
@@ -100,6 +100,21 @@ static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
 static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
 static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
 
+#[cfg(test)]
+type ListPathRawTimeoutSnapshot = (bool, Option<Duration>, Option<Duration>);
+
+fn scanner_abandoned_child_list_options() -> ListPathRawOptions {
+    // A complete heal walk scales with bucket size and may legitimately take
+    // longer than a fixed wall-clock budget. Keep the total duration unbounded;
+    // Retain the scanner's per-read stall budget and keep cancellation controlled
+    // by the scanner cycle token.
+    ListPathRawOptions {
+        skip_walkdir_total_timeout: true,
+        walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT),
+        ..Default::default()
+    }
+}
+
 pub fn data_usage_update_dir_cycles() -> u32 {
     rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
 }
@@ -1281,6 +1296,8 @@ pub struct FolderScanner {
     skip_heal: Arc<std::sync::atomic::AtomicBool>,
     local_disk: Arc<Disk>,
     pending_heals_changed: bool,
+    #[cfg(test)]
+    list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
 }
 
 impl FolderScanner {
@@ -2410,75 +2427,79 @@ impl FolderScanner {
                 let bucket_clone = bucket.clone();
                 let prefix_clone = prefix.clone();
                 let child_ctx_clone = child_ctx.clone();
+                #[cfg(test)]
+                let list_path_raw_options_observer = self.list_path_raw_options_observer.clone();
 
                 tokio::spawn(async move {
-                    if let Err(e) = list_path_raw(
-                        child_ctx_clone.clone(),
-                        ListPathRawOptions {
-                            disks,
-                            bucket: bucket_clone.clone(),
-                            path: prefix_clone.clone(),
-                            recursive: true,
-                            report_not_found: true,
-                            min_disks: disks_quorum,
-                            walkdir_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
-                            walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
-                            agreed: Some(Box::new(move |entry: MetaCacheEntry| {
-                                let entry_name = entry.name.clone();
-                                let agreed_tx = agreed_tx.clone();
-                                Box::pin(async move {
-                                    if let Err(e) = agreed_tx.send(entry_name).await {
-                                        error!(
-                                            target: "rustfs::scanner::folder",
-                                            event = EVENT_SCANNER_FOLDER_STATE,
-                                            component = LOG_COMPONENT_SCANNER,
-                                            subsystem = LOG_SUBSYSTEM_FOLDER,
-                                            entry = %entry.name,
-                                            state = "list_path_agreed_send_failed",
-                                            error = %e,
-                                            "Scanner list_path_raw agreed callback failed"
-                                        );
-                                    }
-                                })
-                            })),
-                            partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
-                                let partial_tx = partial_tx.clone();
-                                Box::pin(async move {
-                                    if let Err(e) = partial_tx.send(entries).await {
-                                        error!(
-                                            target: "rustfs::scanner::folder",
-                                            event = EVENT_SCANNER_FOLDER_STATE,
-                                            component = LOG_COMPONENT_SCANNER,
-                                            subsystem = LOG_SUBSYSTEM_FOLDER,
-                                            state = "list_path_partial_send_failed",
-                                            error = %e,
-                                            "Scanner list_path_raw partial callback failed"
-                                        );
-                                    }
-                                })
-                            })),
-                            finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
-                                let finished_tx = finished_tx.clone();
-                                let errs_clone = errs.to_vec();
-                                Box::pin(async move {
-                                    if let Err(e) = finished_tx.send(errs_clone).await {
-                                        error!(
-                                            target: "rustfs::scanner::folder",
-                                            event = EVENT_SCANNER_FOLDER_STATE,
-                                            component = LOG_COMPONENT_SCANNER,
-                                            subsystem = LOG_SUBSYSTEM_FOLDER,
-                                            state = "list_path_finished_send_failed",
-                                            error = %e,
-                                            "Scanner list_path_raw finished callback failed"
-                                        );
-                                    }
-                                })
-                            })),
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                    {
+                    let options = ListPathRawOptions {
+                        disks,
+                        bucket: bucket_clone.clone(),
+                        path: prefix_clone.clone(),
+                        recursive: true,
+                        report_not_found: true,
+                        min_disks: disks_quorum,
+                        agreed: Some(Box::new(move |entry: MetaCacheEntry| {
+                            let entry_name = entry.name.clone();
+                            let agreed_tx = agreed_tx.clone();
+                            Box::pin(async move {
+                                if let Err(e) = agreed_tx.send(entry_name).await {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        entry = %entry.name,
+                                        state = "list_path_agreed_send_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw agreed callback failed"
+                                    );
+                                }
+                            })
+                        })),
+                        partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
+                            let partial_tx = partial_tx.clone();
+                            Box::pin(async move {
+                                if let Err(e) = partial_tx.send(entries).await {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        state = "list_path_partial_send_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw partial callback failed"
+                                    );
+                                }
+                            })
+                        })),
+                        finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
+                            let finished_tx = finished_tx.clone();
+                            let errs_clone = errs.to_vec();
+                            Box::pin(async move {
+                                if let Err(e) = finished_tx.send(errs_clone).await {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        state = "list_path_finished_send_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw finished callback failed"
+                                    );
+                                }
+                            })
+                        })),
+                        ..scanner_abandoned_child_list_options()
+                    };
+                    #[cfg(test)]
+                    if let Some(observer) = list_path_raw_options_observer {
+                        let _ = observer.send((
+                            options.skip_walkdir_total_timeout,
+                            options.walkdir_timeout,
+                            options.walkdir_stall_timeout,
+                        ));
+                    }
+                    if let Err(e) = list_path_raw(child_ctx_clone.clone(), options).await {
                         if is_missing_path_disk_error(&e) {
                             debug!(
                                 target: "rustfs::scanner::folder",
@@ -2820,6 +2841,8 @@ pub async fn scan_data_folder(
         skip_heal,
         local_disk,
         pending_heals_changed: false,
+        #[cfg(test)]
+        list_path_raw_options_observer: None,
     };
 
     // Check if context is cancelled
@@ -3026,6 +3049,7 @@ mod tests {
             skip_heal: Arc::new(AtomicBool::new(false)),
             local_disk: disk,
             pending_heals_changed: false,
+            list_path_raw_options_observer: None,
         };
 
         (scanner, temp_dir)
@@ -4210,6 +4234,8 @@ mod tests {
         scanner.heal_object_select = 1;
         scanner.disks = disks;
         scanner.disks_quorum = 2;
+        let (options_tx, mut options_rx) = mpsc::unbounded_channel();
+        scanner.list_path_raw_options_observer = Some(options_tx);
         scanner.old_cache.replace(
             &format!("{bucket}/{object}"),
             bucket,
@@ -4231,6 +4257,11 @@ mod tests {
             .expect("scan_folder should not hang after list_path_raw finishes")
             .expect("scan_folder should finish successfully");
 
+        let observed_options = tokio::time::timeout(Duration::from_secs(1), options_rx.recv())
+            .await
+            .expect("abandoned-child listing options should be observed promptly")
+            .expect("abandoned-child listing options channel should remain open");
+        assert_eq!(observed_options, (true, None, Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT)));
         let root = scanner
             .new_cache
             .checked_flatten(bucket)

deny.toml

@@ -43,6 +43,9 @@ allow-git = [
     # Presigned expiry and constant-time authentication fixes pending upstream merge.
     # owner: cxymds  review: 2026-10
     "https://github.com/cxymds/s3s.git",
+    # MiMalloc fork pinned for hotpath allocation counting support.
+    # owner: houseme  review: 2026-10
+    "https://github.com/xonatius/mimalloc_rust.git",
     "https://github.com/apache/datafusion.git",
 ]
 

docs/operations/kms-backend-security.md

@@ -12,6 +12,7 @@ For how the Vault backends authenticate (static token, AppRole, Vault Agent toke
 | Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | No state persisted by RustFS | Rejected (read-only backend) | Simple deployments with an external secret manager |
 | Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Delegated to Vault storage | Versioned retention (immutable per-version records + current pointer) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary |
 | Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs |
+| AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments already rooted in AWS IAM that want AWS as the cryptographic root — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first |
 
 ## Vault KV2: what the backend does and does not do
 
@@ -146,6 +147,25 @@ Use **Vault Transit** (`VaultTransit`) when key material must be cryptographical
 
 Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* the confidentiality boundary and you want the operational simplicity of a single KV mount.
 
+## AWS KMS: deviations from the shared backend contract
+
+Select it with `RUSTFS_KMS_BACKEND=aws`. Credentials and region resolution are delegated entirely to the standard `aws-config` provider chain (environment, shared profile, container/IMDS role), so RustFS never stores, persists, or redacts AWS credential material of its own. Only two non-credential settings are read: `RUSTFS_KMS_AWS_REGION` and `RUSTFS_KMS_AWS_ENDPOINT_URL`. A plaintext (`http://`) endpoint override would expose every KMS request including plaintext data keys, so it is refused unless the development opt-in is set.
+
+AWS owns key state, backing-key rotation, and the deletion window, and this backend mirrors none of it locally. That makes four behaviours differ from every RustFS-managed backend. Verify each against your operational assumptions before switching:
+
+| Behaviour | RustFS-managed backends | AWS KMS backend |
+| --- | --- | --- |
+| Decryption with a `Disabled` or `PendingDeletion` key | Kept working, so disabling a key never breaks reads of objects already encrypted under it | **Refused by AWS.** Objects encrypted under a key that is later disabled become unreadable until it is re-enabled |
+| Key deletion | Physical deletion available | **No physical delete.** `ScheduleKeyDeletion` is the only removal path; AWS destroys the material when the 7-30 day window elapses. RustFS never destroys AWS-held material, and `force_immediate` is refused |
+| Cancelling a scheduled deletion | Key returns to `Enabled` | Key is left **`Disabled`**; enable it explicitly to make it usable again |
+| Creating a key under a caller-chosen name | The requested name becomes the key id | **Refused.** AWS assigns identifiers and this backend does not manage aliases, so a named create would produce a key unreachable by that name |
+
+Two consequences follow from that last row: **SSE-S3 key auto-creation and the synthetic KMS probe are unavailable on this backend**, because both address a key by a name they choose. Pre-create keys in AWS and reference them by AWS key id or ARN.
+
+Key versions are opaque. AWS addresses backing keys internally and picks the right one to decrypt with, so RustFS reports `key_version` as 1 and cannot enumerate versions. Rotation uses `RotateKeyOnDemand`, which retains prior backing keys for decryption; AWS's separate automatic yearly rotation is neither enabled nor reported on by RustFS.
+
+The AWS backend is not configurable through the KMS admin API — use the environment variables above at startup.
+
 ## Local backend durability and deployment support matrix
 
 The Local backend stores one JSON record per key (`<key_id>.key`) plus an Argon2id salt file (`.master-key.salt`) inside the configured `key_dir`. This section documents which deployments that layout supports and how the backend recovers from a crash or power loss. For where the key material lives and who can read it, see the [backend comparison](#backend-comparison) above.

docs/testing/e2e-suite-inventory.md

@@ -9,7 +9,7 @@
 > (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`.
 > 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147
 > repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke
-> lane and 27 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
+> lane and 28 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
 > nightly lane (`.github/workflows/e2e-replication-nightly.yml`).
 > Note: counts exclude `#[ignore]`d tests (nextest lists them separately).
 > The SSE-S3 replication contract is ignored under backlog#1291 until its
@@ -17,11 +17,11 @@
 
 | module | tests | PR smoke |
 |---|---|---|
-| admin_auth_test | 3 | ✅ |
+| admin_auth_test | 4 | ✅ |
 | admin_iam_crud_test | 2 | ✅ |
 | admin_pools_test | 1 | ✅ |
 | admin_timeout_regression_test | 1 |  |
-| anonymous_access_test | 3 | ✅ |
+| anonymous_access_test | 4 | ✅ |
 | api_rate_limit_test | 3 |  |
 | archive_download_integrity_test | 13 |  |
 | bucket_logging_test | 3 |  |
@@ -29,7 +29,7 @@
 | checksum_upload_test | 7 |  |
 | cluster_concurrency_test | 2 |  |
 | cluster_multidrive_pool_test | 2 |  |
-| common | 10 |  |
+| common | 12 |  |
 | compression_test | 1 |  |
 | connection_cap_test | 2 |  |
 | console_smoke_test | 1 | ✅ |
@@ -51,7 +51,9 @@
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
 | heal_erasure_disk_rebuild_test | 3 |  |
-| kms | 40 |  |
+| inline_fast_path_cluster_test | 14 |  |
+| internode_rpc_signature_e2e_test | 5 |  |
+| kms | 41 |  |
 | leading_slash_key_test | 2 | ✅ |
 | list_buckets_double_slash_test | 3 | ✅ |
 | list_object_versions_metadata_extension_test | 1 |  |
@@ -72,7 +74,7 @@
 | protocols | 16 |  |
 | quota_test | 14 |  |
 | reliability_disk_fault_test | 3 |  |
-| reliant | 10 | 5 ✅ |
+| reliant | 24 | 18 ✅ |
 | replication_extension_test | 48 | 20 ✅ +28 🌙 |
 | security_boundary_test | 4 |  |
 | ssec_copy_test | 2 | ✅ |
@@ -81,10 +83,11 @@
 | special_chars_test | 14 | ✅ |
 | stale_multipart_cleanup_cluster_test | 1 |  |
 | storage_class_capability_test | 4 | ✅ |
+| sts_query_compat_test | 3 | ✅ |
 | tls_gen | 3 |  |
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
 `notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
 
-**Total listed: 486 tests across 67 modules · PR smoke subset: 129 tests / 32 modules** (30 full modules + 5 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-07-26.
+**Total listed: 527 tests across 70 modules · PR smoke subset: 147 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-07-30.

rustfs/Cargo.toml

@@ -330,7 +330,7 @@ mimalloc = { workspace = true }
 libsystemd.workspace = true
 
 [target.'cfg(not(target_os = "windows"))'.dependencies]
-libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
+libmimalloc-sys.workspace = true
 
 [dev-dependencies]
 uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }