feat(admin): audit KMS management operations

Every KMS admin endpoint now builds an OperationContext from the
authenticated caller and hands it to the KMS layer, so the record the
manager already produces carries the principal, source address and
canonical request id instead of the internal placeholder.

A new adapter maps those records onto the server's existing AuditEntry
format and installs itself as the KMS audit sink at service assembly, so
KMS activity reaches the targets a deployment already operates. The
handlers emit directly for what the KMS layer cannot see: a request the
authorization gate rejects, and the endpoints with no context-aware KMS
entry point (data-key derivation and service control).

Only the failure class is recorded, never the error message, and the new
module joins the logging guardrail's checked files alongside the handlers
it serves.

.docker/observability/README.md

@@ -60,6 +60,16 @@ The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-con
 | `CodecStreamingFallbackSpike` | Warning | Codec streaming fallback > 10x baseline for 10m |
 | `IoQueueSaturation` | Warning | IO queue utilization > 90% for 5m |
 
+The file `prometheus-rules/rustfs-kms-alerts.yml` contains alerting rules for the KMS backend operation metrics. Thresholds are conservative defaults pending staging baseline calibration; response procedures live in `docs/operations/kms-observability-runbook.md`, and the matching dashboard is `deploy/observability/grafana/rustfs-kms-observability.json`.
+
+| Alert | Severity | Condition |
+|-------|----------|-----------|
+| `KmsBackendFatalErrors` | Critical | Fatal (non-retryable) attempt failures > 0 for 5m |
+| `KmsBackendHighErrorRate` | Critical | Non-success operation ratio > 5% for 10m (with traffic guard) |
+| `KmsBackendP99LatencyHigh` | Warning | Operation p99 duration (incl. retries) > 2s for 10m |
+| `KmsBackendAttemptFailureSpike` | Warning | Attempt failure rate > 0.5/s for 10m |
+| `KmsBackendRetryBudgetExhausted` | Warning | budget_exhausted / deadline_exceeded outcomes > 0.05/s for 10m |
+
 ### Enabling Alert Rules
 
 Add the alert rules file to your Prometheus configuration:

.docker/observability/README_ZH.md

@@ -60,6 +60,16 @@
 | `CodecStreamingFallbackSpike` | 警告 | Codec streaming 回退 > 10x 基线，持续 10 分钟 |
 | `IoQueueSaturation` | 警告 | IO 队列利用率 > 90%，持续 5 分钟 |
 
+文件 `prometheus-rules/rustfs-kms-alerts.yml` 包含 KMS 后端操作指标的告警规则。阈值为保守默认值，待 staging 基线校准；响应流程见 `docs/operations/kms-observability-runbook.md`，配套仪表盘为 `deploy/observability/grafana/rustfs-kms-observability.json`。
+
+| 告警 | 级别 | 条件 |
+|------|------|------|
+| `KmsBackendFatalErrors` | 严重 | fatal（不可重试）尝试失败 > 0，持续 5 分钟 |
+| `KmsBackendHighErrorRate` | 严重 | 非 success 操作占比 > 5%，持续 10 分钟（含流量下限保护） |
+| `KmsBackendP99LatencyHigh` | 警告 | 操作 p99 耗时（含重试）> 2s，持续 10 分钟 |
+| `KmsBackendAttemptFailureSpike` | 警告 | 尝试失败率 > 0.5/s，持续 10 分钟 |
+| `KmsBackendRetryBudgetExhausted` | 警告 | budget_exhausted / deadline_exceeded 结果 > 0.05/s，持续 10 分钟 |
+
 ### 启用告警规则
 
 在 Prometheus 配置中添加告警规则文件：

.docker/observability/prometheus-rules/rustfs-kms-alerts.yml

@@ -0,0 +1,188 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =============================================================================
+# RustFS KMS backend — Prometheus alerting rules
+# =============================================================================
+#
+# Metric source: the KMS operation-policy choke point in
+# crates/kms/src/policy.rs. All label values are static enum strings
+# (operation, op_class, outcome, error_class); key identifiers, key material,
+# and tokens never appear in labels.
+#
+# Response procedures: docs/operations/kms-observability-runbook.md
+#
+# IMPORTANT — threshold status: every numeric threshold below is a
+# conservative default chosen without a production baseline. Calibrate against
+# a staging baseline before relying on these alerts for paging, and prefer
+# loosening over tightening until the baseline exists. Formal SLO targets are
+# deliberately not encoded here (see rustfs/backlog#1584).
+#
+# NOTE: prometheus.yml loads /etc/prometheus/rules/*.yml — keep the .yml
+# extension or the file is silently ignored by the docker-compose stack.
+#
+# Validate: promtool check rules rustfs-kms-alerts.yml
+# =============================================================================
+
+groups:
+  # ==========================================================================
+  # Critical alerts — immediate action required
+  # ==========================================================================
+  - name: rustfs-kms-critical
+    interval: 30s
+    rules:
+      # ------------------------------------------------------------------
+      # 1. KmsBackendFatalErrors
+      #    Any attempt failure classified as fatal (non-retryable): auth
+      #    or permission errors, malformed requests, missing keys. The
+      #    policy never retries these, so even a low rate means real
+      #    operations are failing right now.
+      # ------------------------------------------------------------------
+      - alert: KmsBackendFatalErrors
+        expr: |
+          sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m])) > 0
+        for: 5m
+        labels:
+          severity: critical
+          component: kms
+        annotations:
+          summary: "KMS backend fatal errors on operation {{ $labels.operation }}"
+          description: >-
+            Attempt failures classified as fatal are occurring at
+            {{ $value | printf "%.3f" }}/s on operation
+            {{ $labels.operation }}. Fatal failures are not retried:
+            each one is a KMS backend call that failed permanently
+            (authentication, permissions, malformed request, or a
+            missing key/version).
+          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendfatalerrors"
+
+      # ------------------------------------------------------------------
+      # 2. KmsBackendHighErrorRate
+      #    Sustained share of operations terminating without success
+      #    (fatal, budget_exhausted, deadline_exceeded). The cancelled
+      #    outcome is excluded because shutdowns legitimately produce it.
+      #    The traffic guard keeps a single failure on a near-idle
+      #    cluster from firing the alert.
+      #    Threshold: 5% for 10m — conservative default, calibrate
+      #    against a staging baseline.
+      # ------------------------------------------------------------------
+      - alert: KmsBackendHighErrorRate
+        expr: |
+          (
+            sum(rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))
+            /
+            clamp_min(sum(rate(rustfs_kms_backend_operations_total[5m])), 1e-9)
+          ) > 0.05
+          and
+          sum(rate(rustfs_kms_backend_operations_total[5m])) > 0.02
+        for: 10m
+        labels:
+          severity: critical
+          component: kms
+        annotations:
+          summary: "KMS backend non-success ratio above 5% for 10m"
+          description: >-
+            {{ $value | humanizePercentage }} of KMS backend operations
+            are terminating in fatal, budget_exhausted, or
+            deadline_exceeded. Object encryption and decryption paths
+            depending on the KMS are degraded or failing.
+          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate"
+
+  # ==========================================================================
+  # Warning alerts — investigation needed
+  # ==========================================================================
+  - name: rustfs-kms-warning
+    interval: 30s
+    rules:
+      # ------------------------------------------------------------------
+      # 3. KmsBackendP99LatencyHigh
+      #    p99 wall-clock duration of whole operations (attempts plus
+      #    backoff) is sustained above 2 seconds. Because the histogram
+      #    includes retries, a high p99 usually means the retry policy
+      #    is absorbing backend failures, not that every call is slow.
+      #    Threshold: 2s for 10m — conservative default, calibrate
+      #    against a staging baseline.
+      # ------------------------------------------------------------------
+      - alert: KmsBackendP99LatencyHigh
+        expr: |
+          histogram_quantile(0.99,
+            sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m]))
+          ) > 2
+        for: 10m
+        labels:
+          severity: warning
+          component: kms
+        annotations:
+          summary: "KMS backend operation p99 latency above 2s for 10m"
+          description: >-
+            The 99th-percentile KMS backend operation duration is
+            {{ $value | humanizeDuration }}, including retries and
+            backoff. Encryption and decryption latency is leaking into
+            S3 request latency.
+          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendp99latencyhigh"
+
+      # ------------------------------------------------------------------
+      # 4. KmsBackendAttemptFailureSpike
+      #    Aggregate attempt-failure rate (all error classes) sustained
+      #    above an absolute floor. An absolute threshold is used instead
+      #    of an offset-1d baseline ratio because fresh deployments have
+      #    no baseline and an empty offset vector would keep a ratio
+      #    alert from ever firing; switch to a baseline-relative form
+      #    (see rustfs-get-optimization-alerts.yaml for the pattern)
+      #    once a stable staging baseline exists.
+      #    Threshold: 0.5/s for 10m — conservative default, calibrate
+      #    against a staging baseline.
+      # ------------------------------------------------------------------
+      - alert: KmsBackendAttemptFailureSpike
+        expr: |
+          sum(rate(rustfs_kms_backend_attempt_failures_total[5m])) > 0.5
+        for: 10m
+        labels:
+          severity: warning
+          component: kms
+        annotations:
+          summary: "KMS backend attempt failures above 0.5/s for 10m"
+          description: >-
+            KMS backend attempts are failing at
+            {{ $value | printf "%.2f" }}/s across all error classes.
+            The retry policy may still be masking these from callers —
+            check the error-class breakdown before it stops absorbing
+            them.
+          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendattemptfailurespike"
+
+      # ------------------------------------------------------------------
+      # 5. KmsBackendRetryBudgetExhausted
+      #    Operations are running out of retry budget (budget_exhausted)
+      #    or operation deadline (deadline_exceeded). These surface to
+      #    callers as failed KMS operations even though every individual
+      #    failure was retryable — the backend is unhealthy for longer
+      #    than the policy can bridge.
+      #    Threshold: 0.05/s for 10m — conservative default, calibrate
+      #    against a staging baseline.
+      # ------------------------------------------------------------------
+      - alert: KmsBackendRetryBudgetExhausted
+        expr: |
+          sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m])) > 0.05
+        for: 10m
+        labels:
+          severity: warning
+          component: kms
+        annotations:
+          summary: "KMS backend operations exhausting retry budget ({{ $labels.outcome }})"
+          description: >-
+            KMS backend operations are terminating as
+            {{ $labels.outcome }} at {{ $value | printf "%.3f" }}/s.
+            Retryable failures are outlasting the retry budget, so
+            callers are seeing hard failures.
+          runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted"

.github/actions/setup/action.yml

@@ -25,9 +25,13 @@ inputs:
     required: false
     default: "rustfs-deps"
   cache-save-if:
-    description: "Condition for saving cache"
+    description: >-
+      Whether to save the cache. The fail-safe default is 'false': a caller that
+      wants to populate a cache must opt in explicitly, so a forgotten input
+      costs a cold cache (minutes) rather than silently consuming the
+      repository-wide 10GB Actions cache quota and evicting other lanes.
     required: false
-    default: "true"
+    default: "false"
   install-cross-tools:
     description: "Install cross-compilation tools"
     required: false
@@ -36,28 +40,43 @@ inputs:
     description: "Target architecture to add"
     required: false
     default: ""
-  github-token:
-    description: "GitHub token for API access"
+  install-build-packaging-tools:
+    description: >-
+      Install musl-tools/zip/unzip, needed for musl linking and release
+      packaging. Off for CI test lanes, which use none of them.
     required: false
-    default: ""
+    default: "true"
+  install-test-tools:
+    description: >-
+      Install cargo-nextest and the rustfmt/clippy components. Off for release
+      and audit lanes, which run no tests and no lints.
+    required: false
+    default: "true"
 
 runs:
   using: "composite"
   steps:
+    # protobuf-compiler is deliberately absent: the setup-protoc step below
+    # installs 34.1 into the tool cache and prepends it to PATH, so the apt
+    # build (older, and never version-matched) was shadowed on every run and
+    # simply never used.
     - name: Install system dependencies (Ubuntu)
       if: runner.os == 'Linux'
       shell: bash
       run: |
         sudo apt-get update
         sudo apt-get install -y \
-          musl-tools \
           build-essential \
           pkg-config \
           libssl-dev \
-          ripgrep \
-          unzip \
-          zip \
-          protobuf-compiler
+          ripgrep
+
+    # musl-gcc is needed by the native musl release leg, and zip/unzip by the
+    # release packaging steps. No CI test lane touches any of them.
+    - name: Install packaging and cross-linking dependencies (Ubuntu)
+      if: runner.os == 'Linux' && inputs.install-build-packaging-tools == 'true'
+      shell: bash
+      run: sudo apt-get install -y musl-tools zip unzip
 
     - name: Install protoc
       uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
@@ -75,7 +94,7 @@ runs:
       with:
         toolchain: ${{ inputs.rust-version }}
         targets: ${{ inputs.target }}
-        components: rustfmt, clippy
+        components: ${{ inputs.install-test-tools == 'true' && 'rustfmt, clippy' || '' }}
 
     - name: Install Zig
       if: inputs.install-cross-tools == 'true'
@@ -86,6 +105,7 @@ runs:
       uses: taiki-e/install-action@a21ae4029b089b9ddc45704028756f51ab8abe48 # cargo-zigbuild
 
     - name: Install cargo-nextest
+      if: inputs.install-test-tools == 'true'
       uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
 
     - name: Setup Rust cache

.github/workflows/architecture-migration-rules.yml

@@ -37,6 +37,7 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -45,8 +46,11 @@ jobs:
     name: Architecture Migration Rules
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install ripgrep
         run: |

.github/workflows/audit.yml

@@ -39,7 +39,12 @@ on:
       - 'scripts/security/check_preview_release_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   schedule:
-    - cron: '0 3 * * 0' # Weekly on Sunday 03:00 UTC (staggered after the midnight ci/build crons)
+    # Daily, not weekly. This schedule exists to catch RustSec advisories
+    # published against an unchanged dependency tree; at weekly cadence a new
+    # advisory could sit unnoticed for seven days. The check list is unchanged —
+    # splitting it into a light daily advisories-only run and a weekly full run
+    # would create runs where sources/bans/licenses go unverified.
+    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
   workflow_dispatch:
 
 permissions:
@@ -59,6 +64,7 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -74,11 +80,30 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
         with:
-          cache-shared-key: rustfs-cargo-deny
+          persist-credentials: false
+
+      # cargo-deny compiles nothing, so the full setup composite (apt packages,
+      # protoc, flatc, nextest, rustfmt/clippy) was pure overhead here. It does
+      # still need a real cargo: `cargo deny check` runs `cargo metadata`, and
+      # Cargo.toml pins datafusion and s3s as git dependencies, which must be
+      # materialised into ~/.cargo/git — a cold clone is hundreds of MB, so the
+      # cache stays.
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+      # Was relying on the composite's default, which used to be "true": every
+      # PR touching Cargo.toml/Cargo.lock saved a second, PR-scoped copy of this
+      # cache and pushed the main-scoped lanes out of the 10GB quota. The
+      # default is now "false", but state it explicitly — see
+      # scripts/security/check_cache_save_if.sh.
+      - name: Setup Rust cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          cache-all-crates: true
+          cache-on-failure: true
+          shared-key: rustfs-cargo-deny
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Install cargo-deny
         uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
@@ -96,16 +121,28 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Report unpinned GitHub Actions
         run: ./scripts/security/check_workflow_pins.sh --enforce
 
+      - name: Check setup cache-save-if is explicit
+        run: ./scripts/security/check_cache_save_if.sh
+
+      - name: Check every job declares a timeout
+        run: ./scripts/security/check_job_timeouts.sh
+
+      - name: Check checkouts clear their credentials
+        run: ./scripts/security/check_persist_credentials.sh
+
       - name: Check preview release workflow policy
         run: ./scripts/security/check_preview_release_workflow.sh
 
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     if: github.event_name == 'pull_request' && github.event.action != 'closed'
     permissions:
       contents: read
@@ -113,6 +150,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Dependency Review
         uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5
@@ -125,3 +164,28 @@ jobs:
           # conscious re-review of the license/provenance claim (backlog#1181).
           allow-dependencies-licenses: pkg:cargo/rustfs-uring@0.1.0
           comment-summary-in-pr: always
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # dependency-review is deliberately excluded: it only runs on pull_request,
+    # so it can never contribute a failure to a scheduled run.
+    needs: [cargo-deny, workflow-pin-report]
+    # A scheduled cargo-deny failure usually means the dependency tree just
+    # matched a newly published advisory — the single most important signal this
+    # workflow produces, and until now it was only visible to whoever happened to
+    # open the Actions tab. Same ci-8 mechanism coverage.yml and
+    # e2e-replication-nightly.yml already use.
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build.yml

@@ -50,12 +50,18 @@ on:
       - "**/*.svg"
       - ".gitignore"
       - ".dockerignore"
+      - "flake.lock"
   schedule:
     - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
   workflow_dispatch:
     inputs:
       build_docker:
-        description: "Build and push Docker images after binary build"
+        # Advisory only. docker.yml triggers on workflow_run and its job-level
+        # condition requires the triggering event to be a tag push, so a manual
+        # dispatch of this workflow never produces images regardless of this
+        # value. Kept because the summary step reports it; wiring it up would
+        # mean teaching docker.yml's version parser a second event shape.
+        description: "Build and push Docker images after binary build (ignored: dispatch runs never reach docker.yml)"
         required: false
         default: true
         type: boolean
@@ -83,6 +89,7 @@ jobs:
   build-check:
     name: Build Strategy Check
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     outputs:
       should_build: ${{ steps.check.outputs.should_build }}
       build_type: ${{ steps.check.outputs.build_type }}
@@ -92,6 +99,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Determine build strategy
         id: check
@@ -164,6 +173,7 @@ jobs:
     name: Prepare Platform Matrix
     needs: build-check
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     outputs:
       matrix: ${{ steps.select.outputs.matrix }}
       selected: ${{ steps.select.outputs.selected }}
@@ -171,10 +181,14 @@ jobs:
       - name: Select target platforms
         id: select
         shell: bash
+        env:
+          # via env, not interpolation: a dispatch input is free-form text and
+          # would otherwise be pasted into the script for bash to evaluate.
+          RAW_PLATFORMS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}
         run: |
           set -euo pipefail
 
-          selected="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}"
+          selected="$RAW_PLATFORMS"
           selected="$(echo "${selected}" | tr -d '[:space:]')"
           if [[ -z "${selected}" ]]; then
             selected="all"
@@ -245,6 +259,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           fetch-depth: 0
 
       - name: Setup Rust environment
@@ -253,9 +268,17 @@ jobs:
           rust-version: stable
           target: ${{ matrix.target }}
           cache-shared-key: build-${{ matrix.target }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }}
+          # main only. A cache saved on refs/tags/X is scoped to that tag: no
+          # other tag, no main run and no PR can restore it, so every release
+          # cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
+          # six legs each) that nobody could read, evicting the hot lanes from
+          # the repo-wide 10GB quota. Tag builds still restore the main-scoped
+          # cache, since default-branch caches are readable from every ref.
+          # The one real cost: re-running a failed leg of the same tag no longer
+          # finds that tag's own warm cache and falls back to main's.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
           install-cross-tools: ${{ matrix.cross }}
+          install-test-tools: 'false'
 
       - name: Download static console assets
         shell: bash
@@ -702,9 +725,14 @@ jobs:
     needs: [ build-check, build-rustfs ]
     if: always() && needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Build completion summary
         shell: bash
+        env:
+          # dispatch input via env: free-form text must not be pasted into the
+          # script for bash to evaluate.
+          INPUT_BUILD_DOCKER: ${{ github.event.inputs.build_docker }}
         run: |
           BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
           VERSION="${{ needs.build-check.outputs.version }}"
@@ -746,7 +774,7 @@ jobs:
           echo "🐳 Docker Images:"
           if [[ "$BUILD_TYPE" == "preview" ]]; then
             echo "⏭️  Preview tags do not publish Docker images"
-          elif [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then
+          elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
             echo "⏭️  Docker image build was skipped (binary only build)"
           elif [[ "$BUILD_STATUS" == "success" ]]; then
             echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
@@ -760,6 +788,7 @@ jobs:
     needs: [ build-check, build-rustfs ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
       contents: write
     outputs:
@@ -769,6 +798,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           fetch-depth: 0
 
       - name: Create GitHub Release
@@ -819,12 +849,15 @@ jobs:
     needs: [ build-check, build-rustfs, create-release ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
       contents: write
       actions: read
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download all build artifacts
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -910,6 +943,7 @@ jobs:
     needs: [ build-check, publish-release ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Update latest.json
         env:
@@ -969,6 +1003,7 @@ jobs:
     needs: [ build-check, create-release, upload-release-assets ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
       contents: write
     steps:

.github/workflows/cache-warm.yml

@@ -0,0 +1,231 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sole writer of the Rust dependency caches that ci.yml restores.
+#
+# Why this is a separate workflow rather than steps inside ci.yml: ci.yml's
+# concurrency group cancels in-progress runs on main pushes, and merges land far
+# faster than its 70-minute pipeline. Measured over 15 consecutive main pushes:
+# 12 cancelled, 2 failed, 0 succeeded. A cancelled run never reaches
+# Swatinem/rust-cache's post step (cache-on-failure does not cover cancellation),
+# so the writer lanes were saving nothing and every PR paid a cold restore —
+# 11.8-20.9 minutes of "Setup Rust environment" against 0.7-3.4 warm.
+#
+# Splitting cache writing out of the test pipeline lets ci.yml keep cancelling
+# superseded runs (which is correct — nobody needs test results for a commit
+# that is already three merges behind) while the caches still get written.
+#
+# The group below deliberately does NOT cancel in progress. GitHub keeps at most
+# one running plus one pending run per group, so a burst of merges collapses
+# into "current run finishes, newest queued run follows" rather than a pile-up.
+# That also bounds this workflow to one self-hosted runner at a time.
+#
+# Each job below owns exactly one shared-key and is the only place that sets
+# cache-save-if to anything but 'false' for it; every lane in ci.yml reads.
+# scripts/security/check_cache_save_if.sh keeps the declarations explicit.
+#
+# The builds are supersets of what the reading lanes compile, because a reader
+# restores only what the writer saved. Feature resolution matters here: a lane
+# built with e2e-test-hooks resolves dependency features differently, which
+# changes -Cmetadata, so the plain build does not cover it. See
+# rustfs/backlog#1600.
+
+name: Cache Warm
+
+on:
+  push:
+    branches: [ main ]
+    # Mirrors ci.yml's push paths-ignore: if a commit cannot change what ci.yml
+    # compiles, it cannot change what ci.yml needs restored either.
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "scripts/probe.sh"
+      - "LICENSE*"
+      - ".gitignore"
+      - ".dockerignore"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".github/workflows/build.yml"
+      - ".github/workflows/docker.yml"
+      - ".github/workflows/audit.yml"
+  workflow_dispatch:
+    inputs:
+      emit_timings:
+        description: >-
+          Also emit cargo --timings for the ci-dev build and upload it. Used to
+          decide whether sccache is worth adopting (rustfs/backlog#1601 gate).
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+
+concurrency:
+  group: cache-warm
+  cancel-in-progress: false
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
+  # e2e-tests, e2e-full.
+  warm-ci-dev:
+    name: Warm ci-dev
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      # --all-targets covers the test binaries nextest builds, including
+      # e2e_test, which test-and-lint's own run excludes. The second build adds
+      # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
+      # and that no lint lane enables.
+      # rustfs/backlog#1601 gate. sccache can only cache compilation units whose
+      # --emit includes link, so it covers workspace rlibs and nothing else:
+      # clippy is metadata-only, and the ~100 test binaries, the rustfs bin and
+      # every build script invoke the system linker. Before spending a bucket,
+      # credentials and a supply-chain boundary on it, measure how much of the
+      # build is actually rlib codegen.
+      #
+      # Read from the report: workspace lib codegen as a share of the build, and
+      # s3select-query's own rlib as a share. The plan adopts sccache only above
+      # 50% and 25% respectively; if linking dominates instead, the answer is
+      # mold/lld plus split-debuginfo, which is exactly the part sccache cannot
+      # touch. Off by default — this doubles the ci-dev build.
+      - name: Build ci-dev superset (with --timings)
+        if: inputs.emit_timings
+        env:
+          CARGO_BUILD_JOBS: "2"
+        run: cargo build --workspace --all-targets --timings
+
+      - name: Upload cargo timings report
+        if: inputs.emit_timings
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: cargo-timings-ci-dev
+          path: target/cargo-timings/
+          retention-days: 30
+          if-no-files-found: error
+
+      - name: Build ci-dev superset
+        env:
+          # Same limit ci.yml puts on its nextest step: this builds the same
+          # ~100 workspace test binaries, and three concurrent links saturate the
+          # self-hosted runner's overlay I/O and can wedge Cargo (#5394).
+          CARGO_BUILD_JOBS: "2"
+        run: |
+          cargo build --workspace --all-targets
+          cargo build -p rustfs --bins --features e2e-test-hooks
+
+  # Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
+  warm-ci-feat-rio:
+    name: Warm ci-feat-rio
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-feat-rio
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      - name: Build ci-feat-rio superset
+        run: |
+          cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
+          cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
+
+  # Readers: the swift and sftp legs of test-and-lint-protocols. Built in
+  # sequence rather than as `--features swift,sftp`, which is a combination no
+  # lane actually compiles; running both leaves the union in target/.
+  warm-ci-feat-proto:
+    name: Warm ci-feat-proto
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-feat-proto
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      - name: Build ci-feat-proto superset
+        run: |
+          cargo build -p rustfs -p rustfs-protocols --all-targets --features swift
+          cargo build -p rustfs -p rustfs-protocols --all-targets --features sftp
+
+  # Reader: uring-integration. Runs on ubuntu-latest to match it: rust-cache's
+  # key covers runner.os and arch but not the runner label or image, so a cache
+  # written on sm-standard-4 would be restored by the hosted runner as if it
+  # belonged to it.
+  warm-ci-uring:
+    name: Warm ci-uring
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-uring
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      - name: Install build dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Build ci-uring superset
+        run: cargo build -p rustfs-ecstore --all-targets

.github/workflows/ci-docs-only.yml

@@ -12,18 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Companion to ci.yml for the required "Test and Lint" status check.
+# Companion to ci.yml for required status checks.
 #
-# ci.yml skips docs-only pull requests via paths-ignore, but the branch
-# ruleset requires a check named "Test and Lint" — without this workflow a
-# docs-only PR would wait on that check forever. This workflow triggers on
-# exactly the paths ci.yml ignores and reports an instant success under the
-# same job name. Mixed PRs trigger both workflows and the real check still
-# gates: a required check with any failing run blocks the merge.
+# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
+# requires a check named "Test and Lint" — without this workflow a docs-only PR
+# would wait on it forever. This workflow triggers on exactly the paths ci.yml
+# ignores and reports success under the same job name. Mixed PRs trigger both
+# workflows and the real check still gates: a required check with any failing
+# run blocks the merge.
 # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
 #
+# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
+# required too (rustfs/backlog#1599). Until that change lands this job is
+# inert; mirroring it first is what lets the ruleset change happen without
+# stranding docs-only PRs on a check nobody reports.
+#
 # Keep the paths list below in sync with the pull_request paths-ignore list
-# in ci.yml.
+# in ci.yml, and keep the quick-checks steps below byte-identical to the
+# quick-checks job in ci.yml.
 
 name: Continuous Integration (docs only)
 
@@ -47,17 +53,82 @@ on:
       - ".github/workflows/build.yml"
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
+      - "flake.lock"
 
 permissions:
   contents: read
 
 jobs:
-  test-and-lint:
-    name: Test and Lint
+  # Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
+  # check, ci.yml gates every expensive job behind it, so a mixed PR reports
+  # two check runs with this name: the real one (45-51s) and this companion.
+  # GitHub has no written contract for how it picks between same-named
+  # required check runs ("latest wins" vs "any failure blocks"), so instead of
+  # relying on ordering we make both runs execute the same commands against
+  # the same merge ref — their conclusions are then necessarily identical and
+  # the choice does not matter. Keep these steps byte-identical to the
+  # quick-checks job in ci.yml (a guard script that asserts this, and the paths
+  # sync below, is tracked in rustfs/backlog#1603).
+  #
+  # For a genuinely docs-only PR this adds no strictness (no code changed, so
+  # fmt and the guards always pass) and costs ~50s of ubuntu-latest.
+  quick-checks:
+    name: Quick Checks
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Install ripgrep
+        run: sudo apt-get update && sudo apt-get install -y ripgrep
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          components: rustfmt
+
+      - name: Check code formatting
+        run: cargo fmt --all --check
+
+      - name: Check unsafe code allowances
+        run: ./scripts/check_unsafe_code_allowances.sh
+
+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
+      - name: Check architecture migration rules
+        run: ./scripts/check_architecture_migration_rules.sh
+
+      - name: Check tokio io-uring feature guard
+        run: ./scripts/check_no_tokio_io_uring.sh
+
+      - name: Check extension schema boundaries
+        run: ./scripts/check_extension_schema_boundaries.sh
+
+      - name: Check body-cache whitelist guard
+        run: ./scripts/check_body_cache_whitelist.sh
+
+      - name: Check no planning docs committed
+        run: ./scripts/check_no_planning_docs.sh
+
+      - name: Check CI paths stay in sync
+        run: ./scripts/check_ci_paths_sync.sh
+
+      - name: Check io_uring lane --lib precondition
+        run: ./scripts/check_uring_lane_lib_only.sh
+
+  test-and-lint:
+    name: Test and Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Docs-only PRs skip the full code CI, but they are exactly where a
       # planning-type document could be slipped in (git add -f bypasses

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ on:
       - ".github/workflows/build.yml"
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
+      - "flake.lock"
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
     branches: [ main ]
@@ -54,6 +55,7 @@ on:
       - ".github/workflows/build.yml"
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
+      - "flake.lock"
   merge_group:
     types: [ checks_requested ]
   schedule:
@@ -81,6 +83,7 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -89,13 +92,20 @@ jobs:
     name: Typos
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
   # Fast, compile-free checks that fail early so contributors get feedback in
   # ~1 minute instead of waiting for the full test job.
+  #
+  # These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
+  # PR, which reports two check runs named "Quick Checks", cannot get one red
+  # and one green. Edit both jobs together.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -104,6 +114,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install ripgrep
         run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -137,24 +149,48 @@ jobs:
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
 
+      - name: Check CI paths stay in sync
+        run: ./scripts/check_ci_paths_sync.sh
+
+      - name: Check io_uring lane --lib precondition
+        run: ./scripts/check_uring_lane_lib_only.sh
+
   test-and-lint:
     name: Test and Lint
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 90
+    # Both lines are required. Job-level `permissions` replaces the workflow
+    # block rather than merging with it, so declaring only `actions: write`
+    # would drop `contents: read` and break this job's checkout and the
+    # repo-token the setup action hands to setup-protoc.
+    permissions:
+      contents: read
+      actions: write
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          # This job's token can cancel runs and delete Actions caches. Checkout
+          # otherwise writes it into .git/config, where a PR's own build.rs or
+          # proc-macro could read it back out.
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-test
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          # Every lane in this workflow reads its cache and none writes it.
+          # cache-warm.yml is the sole writer for all four keys: this workflow
+          # cancels superseded runs on main, and a cancelled run never reaches
+          # rust-cache's post step, so writing from here saved nothing (12 of 15
+          # consecutive main-push runs were cancelled). See rustfs/backlog#1600.
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       - name: Prepare test evidence
         run: |
@@ -169,8 +205,14 @@ jobs:
       # Clippy runs before the test pass: lint failures are the most common
       # CI-only breakage and should surface in minutes, not after 20+ minutes
       # of tests.
+      # Sampled too: clippy is the natural control arm for any CARGO_BUILD_JOBS
+      # experiment, since --all-targets is check-only for workspace members and
+      # never links the ~100 test binaries the limit exists to throttle.
       - name: Run clippy lints
-        run: cargo clippy --all-targets -- -D warnings
+        run: |
+          ./scripts/ci/resource_sampler.sh start clippy
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          cargo clippy --all-targets -- -D warnings
 
       - name: Run nextest tests
         env:
@@ -179,33 +221,8 @@ jobs:
           CARGO_BUILD_JOBS: "2"
         run: |
           mkdir -p artifacts/test-and-lint
-          # Evidence sampler for issue #5394: the post-mortem pgrep below runs
-          # only after `timeout` has already TERM'd the whole cargo process
-          # group, so it cannot name a wedged process. Sample system and
-          # process state every 60s instead; the last samples before the
-          # timeout show what was stuck (rustc, linker, build script, memory
-          # pressure, ...). The log rides along in the existing artifact.
-          (
-            while true; do
-              {
-                echo "=== $(date --utc --iso-8601=seconds)"
-                echo "--- load";  cat /proc/loadavg
-                echo "--- psi";   grep -H . /proc/pressure/* 2>/dev/null || true
-                echo "--- mem";   free -m
-                echo "--- disk";  df -h / /home/runner 2>/dev/null || df -h /
-                echo "--- top-rss"
-                ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
-                echo "--- build/test processes"
-                ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
-                echo "--- d-state (uninterruptible IO)"
-                ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
-                echo
-              } >> artifacts/test-and-lint/sampler.log 2>&1 || true
-              sleep 60
-            done
-          ) &
-          sampler_pid=$!
-          trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
+          ./scripts/ci/resource_sampler.sh start nextest
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
           set +e
           NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
             cargo nextest run --profile ci --all --exclude e2e_test \
@@ -277,6 +294,48 @@ jobs:
       - name: Run rebalance/decommission migration proofs
         run: ./scripts/check_migration_gate_count.sh
 
+      # Early stop. Once this job has failed the PR cannot merge, so the sibling
+      # lanes are burning runners on a result nobody can act on: on run
+      # 30674613104 three lanes had already failed while Test and Lint and the
+      # rio-v2 variant kept going past 70 minutes.
+      #
+      # Only this job may cancel. The lanes that are NOT required checks
+      # (protocols, ILM, e2e, s3-tests) must never hold that power: a flake in
+      # one of them would turn the required "Test and Lint" into `cancelled`,
+      # which blocks the merge. Today a maintainer can merge with sftp red, and
+      # that has to stay true.
+      #
+      # These steps run last so the `if: always()` artifact upload above still
+      # captures logs and diagnostics before the run goes away.
+      - name: Annotate early-stop reason
+        if: failure() && github.event_name == 'pull_request'
+        run: |
+          echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
+          echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
+          echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
+
+      # curl rather than `gh`: every existing `gh` call in this repo runs on
+      # ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
+      # ship no C toolchain, see the e2e job below), so `gh` is not known to
+      # exist here.
+      #
+      # Fork PRs are excluded explicitly instead of relying on the error path:
+      # their GITHUB_TOKEN is forced read-only and job-level permissions cannot
+      # raise it, so the call would always 403. Skipping keeps their logs clean.
+      - name: Cancel run on failure (same-repo PR only)
+        if: >-
+          failure() && github.event_name == 'pull_request'
+          && github.event.pull_request.head.repo.full_name == github.repository
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          curl -fsS -X POST \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" || true
+
   # Dedicated serial lane for the ILM / lifecycle integration tests. These tests
   # drive the object layer through process-global singletons (the GLOBAL_ENV
   # ECStore, the global tier-config manager, background-expiry workers) and bind
@@ -290,6 +349,7 @@ jobs:
   test-ilm-integration-serial:
     name: ILM Integration (serial)
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 45
     env:
@@ -297,14 +357,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-ilm-serial
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
       # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
@@ -327,6 +389,7 @@ jobs:
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 60
     env:
@@ -334,14 +397,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-test-rio-v2
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-feat-rio
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       - name: Run rio-v2 clippy lints
         run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
@@ -354,10 +419,17 @@ jobs:
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 60
     strategy:
-      fail-fast: false
+      # On a PR, one failing protocol leg is enough to know the PR is not ready,
+      # so stop the sibling leg instead of paying another ~40 minutes for it.
+      # Everywhere else (main pushes, the merge queue, the weekly schedule) keep
+      # the full signal: there we want to know whether swift AND sftp are broken,
+      # not just whichever failed first. This is the only part of the early-stop
+      # work that also covers fork PRs, since it needs no token.
+      fail-fast: ${{ github.event_name == 'pull_request' }}
       matrix:
         features:
           - name: swift
@@ -369,14 +441,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-test-${{ matrix.features.name }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-feat-proto
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       - name: Run clippy with ${{ matrix.features.name }}
         run: |
@@ -389,6 +463,7 @@ jobs:
   build-rustfs-debug-binary:
     name: Build RustFS Debug Binary
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     env:
@@ -396,14 +471,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-rustfs-debug-binary
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       - name: Build debug binary
         run: cargo build -p rustfs --bins --features e2e-test-hooks
@@ -419,6 +496,7 @@ jobs:
   build-rustfs-debug-binary-rio-v2:
     name: Build RustFS Debug Binary (rio-v2)
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     env:
@@ -426,14 +504,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-rustfs-debug-binary-rio-v2
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-shared-key: ci-feat-rio
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       - name: Build debug binary with rio-v2
         run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
@@ -448,6 +528,14 @@ jobs:
 
   uring-integration:
     name: io_uring Integration (real)
+    # The pull_request trigger includes `closed` purely so the concurrency
+    # group cancels in-flight runs of a closed PR; every other job opts out of
+    # that run with this guard (or is skipped through its `needs` chain). This
+    # job had neither, so each closed/merged PR really ran the whole io_uring
+    # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
+    # 30662728539) and kept the cancellation run in progress for minutes.
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
     # the probe succeeds and the tests exercise the real UringBackend/FdCache/
@@ -458,17 +546,24 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
+          # Keeps its own key rather than joining ci-dev. rust-cache's key is
+          # built from runner.os/arch plus rustc and lockfile fingerprints — it
+          # does NOT include the runner label or image. ubuntu-latest and
+          # sm-standard-4 are therefore indistinguishable to it, so sharing a key
+          # would let two different system images overwrite each other's
+          # artifacts, and would make a 2-core hosted runner unpack ci-dev's ~3GB
+          # instead of this lane's ~1.3GB. cache-warm.yml warms this key on
+          # ubuntu-latest for the same reason.
           cache-shared-key: ci-uring
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Install build dependencies
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       # ext4 supports O_DIRECT; the runner's default TMPDIR may sit on tmpfs or
       # overlayfs, where open(O_DIRECT) returns EINVAL/EOPNOTSUPP and the native
@@ -495,7 +590,17 @@ jobs:
           RUSTFS_IO_URING_READ_ENABLE: "true"
           RUSTFS_URING_TESTS_MUST_RUN: "1"
           TMPDIR: /mnt/rustfs-odirect
-        run: cargo test -p rustfs-ecstore uring_ -- --test-threads=1 --nocapture
+        # --lib narrows what gets compiled, not what gets run: every selected
+        # test lives in the lib target. The 7 integration binaries under
+        # crates/ecstore/tests/ each reported "running 0 tests" here, so they
+        # were compiled and linked for nothing.
+        #
+        # The `uring_` filter must stay exactly as it is. libtest matches on
+        # substring, so it also selects names containing `during_` — 6 of the 18
+        # selected tests are such incidental matches. Narrowing the filter to
+        # `io_uring` would silently drop them, which is a coverage change.
+        # scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
+        run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture
 
   e2e-tests:
     name: End-to-End Tests
@@ -505,6 +610,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Full setup with dependency caching: the smoke-suite step below
       # compiles the e2e_test crate, which pulls in most of the workspace.
@@ -513,9 +620,9 @@ jobs:
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-e2e
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
@@ -589,14 +696,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-e2e
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
 
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
@@ -632,6 +741,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Clean up previous test run
         run: |
@@ -686,6 +797,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download debug binary
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -746,12 +859,20 @@ jobs:
   # evaluates ILM within ~2s of the due time, well inside the poll window.
   s3-lifecycle-behavior-tests:
     name: S3 Lifecycle Behavior Tests
-    needs: [ build-rustfs-debug-binary ]
+    # Also gated on e2e-tests, matching s3-implemented-tests: when the e2e smoke
+    # suite is already red this lane cannot tell us anything new, and it holds a
+    # sm-standard-4 for up to 30 minutes doing so. Both lanes only download the
+    # prebuilt debug binary (no cargo build), and s3-implemented-tests — which
+    # already waits on e2e-tests — finishes later anyway, so a green PR's total
+    # wall clock is unchanged.
+    needs: [ build-rustfs-debug-binary, e2e-tests ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download debug binary
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7

.github/workflows/cla.yml

@@ -22,11 +22,18 @@ on:
   issue_comment:
     types: [created, edited]
 
+# Least privilege at the top, widened per job below. This workflow runs on
+# pull_request_target and issue_comment, so it holds full secrets on every fork
+# PR and on any comment anyone writes — the one place in this repository where a
+# compromised action would be handed a repo-write token. It does not check out
+# or execute PR code, so there is no pwn-request path today, but the blast
+# radius should not depend on that staying true.
+#
+# contents: write in particular was never used: the signature records are
+# written to rustfs/cla through the scoped app token created below, and nothing
+# here writes to this repository's contents.
 permissions:
-  contents: write
-  pull-requests: write
-  issues: write
-  checks: write
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
@@ -36,14 +43,26 @@ jobs:
   cancel-closed-pr-runs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
+    # Echoes one line; the run exists only so the concurrency group cancels the
+    # in-flight run of a closed PR.
+    permissions: {}
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
 
   cla:
     if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }}
+    # checks: write reports the merge-queue check run; pull-requests and issues
+    # let cla-bot comment and label. contents stays read — see the note above.
+    permissions:
+      contents: read
+      checks: write
+      issues: write
+      pull-requests: write
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Report CLA result for merge queue
         if: github.event_name == 'merge_group'

.github/workflows/coverage.yml

@@ -62,14 +62,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
           cache-shared-key: ci-coverage
-          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
 
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
@@ -116,6 +118,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/docker.yml

@@ -85,6 +85,7 @@ jobs:
        github.event.workflow_run.head_branch != 'main' &&
        !contains(github.event.workflow_run.head_branch, '-preview'))
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     outputs:
       should_build: ${{ steps.check.outputs.should_build }}
       should_push: ${{ steps.check.outputs.should_push }}
@@ -97,11 +98,18 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           # For workflow_run events, checkout the specific commit that triggered the workflow
           ref: ${{ github.event.workflow_run.head_sha || github.sha }}
 
       - name: Check build conditions
         id: check
+        env:
+          # dispatch inputs via env, not `${{ }}` interpolation: they are
+          # free-form strings and would otherwise be evaluated by bash.
+          INPUT_VERSION: ${{ github.event.inputs.version }}
+          INPUT_PUSH_IMAGES: ${{ github.event.inputs.push_images }}
+          INPUT_FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
         run: |
           should_build=false
           should_push=false
@@ -202,9 +210,9 @@ jobs:
 
           elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             # Manual trigger
-            input_version="${{ github.event.inputs.version }}"
+            input_version="$INPUT_VERSION"
             version="${input_version}"
-            should_push="${{ github.event.inputs.push_images }}"
+            should_push="$INPUT_PUSH_IMAGES"
             should_build=true
 
             # Get short SHA
@@ -212,7 +220,7 @@ jobs:
 
             echo "🎯 Manual Docker build triggered:"
             echo "   📋 Requested version: $input_version"
-            echo "   🔧 Force rebuild: ${{ github.event.inputs.force_rebuild }}"
+            echo "   🔧 Force rebuild: $INPUT_FORCE_REBUILD"
             echo "   🚀 Push images: $should_push"
 
             case "$input_version" in
@@ -298,6 +306,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Login to Docker Hub
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
@@ -333,32 +343,28 @@ jobs:
           CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
           VARIANT_SUFFIX="${{ matrix.suffix }}"
 
-          # Convert version format for Dockerfile compatibility
+          # Convert version format for Dockerfile compatibility. The former
+          # DOCKER_CHANNEL was "release" down every branch and was passed as a
+          # build-arg no Dockerfile declares, so it is gone.
           case "$VERSION" in
             "latest")
-              # For stable latest, use RELEASE=latest + release CHANNEL
               DOCKER_RELEASE="latest"
-              DOCKER_CHANNEL="release"
               ;;
             v*)
               # For versioned releases (v1.0.0), remove 'v' prefix for Dockerfile
               DOCKER_RELEASE="${VERSION#v}"
-              DOCKER_CHANNEL="release"
               ;;
             *)
               # For other versions, pass as-is
               DOCKER_RELEASE="${VERSION}"
-              DOCKER_CHANNEL="release"
               ;;
           esac
 
           echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT"
-          echo "docker_channel=$DOCKER_CHANNEL" >> "$GITHUB_OUTPUT"
 
           echo "🐳 Docker build parameters:"
           echo "  - Original version: $VERSION"
           echo "  - Docker RELEASE: $DOCKER_RELEASE"
-          echo "  - Docker CHANNEL: $DOCKER_CHANNEL"
 
           # Generate tags based on build type
           # Only support release and prerelease builds (no development builds)
@@ -412,18 +418,24 @@ jobs:
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: |
-            type=gha,scope=docker-${{ matrix.variant }}
-          cache-to: |
-            type=gha,mode=max,scope=docker-${{ matrix.variant }}
+          # No layer cache. This build compiles nothing — it downloads a
+          # release zip and runs apk/apt — so the cache could only save the
+          # minute or two those take, while creating a correctness problem: with
+          # RELEASE=latest the binary URL is resolved by curl *inside* a RUN
+          # layer, and the layer key does not include what that resolved to. A
+          # rebuild at the same RELEASE value (dispatch with version=latest, or
+          # a re-run of the same version) would hit the old layer and ship the
+          # previous release's binary. mode=max also consumed the same 10GB
+          # Actions cache quota the Rust lanes are fighting over.
+          #
+          # Only RELEASE is passed: it is the sole build-arg the Dockerfiles
+          # declare besides TARGETARCH. BUILDTIME, VERSION, BUILD_TYPE, REVISION
+          # and CHANNEL were never read by any stage (and BUILDTIME's $(date ...)
+          # was a literal here, not a shell substitution). BUILD_DATE and VCS_REF
+          # are declared by the Dockerfiles but deliberately left unset —
+          # supplying them would change the published image labels.
           build-args: |
-            BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-            VERSION=${{ needs.build-check.outputs.version }}
-            BUILD_TYPE=${{ needs.build-check.outputs.build_type }}
-            REVISION=${{ github.sha }}
             RELEASE=${{ steps.meta.outputs.docker_release }}
-            CHANNEL=${{ steps.meta.outputs.docker_channel }}
-            BUILDKIT_INLINE_CACHE=1
           provenance: true
           sbom: true
           # Add retry mechanism by splitting the build process
@@ -439,6 +451,7 @@ jobs:
     needs: [ build-check, build-docker ]
     if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     permissions:
       contents: read
       security-events: write
@@ -493,6 +506,7 @@ jobs:
     needs: [ build-check, build-docker ]
     if: always() && needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Docker build completion summary
         run: |

.github/workflows/e2e-replication-nightly.yml

@@ -64,14 +64,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
           cache-shared-key: ci-e2e-repl
-          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
 
       # awscurl lets the STS dual-node test actually exercise its path. Without
       # it the test skips gracefully with a visible log line
@@ -124,6 +126,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/e2e-s3tests.yml

@@ -45,6 +45,13 @@
 # The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
 # via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: e2e-s3tests
 
 on:
@@ -135,6 +142,8 @@ jobs:
       TEST_MODE: ${{ matrix.test-mode }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       # Provision Python explicitly rather than trusting the runner image to
       # ship a working pip (ci-1: a bare python3 without pip is what broke the
@@ -354,6 +363,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/fuzz.yml

@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Fuzz
 
 on:
@@ -59,6 +66,7 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -79,13 +87,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: nightly
           cache-shared-key: fuzz-${{ hashFiles('fuzz/Cargo.lock') }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}
 
       - name: Install cargo-fuzz
@@ -145,6 +154,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -200,6 +211,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -247,6 +260,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/helm-package.yml

@@ -32,6 +32,7 @@ permissions:
 jobs:
   build-helm-package:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (
@@ -49,16 +50,26 @@ jobs:
     steps:
       - name: Checkout helm chart repo
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
+      # Both inputs reach the shell through env rather than `${{ }}`
+      # interpolation. A git ref name may contain `$(...)` — anything without a
+      # space is a legal tag — and interpolation pastes it into the script
+      # verbatim, where bash would run it. Reading "$RAW_INPUT" instead makes it
+      # data.
       - name: Normalize release version
         id: version
+        env:
+          RAW_INPUT: ${{ github.event.inputs.version }}
+          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
         run: |
           set -eux
 
           if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            RAW="${{ github.event.inputs.version }}"
+            RAW="$RAW_INPUT"
           else
-            RAW="${{ github.event.workflow_run.head_branch }}"
+            RAW="$RAW_BRANCH"
           fi
 
           case "$RAW" in
@@ -73,10 +84,13 @@ jobs:
           ./scripts/helm_chart_version.sh "$RAW_TAG"
 
       - name: Replace chart version and app version
+        env:
+          CHART_VERSION: ${{ steps.version.outputs.chart_version }}
+          APP_VERSION: ${{ steps.version.outputs.app_version }}
         run: |
           set -eux
-          sed -i -E 's/^version:.*/version: "${{ steps.version.outputs.chart_version }}"/' helm/rustfs/Chart.yaml
-          sed -i -E 's/^appVersion:.*/appVersion: "${{ steps.version.outputs.app_version }}"/' helm/rustfs/Chart.yaml
+          sed -i -E "s/^version:.*/version: \"${CHART_VERSION}\"/" helm/rustfs/Chart.yaml
+          sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" helm/rustfs/Chart.yaml
 
       - name: Set up Helm
         uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
@@ -101,6 +115,7 @@ jobs:
 
   publish-helm-package:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     needs: [ build-helm-package ]
     if: needs.build-helm-package.result == 'success'
 
@@ -108,6 +123,8 @@ jobs:
       - name: Checkout helm package repo
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          # persist-credentials-exempt: this checkout's token IS the push credential —
+          # the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
           repository: rustfs/helm
           token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
 
@@ -123,11 +140,19 @@ jobs:
       - name: Generate index
         run: helm repo index . --url https://charts.rustfs.com
 
+      # app_version is derived from the triggering tag name, and this job holds
+      # the cross-repository push token with rustfs/helm already checked out —
+      # the worst place in the repo to paste an attacker-influenced string into
+      # a shell line. Passed through env so bash treats it as data.
       - name: Push helm package and index file
+        env:
+          GIT_USERNAME: ${{ secrets.USERNAME }}
+          GIT_EMAIL: ${{ secrets.EMAIL_ADDRESS }}
+          APP_VERSION: ${{ needs.build-helm-package.outputs.app_version }}
         run: |
           set -eux
-          git config --global user.name "${{ secrets.USERNAME }}"
-          git config --global user.email "${{ secrets.EMAIL_ADDRESS }}"
+          git config --global user.name "${GIT_USERNAME}"
+          git config --global user.email "${GIT_EMAIL}"
           git add .
-          git commit -m "Update rustfs helm package with ${{ needs.build-helm-package.outputs.app_version }}." || echo "No changes to commit"
+          git commit -m "Update rustfs helm package with ${APP_VERSION}." || echo "No changes to commit"
           git push origin main

.github/workflows/issue-translator.yml

@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: "issue-translator"
 on:
   issue_comment:
@@ -26,6 +33,7 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - uses: usthe/issues-translate-action@b41f55ddc81d7d54bd542a4f289fe28ec081898e # v2.7
         with:

.github/workflows/minio-interop.yml

@@ -23,6 +23,13 @@
 # Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
 # unlike the self-hosted fleet, whose pods drift in Docker/pip availability
 # (see the infra note in e2e-s3tests.yml). Nightly + manual only.
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: minio-interop
 
 on:
@@ -51,13 +58,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
           cache-shared-key: ci-minio-interop
-          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Generate real MinIO fixtures via Docker

.github/workflows/mint.yml

@@ -45,6 +45,13 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: mint
 
 on:
@@ -118,6 +125,8 @@ jobs:
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Enable buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -263,6 +272,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/nix-flake-update.yml

@@ -19,9 +19,12 @@ on:
   schedule:
     - cron: '0 5 * * 0' # Weekly on Sunday 05:00 UTC (staggered after the midnight ci/build crons)
 
+# GITHUB_TOKEN only needs to read the repository here: the branch push and the
+# pull request are both created by update-flake-lock using the
+# FLAKE_UPDATE_TOKEN PAT below, not by this token. Leaving write on it hands a
+# repo-write credential to an unattended weekly job that does not use it.
 permissions:
-  contents: write
-  pull-requests: write
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -37,6 +40,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        # persist-credentials-exempt: update-flake-lock pushes the branch and opens
+        # the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
+        # than reusing .git/config, but that is unverified — exempt until a
+        # workflow_dispatch run confirms it (rustfs/backlog#1602).
 
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3

.github/workflows/nix.yml

@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Nix CI
 
 on:
@@ -46,6 +53,7 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -63,6 +71,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0

.github/workflows/performance-ab.yml

@@ -22,6 +22,13 @@
 # a deliberate correctness cost (e.g. the #4221 fsync durability fix) is
 # recorded but does not block (rustfs/backlog#935 correction 1).
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Performance A/B
 
 on:
@@ -92,6 +99,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -99,7 +108,6 @@ jobs:
           rust-version: stable
           cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build release rustfs
         run: cargo build --release --bin rustfs
@@ -142,6 +150,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
+          persist-credentials: false
           fetch-depth: 0 # baseline is built from origin/main
 
       - name: Setup Rust environment
@@ -150,7 +159,6 @@ jobs:
           rust-version: stable
           cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install warp
         run: |
@@ -162,13 +170,15 @@ jobs:
 
       - name: Decide exemption
         id: exempt
+        env:
+          INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
         run: |
           allow="false"
           if [[ "${{ github.event_name }}" == "pull_request" ]] \
             && ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then
             allow="true"
           fi
-          if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then
+          if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
             allow="true"
           fi
           echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
@@ -224,6 +234,8 @@ jobs:
 
       - name: Run warp A/B and gate
         id: ab
+        env:
+          INPUT_DURATION: ${{ github.event.inputs.duration }}
         run: |
           set -euo pipefail
           # Budget note: with perf-3's cached baseline the nightly does no source
@@ -234,7 +246,7 @@ jobs:
           # budget, which the rig's previous 60s health poll undershot (the first
           # two nightly failures). perf-6 recalibrates these once the noise study
           # lands.
-          duration="${{ github.event.inputs.duration || '12s' }}"
+          duration="${INPUT_DURATION:-12s}"
           baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
           candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
           baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
@@ -385,6 +397,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/schedule-failure-alert-drill.yml

@@ -24,6 +24,13 @@
 # The run itself is expected to end red (the forced failure); only the
 # alert-on-failure job result matters.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Schedule Failure Alert Drill
 
 on:
@@ -56,6 +63,8 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/stale.yml

@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: "Mark stale issues"
 on:
   schedule:
@@ -20,6 +27,7 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
         with:

.github/workflows/star-history.yml

@@ -15,6 +15,7 @@ concurrency:
 jobs:
   update:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     steps:
       - uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0
         with:

AGENTS.md

@@ -105,33 +105,78 @@ CI) fails the build if anything is committed under `docs/superpowers/`, even via
 
 ## Verification Before PR
 
-Convert changes into independently verifiable outcomes. Prefer focused tests for behavior changes and run the relevant checks before declaring completion.
-Non-exempt changes must also pass Adversarial Validation (next section) before the checks below count as completion.
+Convert changes into independently verifiable outcomes. This section controls
+agent-run local validation; preparing a commit or PR does not by itself require
+the broadest gate. Inspect only the final task-owned diff, classify it by
+behavioral impact rather than line count or path alone, and run the smallest
+set of checks that provides meaningful coverage. Do not let unrelated
+worktree changes or a generic contributor checklist expand the scope.
+Non-exempt changes must also pass Adversarial Validation (next section) before
+the checks below count as completion.
 
-For code changes, run and pass the following before opening a PR:
+### Validation floor
 
-```bash
-make pre-pr
-```
+- Every change that is not documentation-only must finish with
+  `cargo fmt --all --check` passing. An umbrella gate that runs this exact
+  check satisfies the requirement; do not run it twice. Use `cargo fmt --all`
+  only when formatting needs to be fixed. Run the configured formatter or
+  validator for other changed languages when one exists.
+- Documentation-only or instruction-only means all task-owned changes are
+  prose or documentation assets and cannot affect runtime, builds, CI,
+  dependencies, generated code, or tests. Run `git diff --check` and any
+  relevant documentation guard, but skip Cargo formatting, compilation,
+  Clippy, tests, `make pre-commit`, and `make pre-pr`.
+- Behavior changes require relevant existing or new tests. Prefer the most
+  focused test or affected package. A passing targeted test can also provide
+  sufficient compilation coverage when it builds every changed target and
+  feature involved; do not add a redundant `cargo check` in that case.
+- `cargo check` supplements compilation coverage; it never substitutes for a
+  behavioral test. If a relevant test cannot reasonably be added or run, use
+  the narrowest compilation check and report the reason and remaining risk.
 
-Before committing code changes, prefer focused verification for the touched
-surface and use the faster local gate when a broad smoke check is needed:
+### Validation tiers
 
-```bash
-make pre-commit
-```
+1. **Documentation/instruction-only:** Apply the exemption above. Run a guard
+   such as `make doc-paths-check` only when it is relevant to the edited text.
+2. **Non-behavioral source change:** For comments, formatting, or another
+   demonstrably non-executable change, run the formatting floor. Compilation,
+   Clippy, and tests may be skipped only when the edit cannot affect
+   compilation or runtime behavior; run targeted doctests if executable
+   documentation examples changed.
+3. **Localized or bounded behavior change:** Run the formatting floor and the
+   narrowest relevant tests. Add package-scoped `cargo check` or Clippy only
+   for changed targets, features, APIs, error handling, async behavior, or
+   control flow not already covered. When several crates are affected but the
+   dependency set is identifiable, validate those packages and known
+   dependents instead of the whole workspace. Use `make pre-commit` only when
+   a repository-wide fast gate adds useful confidence beyond those checks.
+4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage
+   cannot bound the impact, including:
+   - dependency, feature, build-script, procedural-macro, code-generation,
+     toolchain, or CI changes that alter compilation or the test matrix;
+   - cross-crate public APIs, shared foundational code, or broad refactors with
+     an unbounded dependent set;
+   - locking, storage durability or formats, erasure coding, replication,
+     RPC/protocol compatibility, IAM/KMS/auth, cryptography, or other
+     security-sensitive behavior;
+   - a targeted check that reveals wider impact, an explicit user request, or
+     a release policy that requires the full gate.
 
-For migration batches, do not run the full `make pre-pr` gate before every
-intermediate commit. Use focused tests and `make pre-commit` during
-development, then reserve `make pre-pr` for the final PR-ready branch.
+Documentation-only and non-behavioral classifications take precedence over
+path-based triggers. A small diff can still be high-risk, while a CI comment,
+manifest comment, or release-note edit does not require full validation.
 
-Before pushing code changes, make sure formatting is clean:
+`make pre-pr` includes `make pre-commit` coverage. Never run both for the same
+unchanged diff, and do not repeat equivalent checks during PR preparation or
+because a local hook already ran them. Rerun only checks whose scope is affected
+by later edits. Full workspace checks do not replace a relevant integration or
+E2E test for changed behavior; run that focused test when required and
+available, or report why it was not run and the remaining risk.
 
-- Run `cargo fmt --all`.
-- Run `cargo fmt --all --check` and ensure no files are modified unexpectedly.
+If `make` is unavailable, run the equivalent checks defined under
+`.config/make/`. At handoff, list the checks actually run, checks intentionally
+skipped, and the reason for the selected tier.
 
-If `make` is unavailable, run the equivalent checks defined under `.config/make/`.
-Documentation-only or instruction-only changes are exempt from the verification commands above (including the `.config/make/` equivalents), though any locally installed git pre-commit hooks may still run on commit unless explicitly skipped.
 After build-based verification completes, clean generated build artifacts before wrapping up to avoid unnecessary disk usage.
 Do not open a PR with code changes when the required checks fail.
 Make a failing check pass by fixing the cause, never by weakening the gate:

Cargo.lock

@@ -9523,6 +9523,7 @@ dependencies = [
  "moka",
  "rand 0.10.2",
  "reqwest",
+ "rustfs-s3-types",
  "rustfs-security-governance",
  "rustfs-utils",
  "rustify",
@@ -10027,6 +10028,7 @@ dependencies = [
  "rustfs-data-usage",
  "rustfs-ecstore",
  "rustfs-filemeta",
+ "rustfs-lock",
  "rustfs-storage-api",
  "rustfs-utils",
  "s3s",

crates/config/src/constants/app.rs

@@ -230,6 +230,19 @@ pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
 /// Default value: false
 pub const DEFAULT_KMS_ENABLE: bool = false;
 
+/// Environment variable enabling per-key KMS authorization on the SSE-KMS data path.
+///
+/// When enabled, an SSE-KMS write additionally requires `kms:GenerateDataKey` and an
+/// SSE-KMS read additionally requires `kms:Decrypt` on the resolved key, evaluated as
+/// the requesting identity. SSE-S3 and SSE-C are unaffected.
+pub const ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY: &str = "RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY";
+
+/// Default per-key KMS authorization mode for the SSE-KMS data path.
+///
+/// Off for now so deployments whose identity policies only grant s3 actions keep
+/// working; the default flips to on in a later release.
+pub const DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY: bool = false;
+
 /// Environment variable for server KMS backend.
 pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
 

crates/e2e_test/src/admin_iam_crud_test.rs

@@ -26,75 +26,16 @@
 //! Later batches tracked on backlog#1154: config get/set, info, pools status,
 //! group lifecycle, import/export IAM.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::{Client, Config};
-use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use serial_test::serial;
 use std::error::Error;
 use tokio::time::{Duration, sleep};
 
 type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-type BoxError = Box<dyn Error + Send + Sync>;
-
-/// Signs and sends an admin HTTP request with the given credential, returning
-/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
-/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
-async fn admin_request(
-    base_url: &str,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(StatusCode, String), BoxError> {
-    let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut builder = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        builder = builder.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
-    let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
-
-    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
-    let mut request = local_http_client().request(reqwest_method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
-    let status = response.status();
-    let text = response.text().await.unwrap_or_default();
-    Ok((status, text))
-}
-
-/// Root-credential admin request that must succeed; returns the response body.
-async fn admin_ok(
-    env: &RustFSTestEnvironment,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-) -> Result<String, BoxError> {
-    let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
-    if !status.is_success() {
-        return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
-    }
-    Ok(text)
-}
 
 fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
     let config = Config::builder()

crates/e2e_test/src/common.rs

@@ -24,7 +24,12 @@
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use http::header::{CONTENT_TYPE, HOST};
 use reqwest::Client as HttpClient;
+use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::path::{Path, PathBuf};
@@ -75,6 +80,58 @@ pub fn local_http_client() -> HttpClient {
         .expect("failed to build local reqwest client")
 }
 
+/// Signs and sends an admin HTTP request with the given credentials.
+pub(crate) async fn admin_request(
+    base_url: &str,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
+    let url = format!("{base_url}{path_and_query}");
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let mut request = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if body.is_some() {
+        request = request.header(CONTENT_TYPE, "application/json");
+    }
+
+    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
+    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
+
+    let mut request = local_http_client().request(method, &url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    let response = request.send().await?;
+    let status = response.status();
+    let body = response.text().await?;
+    Ok((status, body))
+}
+
+/// Sends a root-credential admin request and returns its successful response body.
+pub(crate) async fn admin_ok(
+    env: &RustFSTestEnvironment,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+    let (status, response_body) =
+        admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
+    if !status.is_success() {
+        return Err(format!("{method} {path_and_query} failed: {status} {response_body}").into());
+    }
+    Ok(response_body)
+}
+
 /// Resolve the RustFS binary relative to the workspace.
 pub fn rustfs_binary_path() -> PathBuf {
     rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())

crates/e2e_test/src/copy_object_metadata_test.rs

@@ -117,9 +117,14 @@ mod tests {
             .key("assets/explicit-copy.js")
             .copy_source(format!("{bucket}/{key}"))
             .metadata_directive(MetadataDirective::Copy)
+            .customize()
+            .mutate_request(|request| {
+                request.headers_mut().insert("content-type", "application/octet-stream");
+                request.headers_mut().insert("x-amz-meta-request-only", "ignored");
+            })
             .send()
             .await
-            .expect("explicit COPY directive failed");
+            .expect("explicit COPY directive with request metadata failed");
         let explicit_copy_head = client
             .head_object()
             .bucket(bucket)
@@ -128,6 +133,18 @@ mod tests {
             .await
             .expect("HEAD failed after explicit COPY");
         assert_eq!(explicit_copy_head.cache_control(), Some("max-age=60"));
+        assert_eq!(explicit_copy_head.content_type(), Some("text/javascript; charset=utf-8"));
+        assert_eq!(
+            explicit_copy_head.metadata().and_then(|metadata| metadata.get("mtime")),
+            Some(&"1777992333".to_string())
+        );
+        assert_eq!(
+            explicit_copy_head
+                .metadata()
+                .and_then(|metadata| metadata.get("request-only")),
+            None,
+            "COPY must ignore request metadata"
+        );
         assert_eq!(
             explicit_copy_head.website_redirect_location(),
             None,
@@ -571,20 +588,6 @@ mod tests {
             Some("InvalidArgument")
         );
 
-        let ignored_replacement = client
-            .copy_object()
-            .bucket(bucket)
-            .key(key)
-            .copy_source(format!("{bucket}/{key}"))
-            .content_type("application/ignored")
-            .send()
-            .await
-            .expect_err("Replacement fields without REPLACE should be rejected");
-        assert_eq!(
-            ignored_replacement.as_service_error().and_then(|error| error.code()),
-            Some("InvalidRequest")
-        );
-
         let unchanged = client
             .get_object()
             .bucket(bucket)

crates/e2e_test/src/delete_marker_migration_semantics_test.rs

@@ -56,6 +56,21 @@ mod tests {
         );
     }
 
+    async fn assert_current_list_hides_delete_marker(client: &Client, bucket: &str, key: &str) {
+        let listed = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("list current objects after delete marker");
+
+        assert!(
+            listed.contents().iter().all(|object| object.key() != Some(key)),
+            "ListObjectsV2 must hide an object whose latest version is a delete marker"
+        );
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() {
@@ -94,6 +109,7 @@ mod tests {
         assert_eq!(markers[0].version_id(), Some(delete_marker_version_id));
         assert_eq!(markers[0].is_latest(), Some(true));
         assert_current_get_is_delete_marker_not_found(&client, bucket, key).await;
+        assert_current_list_hides_delete_marker(&client, bucket, key).await;
     }
 
     #[tokio::test]
@@ -118,6 +134,17 @@ mod tests {
             .await
             .expect("put historical version");
         let data_version_id = put.version_id().expect("put should return data version id");
+        let listed_before_delete = client
+            .list_objects_v2()
+            .bucket(bucket)
+            .prefix(key)
+            .send()
+            .await
+            .expect("list current object before creating delete marker");
+        assert!(
+            listed_before_delete.contents().iter().any(|object| object.key() == Some(key)),
+            "ListObjectsV2 must include the current object before it is deleted"
+        );
 
         let delete_marker = client
             .delete_object()
@@ -145,6 +172,7 @@ mod tests {
         assert_eq!(markers[0].version_id(), Some(delete_marker_version_id));
         assert_eq!(markers[0].is_latest(), Some(true));
         assert_current_get_is_delete_marker_not_found(&client, bucket, key).await;
+        assert_current_list_hides_delete_marker(&client, bucket, key).await;
 
         let historical = client
             .get_object()

crates/e2e_test/src/object_lock/object_lock_test.rs

@@ -26,6 +26,7 @@
 
 use super::common::*;
 use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::{ByteStream, DateTimeFormat};
 use aws_sdk_s3::types::{
     CompletedMultipartUpload, CompletedPart, Delete, MetadataDirective, ObjectIdentifier, ObjectLockLegalHoldStatus,
@@ -2120,6 +2121,127 @@ async fn test_multipart_default_retention_fixed_at_create() {
 // Versioning Auto-Enable Tests
 // ============================================================================
 
+#[tokio::test]
+#[serial]
+async fn test_unretained_object_lock_object_delete_and_bucket_cleanup() {
+    init_logging();
+    info!("🧪 Test: Unretained Object Lock object delete and bucket cleanup (Issue #5339)");
+
+    let mut env = ObjectLockTestEnvironment::new()
+        .await
+        .expect("failed to create Object Lock test environment");
+    env.start_rustfs().await.expect("failed to start RustFS");
+
+    let bucket = "test-object-lock-delete-cleanup";
+    let key = "unretained-object";
+
+    env.create_object_lock_bucket(bucket)
+        .await
+        .expect("failed to create Object Lock bucket");
+    let client = env.s3_client();
+
+    let put_response = client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"unretained data"))
+        .send()
+        .await
+        .expect("failed to upload unretained object");
+    let object_version_id = put_response
+        .version_id()
+        .expect("Object Lock buckets must create versioned objects")
+        .to_string();
+
+    let delete_response = client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .send()
+        .await
+        .expect("failed to create delete marker");
+    assert_eq!(delete_response.delete_marker(), Some(true));
+    let delete_marker_version_id = delete_response
+        .version_id()
+        .expect("Deleting without a version ID must create a delete marker")
+        .to_string();
+
+    let get_error = client
+        .get_object()
+        .bucket(bucket)
+        .key(key)
+        .send()
+        .await
+        .expect_err("GET must not return an object hidden by a delete marker");
+    assert_eq!(get_error.raw_response().map(|response| response.status().as_u16()), Some(404));
+    assert_eq!(get_error.as_service_error().and_then(|error| error.code()), Some("NoSuchKey"));
+
+    let listed_objects = client
+        .list_objects_v2()
+        .bucket(bucket)
+        .send()
+        .await
+        .expect("failed to list current objects");
+    assert!(
+        listed_objects.contents().iter().all(|object| object.key() != Some(key)),
+        "ListObjectsV2 must hide objects whose latest version is a delete marker"
+    );
+
+    let listed_versions = client
+        .list_object_versions()
+        .bucket(bucket)
+        .send()
+        .await
+        .expect("failed to list object versions");
+    assert!(
+        listed_versions
+            .versions()
+            .iter()
+            .any(|version| version.key() == Some(key) && version.version_id() == Some(object_version_id.as_str())),
+        "The data version must remain until it is explicitly deleted"
+    );
+    assert!(
+        listed_versions
+            .delete_markers()
+            .iter()
+            .any(|marker| marker.key() == Some(key) && marker.version_id() == Some(delete_marker_version_id.as_str())),
+        "ListObjectVersions must expose the delete marker"
+    );
+
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(object_version_id)
+        .send()
+        .await
+        .expect("failed to delete the data version");
+    client
+        .delete_object()
+        .bucket(bucket)
+        .key(key)
+        .version_id(delete_marker_version_id)
+        .send()
+        .await
+        .expect("failed to delete the delete marker");
+
+    let remaining_versions = client
+        .list_object_versions()
+        .bucket(bucket)
+        .send()
+        .await
+        .expect("failed to list versions after cleanup");
+    assert!(remaining_versions.versions().is_empty());
+    assert!(remaining_versions.delete_markers().is_empty());
+
+    client
+        .delete_bucket()
+        .bucket(bucket)
+        .send()
+        .await
+        .expect("Deleting every version must remove xl.meta so the bucket can be deleted normally");
+}
+
 #[tokio::test]
 #[serial]
 async fn test_versioning_auto_enabled_with_object_lock() {

crates/e2e_test/src/reliant/tiering.rs

@@ -382,7 +382,6 @@ struct ManualTransitionRunReport {
     skipped_delete_marker: u64,
     skipped_directory: u64,
     skipped_replication: u64,
-    skipped_already_transitioned: u64,
     skipped_already_in_flight: u64,
     skipped_queue_full: u64,
     skipped_queue_closed: u64,
@@ -408,41 +407,6 @@ fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionR
     }
 }
 
-fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) {
-    assert_completed_or_in_flight_partial(state, report, context);
-    if report.skipped_already_in_flight > 0 {
-        assert!(
-            report.scanned <= expected_objects,
-            "{context}: scanned more objects than the conflict scope contains: {report:#?}"
-        );
-        assert!(
-            report.eligible <= expected_objects,
-            "{context}: marked more objects eligible than the conflict scope contains: {report:#?}"
-        );
-        assert_eq!(
-            report.enqueued + report.skipped_already_in_flight,
-            report.eligible,
-            "{context}: partial in-flight accounting must cover every eligible object: {report:#?}"
-        );
-    } else {
-        assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}");
-        assert_eq!(
-            report.eligible + report.skipped_already_transitioned,
-            expected_objects,
-            "{context}: {report:#?}"
-        );
-        assert_eq!(
-            report.enqueued + report.skipped_already_in_flight,
-            expected_objects,
-            "{context}: {report:#?}"
-        );
-    }
-    assert_eq!(
-        report.transition_completed, report.enqueued,
-        "{context}: winner must wait for all queued transitions: {report:#?}"
-    );
-}
-
 #[derive(Debug, Deserialize)]
 struct ManualTransitionQueueSnapshot {
     queue_capacity: u64,
@@ -1276,8 +1240,15 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
-        .await?;
+    hot.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_SCANNER_ENABLED", "false"),
+            ("RUSTFS_SCANNER_CYCLE", "3600"),
+            (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
+        ],
+    )
+    .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1347,20 +1318,21 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     assert_eq!(conflict.cancel_endpoint, status_endpoint);
     assert!(!conflict.scope_key.is_empty());
 
+    manual_transition_job_cancel(&hot, cancel_endpoint).await?;
+
     let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
     assert_eq!(terminal.job_id, job_id);
+    assert_eq!(terminal.status, "cancelled", "terminal conflict winner response: {terminal:#?}");
     assert!(!terminal.report.dry_run);
     assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
     assert_eq!(terminal.report.prefix, accepted.report.prefix);
-    assert_conflict_winner_report(
-        &terminal.status,
-        &terminal.report,
-        MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
-        "terminal conflict winner response",
+    assert!(terminal.report.cancelled, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(terminal.report.scanned, 0, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(terminal.report.enqueued, 0, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(
+        terminal.report.transition_completed, 0,
+        "terminal conflict winner response: {terminal:#?}"
     );
-    assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}");
     let after_remote_count = cold_tier_object_count(&cold_client).await?;
     assert!(after_remote_count >= before_remote_count);
     assert!(after_remote_count <= before_remote_count + MANUAL_ASYNC_CONFLICT_OBJECTS);

crates/e2e_test/src/sts_query_compat_test.rs

@@ -12,22 +12,35 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
 use aws_sdk_sts::config::retry::RetryConfig;
 use aws_sdk_sts::config::{Credentials, Region};
 use aws_sdk_sts::error::ProvideErrorMetadata;
 use aws_sdk_sts::operation::RequestId;
 use aws_sdk_sts::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use http::header::{CONTENT_TYPE, HOST};
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
+use bytes::Bytes;
+use http::header::{AUTHORIZATION, CONTENT_TYPE};
+use http::{Request, Response};
+use http_body_util::{BodyExt, Full};
+use hyper::body::Incoming;
+use hyper::server::conn::http1;
+use hyper::service::service_fn;
+use hyper_util::rt::TokioIo;
+use serde_json::Value;
 use serial_test::serial;
+use std::collections::BTreeSet;
+use std::convert::Infallible;
 use std::error::Error;
+use std::sync::Arc;
+use tokio::net::TcpListener;
+use tokio::sync::{Notify, mpsc};
+use tokio::task::{JoinHandle, JoinSet};
+use tokio::time::{Duration, timeout};
 
 type BoxError = Box<dyn Error + Send + Sync>;
 type TestResult = Result<(), BoxError>;
+const OPA_AUTH_TOKEN: &str = "sts-opa-token";
 
 fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
     let mut config = Config::builder()
@@ -49,32 +62,14 @@ fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Opti
 }
 
 async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
-    let path = "/rustfs/admin/v3/add-service-accounts";
-    let url = format!("{}{path}", env.url);
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
-    let request = http::Request::builder()
-        .method(http::Method::PUT)
-        .uri(uri)
-        .header(HOST, authority)
-        .header(CONTENT_TYPE, "application/json")
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
-        .body(Body::empty())?;
-    let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
-    let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
-    let mut request = local_http_client().put(&url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    let response = request.body(body).send().await?;
-    let status = response.status();
-    let body = response.text().await?;
-    if !status.is_success() {
-        return Err(format!("create service account failed: {status} {body}").into());
-    }
-
-    let response: serde_json::Value = serde_json::from_str(&body)?;
+    let body = admin_ok(
+        env,
+        http::Method::PUT,
+        "/rustfs/admin/v3/add-service-accounts",
+        Some(serde_json::json!({ "targetUser": env.access_key.clone() }).to_string()),
+    )
+    .await?;
+    let response: Value = serde_json::from_str(&body)?;
     let access_key = response["credentials"]["accessKey"]
         .as_str()
         .ok_or("service account response should contain credentials.accessKey")?
@@ -86,28 +81,237 @@ async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(Str
     Ok((access_key, secret_key))
 }
 
-async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
+async fn create_user_with_policy(
+    env: &RustFSTestEnvironment,
+    user: &str,
+    secret: &str,
+    policy_name: &str,
+    statements: Value,
+) -> TestResult {
+    create_user(env, user, secret).await?;
+    admin_ok(
+        env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
+        Some(
+            serde_json::json!({
+                "Version": "2012-10-17",
+                "Statement": statements,
+            })
+            .to_string(),
+        ),
+    )
+    .await?;
+    admin_ok(
+        env,
+        http::Method::POST,
+        "/rustfs/admin/v3/idp/builtin/policy/attach",
+        Some(serde_json::json!({ "policies": [policy_name], "user": user }).to_string()),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn create_user(env: &RustFSTestEnvironment, user: &str, secret: &str) -> TestResult {
+    admin_ok(
+        env,
+        http::Method::PUT,
+        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
+        Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
+    )
+    .await?;
+    Ok(())
+}
+
+async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
     let error = client
         .assume_role()
         .role_arn("arn:aws:iam::123456789012:role/test")
         .role_session_name("sts-query-compat-e2e")
         .send()
         .await
-        .expect_err("credential chaining must be denied");
+        .expect_err("AssumeRole must be denied");
     let service_error = error
         .as_service_error()
-        .ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
+        .ok_or_else(|| format!("{context} should deserialize as an STS service error: {error:?}"))?;
 
     assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
     assert_eq!(service_error.code(), Some("AccessDenied"));
     assert_eq!(service_error.message(), Some("Access Denied"));
     assert!(
         error.request_id().is_some_and(|request_id| !request_id.is_empty()),
-        "{credential_kind} denial should include a request ID"
+        "{context} should include a request ID"
     );
     Ok(())
 }
 
+async fn handle_opa_request(
+    request: Request<Incoming>,
+    requests: mpsc::UnboundedSender<Value>,
+    validation_started: mpsc::UnboundedSender<()>,
+    validation_mode: OpaValidationMode,
+    expected_authorization: Option<String>,
+) -> Result<Response<Full<Bytes>>, Infallible> {
+    if let Some(expected_authorization) = expected_authorization
+        && request.headers().get(AUTHORIZATION).and_then(|value| value.to_str().ok()) != Some(expected_authorization.as_str())
+    {
+        return Ok(Response::builder()
+            .status(401)
+            .body(Full::new(Bytes::new()))
+            .expect("static OPA unauthorized response must be valid"));
+    }
+
+    let body = match request.into_body().collect().await {
+        Ok(body) => body.to_bytes(),
+        Err(error) => {
+            return Ok(Response::builder()
+                .status(400)
+                .body(Full::new(Bytes::from(error.to_string())))
+                .expect("static OPA error response must be valid"));
+        }
+    };
+
+    let payload = if body.is_empty() {
+        None
+    } else {
+        match serde_json::from_slice::<Value>(&body) {
+            Ok(payload) => Some(payload),
+            Err(error) => {
+                return Ok(Response::builder()
+                    .status(400)
+                    .body(Full::new(Bytes::from(error.to_string())))
+                    .expect("static OPA error response must be valid"));
+            }
+        }
+    };
+    if payload.is_none() {
+        let _ = validation_started.send(());
+        if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
+            release.notified().await;
+            return Ok(Response::builder()
+                .status(503)
+                .body(Full::new(Bytes::new()))
+                .expect("static OPA unavailable response must be valid"));
+        }
+    }
+    let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
+        Some(Value::String(account)) if account == "opaallow" => payload
+            .as_ref()
+            .and_then(|value| value.pointer("/input/context/deny_only"))
+            .and_then(Value::as_bool)
+            .unwrap_or(false),
+        Some(Value::String(account)) if account == "opadeny" => false,
+        None => true,
+        _ => false,
+    };
+    if let Some(payload) = payload {
+        let _ = requests.send(payload);
+    }
+    let body =
+        serde_json::to_vec(&serde_json::json!({ "result": { "allow": allow } })).expect("static OPA response must serialize");
+    Ok(Response::builder()
+        .header(CONTENT_TYPE, "application/json")
+        .body(Full::new(Bytes::from(body)))
+        .expect("static OPA response must be valid"))
+}
+
+#[derive(Clone)]
+enum OpaValidationMode {
+    Ready,
+    DelayedUnavailable(Arc<Notify>),
+}
+
+struct OpaMock {
+    url: String,
+    requests: mpsc::UnboundedReceiver<Value>,
+    validation_started: mpsc::UnboundedReceiver<()>,
+    validation_release: Option<Arc<Notify>>,
+    task: JoinHandle<()>,
+}
+
+impl OpaMock {
+    async fn start() -> Result<Self, BoxError> {
+        Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
+    }
+
+    async fn start_delayed_unavailable() -> Result<Self, BoxError> {
+        let release = Arc::new(Notify::new());
+        Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
+    }
+
+    async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
+        let listener = TcpListener::bind("127.0.0.1:0").await?;
+        let url = format!("http://{}/v1/data/rustfs/authz/allow", listener.local_addr()?);
+        let (requests_tx, requests) = mpsc::unbounded_channel();
+        let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
+        let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
+        let validation_release = match &validation_mode {
+            OpaValidationMode::Ready => None,
+            OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
+        };
+        let task = tokio::spawn(async move {
+            let mut connections = JoinSet::new();
+            loop {
+                tokio::select! {
+                    accepted = listener.accept() => {
+                        let Ok((stream, _)) = accepted else { break };
+                        let requests = requests_tx.clone();
+                        let validation_started = validation_started_tx.clone();
+                        let validation_mode = validation_mode.clone();
+                        let expected_authorization = expected_authorization.clone();
+                        connections.spawn(async move {
+                            let handler = service_fn(move |request| {
+                                handle_opa_request(
+                                    request,
+                                    requests.clone(),
+                                    validation_started.clone(),
+                                    validation_mode.clone(),
+                                    expected_authorization.clone(),
+                                )
+                            });
+                            let _ = http1::Builder::new()
+                                .serve_connection(TokioIo::new(stream), handler)
+                                .await;
+                        });
+                    }
+                    _ = connections.join_next(), if !connections.is_empty() => {}
+                }
+            }
+        });
+        Ok(Self {
+            url,
+            requests,
+            validation_started,
+            validation_release,
+            task,
+        })
+    }
+
+    async fn next_request(&mut self) -> Result<Value, BoxError> {
+        timeout(Duration::from_secs(5), self.requests.recv())
+            .await?
+            .ok_or_else(|| "OPA request channel closed".into())
+    }
+
+    async fn wait_for_validation(&mut self) -> TestResult {
+        timeout(Duration::from_secs(5), self.validation_started.recv())
+            .await?
+            .ok_or_else(|| "OPA validation channel closed".into())
+    }
+
+    fn release_validation(&self) {
+        if let Some(release) = &self.validation_release {
+            release.notify_one();
+        }
+    }
+}
+
+impl Drop for OpaMock {
+    fn drop(&mut self) {
+        self.task.abort();
+    }
+}
+
 #[tokio::test]
 #[serial]
 async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
@@ -155,7 +359,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
         "signature rejection should include a request ID"
     );
 
-    assert_chaining_denied(
+    assert_access_denied(
         &sts_client(
             &env.url,
             temporary.access_key_id(),
@@ -167,7 +371,187 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
     .await?;
 
     let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
-    assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
+    assert_access_denied(
+        &sts_client(&env.url, &service_access_key, &service_secret_key, None),
+        "service-account denial",
+    )
+    .await?;
+
+    let implicit_user = "stsimplicit";
+    let explicit_allow_user = "stsallow";
+    let explicit_deny_user = "stsdeny";
+    let policyless_user = "stspolicyless";
+    let secret = "stsAuthzSecret123";
+    create_user(&env, policyless_user, secret).await?;
+    assert_access_denied(&sts_client(&env.url, policyless_user, secret, None), "policyless user").await?;
+    create_user_with_policy(
+        &env,
+        implicit_user,
+        secret,
+        "sts-implicit-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+    create_user_with_policy(
+        &env,
+        explicit_allow_user,
+        secret,
+        "sts-allow-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["sts:AssumeRole"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+    create_user_with_policy(
+        &env,
+        explicit_deny_user,
+        secret,
+        "sts-deny-policy",
+        serde_json::json!([
+            {
+                "Effect": "Allow",
+                "Action": ["sts:AssumeRole"],
+                "Resource": ["arn:aws:s3:::*"],
+            },
+            {
+                "Effect": "Deny",
+                "Action": ["sts:AssumeRole"],
+                "Resource": ["arn:aws:s3:::*"],
+            }
+        ]),
+    )
+    .await?;
+
+    for user in [implicit_user, explicit_allow_user] {
+        let output = sts_client(&env.url, user, secret, None)
+            .assume_role()
+            .role_arn("arn:aws:iam::123456789012:role/test")
+            .role_session_name("sts-authz-e2e")
+            .send()
+            .await
+            .map_err(|error| format!("{user} should be allowed to call AssumeRole: {error:?}"))?;
+        let credentials = output
+            .credentials()
+            .ok_or_else(|| format!("{user} AssumeRole response should contain credentials"))?;
+        assert!(!credentials.access_key_id().is_empty());
+        assert!(!credentials.secret_access_key().is_empty());
+        assert!(!credentials.session_token().is_empty());
+    }
+    assert_access_denied(&sts_client(&env.url, explicit_deny_user, secret, None), "explicit sts:AssumeRole Deny").await?;
+
+    env.stop_server();
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_assume_role_opa_contract() -> TestResult {
+    init_logging();
+
+    let mut opa = OpaMock::start().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(
+        vec![],
+        &[
+            ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
+            ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
+        ],
+    )
+    .await?;
+
+    let secret = "stsOpaSecret123";
+    create_user_with_policy(
+        &env,
+        "opaallow",
+        secret,
+        "sts-opa-local-deny-policy",
+        serde_json::json!([{
+            "Effect": "Deny",
+            "Action": ["sts:AssumeRole"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+    create_user_with_policy(
+        &env,
+        "opadeny",
+        secret,
+        "sts-opa-local-allow-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["sts:AssumeRole"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+
+    sts_client(&env.url, "opaallow", secret, None)
+        .assume_role()
+        .role_arn("arn:aws:iam::123456789012:role/test")
+        .role_session_name("sts-opa-contract")
+        .send()
+        .await
+        .map_err(|error| format!("OPA allow should override the local explicit Deny: {error:?}"))?;
+    assert_access_denied(
+        &sts_client(&env.url, "opadeny", secret, None),
+        "OPA denial despite local sts:AssumeRole Allow",
+    )
+    .await?;
+
+    let mut accounts = BTreeSet::new();
+    for _ in 0..2 {
+        let request = opa.next_request().await?;
+        assert_eq!(request.pointer("/input/action").and_then(Value::as_str), Some("sts:AssumeRole"));
+        assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(true));
+        let account = request
+            .pointer("/input/identity/account")
+            .and_then(Value::as_str)
+            .ok_or("OPA input should include identity.account")?;
+        accounts.insert(account.to_owned());
+    }
+    assert_eq!(accounts, BTreeSet::from(["opaallow".to_owned(), "opadeny".to_owned()]));
+
+    env.stop_server();
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
+    init_logging();
+
+    let mut opa = OpaMock::start_delayed_unavailable().await?;
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
+        .await?;
+    opa.wait_for_validation().await?;
+
+    let user = "opaunavailable";
+    let secret = "stsOpaUnavailableSecret123";
+    create_user_with_policy(
+        &env,
+        user,
+        secret,
+        "sts-opa-unavailable-local-policy",
+        serde_json::json!([{
+            "Effect": "Allow",
+            "Action": ["s3:ListAllMyBuckets"],
+            "Resource": ["arn:aws:s3:::*"],
+        }]),
+    )
+    .await?;
+
+    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
+
+    opa.release_validation();
+    tokio::time::sleep(Duration::from_millis(200)).await;
+    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
 
     env.stop_server();
     Ok(())

crates/ecstore/src/api/mod.rs

@@ -267,12 +267,13 @@ pub mod config {
     pub mod com {
         pub use crate::config::com::{
             COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
-            ServerConfigCorruptError, ServerConfigSnapshot, delete_config, is_server_config_corrupt_error, lookup_configs,
-            read_config, read_config_no_lock, read_config_with_metadata, read_config_without_migrate,
-            read_config_without_migrate_no_lock, read_existing_server_config_no_lock, read_server_config_snapshot, save_config,
-            save_config_no_lock, save_config_with_opts, save_server_config, save_server_config_no_lock,
-            save_server_config_snapshot, server_config_path, try_migrate_server_config, with_config_object_read_lock,
-            with_config_object_write_lock, with_server_config_read_lock, with_server_config_write_lock,
+            ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config,
+            is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
+            read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
+            read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
+            save_server_config_no_lock, save_server_config_snapshot, save_server_config_snapshot_with_generation,
+            server_config_path, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
+            with_server_config_read_lock, with_server_config_write_lock,
         };
     }
 
@@ -418,15 +419,15 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
-        PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
-        ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
-        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
-        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
-        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
-        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
-        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
-        verify_tonic_rpc_signature_with_bootstrap,
+        AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
+        PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
+        ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
+        gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
+        node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
+        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
+        tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
+        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
+        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/cluster/rpc/mod.rs

@@ -44,7 +44,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub(crate) use peer_rest_client::TierConfigReloadOutcome;
 pub use peer_rest_client::{
-    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
     SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -45,9 +45,9 @@ use rustfs_protos::proto_gen::node_service::{
     HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
     LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
-    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
-    StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
-    TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
+    StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
+    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -71,6 +71,12 @@ use uuid::Uuid;
 
 pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
 pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
+/// Dynamic config subsystem for the cluster-persisted KMS configuration.
+///
+/// KMS configuration lives in its own cluster object rather than in the server
+/// config document, so it is not a `ServerConfig` subsystem; it only shares the
+/// reload signal transport.
+pub const KMS_SIGNAL_SUBSYSTEM: &str = "kms";
 const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
 const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
 const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
@@ -99,8 +105,13 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
 }
 
 fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
+    // The version stays pinned to DYNAMIC_CONFIG_PROTOCOL_VERSION rather than
+    // being bumped per subsystem: the comparison is shared, so raising it would
+    // retire peers that already converge scanner and heal config correctly.
+    // Subsystems added after a peer was built are rejected by that peer's own
+    // subsystem allow-list, which surfaces as an explicit failed signal.
     if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
-        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
+        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS | KMS_SIGNAL_SUBSYSTEM)
         && protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
     {
         return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
@@ -1500,6 +1511,22 @@ impl PeerRestClient {
     }
 
     pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
+        self.signal_service_checked(sig, sub_sys, dry_run).await.map(|_| ())
+    }
+
+    /// Report the KMS configuration fingerprint the peer is currently running.
+    ///
+    /// Sent as a dry-run reload signal so the peer answers without swapping its
+    /// own configuration. `None` means the peer has no KMS configuration. The
+    /// fingerprint is advisory and feeds cluster status reporting only, so the
+    /// response is not proof-signed.
+    pub async fn kms_config_fingerprint(&self) -> Result<Option<String>> {
+        self.signal_service_checked(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, true)
+            .await
+            .map(|response| response.config_fingerprint)
+    }
+
+    async fn signal_service_checked(&self, sig: u64, sub_sys: &str, dry_run: bool) -> Result<SignalServiceResponse> {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
@@ -1520,7 +1547,7 @@ impl PeerRestClient {
                     return Err(Error::other(""));
                 }
                 validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
-                Ok(())
+                Ok(response)
             }
             .await,
         )
@@ -2347,6 +2374,19 @@ mod tests {
             .expect("full refresh compatibility is guarded by its scanner preflight");
     }
 
+    #[test]
+    fn dynamic_kms_config_requires_versioned_peer_acknowledgement() {
+        let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, 0)
+            .expect_err("an unversioned peer must not claim KMS config convergence");
+        assert!(err.to_string().contains("does not support dynamic"));
+        validate_signal_service_protocol(
+            SERVICE_SIGNAL_RELOAD_DYNAMIC,
+            KMS_SIGNAL_SUBSYSTEM,
+            rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
+        )
+        .expect("a current peer should support dynamic KMS config");
+    }
+
     #[test]
     fn peer_rest_client_marks_network_like_errors() {
         assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));

crates/ecstore/src/config/com.rs

@@ -53,12 +53,14 @@ use std::sync::LazyLock;
 use std::sync::{Arc, RwLock};
 use tokio::sync::{OwnedRwLockWriteGuard, RwLock as AsyncRwLock};
 use tracing::{debug, error, info, instrument, warn};
+use uuid::Uuid;
 
 pub const CONFIG_PREFIX: &str = "config";
 const SERVER_CONFIG_OBJECT: &str = "config/config.json";
+const CONFIG_TRANSACTION_LOCK_SUFFIX: &str = ".transaction.lock";
 
-// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock
-// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
+// Server-config lock order: SERVER_CONFIG_LOCK -> transaction lock ->
+// SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
 static SERVER_CONFIG_LOCK: LazyLock<Arc<AsyncRwLock<()>>> = LazyLock::new(|| Arc::new(AsyncRwLock::new(())));
 
 fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error {
@@ -76,8 +78,11 @@ where
     T: Send + 'static,
 {
     tokio::spawn(async move {
-        // Lock order: SERVER_CONFIG_LOCK -> namespace write lock.
+        // Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock.
         let _local_guard = SERVER_CONFIG_LOCK.write().await;
+        let transaction_lock = server_config_transaction_lock_path();
+        let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
+        let _transaction_guard = transaction_lock.get_write_lock(get_lock_acquire_timeout()).await?;
         let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
         let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
         Ok(operation().await)
@@ -96,8 +101,11 @@ where
     T: Send + 'static,
 {
     tokio::spawn(async move {
-        // Lock order: SERVER_CONFIG_LOCK -> namespace read lock.
+        // Lock order: SERVER_CONFIG_LOCK -> transaction lock -> object lock.
         let _local_guard = SERVER_CONFIG_LOCK.read().await;
+        let transaction_lock = server_config_transaction_lock_path();
+        let transaction_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
+        let _transaction_guard = transaction_lock.get_read_lock(get_lock_acquire_timeout()).await?;
         let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
         let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
         Ok(operation().await)
@@ -567,6 +575,21 @@ where
 }
 
 pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_config_with_opts_and_metadata(api, file, data, opts).await.map(|_| ())
+}
+
+async fn save_config_with_opts_and_metadata<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<ObjectInfo>
 where
     S: ObjectIO<
             Error = Error,
@@ -579,11 +602,13 @@ where
         >,
 {
     let mut put_data = PutObjReader::from_vec(data);
-    if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
-        error!("save_config_with_opts: err: {:?}, file: {}", err, file);
-        return Err(err);
+    match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
+        Ok(object_info) => Ok(object_info),
+        Err(err) => {
+            error!("save_config_with_opts: err: {:?}, file: {}", err, file);
+            Err(err)
+        }
     }
-    Ok(())
 }
 
 fn new_server_config() -> Config {
@@ -594,8 +619,12 @@ async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config>
 where
     S: EcstoreObjectIO + StorageAdminApi + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
 {
+    let snapshot = read_server_config_snapshot(api.clone()).await?;
+    if snapshot.object_exists() {
+        return Ok(snapshot.config.clone());
+    }
     let cfg = new_server_config();
-    save_server_config(api, &cfg).await?;
+    save_server_config_snapshot(api, &cfg, &snapshot).await?;
 
     Ok(cfg)
 }
@@ -617,6 +646,10 @@ pub fn server_config_path() -> String {
     SERVER_CONFIG_OBJECT.to_string()
 }
 
+fn server_config_transaction_lock_path() -> String {
+    format!("{}{CONFIG_TRANSACTION_LOCK_SUFFIX}", server_config_path())
+}
+
 fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS {
     let sub_cfg = cfg.0.entry(STORAGE_CLASS_SUB_SYS.to_string()).or_insert_with(|| {
         let mut section = HashMap::new();
@@ -819,6 +852,9 @@ fn apply_external_scalar_config_map(
     let Some(config_value) = root.get(descriptor.subsystem_key) else {
         return Ok(false);
     };
+    if descriptor.subsystem_key == HEAL_SUB_SYS && config_value.is_null() {
+        return Ok(false);
+    }
     let overrides = decode_scalar_config_value(config_value, descriptor)?;
 
     if overrides.is_empty() {
@@ -1463,21 +1499,142 @@ fn build_audit_object(cfg: &Config) -> Map<String, Value> {
     build_target_object(cfg, &audit_target_descriptors())
 }
 
+fn sync_rendered_target_instance(existing: Value, rendered: Option<&Value>, valid_keys: &[&str]) -> Option<Value> {
+    match existing {
+        Value::Object(mut instance) => {
+            for key in valid_keys {
+                instance.remove(*key);
+            }
+            if let Some(Value::Object(rendered)) = rendered {
+                instance.extend(rendered.clone());
+            }
+            (!instance.is_empty()).then_some(Value::Object(instance))
+        }
+        Value::Array(entries) => {
+            let mut pending = rendered
+                .and_then(Value::as_object)
+                .map(|rendered| {
+                    rendered
+                        .iter()
+                        .filter_map(|(key, value)| parse_target_scalar_value(key, value).map(|value| (key.clone(), value)))
+                        .collect::<HashMap<_, _>>()
+                })
+                .unwrap_or_default();
+            let mut updated = Vec::with_capacity(entries.len().saturating_add(pending.len()));
+            for entry in entries {
+                let Some(entry_obj) = entry.as_object() else {
+                    updated.push(entry);
+                    continue;
+                };
+                let Some(key) = entry_obj.get("key").and_then(Value::as_str) else {
+                    updated.push(entry);
+                    continue;
+                };
+                if !valid_keys.contains(&key) {
+                    updated.push(entry);
+                    continue;
+                }
+                let Some(value) = pending.remove(key) else {
+                    continue;
+                };
+                let mut entry_obj = entry_obj.clone();
+                entry_obj.insert("value".to_string(), Value::String(value));
+                updated.push(Value::Object(entry_obj));
+            }
+            updated.extend(rendered_scalar_config_kvs_entries(&pending));
+            (!updated.is_empty()).then_some(Value::Array(updated))
+        }
+        value if rendered.is_none() => Some(value),
+        _ => rendered.cloned(),
+    }
+}
+
 fn sync_rendered_target_object(
     target_obj: &mut Map<String, Value>,
     rendered_target: &Map<String, Value>,
     descriptors: &[TargetConfigDescriptor],
 ) {
     for descriptor in descriptors {
-        match rendered_target.get(descriptor.external_key) {
-            Some(Value::Object(v)) => {
-                target_obj.insert(descriptor.external_key.to_string(), Value::Object(v.clone()));
-                target_obj.remove(descriptor.subsystem_key);
+        let existing = target_obj.remove(descriptor.external_key);
+        let alias = target_obj.remove(descriptor.subsystem_key);
+        let mut section = existing
+            .or(alias)
+            .and_then(|value| value.as_object().cloned())
+            .unwrap_or_default();
+        let rendered = rendered_target.get(descriptor.external_key).and_then(Value::as_object);
+
+        if is_target_instance_shorthand(&section, descriptor.valid_keys) {
+            let has_named_instances = rendered.is_some_and(|instances| instances.keys().any(|name| name != "default"));
+            if !has_named_instances {
+                if let Some(section) = sync_rendered_target_instance(
+                    Value::Object(section),
+                    rendered.and_then(|instances| instances.get("default")),
+                    descriptor.valid_keys,
+                ) {
+                    target_obj.insert(descriptor.external_key.to_string(), section);
+                }
+                continue;
             }
-            _ => {
-                target_obj.remove(descriptor.external_key);
-                target_obj.remove(descriptor.subsystem_key);
+
+            let mut nested = Map::new();
+            if let Some(default) = sync_rendered_target_instance(
+                Value::Object(section),
+                rendered.and_then(|instances| instances.get("default")),
+                descriptor.valid_keys,
+            ) {
+                nested.insert("default".to_string(), default);
             }
+            if let Some(rendered) = rendered {
+                for (instance_name, instance) in rendered {
+                    if instance_name != "default" {
+                        nested.insert(instance_name.clone(), instance.clone());
+                    }
+                }
+            }
+            if !nested.is_empty() {
+                target_obj.insert(descriptor.external_key.to_string(), Value::Object(nested));
+            }
+            continue;
+        }
+
+        if let Some(default_alias) = section.remove(DEFAULT_DELIMITER) {
+            if let Some(default) = section.get_mut("default") {
+                if let Some(alias) = sync_rendered_target_instance(default_alias, None, descriptor.valid_keys) {
+                    match (default, alias) {
+                        (Value::Object(default), Value::Object(alias)) => {
+                            for (key, value) in alias {
+                                default.entry(key).or_insert(value);
+                            }
+                        }
+                        (Value::Array(default), Value::Array(alias)) => default.extend(alias),
+                        _ => {}
+                    }
+                }
+            } else {
+                section.insert("default".to_string(), default_alias);
+            }
+        }
+
+        let mut merged = Map::new();
+        for (instance_name, instance) in section {
+            if let Some(instance) = sync_rendered_target_instance(
+                instance,
+                rendered.and_then(|instances| instances.get(&instance_name)),
+                descriptor.valid_keys,
+            ) {
+                merged.insert(instance_name, instance);
+            }
+        }
+        if let Some(rendered) = rendered {
+            for (instance_name, instance) in rendered {
+                if !merged.contains_key(instance_name) {
+                    merged.insert(instance_name.clone(), instance.clone());
+                }
+            }
+        }
+
+        if !merged.is_empty() {
+            target_obj.insert(descriptor.external_key.to_string(), Value::Object(merged));
         }
     }
 }
@@ -1496,6 +1653,14 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
         Some(Value::Object(v)) => v,
         _ => Map::new(),
     };
+    for key in [
+        storageclass::CLASS_STANDARD,
+        storageclass::CLASS_RRS,
+        storageclass::OPTIMIZE,
+        storageclass::INLINE_BLOCK,
+    ] {
+        sc_obj.remove(key);
+    }
     for (k, v) in build_storageclass_object(cfg) {
         sc_obj.insert(k, v);
     }
@@ -1503,7 +1668,10 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
     root.remove("storage_class");
 
     for descriptor in [scanner_config_descriptor(), heal_config_descriptor()] {
-        let existing = root.remove(descriptor.subsystem_key);
+        let mut existing = root.remove(descriptor.subsystem_key);
+        if descriptor.subsystem_key == HEAL_SUB_SYS && existing.as_ref().is_some_and(Value::is_null) {
+            existing = None;
+        }
         let rendered = build_scalar_config_object(cfg, descriptor);
         if let Some(config_value) = sync_rendered_scalar_config_value(existing, &rendered, descriptor)? {
             root.insert(descriptor.subsystem_key.to_string(), config_value);
@@ -1560,6 +1728,7 @@ fn is_standard_object_server_config(data: &[u8]) -> bool {
     matches!(root.get("version"), Some(Value::String(v)) if !v.trim().is_empty())
         && matches!(root.get("storageclass"), Some(Value::Object(_)))
         && !root.contains_key("storage_class")
+        && !matches!(root.get(HEAL_SUB_SYS), Some(Value::Null))
 }
 
 fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
@@ -1593,7 +1762,7 @@ where
             FileInfo = FileInfo,
             ObjectToDelete = ObjectToDelete,
             DeletedObject = DeletedObject,
-        >,
+        > + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
 {
     if let Some(decrypt) = &decrypt_fn {
         register_server_config_decrypt_fn(decrypt.clone());
@@ -1601,14 +1770,7 @@ where
 
     let config_file = server_config_path();
     match api
-        .get_object_info(
-            RUSTFS_META_BUCKET,
-            &config_file,
-            &ObjectOptions {
-                no_lock: true,
-                ..Default::default()
-            },
-        )
+        .get_object_info(RUSTFS_META_BUCKET, &config_file, &ObjectOptions::default())
         .await
     {
         Ok(_) => {
@@ -1624,7 +1786,6 @@ where
 
     let opts = ObjectOptions {
         max_parity: true,
-        no_lock: true,
         ..Default::default()
     };
 
@@ -1677,7 +1838,33 @@ where
         }
     };
 
-    match save_config(api, &config_file, normalized).await {
+    let snapshot = match read_server_config_snapshot(api.clone()).await {
+        Ok(snapshot) => snapshot,
+        Err(err) => {
+            warn!("recheck target server config failed, skip migration: {:?}", err);
+            return;
+        }
+    };
+    if snapshot.object_exists() {
+        debug!("server config was created while legacy migration was preparing, skip migration");
+        return;
+    }
+
+    match save_config_with_opts(
+        api,
+        &config_file,
+        normalized,
+        &ObjectOptions {
+            max_parity: true,
+            http_preconditions: Some(HTTPPreconditions {
+                if_none_match: Some("*".to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        },
+    )
+    .await
+    {
         Ok(()) => {
             info!("Migrated compatible server config from legacy metadata bucket");
         }
@@ -1769,8 +1956,13 @@ where
 {
     let config_file = server_config_path();
 
-    // Try to read the configuration file
-    match read_config_no_lock(api.clone(), &config_file).await {
+    // Try to read the configuration file.
+    let data = if namespace_lock_held {
+        read_config_no_lock(api.clone(), &config_file).await
+    } else {
+        read_config(api.clone(), &config_file).await
+    };
+    match data {
         Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
         Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
         Err(err) => handle_config_read_error(err, &config_file),
@@ -1787,7 +1979,12 @@ where
         warn!("Received empty configuration data, try to reread from '{}'", config_file);
 
         // Try to read the configuration again
-        match read_config_no_lock(api.clone(), &config_file).await {
+        let data = if namespace_lock_held {
+            read_config_no_lock(api.clone(), &config_file).await
+        } else {
+            read_config(api.clone(), &config_file).await
+        };
+        match data {
             Ok(cfg_data) => {
                 let cfg = decode_persisted_server_config(&cfg_data)?;
                 return Ok(cfg.merge());
@@ -2036,11 +2233,16 @@ pub struct ServerConfigSnapshot {
     raw: Option<Vec<u8>>,
     seed: Option<Vec<u8>>,
     etag: Option<String>,
+    generation: Option<Uuid>,
     _local_guard: OwnedRwLockWriteGuard<()>,
     _guard: rustfs_lock::NamespaceLockGuard,
 }
 
 impl ServerConfigSnapshot {
+    pub fn object_exists(&self) -> bool {
+        self.raw.is_some()
+    }
+
     pub fn ensure_lock_held(&self) -> Result<()> {
         if self._guard.is_lock_lost() {
             return Err(Error::other("server config transaction lock was lost"));
@@ -2051,12 +2253,34 @@ impl ServerConfigSnapshot {
     pub fn is_lock_lost(&self) -> bool {
         self._guard.is_lock_lost()
     }
+
+    pub fn generation(&self) -> Option<Uuid> {
+        self.generation
+    }
 }
 
-/// Read a server config transaction snapshot while holding the same local and
-/// distributed write locks used by every other server-config writer. Internal
-/// reads and the later conditional write use no-lock object I/O; the guards
-/// remain live until the snapshot is dropped.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ServerConfigSaveResult {
+    persisted: bool,
+    generation: Option<Uuid>,
+}
+
+impl ServerConfigSaveResult {
+    pub fn persisted(&self) -> bool {
+        self.persisted
+    }
+
+    pub fn generation(&self) -> Option<Uuid> {
+        self.generation
+    }
+}
+
+/// Read a server config transaction snapshot while holding a dedicated
+/// transaction lock. The config object's normal namespace lock remains
+/// available to fence reads and the conditional write at commit time.
+/// The transaction guard remains live until the snapshot is dropped,
+/// serializing persistence and history ordering across admin nodes. Runtime
+/// state is reloaded from the durable object after this guard is released.
 pub async fn read_server_config_snapshot<S>(api: Arc<S>) -> Result<ServerConfigSnapshot>
 where
     S: ObjectIO<
@@ -2071,12 +2295,10 @@ where
 {
     let config_file = server_config_path();
     let local_guard = SERVER_CONFIG_LOCK.clone().write_owned().await;
-    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &config_file).await?;
+    let transaction_lock = server_config_transaction_lock_path();
+    let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &transaction_lock).await?;
     let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
-    let read_options = ObjectOptions {
-        no_lock: true,
-        ..Default::default()
-    };
+    let read_options = ObjectOptions::default();
     match read_config_with_metadata_inner(api, &config_file, &read_options, true).await {
         Ok((raw, object_info)) => {
             let (config, seed) = decode_persisted_server_config_with_seed(&raw)?;
@@ -2085,6 +2307,7 @@ where
                 raw: Some(raw),
                 seed: Some(seed),
                 etag: object_info.etag,
+                generation: object_info.data_dir.filter(|generation| !generation.is_nil()),
                 _local_guard: local_guard,
                 _guard: guard,
             })
@@ -2094,6 +2317,7 @@ where
             raw: None,
             seed: None,
             etag: None,
+            generation: None,
             _local_guard: local_guard,
             _guard: guard,
         }),
@@ -2108,6 +2332,27 @@ where
 /// lock, so a concurrent update or transaction lease loss cannot commit an
 /// unfenced overwrite.
 pub async fn save_server_config_snapshot<S>(api: Arc<S>, cfg: &Config, snapshot: &ServerConfigSnapshot) -> Result<bool>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
+{
+    save_server_config_snapshot_with_generation(api, cfg, snapshot)
+        .await
+        .map(|result| result.persisted())
+}
+
+pub async fn save_server_config_snapshot_with_generation<S>(
+    api: Arc<S>,
+    cfg: &Config,
+    snapshot: &ServerConfigSnapshot,
+) -> Result<ServerConfigSaveResult>
 where
     S: ObjectIO<
             Error = Error,
@@ -2126,13 +2371,19 @@ where
         && configs_semantically_equal(&snapshot.config, cfg)
     {
         debug!("server config unchanged and already in standard object shape, skip write");
-        return Ok(false);
+        return Ok(ServerConfigSaveResult {
+            persisted: false,
+            generation: snapshot.generation(),
+        });
     }
 
     let data = encode_server_config_blob(cfg, snapshot.seed.as_deref())?;
     if snapshot.raw.as_deref().is_some_and(|current| current == data.as_slice()) {
         debug!("server config bytes unchanged after encode, skip write");
-        return Ok(false);
+        return Ok(ServerConfigSaveResult {
+            persisted: false,
+            generation: snapshot.generation(),
+        });
     }
 
     let http_preconditions = if snapshot.raw.is_some() {
@@ -2152,19 +2403,22 @@ where
         }
     };
 
-    save_config_with_opts(
+    snapshot.ensure_lock_held()?;
+    let object_info = save_config_with_opts_and_metadata(
         api,
         &config_file,
         data,
         &ObjectOptions {
             max_parity: true,
-            no_lock: true,
             http_preconditions: Some(http_preconditions),
             ..Default::default()
         },
     )
     .await?;
-    Ok(true)
+    Ok(ServerConfigSaveResult {
+        persisted: true,
+        generation: object_info.data_dir.filter(|generation| !generation.is_nil()),
+    })
 }
 
 /// Saves the server config while an upper layer holds the namespace write
@@ -2301,8 +2555,9 @@ mod tests {
     use super::{
         SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, config_task_join_error,
         configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
-        lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata, read_config_without_migrate,
-        read_server_config_snapshot, save_server_config, save_server_config_snapshot, server_config_path, storage_class_kvs_mut,
+        lookup_configs, new_and_save_server_config, read_config, read_config_preserve_empty, read_config_with_metadata,
+        read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
+        save_server_config_snapshot_with_generation, server_config_transaction_lock_path, storage_class_kvs_mut,
     };
     use crate::config::{audit, heal, notify, oidc, scanner};
     use crate::disk::endpoint::Endpoint;
@@ -2311,7 +2566,9 @@ mod tests {
     use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
     use crate::runtime::sources as runtime_sources;
     use crate::set_disk::SetDisks;
-    use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, range::HTTPRangeSpec};
+    use crate::storage_api_contracts::{
+        admin::StorageAdminApi, namespace::NamespaceLocking as _, object::HTTPPreconditions, range::HTTPRangeSpec,
+    };
     use http::HeaderMap;
     use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
     use rustfs_config::notify::{
@@ -3104,6 +3361,85 @@ mod tests {
         );
     }
 
+    #[test]
+    fn root_heal_null_decodes_as_no_override_and_is_canonicalized_on_save() {
+        let seed = br#"{
+          "version":"33",
+          "storageclass":{"standard":"","rrs":""},
+          "heal":null,
+          "future_root":{"mode":"keep"},
+          "openid":{"default":{
+            "config_url":"https://issuer.example/.well-known/openid-configuration",
+            "client_id":"console",
+            "client_secret":"oidc-secret",
+            "future_provider_control":"keep"
+          }},
+          "notify":{"webhook":{"primary":{
+            "enable":true,
+            "endpoint":"https://notify.example/hook",
+            "auth_token":"notify-secret",
+            "future_notify_control":"keep"
+          }}},
+          "logger":{"webhook":{"primary":{
+            "enable":true,
+            "endpoint":"https://audit.example/hook",
+            "auth_token":"audit-secret",
+            "future_audit_control":"keep"
+          }}}
+        }"#;
+
+        let cfg = decode_server_config_blob(seed).expect("root heal null should mean no persisted override");
+        assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none());
+        assert!(!is_standard_object_server_config(seed));
+
+        let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("legacy seed should canonicalize on an authorized save");
+        let value: Value = serde_json::from_slice(&encoded).expect("canonical config should be valid JSON");
+        assert!(value.get(HEAL_SUB_SYS).is_none());
+        assert_eq!(value["future_root"]["mode"].as_str(), Some("keep"));
+        assert_eq!(value["openid"]["default"]["client_secret"].as_str(), Some("oidc-secret"));
+        assert_eq!(value["openid"]["default"]["future_provider_control"].as_str(), Some("keep"));
+        assert_eq!(value["notify"]["webhook"]["primary"]["auth_token"].as_str(), Some("notify-secret"));
+        assert_eq!(value["notify"]["webhook"]["primary"]["future_notify_control"].as_str(), Some("keep"));
+        assert_eq!(value["logger"]["webhook"]["primary"]["auth_token"].as_str(), Some("audit-secret"));
+        assert_eq!(value["logger"]["webhook"]["primary"]["future_audit_control"].as_str(), Some("keep"));
+        assert!(is_standard_object_server_config(&encoded));
+    }
+
+    #[test]
+    fn invalid_scalar_and_nested_null_config_shapes_remain_rejected() {
+        let invalid_sections = [
+            r#""scanner":null"#,
+            r#""heal":"""#,
+            r#""heal":false"#,
+            r#""heal":0"#,
+            r#""heal":{"default":null}"#,
+            r#""heal":{"_":null}"#,
+            r#""heal":{"bitrot_cycle":null}"#,
+            r#""heal":[{"key":"bitrot_cycle","value":null}]"#,
+        ];
+
+        for section in invalid_sections {
+            let input = format!(r#"{{"version":"33","storageclass":{{"standard":"","rrs":""}},{section}}}"#);
+            let err = decode_server_config_blob(input.as_bytes()).expect_err("invalid scalar shape must remain rejected");
+            assert!(
+                err.to_string().contains("expected"),
+                "invalid section {section} returned an unrelated error: {err}"
+            );
+        }
+    }
+
+    #[test]
+    fn valid_heal_object_and_kvs_array_shapes_remain_accepted() {
+        let empty_object = br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":{}}"#;
+        let cfg = decode_server_config_blob(empty_object).expect("empty heal object should decode as no override");
+        assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_none());
+
+        let kvs_array =
+            br#"{"version":"33","storageclass":{"standard":"","rrs":""},"heal":[{"key":"bitrot_cycle","value":"off"}]}"#;
+        let cfg = decode_server_config_blob(kvs_array).expect("heal KVS array should decode");
+        assert!(cfg.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER).is_some());
+    }
+
     #[test]
     fn scanner_update_preserves_unknown_root_and_oidc_provider_fields() {
         let seed = br#"{
@@ -3131,6 +3467,171 @@ mod tests {
         assert_eq!(value["openid"]["default"]["client_id"].as_str(), Some("console"));
     }
 
+    #[test]
+    fn storageclass_reset_removes_stale_inline_block_from_seed() {
+        let seed = br#"{
+          "version":"33",
+          "storageclass":{
+            "standard":"EC:2",
+            "rrs":"EC:1",
+            "optimize":"availability",
+            "inline_block":"64KiB",
+            "future_storage_control":"keep"
+          }
+        }"#;
+
+        let encoded = encode_server_config_blob(&Config::new(), Some(seed)).expect("storageclass reset should encode");
+        let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
+        let storageclass = value["storageclass"].as_object().expect("storageclass object");
+
+        assert!(storageclass.get(crate::config::storageclass::INLINE_BLOCK).is_none());
+        assert_eq!(storageclass["future_storage_control"].as_str(), Some("keep"));
+    }
+
+    #[test]
+    fn target_update_preserves_unknown_fields_without_restoring_removed_instances() {
+        let seed = br#"{
+          "version":"33",
+          "storageclass":{"standard":"","rrs":""},
+          "notify":{"webhook":{
+            "primary":{
+              "enable":true,
+              "endpoint":"https://notify.example/old",
+              "auth_token":"notify-secret",
+              "future_control":"keep"
+            },
+            "removed":{"enable":true,"endpoint":"https://notify.example/removed"},
+            "retained":{"enable":true,"endpoint":"https://notify.example/retained","future_control":"keep"},
+            "enable":{"enable":true,"endpoint":"https://notify.example/named-enable","future_control":"keep"}
+          }}
+        }"#;
+        let mut cfg = decode_server_config_blob(seed).expect("target seed should decode");
+        let webhook = cfg
+            .0
+            .get_mut(NOTIFY_WEBHOOK_SUB_SYS)
+            .expect("notify webhook subsystem should exist");
+        webhook
+            .get_mut("primary")
+            .expect("primary target should exist")
+            .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string());
+        webhook.remove("removed");
+        webhook.remove("retained");
+
+        let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target update should encode");
+        let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
+        let webhook = value["notify"]["webhook"].as_object().expect("webhook section");
+
+        assert_eq!(webhook["primary"]["endpoint"].as_str(), Some("https://notify.example/new"));
+        assert_eq!(webhook["primary"]["auth_token"].as_str(), Some("notify-secret"));
+        assert_eq!(webhook["primary"]["future_control"].as_str(), Some("keep"));
+        assert!(webhook.get("removed").is_none());
+        assert_eq!(webhook["retained"]["future_control"].as_str(), Some("keep"));
+        assert!(webhook["retained"].get("enable").is_none());
+        assert!(webhook["retained"].get("endpoint").is_none());
+        assert_eq!(webhook["enable"]["endpoint"].as_str(), Some("https://notify.example/named-enable"));
+        assert_eq!(webhook["enable"]["future_control"].as_str(), Some("keep"));
+    }
+
+    #[test]
+    fn shorthand_target_update_preserves_shape_and_unknown_nested_fields() {
+        let seed = br#"{
+          "version":"33",
+          "storageclass":{"standard":"","rrs":""},
+          "notify":{"webhook":{
+            "enable":true,
+            "endpoint":"https://notify.example/old",
+            "future_control":{"endpoint":"leave-untouched","mode":"keep"}
+          }}
+        }"#;
+        let mut cfg = decode_server_config_blob(seed).expect("shorthand target should decode");
+        cfg.0
+            .get_mut(NOTIFY_WEBHOOK_SUB_SYS)
+            .and_then(|targets| targets.get_mut(DEFAULT_DELIMITER))
+            .expect("default webhook target should exist")
+            .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string());
+
+        let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("shorthand target update should encode");
+        let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
+        let webhook = value["notify"]["webhook"].as_object().expect("webhook shorthand object");
+
+        assert_eq!(webhook["endpoint"].as_str(), Some("https://notify.example/new"));
+        assert!(webhook.get("default").is_none());
+        assert_eq!(webhook["future_control"]["endpoint"].as_str(), Some("leave-untouched"));
+        assert_eq!(webhook["future_control"]["mode"].as_str(), Some("keep"));
+        let decoded = decode_server_config_blob(&encoded).expect("updated shorthand target should remain decodable");
+        assert_eq!(
+            decoded
+                .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER)
+                .expect("updated default webhook target")
+                .get(rustfs_config::WEBHOOK_ENDPOINT),
+            "https://notify.example/new"
+        );
+    }
+
+    #[test]
+    fn target_kvs_update_preserves_unknown_entries_and_attributes() {
+        let seed = br#"{
+          "version":"33",
+          "storageclass":{"standard":"","rrs":""},
+          "notify":{"webhook":{"primary":[
+            {"key":"enable","value":"on","hidden_if_empty":false},
+            {"key":"endpoint","value":"https://notify.example/old","future_attribute":"keep-endpoint"},
+            {"key":"future_control","value":"keep","future_attribute":"keep-control"}
+          ]}}
+        }"#;
+        let mut cfg = decode_server_config_blob(seed).expect("target KVS seed should decode");
+        cfg.0
+            .get_mut(NOTIFY_WEBHOOK_SUB_SYS)
+            .and_then(|targets| targets.get_mut("primary"))
+            .expect("primary webhook target should exist")
+            .insert(rustfs_config::WEBHOOK_ENDPOINT.to_string(), "https://notify.example/new".to_string());
+
+        let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("target KVS update should encode");
+        let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
+        let entries = value["notify"]["webhook"]["primary"]
+            .as_array()
+            .expect("target KVS shape should be preserved");
+        let endpoint = entries
+            .iter()
+            .find(|entry| entry["key"].as_str() == Some(rustfs_config::WEBHOOK_ENDPOINT))
+            .expect("endpoint entry should remain");
+        let future = entries
+            .iter()
+            .find(|entry| entry["key"].as_str() == Some("future_control"))
+            .expect("unknown target entry should remain");
+
+        assert_eq!(endpoint["value"].as_str(), Some("https://notify.example/new"));
+        assert_eq!(endpoint["future_attribute"].as_str(), Some("keep-endpoint"));
+        assert_eq!(future["value"].as_str(), Some("keep"));
+        assert_eq!(future["future_attribute"].as_str(), Some("keep-control"));
+    }
+
+    #[test]
+    fn target_default_alias_is_canonicalized_without_losing_unknown_fields() {
+        let seed = br#"{
+          "version":"33",
+          "storageclass":{"standard":"","rrs":""},
+          "notify":{"webhook":{
+            "_":{"enable":false,"endpoint":"https://notify.example/alias","future_alias":"keep"},
+            "default":{"enable":true,"endpoint":"https://notify.example/default","future_default":"keep"}
+          }}
+        }"#;
+        let cfg = decode_server_config_blob(seed).expect("dual default aliases should decode");
+        let expected_endpoint = cfg
+            .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER)
+            .expect("default webhook target should exist")
+            .get(rustfs_config::WEBHOOK_ENDPOINT);
+
+        let encoded = encode_server_config_blob(&cfg, Some(seed)).expect("default alias should canonicalize");
+        let value: Value = serde_json::from_slice(&encoded).expect("encoded config should be valid json");
+        let webhook = value["notify"]["webhook"].as_object().expect("webhook section");
+
+        assert!(webhook.get(DEFAULT_DELIMITER).is_none());
+        assert_eq!(webhook["default"]["endpoint"].as_str(), Some(expected_endpoint.as_str()));
+        assert_eq!(webhook["default"]["future_alias"].as_str(), Some("keep"));
+        assert_eq!(webhook["default"]["future_default"].as_str(), Some("keep"));
+    }
+
     #[test]
     fn test_scanner_config_changes_are_semantically_significant() {
         let baseline = Config::new();
@@ -4124,6 +4625,7 @@ mod tests {
 
     /// What reads of the config object currently return.
     enum RecoveryReadState {
+        Missing,
         Blob(Vec<u8>),
         QuorumError,
     }
@@ -4136,6 +4638,7 @@ mod tests {
         heal_calls: AtomicUsize,
         write_calls: AtomicUsize,
         last_put_no_lock: AtomicBool,
+        last_put_preconditions: Mutex<Option<HTTPPreconditions>>,
         revision: AtomicUsize,
         drive_counts: Vec<usize>,
         lock_manager: Arc<rustfs_lock::GlobalLockManager>,
@@ -4150,6 +4653,7 @@ mod tests {
                 heal_calls: AtomicUsize::new(0),
                 write_calls: AtomicUsize::new(0),
                 last_put_no_lock: AtomicBool::new(false),
+                last_put_preconditions: Mutex::new(None),
                 revision: AtomicUsize::new(1),
                 drive_counts: vec![2],
                 lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
@@ -4206,6 +4710,7 @@ mod tests {
             _opts: &ObjectOptions,
         ) -> Result<GetObjectReader> {
             let data = match &*self.state.lock().expect("state lock poisoned") {
+                RecoveryReadState::Missing => return Err(Error::ConfigNotFound),
                 RecoveryReadState::Blob(data) => data.clone(),
                 RecoveryReadState::QuorumError => return Err(Error::ErasureReadQuorum),
             };
@@ -4213,6 +4718,9 @@ mod tests {
                 size: data.len() as i64,
                 actual_size: data.len() as i64,
                 etag: Some(format!("config-{}", self.revision.load(Ordering::SeqCst))),
+                data_dir: Some(uuid::Uuid::from_u128(
+                    u128::try_from(self.revision.load(Ordering::SeqCst)).expect("test revision should fit in u128"),
+                )),
                 ..Default::default()
             };
             Ok(GetObjectReader {
@@ -4231,15 +4739,19 @@ mod tests {
             opts: &ObjectOptions,
         ) -> Result<ObjectInfo> {
             let current_etag = format!("config-{}", self.revision.load(Ordering::SeqCst));
+            let object_exists = matches!(&*self.state.lock().expect("state lock poisoned"), RecoveryReadState::Blob(_));
             if let Some(preconditions) = &opts.http_preconditions
-                && (preconditions.if_match_value().is_some_and(|etag| etag != current_etag)
-                    || preconditions.if_none_match_value() == Some("*"))
+                && (preconditions
+                    .if_match_value()
+                    .is_some_and(|etag| !object_exists || etag != current_etag)
+                    || (object_exists && preconditions.if_none_match_value() == Some("*")))
             {
                 return Err(Error::PreconditionFailed);
             }
             let mut body = Vec::new();
             data.stream.read_to_end(&mut body).await?;
             self.last_put_no_lock.store(opts.no_lock, Ordering::SeqCst);
+            *self.last_put_preconditions.lock().expect("preconditions lock poisoned") = opts.http_preconditions.clone();
             self.write_calls.fetch_add(1, Ordering::SeqCst);
             *self.state.lock().expect("state lock poisoned") = RecoveryReadState::Blob(body.clone());
             let revision = self.revision.fetch_add(1, Ordering::SeqCst) + 1;
@@ -4247,6 +4759,7 @@ mod tests {
                 size: i64::try_from(body.len()).expect("test config should fit in i64"),
                 actual_size: i64::try_from(body.len()).expect("test config should fit in i64"),
                 etag: Some(format!("config-{revision}")),
+                data_dir: Some(uuid::Uuid::from_u128(u128::try_from(revision).expect("test revision should fit in u128"))),
                 ..Default::default()
             })
         }
@@ -4284,10 +4797,18 @@ mod tests {
             .expect("scanner-only config change should be persisted");
 
         assert_eq!(store.write_calls.load(Ordering::SeqCst), 1);
-        assert!(store.last_put_no_lock.load(Ordering::SeqCst));
+        assert!(!store.last_put_no_lock.load(Ordering::SeqCst));
+        let preconditions = store
+            .last_put_preconditions
+            .lock()
+            .expect("preconditions lock poisoned")
+            .clone()
+            .expect("existing config update must be conditional");
+        assert_eq!(preconditions.if_match_value(), Some("config-1"));
+        assert_eq!(preconditions.if_none_match_value(), None);
         assert_eq!(
             store.lock_resources.lock().expect("lock resources mutex poisoned").as_slice(),
-            &[server_config_path()]
+            &[server_config_transaction_lock_path()]
         );
         let decoded = read_config_without_migrate(store)
             .await
@@ -4301,6 +4822,73 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn server_config_snapshot_save_returns_committed_generation() {
+        let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode");
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None));
+        let snapshot = read_server_config_snapshot(store.clone())
+            .await
+            .expect("server config snapshot");
+
+        let result = save_server_config_snapshot_with_generation(store, &config_with_scanner_cycle("61"), &snapshot)
+            .await
+            .expect("conditional config save");
+
+        assert!(result.persisted());
+        assert_eq!(result.generation(), Some(uuid::Uuid::from_u128(2)));
+    }
+
+    #[tokio::test]
+    async fn missing_server_config_is_created_with_if_none_match() {
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Missing, None));
+        let cfg = config_with_scanner_cycle("61");
+
+        save_server_config(store.clone(), &cfg)
+            .await
+            .expect("missing config should be created conditionally");
+
+        assert_eq!(store.write_calls.load(Ordering::SeqCst), 1);
+        assert!(!store.last_put_no_lock.load(Ordering::SeqCst));
+        let preconditions = store
+            .last_put_preconditions
+            .lock()
+            .expect("preconditions lock poisoned")
+            .clone()
+            .expect("missing config create must be conditional");
+        assert_eq!(preconditions.if_match_value(), None);
+        assert_eq!(preconditions.if_none_match_value(), Some("*"));
+        let persisted = read_config_without_migrate(store)
+            .await
+            .expect("created config should reload");
+        assert_eq!(
+            persisted
+                .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
+                .expect("persisted scanner config")
+                .get(SCANNER_CYCLE),
+            "61"
+        );
+    }
+
+    #[tokio::test]
+    async fn missing_config_initialization_recheck_preserves_concurrent_config() {
+        let existing = config_with_scanner_cycle("73");
+        let baseline = encode_server_config_blob(&existing, None).expect("existing config should encode");
+        let store = Arc::new(RecoveryMockStore::new(RecoveryReadState::Blob(baseline), None));
+
+        let observed = new_and_save_server_config(store.clone())
+            .await
+            .expect("initialization recheck should return the config created by another writer");
+
+        assert_eq!(store.write_calls.load(Ordering::SeqCst), 0);
+        assert_eq!(
+            observed
+                .get_value(SCANNER_SUB_SYS, DEFAULT_DELIMITER)
+                .expect("concurrent scanner config")
+                .get(SCANNER_CYCLE),
+            "73"
+        );
+    }
+
     #[tokio::test]
     async fn stale_server_config_snapshot_cannot_overwrite_newer_update() {
         let baseline = encode_server_config_blob(&Config::new(), None).expect("baseline config should encode");
@@ -4357,7 +4945,7 @@ mod tests {
         let lock = rustfs_lock::NamespaceLock::new("server-config-lease-loss".to_string(), client.clone());
         let guard = lock
             .lock_guard(
-                rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_path()),
+                rustfs_lock::ObjectKey::new(crate::disk::RUSTFS_META_BUCKET, server_config_transaction_lock_path()),
                 "server-config-lease-loss",
                 std::time::Duration::from_secs(1),
                 std::time::Duration::from_millis(120),
@@ -4372,6 +4960,7 @@ mod tests {
             raw: Some(baseline.clone()),
             seed: None,
             etag: Some("config-0".to_string()),
+            generation: Some(uuid::Uuid::from_u128(1)),
             _local_guard: local_guard,
             _guard: guard,
         };

crates/ecstore/src/disk/local.rs

@@ -6641,6 +6641,49 @@ impl LocalDisk {
         let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
         restore_delete_rollback_after_error(object_dir, &xl_path, Some(rollback_dir), volume, object, stage, err).await
     }
+
+    /// Execute every deferred data-dir deletion pending on `volume` right now,
+    /// even while snapshot leases are still held. Bucket deletion requires it:
+    /// a streaming reader defers the physical cleanup of an already-deleted
+    /// version, and a non-force `delete_volume` would otherwise fail closed
+    /// with `VolumeNotEmpty` on those remnants even though the bucket is
+    /// logically empty. The still-active readers keep their open descriptors;
+    /// only path-based reopens observe the removal.
+    async fn settle_pending_snapshot_deletes(&self, volume: &str) {
+        let pending: Vec<(SnapshotLeaseKey, DeleteOptions)> = {
+            let mut registry = self.snapshot_leases.lock().await;
+            registry
+                .entries
+                .iter_mut()
+                .filter(|(key, entry)| key.volume == volume && !entry.deleting && entry.pending_delete.is_some())
+                .map(|(key, entry)| {
+                    entry.deleting = true;
+                    (key.clone(), entry.pending_delete.clone().expect("filtered on Some"))
+                })
+                .collect()
+        };
+
+        for (key, opts) in pending {
+            let result = self.delete_unleased(&key.volume, &key.path, &opts).await;
+            let mut registry = self.snapshot_leases.lock().await;
+            match result {
+                Ok(()) => {
+                    registry.entries.remove(&key);
+                }
+                Err(err) => {
+                    if let Some(entry) = registry.entries.get_mut(&key) {
+                        entry.deleting = false;
+                    }
+                    warn!(
+                        volume = %key.volume,
+                        path = %key.path,
+                        error = %err,
+                        "failed to settle deferred data-dir deletion before volume removal"
+                    );
+                }
+            }
+        }
+    }
 }
 
 #[async_trait::async_trait]
@@ -9122,6 +9165,14 @@ impl DiskAPI for LocalDisk {
         let p = self.get_bucket_path(volume)?;
         let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await;
 
+        // A streaming reader's snapshot lease defers the physical cleanup of
+        // data dirs whose version delete already committed. Those remnants are
+        // logically deleted, so run the parked cleanups now instead of letting
+        // the non-force removal below fail closed on them (the s3-tests SSE-C
+        // teardown races exactly this way: DeleteObjects, then DeleteBucket
+        // while an abandoned GET body still pins the lease).
+        self.settle_pending_snapshot_deletes(volume).await;
+
         // Non-force removes empty directory remnants children-first with
         // non-recursive rmdir calls. A file that exists during the scan, or
         // appears before its parent is removed, fails closed with
@@ -15981,6 +16032,74 @@ mod test {
         assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
     }
 
+    #[tokio::test]
+    async fn delete_volume_settles_lease_deferred_cleanup() {
+        use tempfile::tempdir;
+
+        let root_dir = tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        let volume = "snapshot-lease-bucket-delete";
+        let object = "multipart_enc";
+        let version_id = Uuid::new_v4();
+        let data_dir = Uuid::new_v4();
+        let rollback_dir = Uuid::new_v4();
+        let data_path = path_join_buf(&[object, &data_dir.to_string()]);
+        let part = path_join_buf(&[&data_path, "part.1"]);
+        ensure_test_volume(&disk, volume).await;
+        disk.write_all(volume, &part, Bytes::from_static(b"payload"))
+            .await
+            .expect("shard should be written");
+        let fi = test_file_info(object, version_id, Some(data_dir), None);
+        disk.write_all(volume, &path_join_buf(&[object, STORAGE_FORMAT_FILE]), test_meta(fi.clone()).into())
+            .await
+            .expect("metadata should be written");
+
+        // An abandoned streaming GET pins the data dir with a snapshot lease.
+        let snapshot = disk
+            .acquire_snapshot_lease(volume, &data_path)
+            .await
+            .expect("snapshot lease should be acquired");
+        disk.delete_version(
+            volume,
+            object,
+            fi.clone(),
+            false,
+            DeleteOptions {
+                old_data_dir: Some(rollback_dir),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("version delete should commit metadata");
+        disk.delete(
+            volume,
+            &format!("{object}/{rollback_dir}"),
+            DeleteOptions {
+                recursive: true,
+                immediate: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("version delete should schedule physical cleanup");
+
+        // The bucket is logically empty; a non-force volume delete must settle
+        // the deferred data-dir cleanup instead of failing with VolumeNotEmpty.
+        disk.delete_volume(volume, false)
+            .await
+            .expect("bucket delete must not observe lease-deferred remnants");
+        assert!(matches!(
+            disk.read_all(volume, &part).await,
+            Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
+        ));
+
+        // The late lease release finds nothing pending and stays idempotent.
+        disk.release_snapshot_lease(volume, &data_path, snapshot)
+            .await
+            .expect("releasing the lease after bucket deletion should be a no-op");
+    }
+
     #[tokio::test]
     async fn version_delete_cleanup_intent_survives_local_disk_restart() {
         use tempfile::tempdir;

crates/ecstore/src/erasure/coding/decode.rs

@@ -1645,15 +1645,28 @@ impl Erasure {
             }
             Err(e) => {
                 record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
-                error!(
-                    block_offset,
-                    block_length,
-                    bytes_written = *written,
-                    stage = GET_STAGE_EMIT,
-                    reason = classify_io_error(&e).as_str(),
-                    error = ?e,
-                    "Erasure decode failed to emit reconstructed data"
-                );
+                let reason = classify_io_error(&e);
+                if reason == GetObjectFailureReason::DownstreamClosed {
+                    debug!(
+                        block_offset,
+                        block_length,
+                        bytes_written = *written,
+                        stage = GET_STAGE_EMIT,
+                        reason = reason.as_str(),
+                        error = ?e,
+                        "Erasure decode stopped after downstream closed"
+                    );
+                } else {
+                    error!(
+                        block_offset,
+                        block_length,
+                        bytes_written = *written,
+                        stage = GET_STAGE_EMIT,
+                        reason = reason.as_str(),
+                        error = ?e,
+                        "Erasure decode failed to emit reconstructed data"
+                    );
+                }
                 *ret_err = Some(e);
                 return StripeFlow::Stop;
             }
@@ -1945,7 +1958,7 @@ mod tests {
     use std::io::Cursor;
     use std::pin::Pin;
     use std::sync::{
-        Arc,
+        Arc, Mutex,
         atomic::{AtomicUsize, Ordering},
     };
     use std::task::{Context, Poll};
@@ -2120,6 +2133,59 @@ mod tests {
         }
     }
 
+    struct DownstreamClosedWriter;
+
+    impl AsyncWrite for DownstreamClosedWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
+            Poll::Ready(Err(crate::diagnostics::get::mark_get_object_downstream_closed(io::Error::new(
+                ErrorKind::BrokenPipe,
+                "injected downstream close",
+            ))))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[derive(Clone, Default)]
+    struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
+
+    struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
+
+    impl CapturedLogs {
+        fn contents(&self) -> String {
+            String::from_utf8(self.0.lock().expect("captured logs mutex should not be poisoned").clone())
+                .expect("captured logs should be valid UTF-8")
+        }
+    }
+
+    impl std::io::Write for CapturedLogWriter {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            self.0
+                .lock()
+                .expect("captured logs mutex should not be poisoned")
+                .extend_from_slice(buf);
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
+        type Writer = CapturedLogWriter;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            CapturedLogWriter(Arc::clone(&self.0))
+        }
+    }
+
     #[test]
     fn parallel_reader_constructor_variants_preserve_read_cost_and_verification_flags() {
         let erasure = Erasure::new(2, 1, 64);
@@ -2215,6 +2281,47 @@ mod tests {
         assert_eq!(err.to_string(), "injected emit failure");
     }
 
+    #[tokio::test(flavor = "current_thread")]
+    async fn erasure_decode_logs_reconstructed_downstream_close_at_debug() {
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_max_level(tracing::Level::DEBUG)
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _guard = tracing::subscriber::set_default(subscriber);
+
+        let erasure = Erasure::new(2, 1, 64);
+        let data: Vec<u8> = (0..64).collect();
+        let shard_size = erasure.shard_size();
+        let encoded = erasure.encode_data(&data).expect("test data should encode");
+        let readers = vec![
+            None,
+            Some(BitrotReader::new(
+                Cursor::new(encoded[1].to_vec()),
+                shard_size,
+                HashAlgorithm::None,
+                false,
+            )),
+            Some(BitrotReader::new(
+                Cursor::new(encoded[2].to_vec()),
+                shard_size,
+                HashAlgorithm::None,
+                false,
+            )),
+        ];
+
+        let mut writer = DownstreamClosedWriter;
+        let (written, err) = erasure.decode(&mut writer, readers, 0, data.len(), data.len()).await;
+
+        assert_eq!(written, 0);
+        assert_eq!(err.expect("downstream close must still terminate the GET").kind(), ErrorKind::BrokenPipe);
+        let captured = logs.contents();
+        assert!(captured.contains("Erasure decode stopped after downstream closed"));
+        assert!(!captured.contains("Erasure decode failed to emit reconstructed data"));
+    }
+
     #[tokio::test]
     async fn test_erasure_decode_rejects_reader_count_and_range_overflow() {
         let erasure = Erasure::new(2, 1, 64);

crates/ecstore/src/layout/endpoints.rs

@@ -22,6 +22,8 @@ use rustfs_config::{
     ENV_STARTUP_TOPOLOGY_WAIT_MODE, ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, ENV_UNSAFE_BYPASS_DISK_CHECK,
 };
 use rustfs_utils::{XHost, check_local_server_addr, get_env_opt_str, get_host_ip, is_local_host};
+#[cfg(test)]
+use std::sync::{LazyLock, Mutex};
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
     future::Future,
@@ -598,6 +600,58 @@ const DNS_RETRY_JITTER_PERCENT: u64 = 20;
 /// wait does not flood the log with one line per backoff tick.
 const TOPOLOGY_WARN_THROTTLE: Duration = Duration::from_secs(30);
 
+#[cfg(test)]
+static FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS: LazyLock<Mutex<HashSet<String>>> = LazyLock::new(|| Mutex::new(HashSet::new()));
+
+#[cfg(test)]
+struct LocalHostResolutionTimeoutGuard {
+    hosts: Vec<String>,
+}
+
+#[cfg(test)]
+impl Drop for LocalHostResolutionTimeoutGuard {
+    fn drop(&mut self) {
+        let mut forced_hosts = FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
+            .lock()
+            .expect("local-host test resolver mutex poisoned");
+        for host in &self.hosts {
+            forced_hosts.remove(host);
+        }
+    }
+}
+
+#[cfg(test)]
+fn force_local_host_resolution_timeout_for_test(hosts: &[&str]) -> LocalHostResolutionTimeoutGuard {
+    let hosts = hosts.iter().map(|host| (*host).to_string()).collect::<Vec<_>>();
+    let mut forced_hosts = FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
+        .lock()
+        .expect("local-host test resolver mutex poisoned");
+    forced_hosts.extend(hosts.iter().cloned());
+    LocalHostResolutionTimeoutGuard { hosts }
+}
+
+#[cfg(test)]
+fn local_host_resolution_timeout_forced(host: &Host<&str>) -> bool {
+    let host = match host {
+        Host::Domain(domain) => (*domain).to_string(),
+        Host::Ipv4(ip) => ip.to_string(),
+        Host::Ipv6(ip) => ip.to_string(),
+    };
+    FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
+        .lock()
+        .expect("local-host test resolver mutex poisoned")
+        .contains(&host)
+}
+
+fn endpoint_is_local_host(host: Host<&str>, port: u16, local_port: u16) -> Result<bool> {
+    #[cfg(test)]
+    if local_host_resolution_timeout_forced(&host) {
+        return Err(Error::new(ErrorKind::TimedOut, "resolver timeout"));
+    }
+
+    is_local_host(host, port, local_port)
+}
+
 struct DnsRetryDeadline {
     started: Instant,
     timeout: Duration,
@@ -694,7 +748,7 @@ async fn resolve_local_host_with_retry(
     retry_dns_operation(
         || {
             let host = host.clone();
-            async move { is_local_host(host, port, local_port) }
+            async move { endpoint_is_local_host(host, port, local_port) }
         },
         async_sleep,
         dns_retry_deadline,
@@ -2231,6 +2285,9 @@ mod test {
     #[serial]
     #[tokio::test]
     async fn create_server_endpoints_bounds_kubernetes_alias_dns_fallback() {
+        let _resolution_timeout =
+            force_local_host_resolution_timeout_for_test(&["unrelated-0.example.invalid", "unrelated-1.example.invalid"]);
+
         async_with_vars(
             [
                 (ENV_LOCAL_ENDPOINT_HOST, None),

crates/ecstore/src/services/notification_sys.rs

@@ -337,6 +337,15 @@ pub struct NotificationPeerErr {
     pub err: Option<Error>,
 }
 
+/// One peer's answer to a KMS configuration fingerprint probe.
+pub struct PeerKmsConfigFingerprint {
+    pub host: String,
+    /// `None` when the peer has no KMS configuration of its own, or could not
+    /// be asked at all, in which case `err` carries the reason.
+    pub fingerprint: Option<String>,
+    pub err: Option<Error>,
+}
+
 fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
     NotificationPeerErr { host, err: result.err() }
 }
@@ -518,6 +527,47 @@ impl NotificationSys {
         self.signal_dynamic_config(sub_sys, false).await
     }
 
+    /// Ask every peer to re-read the cluster-persisted KMS configuration.
+    ///
+    /// Best-effort by contract: the caller has already switched locally, so a
+    /// peer that fails is reported rather than rolled back. Peers built before
+    /// the KMS subsystem existed reject the signal with an explicit error.
+    pub async fn reload_kms_config(&self) -> Vec<NotificationPeerErr> {
+        self.reload_dynamic_config(crate::cluster::rpc::KMS_SIGNAL_SUBSYSTEM).await
+    }
+
+    /// Collect the KMS configuration fingerprint each peer is running.
+    ///
+    /// A peer whose build predates the KMS subsystem rejects the probe, so it
+    /// is reported as an error rather than silently agreeing with this node.
+    pub async fn kms_config_fingerprints(&self) -> Vec<PeerKmsConfigFingerprint> {
+        let mut futures = Vec::with_capacity(self.peer_clients.len());
+        for client in self.peer_clients.iter() {
+            futures.push(async move {
+                let Some(client) = client else {
+                    return PeerKmsConfigFingerprint {
+                        host: String::new(),
+                        fingerprint: None,
+                        err: Some(Error::other("peer is not reachable")),
+                    };
+                };
+                match client.kms_config_fingerprint().await {
+                    Ok(fingerprint) => PeerKmsConfigFingerprint {
+                        host: client.host.to_string(),
+                        fingerprint,
+                        err: None,
+                    },
+                    Err(e) => PeerKmsConfigFingerprint {
+                        host: client.host.to_string(),
+                        fingerprint: None,
+                        err: Some(e),
+                    },
+                }
+            });
+        }
+        join_all(futures).await
+    }
+
     pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {

crates/ecstore/src/set_disk/mod.rs

@@ -4714,6 +4714,7 @@ mod tests {
     use crate::layout::endpoints::SetupType;
     use crate::object_api::BLOCK_SIZE_V2;
     use crate::object_api::ObjectInfo;
+    use crate::set_disk::core::io_primitives::rename_fanout_barrier;
     use crate::storage_api_contracts::{
         heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
         namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _,
@@ -9565,6 +9566,15 @@ mod tests {
         make_local_bucket_test_set_disks_with_drive_count(2).await
     }
 
+    fn assert_exclusive_object_lock_held(set_disks: &SetDisks, bucket: &str, object: &str) {
+        let lock = set_disks
+            .local_lock_manager_for_test()
+            .get_lock_info(&ObjectKey::new(bucket, object))
+            .expect("object lock should be visible while rename is paused");
+        assert!(matches!(lock.mode, rustfs_lock::LockMode::Exclusive));
+        assert_eq!(lock.owner.as_ref(), set_disks.locker_owner.as_str());
+    }
+
     async fn make_local_bucket_test_set_disks_with_drive_count(drive_count: usize) -> Arc<SetDisks> {
         let format = FormatV3::new(1, drive_count);
         let mut endpoints = Vec::new();
@@ -9599,7 +9609,9 @@ mod tests {
             disks.push(Some(disk));
         }
 
-        let set_disks = SetDisks::new(
+        let instance_ctx = Arc::new(InstanceContext::new());
+        instance_ctx.update_erasure_type(SetupType::Erasure).await;
+        let set_disks = SetDisks::new_with_instance_ctx(
             "test-owner".to_string(),
             Arc::new(RwLock::new(disks)),
             drive_count,
@@ -9609,6 +9621,7 @@ mod tests {
             endpoints,
             format,
             Vec::new(),
+            instance_ctx,
         )
         .await;
         set_disks.set_test_storage_class_config(
@@ -10262,6 +10275,216 @@ mod tests {
         ));
     }
 
+    #[tokio::test]
+    async fn conditional_replace_holds_object_lock_through_rename() {
+        let set_disks = make_local_bucket_test_set_disks().await;
+        let bucket = "bucket-conditional-replace-fence";
+        let object = "config/conditional-replace.json";
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        let mut initial_reader = PutObjReader::from_vec(b"initial config".to_vec());
+        let initial = set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut initial_reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("initial config should be written");
+        let initial_etag = initial.etag.expect("initial config should have an ETag");
+
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+        let writer_store = set_disks.clone();
+        let expected_etag = initial_etag.clone();
+        let writer = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(b"replacement config".to_vec());
+            writer_store
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        preserve_etag: Some("replacement-etag".to_string()),
+                        http_preconditions: Some(HTTPPreconditions {
+                            if_match: Some(expected_etag),
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("conditional replace should reach the rename barrier");
+        assert_exclusive_object_lock_held(&set_disks, bucket, object);
+        barrier.release();
+        writer
+            .await
+            .expect("conditional writer task should finish")
+            .expect("matching conditional replace should commit");
+        assert!(
+            set_disks
+                .local_lock_manager_for_test()
+                .get_lock_info(&ObjectKey::new(bucket, object))
+                .is_none(),
+            "conditional replace should release the object lock after commit"
+        );
+        let contender = set_disks
+            .new_ns_lock(bucket, object)
+            .await
+            .expect("contender namespace lock should be created");
+        let contender_guard = contender
+            .get_write_lock(std::time::Duration::from_secs(30))
+            .await
+            .expect("contender should acquire after conditional replace commits");
+        drop(contender_guard);
+
+        let mut stale_reader = PutObjReader::from_vec(b"stale config".to_vec());
+        let err = set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut stale_reader,
+                &ObjectOptions {
+                    http_preconditions: Some(HTTPPreconditions {
+                        if_match: Some(initial_etag),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("the old ETag must fail after the fenced replacement commits");
+        assert_eq!(err, StorageError::PreconditionFailed);
+    }
+
+    #[tokio::test]
+    async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() {
+        let set_disks = make_local_bucket_test_set_disks().await;
+        let bucket = "bucket-write-generation";
+        let object = "config/write-generation.json";
+        let body = b"identical config body".to_vec();
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        let mut first_reader = PutObjReader::from_vec(body.clone());
+        let first = set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut first_reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("first config body should be written");
+        let mut second_reader = PutObjReader::from_vec(body);
+        let second = set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut second_reader,
+                &ObjectOptions {
+                    no_lock: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("identical config body should be rewritten");
+
+        assert_eq!(first.etag, second.etag, "content ETag should expose the ABA collision");
+        assert_ne!(first.data_dir, second.data_dir, "each committed body write needs a unique generation");
+        assert!(first.data_dir.is_some() && second.data_dir.is_some());
+    }
+
+    #[tokio::test]
+    async fn conditional_create_holds_object_lock_through_rename() {
+        let set_disks = make_local_bucket_test_set_disks().await;
+        let bucket = "bucket-conditional-create-fence";
+        let object = "config/conditional-create.json";
+        set_disks
+            .make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("bucket should be created");
+
+        let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
+        let writer_store = set_disks.clone();
+        let writer = tokio::spawn(async move {
+            let mut reader = PutObjReader::from_vec(b"created config".to_vec());
+            writer_store
+                .put_object(
+                    bucket,
+                    object,
+                    &mut reader,
+                    &ObjectOptions {
+                        http_preconditions: Some(HTTPPreconditions {
+                            if_none_match: Some("*".to_string()),
+                            ..Default::default()
+                        }),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+
+        tokio::time::timeout(std::time::Duration::from_secs(30), barrier.wait_until_paused())
+            .await
+            .expect("conditional create should reach the rename barrier");
+        assert_exclusive_object_lock_held(&set_disks, bucket, object);
+        barrier.release();
+        writer
+            .await
+            .expect("conditional writer task should finish")
+            .expect("first conditional create should commit");
+        assert!(
+            set_disks
+                .local_lock_manager_for_test()
+                .get_lock_info(&ObjectKey::new(bucket, object))
+                .is_none(),
+            "conditional create should release the object lock after commit"
+        );
+        let contender = set_disks
+            .new_ns_lock(bucket, object)
+            .await
+            .expect("contender namespace lock should be created");
+        let contender_guard = contender
+            .get_write_lock(std::time::Duration::from_secs(30))
+            .await
+            .expect("contender should acquire after conditional create commits");
+        drop(contender_guard);
+
+        let mut duplicate_reader = PutObjReader::from_vec(b"duplicate config".to_vec());
+        let err = set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut duplicate_reader,
+                &ObjectOptions {
+                    http_preconditions: Some(HTTPPreconditions {
+                        if_none_match: Some("*".to_string()),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("a second create-only write must not replace the committed config");
+        assert_eq!(err, StorageError::PreconditionFailed);
+    }
+
     #[tokio::test]
     async fn set_level_if_none_match_fails_closed_without_read_quorum() {
         let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;

crates/ecstore/src/store/list_objects.rs

@@ -4861,10 +4861,7 @@ async fn poll_merge_head(rx: &CancellationToken, in_channels: &mut [Receiver<Met
 
 async fn send_or_cancel(rx: &CancellationToken, out_channel: &Sender<MetaCacheEntry>, entry: MetaCacheEntry) -> Result<bool> {
     tokio::select! {
-        result = out_channel.send(entry) => {
-            result.map_err(Error::other)?;
-            Ok(true)
-        }
+        result = out_channel.send(entry) => Ok(result.is_ok()),
         _ = rx.cancelled() => Ok(false),
     }
 }
@@ -9858,6 +9855,28 @@ mod test {
         assert_eq!(results, vec!["obj-a", "obj-b"]);
     }
 
+    #[tokio::test]
+    async fn merge_entry_channels_treats_dropped_output_receiver_as_completion() {
+        let (tx_a, rx_a) = mpsc::channel(4);
+        let (tx_b, rx_b) = mpsc::channel(4);
+        let (out_tx, out_rx) = mpsc::channel(1);
+
+        tx_a.send(test_meta_entry("obj-a")).await.unwrap();
+        tx_b.send(test_meta_entry("obj-b")).await.unwrap();
+        drop(tx_a);
+        drop(tx_b);
+        drop(out_rx);
+
+        let result = timeout(
+            Duration::from_secs(1),
+            merge_entry_channels(CancellationToken::new(), vec![rx_a, rx_b], out_tx, 1),
+        )
+        .await
+        .expect("merge should stop promptly when its consumer disconnects");
+
+        assert!(result.is_ok(), "consumer disconnect must not surface as a merge worker error");
+    }
+
     #[test]
     fn walk_ascending_versions_contract_reverses_newest_first_metadata() {
         // Documents the invariant the walk `versions_sort` handling relies on:

crates/iam/src/sys.rs

@@ -41,7 +41,7 @@ use rustfs_policy::policy::Args;
 use rustfs_policy::policy::opa;
 use rustfs_policy::policy::{Policy, PolicyDoc, iam_policy_claim_name_sa, policy_needs_existing_object_tag_for_args};
 use serde_json::Value;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 use std::sync::OnceLock;
 use time::OffsetDateTime;
@@ -61,10 +61,45 @@ const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-polic
 const FEDERATED_POLICY_BOUNDARY_CLAIM: &str = "x-rustfs-internal-federated-boundary";
 pub(crate) const SITE_REPLICATOR_CLAIM: &str = "site-replicator";
 
-static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
+#[derive(Clone)]
+enum PolicyPluginState {
+    Disabled,
+    Initializing,
+    Ready(opa::AuthZPlugin),
+    Failed,
+}
 
-fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
-    POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
+static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
+
+fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
+    POLICY_PLUGIN_STATE
+        .get_or_init(|| {
+            let configured = opa::is_configured();
+            let state = Arc::new(RwLock::new(if configured {
+                PolicyPluginState::Initializing
+            } else {
+                PolicyPluginState::Disabled
+            }));
+            if configured {
+                let state = Arc::clone(&state);
+                tokio::spawn(async move {
+                    let next_state = match opa::lookup_config().await {
+                        Ok(conf) if conf.enable() => {
+                            info!("OPA plugin enabled");
+                            PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
+                        }
+                        Ok(_) => PolicyPluginState::Failed,
+                        Err(e) => {
+                            error!("Error loading OPA configuration err:{}", e);
+                            PolicyPluginState::Failed
+                        }
+                    };
+                    *state.write().await = next_state;
+                });
+            }
+            state
+        })
+        .clone()
 }
 
 pub struct IamSys<T> {
@@ -173,19 +208,7 @@ impl<T: Store> IamSys<T> {
     /// # Returns
     /// A new instance of IamSys
     pub fn new(store: Arc<IamCache<T>>) -> Self {
-        tokio::spawn(async move {
-            match opa::lookup_config().await {
-                Ok(conf) => {
-                    if conf.enable() {
-                        Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
-                        info!("OPA plugin enabled");
-                    }
-                }
-                Err(e) => {
-                    error!("Error loading OPA configuration err:{}", e);
-                }
-            };
-        });
+        get_policy_plugin_state();
 
         Self {
             store,
@@ -206,15 +229,22 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
-        let policy_plugin_client = get_policy_plugin_client();
-        let mut guard = policy_plugin_client.write().await;
-        *guard = Some(client);
+        let policy_plugin_state = get_policy_plugin_state();
+        let mut guard = policy_plugin_state.write().await;
+        *guard = PolicyPluginState::Ready(client);
     }
 
     pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
-        let policy_plugin_client = get_policy_plugin_client();
-        let guard = policy_plugin_client.read().await;
-        guard.clone()
+        let policy_plugin_state = get_policy_plugin_state();
+        let guard = policy_plugin_state.read().await;
+        match &*guard {
+            PolicyPluginState::Ready(client) => Some(client.clone()),
+            PolicyPluginState::Disabled | PolicyPluginState::Initializing | PolicyPluginState::Failed => None,
+        }
+    }
+
+    async fn policy_plugin_state() -> PolicyPluginState {
+        get_policy_plugin_state().read().await.clone()
     }
 
     pub async fn load_group(&self, name: &str) -> Result<()> {
@@ -1060,11 +1090,20 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        if Self::get_policy_plugin_client().await.is_some() {
-            return PreparedIamAuth {
-                needs_existing_object_tag: false,
-                mode: PreparedIamMode::Opa,
-            };
+        match Self::policy_plugin_state().await {
+            PolicyPluginState::Ready(_) => {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::Opa,
+                };
+            }
+            PolicyPluginState::Initializing | PolicyPluginState::Failed => {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::Deny,
+                };
+            }
+            PolicyPluginState::Disabled => {}
         }
 
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
@@ -1163,7 +1202,22 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        let combined_policy = self.get_combined_policy(&policies).await;
+        let (resolved_policies, combined_policy) = self.store.merge_policies(&policies.join(",")).await;
+        if args.deny_only {
+            let resolved_policy_names: HashSet<&str> = resolved_policies
+                .split(',')
+                .filter(|policy_name| !policy_name.trim().is_empty())
+                .collect();
+            if policies
+                .iter()
+                .any(|policy_name| !resolved_policy_names.contains(policy_name.as_str()))
+            {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::Deny,
+                };
+            }
+        }
         PreparedIamAuth {
             needs_existing_object_tag: policy_needs_existing_object_tag_for_args(&combined_policy, args).await,
             mode: PreparedIamMode::Regular { combined_policy },
@@ -1703,7 +1757,7 @@ mod tests {
     fn deprecated_list_polices_api_is_available() {
         let _ = IamSys::<StsTestMockStore>::list_polices;
     }
-    use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
+    use rustfs_policy::policy::action::{Action, AdminAction, S3Action, StsAction};
     use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
     use serde_json::Value;
     use std::{
@@ -3309,6 +3363,155 @@ mod tests {
         assert!(!iam_sys.eval_prepared(&prepared, &args).await);
     }
 
+    #[tokio::test]
+    async fn regular_deny_only_rejects_unresolved_mapped_policy() {
+        let iam_sys = test_iam_sys().await;
+        let user = "regular-unresolved-policy";
+        let identity = UserIdentity::from(Credentials {
+            access_key: user.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(user, &identity, now);
+            cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
+        });
+
+        let groups = None;
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: user,
+            groups: &groups,
+            action: Action::StsAction(StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Deny));
+        assert!(
+            !iam_sys.eval_prepared(&prepared, &args).await,
+            "deny-only evaluation must fail closed when a mapped policy document is unresolved"
+        );
+    }
+
+    #[tokio::test]
+    async fn regular_full_evaluation_keeps_resolved_part_of_partial_mapping() {
+        let iam_sys = test_iam_sys().await;
+        let user = "regular-partial-policy";
+        let identity = UserIdentity::from(Credentials {
+            access_key: user.to_string(),
+            secret_key: "longenoughsecret".to_string(),
+            status: ACCOUNT_ON.to_string(),
+            ..Default::default()
+        });
+        iam_sys.store.cache.with_write_lock(|cache| {
+            let now = OffsetDateTime::now_utc();
+            cache.add_or_update_user(user, &identity, now);
+            cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
+        });
+
+        let groups = None;
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: user,
+            groups: &groups,
+            action: Action::S3Action(S3Action::GetObjectAction),
+            bucket: "bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "object",
+            claims: &claims,
+            deny_only: false,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
+        assert!(
+            iam_sys.eval_prepared(&prepared, &args).await,
+            "full evaluation must preserve the historical partial-resolution behavior"
+        );
+    }
+
+    #[tokio::test]
+    async fn regular_deny_only_honors_group_derived_explicit_deny() {
+        let iam_sys = test_iam_sys().await;
+        let deny_policy =
+            Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["sts:AssumeRole"]}]}"#)
+                .expect("group deny policy should parse");
+        let now = OffsetDateTime::now_utc();
+        iam_sys
+            .store
+            .cache
+            .add_or_update_policy_doc("deny-assume-role", &PolicyDoc::new(deny_policy), now);
+        iam_sys
+            .store
+            .cache
+            .add_or_update_group_policy("testgroup", &MappedPolicy::new("deny-assume-role"), now);
+
+        let groups = Some(vec!["testgroup".to_string()]);
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: "sts-fallback-test-parent",
+            groups: &groups,
+            action: Action::StsAction(StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
+        assert!(
+            !iam_sys.eval_prepared(&prepared, &args).await,
+            "group-derived explicit Deny must remain authoritative"
+        );
+    }
+
+    #[tokio::test]
+    async fn regular_deny_only_rejects_unresolved_group_mapping() {
+        let iam_sys = test_iam_sys().await;
+        iam_sys.store.cache.add_or_update_group_policy(
+            "testgroup",
+            &MappedPolicy::new("readwrite,missing-policy"),
+            OffsetDateTime::now_utc(),
+        );
+
+        let groups = Some(vec!["testgroup".to_string()]);
+        let claims = HashMap::new();
+        let conditions = HashMap::new();
+        let args = Args {
+            account: "sts-fallback-test-parent",
+            groups: &groups,
+            action: Action::StsAction(StsAction::AssumeRoleAction),
+            bucket: "",
+            conditions: &conditions,
+            is_owner: false,
+            object: "",
+            claims: &claims,
+            deny_only: true,
+        };
+
+        let prepared = iam_sys.prepare_regular_auth(&args).await;
+        assert!(matches!(prepared.mode, PreparedIamMode::Deny));
+        assert!(
+            !iam_sys.eval_prepared(&prepared, &args).await,
+            "deny-only evaluation must fail closed for an unresolved group-derived mapping"
+        );
+    }
+
     #[tokio::test]
     async fn test_sts_claim_policy_ignores_unsafe_and_missing_policy_names() {
         let store = StsTestMockStore::new(true);

crates/kms/Cargo.toml

@@ -64,6 +64,10 @@ md-5 = { workspace = true }
 arc-swap = { workspace = true }
 rustfs-utils = { workspace = true }
 rustfs-security-governance = { workspace = true }
+# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
+# so the audit sink can live outside this crate without a second, drifting
+# copy of the event vocabulary.
+rustfs-s3-types = { workspace = true }
 
 # HTTP client for Vault
 reqwest = { workspace = true }

crates/kms/examples/kms_local_demo.rs

@@ -227,10 +227,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Step 12: Check cache statistics
     println!("12. Checking cache statistics...");
-    if let Some((hits, misses)) = encryption_service.cache_stats().await {
+    if let Some(stats) = encryption_service.cache_stats().await {
         println!("   ✓ Cache statistics:");
-        println!("     - Cache hits: {}", hits);
-        println!("     - Cache misses: {}\n", misses);
+        println!("     - Cached entries: {}", stats.entries);
+        println!("     - Cache hits: {}", stats.hits);
+        println!("     - Cache misses: {}", stats.misses);
+        println!("     - Entries evicted: {}\n", stats.evictions);
     } else {
         println!("   - Cache is disabled\n");
     }

crates/kms/examples/kms_vault_kv_demo.rs

@@ -263,10 +263,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Step 12: Check cache statistics
     println!("12. Checking cache statistics...");
-    if let Some((hits, misses)) = encryption_service.cache_stats().await {
+    if let Some(stats) = encryption_service.cache_stats().await {
         println!("   ✓ Cache statistics:");
-        println!("     - Cache hits: {}", hits);
-        println!("     - Cache misses: {}\n", misses);
+        println!("     - Cached entries: {}", stats.entries);
+        println!("     - Cache hits: {}", stats.hits);
+        println!("     - Cache misses: {}", stats.misses);
+        println!("     - Entries evicted: {}\n", stats.evictions);
     } else {
         println!("   - Cache is disabled\n");
     }

crates/kms/src/audit.rs

@@ -0,0 +1,423 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Audit contract for KMS management operations.
+//!
+//! [`KmsManager`](crate::manager::KmsManager) builds a [`KmsAuditRecord`] for
+//! every management operation it serves and hands it to the installed
+//! [`KmsAuditSink`]. No sink is installed by default, so a deployment that
+//! does not consume KMS audit records pays nothing beyond the `Option` check.
+//!
+//! The record is deliberately a KMS-side type rather than an audit-crate
+//! entry: keeping the dependency edge out of this crate lets the server map
+//! records onto its existing audit pipeline (and its delivery semantics)
+//! without KMS having to know that pipeline exists.
+//!
+//! # Redaction
+//!
+//! A record has no field that can hold key material, and every value that
+//! originates from a caller passes through [`redact_encryption_context`]
+//! before it is stored. Adding a field here means re-checking that invariant.
+
+use crate::error::KmsError;
+use crate::types::OperationContext;
+use rustfs_s3_types::EventName;
+use sha2::{Digest, Sha256};
+use std::collections::{BTreeMap, HashMap};
+use std::time::Duration;
+use uuid::Uuid;
+
+/// Encryption-context keys that describe *where* an object lives rather than
+/// anything secret about it. Their values are recorded verbatim; every other
+/// key is reduced to a digest.
+const ENCRYPTION_CONTEXT_ALLOWLIST: [&str; 5] = ["bucket", "object", "object_key", "algorithm", "sse_type"];
+
+/// Prefix marking a value that was replaced by a digest of itself.
+const DIGEST_PREFIX: &str = "sha256:";
+
+/// Hex characters of the digest that are kept. Enough to correlate repeated
+/// values across records without being reversible in practice.
+const DIGEST_LEN: usize = 16;
+
+/// A KMS management operation that produces an audit record.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum KmsAuditOperation {
+    /// Creation of a new master key.
+    CreateKey,
+    /// Metadata lookup for a single key.
+    DescribeKey,
+    /// Enumeration of keys.
+    ListKeys,
+    /// Scheduling a key for deletion after the pending window.
+    ScheduleKeyDeletion,
+    /// Cancelling a previously scheduled deletion.
+    CancelKeyDeletion,
+    /// Returning a disabled key to service.
+    EnableKey,
+    /// Taking a key out of service without destroying it.
+    DisableKey,
+    /// Rotating a key to a new version.
+    RotateKey,
+    /// Irreversible removal of key material once the pending window expired.
+    DeleteKey,
+}
+
+impl KmsAuditOperation {
+    /// Stable operation name for audit consumers.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            KmsAuditOperation::CreateKey => "CreateKey",
+            KmsAuditOperation::DescribeKey => "DescribeKey",
+            KmsAuditOperation::ListKeys => "ListKeys",
+            KmsAuditOperation::ScheduleKeyDeletion => "ScheduleKeyDeletion",
+            KmsAuditOperation::CancelKeyDeletion => "CancelKeyDeletion",
+            KmsAuditOperation::EnableKey => "EnableKey",
+            KmsAuditOperation::DisableKey => "DisableKey",
+            KmsAuditOperation::RotateKey => "RotateKey",
+            KmsAuditOperation::DeleteKey => "DeleteKey",
+        }
+    }
+
+    /// Notification/audit event name carried by records for this operation.
+    pub fn event_name(&self) -> EventName {
+        match self {
+            KmsAuditOperation::CreateKey => EventName::KmsKeyCreated,
+            KmsAuditOperation::DescribeKey | KmsAuditOperation::ListKeys => EventName::KmsKeyAccessed,
+            KmsAuditOperation::ScheduleKeyDeletion => EventName::KmsKeyDeletionScheduled,
+            KmsAuditOperation::CancelKeyDeletion => EventName::KmsKeyDeletionCancelled,
+            KmsAuditOperation::EnableKey => EventName::KmsKeyEnabled,
+            KmsAuditOperation::DisableKey => EventName::KmsKeyDisabled,
+            KmsAuditOperation::RotateKey => EventName::KmsKeyRotated,
+            KmsAuditOperation::DeleteKey => EventName::KmsKeyDeleted,
+        }
+    }
+}
+
+/// Whether the audited operation completed.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum KmsAuditOutcome {
+    /// The operation completed and its effect is durable.
+    Success,
+    /// The operation failed; `error_class` carries the reason category.
+    Failure,
+}
+
+impl KmsAuditOutcome {
+    /// Stable outcome name for audit consumers.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            KmsAuditOutcome::Success => "success",
+            KmsAuditOutcome::Failure => "failure",
+        }
+    }
+}
+
+/// Coarse, stable classification of a failed KMS operation.
+///
+/// Audit consumers alert on these, so the strings are a wire contract: add
+/// new classes rather than renaming existing ones.
+pub fn error_class(error: &KmsError) -> &'static str {
+    match error {
+        KmsError::ConfigurationError { .. } => "configuration",
+        KmsError::KeyNotFound { .. } => "key_not_found",
+        KmsError::InvalidKey { .. } => "invalid_key",
+        KmsError::CryptographicError { .. } => "cryptographic",
+        KmsError::BackendError { .. } => "backend",
+        KmsError::AccessDenied { .. } => "access_denied",
+        KmsError::KeyAlreadyExists { .. } => "key_already_exists",
+        KmsError::InvalidOperation { .. } => "invalid_operation",
+        KmsError::InternalError { .. } => "internal",
+        KmsError::SerializationError { .. } => "serialization",
+        KmsError::IoError { .. } => "io",
+        KmsError::CacheError { .. } => "cache",
+        KmsError::ValidationError { .. } => "validation",
+        KmsError::UnsupportedAlgorithm { .. } => "unsupported_algorithm",
+        KmsError::InvalidKeySize { .. } => "invalid_key_size",
+        KmsError::ContextMismatch { .. } => "context_mismatch",
+        KmsError::OperationTimedOut { .. } => "timeout",
+        KmsError::OperationCancelled { .. } => "cancelled",
+        KmsError::MaterialMissing { .. } => "material_missing",
+        KmsError::MaterialCorrupt { .. } => "material_corrupt",
+        KmsError::MaterialAuthenticationFailed { .. } => "material_authentication_failed",
+        KmsError::UnsupportedFormatVersion { .. } => "unsupported_format_version",
+        KmsError::KeyVersionNotFound { .. } => "key_version_not_found",
+        KmsError::Backup(_) => "backup",
+        KmsError::UnsupportedCapability { .. } => "unsupported_capability",
+        KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
+    }
+}
+
+/// One audited KMS management operation.
+///
+/// Everything an audit consumer needs to answer "who did what to which key,
+/// against which backend, and did it work" — and nothing that could carry key
+/// material. See the module docs for the redaction rules.
+///
+/// Tenant attribution is intentionally absent: multi-tenancy is not modelled
+/// yet, and an invented tenant value is worse than a missing one. It will
+/// arrive as an entry in [`Self::context`] once tenancy lands.
+#[derive(Debug, Clone)]
+pub struct KmsAuditRecord {
+    /// Correlates every record emitted for one logical request.
+    pub operation_id: Uuid,
+    /// The audited operation.
+    pub operation: KmsAuditOperation,
+    /// Event name published to audit consumers.
+    pub event: EventName,
+    /// Authenticated identity that requested the operation, or
+    /// [`OperationContext::INTERNAL_PRINCIPAL`] for server-initiated work.
+    pub principal: String,
+    /// Client address, when the caller supplied one.
+    pub source_ip: Option<String>,
+    /// Client user agent, when the caller supplied one.
+    pub user_agent: Option<String>,
+    /// Key the operation acted on. `None` for operations that span keys, such
+    /// as listing.
+    pub key_id: Option<String>,
+    /// Key version the operation resolved to, when the result identifies one.
+    /// Operations on a key as a whole leave this unset.
+    pub key_version: Option<u32>,
+    /// Whether the operation succeeded.
+    pub outcome: KmsAuditOutcome,
+    /// Failure category; `None` on success. See [`error_class`].
+    pub error_class: Option<&'static str>,
+    /// Backend that served the operation, e.g. `local` or `vault-transit`.
+    pub backend: &'static str,
+    /// Wall-clock time spent in the audited operation.
+    pub latency: Duration,
+    /// Retries performed above the audit point. `None` means the number is
+    /// not observable here: backend-internal retries are accounted for by the
+    /// `rustfs_kms_backend_operation_attempts` metric instead of being
+    /// duplicated (and possibly contradicted) in the audit trail.
+    pub retry_count: Option<u32>,
+    /// Server-supplied correlation values carried by the operation context.
+    /// Also the reserved slot for tenant attribution.
+    pub context: BTreeMap<String, String>,
+    /// Caller-supplied encryption context, after [`redact_encryption_context`].
+    pub encryption_context: BTreeMap<String, String>,
+}
+
+impl KmsAuditRecord {
+    /// Start a record for `operation` performed under `context`.
+    ///
+    /// The outcome defaults to failure so that a record which somehow escapes
+    /// without [`Self::with_result`] under-reports success rather than
+    /// inventing it.
+    pub fn new(operation: KmsAuditOperation, context: &OperationContext, backend: &'static str) -> Self {
+        Self {
+            operation_id: context.operation_id,
+            operation,
+            event: operation.event_name(),
+            principal: context.principal.clone(),
+            source_ip: context.source_ip.clone(),
+            user_agent: context.user_agent.clone(),
+            key_id: None,
+            key_version: None,
+            outcome: KmsAuditOutcome::Failure,
+            error_class: None,
+            backend,
+            latency: Duration::ZERO,
+            retry_count: None,
+            context: context
+                .additional_context
+                .iter()
+                .map(|(k, v)| (k.clone(), v.clone()))
+                .collect(),
+            encryption_context: BTreeMap::new(),
+        }
+    }
+
+    /// Attach the key the operation acted on.
+    pub fn with_key_id(mut self, key_id: Option<impl Into<String>>) -> Self {
+        self.key_id = key_id.map(Into::into);
+        self
+    }
+
+    /// Attach the key version the operation resolved to.
+    pub fn with_key_version(mut self, key_version: Option<u32>) -> Self {
+        self.key_version = key_version;
+        self
+    }
+
+    /// Attach the measured duration of the operation.
+    pub fn with_latency(mut self, latency: Duration) -> Self {
+        self.latency = latency;
+        self
+    }
+
+    /// Derive outcome and error class from the operation result.
+    pub fn with_result<T>(mut self, result: &crate::error::Result<T>) -> Self {
+        match result {
+            Ok(_) => {
+                self.outcome = KmsAuditOutcome::Success;
+                self.error_class = None;
+            }
+            Err(error) => {
+                self.outcome = KmsAuditOutcome::Failure;
+                self.error_class = Some(error_class(error));
+            }
+        }
+        self
+    }
+
+    /// Attach the caller-supplied encryption context, redacted.
+    ///
+    /// Redaction happens here rather than at the call site so no caller can
+    /// place a raw context value into a record by accident.
+    pub fn with_encryption_context(mut self, encryption_context: &HashMap<String, String>) -> Self {
+        self.encryption_context = redact_encryption_context(encryption_context);
+        self
+    }
+}
+
+/// Reduce a caller-supplied encryption context to something safe to persist.
+///
+/// Keys naming the object's location are kept verbatim because they are the
+/// reason the context is audited at all. Every other value is replaced by a
+/// truncated digest, which still correlates repeated values across records
+/// but does not reproduce whatever the caller put there.
+pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -> BTreeMap<String, String> {
+    encryption_context
+        .iter()
+        .map(|(key, value)| {
+            let recorded = if ENCRYPTION_CONTEXT_ALLOWLIST.contains(&key.as_str()) {
+                value.clone()
+            } else {
+                digest_value(value)
+            };
+            (key.clone(), recorded)
+        })
+        .collect()
+}
+
+fn digest_value(value: &str) -> String {
+    let digest = hex::encode(Sha256::digest(value.as_bytes()));
+    format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
+}
+
+/// Receives KMS audit records.
+///
+/// Implementations must not block: they are called on the task that served
+/// the KMS operation. Delivery failures are the sink's problem — the KMS
+/// operation has already completed by the time a record is emitted, and its
+/// result is never changed by what the sink does.
+pub trait KmsAuditSink: Send + Sync {
+    /// Handle one audit record.
+    fn emit(&self, record: KmsAuditRecord);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn every_operation_maps_to_a_kms_event() {
+        let operations = [
+            KmsAuditOperation::CreateKey,
+            KmsAuditOperation::DescribeKey,
+            KmsAuditOperation::ListKeys,
+            KmsAuditOperation::ScheduleKeyDeletion,
+            KmsAuditOperation::CancelKeyDeletion,
+            KmsAuditOperation::EnableKey,
+            KmsAuditOperation::DisableKey,
+            KmsAuditOperation::RotateKey,
+            KmsAuditOperation::DeleteKey,
+        ];
+
+        for operation in operations {
+            let event = operation.event_name();
+            assert!(event.is_kms(), "{} must map to a KMS event, got {event}", operation.as_str());
+        }
+    }
+
+    #[test]
+    fn record_defaults_to_failure_until_a_result_is_attached() {
+        let context = OperationContext::new("tester".to_string());
+        let record = KmsAuditRecord::new(KmsAuditOperation::CreateKey, &context, "local");
+
+        assert_eq!(record.outcome, KmsAuditOutcome::Failure);
+        assert!(record.error_class.is_none());
+
+        let ok: crate::error::Result<()> = Ok(());
+        assert_eq!(record.clone().with_result(&ok).outcome, KmsAuditOutcome::Success);
+
+        let denied: crate::error::Result<()> = Err(KmsError::access_denied("nope"));
+        let failed = record.with_result(&denied);
+        assert_eq!(failed.outcome, KmsAuditOutcome::Failure);
+        assert_eq!(failed.error_class, Some("access_denied"));
+    }
+
+    #[test]
+    fn record_carries_the_full_operation_context() {
+        let context = OperationContext::new("arn:aws:iam::user/alice".to_string())
+            .with_source_ip("10.0.0.7".to_string())
+            .with_user_agent("aws-cli/2".to_string())
+            .with_context("requestID".to_string(), "req-1".to_string());
+
+        let record = KmsAuditRecord::new(KmsAuditOperation::RotateKey, &context, "vault-transit")
+            .with_key_id(Some("key-1"))
+            .with_key_version(Some(3))
+            .with_latency(Duration::from_millis(12));
+
+        assert_eq!(record.operation_id, context.operation_id);
+        assert_eq!(record.principal, "arn:aws:iam::user/alice");
+        assert_eq!(record.source_ip.as_deref(), Some("10.0.0.7"));
+        assert_eq!(record.user_agent.as_deref(), Some("aws-cli/2"));
+        assert_eq!(record.key_id.as_deref(), Some("key-1"));
+        assert_eq!(record.key_version, Some(3));
+        assert_eq!(record.backend, "vault-transit");
+        assert_eq!(record.latency, Duration::from_millis(12));
+        assert_eq!(record.event, EventName::KmsKeyRotated);
+        assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-1"));
+    }
+
+    #[test]
+    fn encryption_context_keeps_only_allowlisted_values_verbatim() {
+        let secret = "eyJhbGciOiJIUzI1NiJ9.super-secret-grant-token";
+        let context = HashMap::from([
+            ("bucket".to_string(), "photos".to_string()),
+            ("object_key".to_string(), "2026/cat.png".to_string()),
+            ("algorithm".to_string(), "AES256".to_string()),
+            ("grant_token".to_string(), secret.to_string()),
+            ("customer_reference".to_string(), "acct-4711".to_string()),
+        ]);
+
+        let redacted = redact_encryption_context(&context);
+
+        assert_eq!(redacted.get("bucket").map(String::as_str), Some("photos"));
+        assert_eq!(redacted.get("object_key").map(String::as_str), Some("2026/cat.png"));
+        assert_eq!(redacted.get("algorithm").map(String::as_str), Some("AES256"));
+
+        for key in ["grant_token", "customer_reference"] {
+            let value = redacted.get(key).expect("non-allowlisted key should be kept as a digest");
+            assert!(value.starts_with(DIGEST_PREFIX), "{key} should be digested, got {value}");
+        }
+
+        let rendered = format!("{redacted:?}");
+        assert!(!rendered.contains(secret), "redacted context must not reproduce the raw value");
+        assert!(!rendered.contains("acct-4711"), "redacted context must not reproduce the raw value");
+    }
+
+    #[test]
+    fn identical_values_digest_identically_across_records() {
+        // Correlation is the reason a digest is preferable to a fixed
+        // placeholder; assert it actually holds.
+        let first = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
+        let second = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
+        let other = redact_encryption_context(&HashMap::from([("tenant".to_string(), "beta".to_string())]));
+
+        assert_eq!(first.get("tenant"), second.get("tenant"));
+        assert_ne!(first.get("tenant"), other.get("tenant"));
+    }
+}

crates/kms/src/backends/local.rs

@@ -46,7 +46,10 @@ use zeroize::Zeroizing;
 /// `key_dir` is accepted, so identifiers already in use by existing deployments keep
 /// resolving. Only separators, traversal and the degenerate cases are refused, which is
 /// what stops `key_dir.join(...)` from escaping.
-fn validate_key_id(key_id: &str) -> Result<()> {
+///
+/// pub(crate) because the backup restore path applies the same containment
+/// rule to key identifiers recovered from bundle artifacts.
+pub(crate) fn validate_key_id(key_id: &str) -> Result<()> {
     if key_id.is_empty() {
         return Err(KmsError::invalid_key("key identifier must not be empty"));
     }
@@ -68,7 +71,15 @@ fn validate_key_id(key_id: &str) -> Result<()> {
     }
 }
 
-const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt";
+// The salt and restore-marker file names are pub(crate) so the backup/restore
+// modules (`crate::backup`) address the exact on-disk names instead of copies
+// that could drift.
+pub(crate) const LOCAL_KMS_MASTER_KEY_SALT_FILE: &str = ".master-key.salt";
+/// Commit marker of an in-progress Local restore cutover (see
+/// `crate::backup::local_restore`). Its presence means the key directory is
+/// mid-cutover: startup must fail closed until the restore is rolled forward
+/// or explicitly aborted.
+pub(crate) const LOCAL_RESTORE_COMMIT_MARKER_FILE: &str = ".restore-commit.json";
 // The KDF parameters are pub(crate) so the backup manifest contract
 // (`crate::backup`) records the exact compiled-in derivation instead of a
 // copy that could drift.
@@ -87,7 +98,11 @@ pub(crate) const LOCAL_KMS_ARGON2_P_COST: u32 = 1;
 /// `foo.tmp-<uuid>` is stored as `foo.tmp-<uuid>.key` — so the `.key` guard
 /// plus the exact hyphenated-UUID check makes it impossible to match an
 /// authoritative file.
-fn is_orphan_commit_temp_name(file_name: &str) -> bool {
+///
+/// pub(crate) because the backup restore path applies the same classification
+/// when it re-enters an interrupted run: a leftover commit temp is never
+/// authoritative state, so it does not make a target non-empty.
+pub(crate) fn is_orphan_commit_temp_name(file_name: &str) -> bool {
     if file_name.ends_with(".key") {
         return false;
     }
@@ -110,12 +125,15 @@ fn is_orphan_commit_temp_name(file_name: &str) -> bool {
 ///
 /// This intentionally mirrors ecstore's fsync helpers without depending on the
 /// ecstore crate: the KMS backend stays decoupled from storage internals.
-mod durable_file {
+///
+/// pub(crate) because the backup restore path (`crate::backup::local_restore`)
+/// commits staged files and its cutover marker through the same protocol.
+pub(crate) mod durable_file {
     use std::io::{self, Write};
     use std::path::{Path, PathBuf};
 
     /// How the fully written temp file becomes visible under its final name.
-    pub(super) enum Publish {
+    pub(crate) enum Publish {
         /// Atomically replace whatever is at the destination via `rename`.
         Replace,
         /// Publish via `hard_link`, failing with [`CommitError::AlreadyExists`]
@@ -124,7 +142,7 @@ mod durable_file {
     }
 
     #[derive(Debug)]
-    pub(super) enum CommitError {
+    pub(crate) enum CommitError {
         AlreadyExists,
         Io(io::Error),
         /// Test-only simulated crash: the protocol stops after the given step
@@ -158,14 +176,14 @@ mod durable_file {
     /// to prove that every interrupted prefix recovers to either the complete
     /// old state or the complete new state.
     #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-    pub(super) enum CommitStep {
+    pub(crate) enum CommitStep {
         TempWritten,
         FileSynced,
         Published,
         DirSynced,
     }
 
-    pub(super) async fn commit(
+    pub(crate) async fn commit(
         temp_path: PathBuf,
         final_path: PathBuf,
         content: Vec<u8>,
@@ -179,7 +197,7 @@ mod durable_file {
 
     /// Remove a published file durably: without the parent directory fsync a
     /// deleted key could resurface after power loss.
-    pub(super) async fn remove_durably(path: PathBuf) -> io::Result<()> {
+    pub(crate) async fn remove_durably(path: PathBuf) -> io::Result<()> {
         tokio::task::spawn_blocking(move || {
             std::fs::remove_file(&path)?;
             let parent = path
@@ -191,6 +209,41 @@ mod durable_file {
         .map_err(io::Error::other)?
     }
 
+    /// Publish an already-durable file under a second name via `hard_link`,
+    /// then fsync the destination's parent directory.
+    ///
+    /// This is the restore cutover primitive: the source (a staged file that
+    /// went through [`commit`]) is already durable, so linking plus a parent
+    /// fsync is a complete publish. `AlreadyExists` is idempotent success only
+    /// when the destination content is byte-identical to the source — that is
+    /// exactly the re-entry case of a cutover interrupted after this link —
+    /// and a hard failure otherwise, so the primitive can never clobber or
+    /// silently accept foreign state.
+    pub(crate) async fn link_durably(source: PathBuf, dest: PathBuf) -> io::Result<()> {
+        tokio::task::spawn_blocking(move || {
+            match std::fs::hard_link(&source, &dest) {
+                Ok(()) => {}
+                Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
+                    let existing = std::fs::read(&dest)?;
+                    let staged = std::fs::read(&source)?;
+                    if existing != staged {
+                        return Err(io::Error::new(
+                            io::ErrorKind::AlreadyExists,
+                            format!("destination {} already exists with different content", dest.display()),
+                        ));
+                    }
+                }
+                Err(error) => return Err(error),
+            }
+            let parent = dest
+                .parent()
+                .ok_or_else(|| io::Error::other("destination has no parent directory"))?;
+            fsync_dir(parent)
+        })
+        .await
+        .map_err(io::Error::other)?
+    }
+
     fn commit_blocking(
         temp_path: &Path,
         final_path: &Path,
@@ -355,7 +408,7 @@ mod durable_file {
     /// Test-only failpoints simulating a crash after a given commit step.
     /// Armed per directory so parallel tests never affect each other.
     #[cfg(test)]
-    pub(super) mod failpoint {
+    pub(crate) mod failpoint {
         use super::CommitStep;
         use std::path::{Path, PathBuf};
         use std::sync::Mutex;
@@ -397,6 +450,20 @@ pub struct LocalKmsClient {
     /// Per-key write locks serializing read-modify-write updates within this
     /// process (see [`Self::lock_key_for_write`]).
     key_write_locks: Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
+    /// Directory-wide writer fence for backup export (see
+    /// [`Self::acquire_export_fence`]). Writers hold the read side; an export
+    /// snapshot holds the write side so it observes a single-generation view.
+    export_fence: Arc<tokio::sync::RwLock<()>>,
+}
+
+/// Guard pairing the export-fence read lock with a per-key write mutex.
+///
+/// Dropping it releases both, so every existing `lock_key_for_write` call
+/// site participates in the export fence without changes.
+#[must_use]
+struct KeyWriteGuard {
+    _fence: tokio::sync::OwnedRwLockReadGuard<()>,
+    _key: tokio::sync::OwnedMutexGuard<()>,
 }
 
 // pub(crate) so the backup contract tests can anchor the manifest's
@@ -446,6 +513,11 @@ impl LocalKmsClient {
             debug!(path = ?config.key_dir, "KMS key directory created");
         }
 
+        // The restore-marker guard must run before anything else touches the
+        // directory (in particular before salt load/creation): a directory
+        // mid-cutover holds an arbitrary mix of old and new state.
+        Self::ensure_no_restore_marker(&config).await?;
+
         // Initialize master cipher if master key is provided
         let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key {
             let salt = Self::load_or_create_master_key_salt(&config).await?;
@@ -463,6 +535,7 @@ impl LocalKmsClient {
             legacy_master_cipher,
             dek_crypto: AesDekCrypto::new(),
             key_write_locks: Mutex::new(HashMap::new()),
+            export_fence: Arc::new(tokio::sync::RwLock::new(())),
         };
         client.validate_existing_keys().await?;
         Ok(client)
@@ -476,6 +549,7 @@ impl LocalKmsClient {
         if !fs::try_exists(&config.key_dir).await? {
             return Err(KmsError::configuration_error("Local KMS key directory does not exist"));
         }
+        Self::ensure_no_restore_marker(&config).await?;
 
         let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key {
             let legacy_key = Self::derive_legacy_master_key(master_key)?;
@@ -505,6 +579,7 @@ impl LocalKmsClient {
             legacy_master_cipher,
             dek_crypto: AesDekCrypto::new(),
             key_write_locks: Mutex::new(HashMap::new()),
+            export_fence: Arc::new(tokio::sync::RwLock::new(())),
         })
     }
 
@@ -515,16 +590,72 @@ impl LocalKmsClient {
     /// delete with a rewrite. Cross-process writers sharing a key directory
     /// remain unsupported. Entries live for the client's lifetime; the table
     /// is bounded by the number of distinct key ids this process touches.
-    async fn lock_key_for_write(&self, key_id: &str) -> tokio::sync::OwnedMutexGuard<()> {
+    async fn lock_key_for_write(&self, key_id: &str) -> KeyWriteGuard {
+        // Fence first, per-key mutex second: the ordering is uniform across
+        // all writers, so an export waiting on the write side can never
+        // deadlock with a writer holding a key mutex.
+        let fence = Arc::clone(&self.export_fence).read_owned().await;
         let lock = {
             let mut locks = self.key_write_locks.lock().expect("Local KMS key write lock table poisoned");
             Arc::clone(locks.entry(key_id.to_string()).or_default())
         };
-        lock.lock_owned().await
+        KeyWriteGuard {
+            _fence: fence,
+            _key: lock.lock_owned().await,
+        }
+    }
+
+    /// Block every key-directory writer while a backup export collects its
+    /// snapshot, so all records belong to one generation.
+    ///
+    /// Mutating operations hold the read side (via [`Self::lock_key_for_write`]
+    /// or [`Self::save_new_master_key`]); the export holds the write side only
+    /// for the collection phase, never while encrypting or writing the bundle.
+    pub(crate) async fn acquire_export_fence(&self) -> tokio::sync::OwnedRwLockWriteGuard<()> {
+        Arc::clone(&self.export_fence).write_owned().await
+    }
+
+    /// Key directory root, exposed for the backup export module.
+    pub(crate) fn key_directory(&self) -> &Path {
+        &self.config.key_dir
+    }
+
+    /// Absolute path of the master-key KDF salt file, exposed for the backup
+    /// export module.
+    pub(crate) fn master_key_salt_file(&self) -> PathBuf {
+        Self::master_key_salt_path(&self.config)
+    }
+
+    /// Operator-configured master key string, exposed for the backup export
+    /// module so it can record a one-way verifier in the bundle manifest.
+    /// Never log or persist this value.
+    pub(crate) fn configured_master_key(&self) -> Option<&str> {
+        self.config.master_key.as_deref()
+    }
+
+    /// Fail closed while a restore cutover marker is present: the directory
+    /// then holds an arbitrary mix of pre-restore and restored state, and the
+    /// only valid next steps are re-running the restore with the same bundle
+    /// (roll forward) or explicitly aborting it. This mirrors the missing-salt
+    /// guard: startup must never paper over a half-applied restore.
+    async fn ensure_no_restore_marker(config: &LocalConfig) -> Result<()> {
+        let marker = config.key_dir.join(LOCAL_RESTORE_COMMIT_MARKER_FILE);
+        if fs::try_exists(&marker).await? {
+            return Err(KmsError::configuration_error(format!(
+                "Local KMS key directory has an unfinished restore (marker {} present); \
+                 re-run the restore with the same bundle to roll it forward or abort it explicitly",
+                marker.display()
+            )));
+        }
+        Ok(())
     }
 
     /// Derive a 256-bit key from the master key string using a persistent Argon2id salt.
-    fn derive_master_key(master_key: &str, salt: &[u8]) -> Result<Key<Aes256Gcm>> {
+    ///
+    /// pub(crate) because the backup restore path derives the same key from
+    /// the operator-supplied master key and the bundled salt for its verifier
+    /// check and staged decryption probe.
+    pub(crate) fn derive_master_key(master_key: &str, salt: &[u8]) -> Result<Key<Aes256Gcm>> {
         let params = Params::new(
             LOCAL_KMS_ARGON2_M_COST_KIB,
             LOCAL_KMS_ARGON2_T_COST,
@@ -541,7 +672,7 @@ impl LocalKmsClient {
         Ok(key)
     }
 
-    fn derive_legacy_master_key(master_key: &str) -> Result<Key<Aes256Gcm>> {
+    pub(crate) fn derive_legacy_master_key(master_key: &str) -> Result<Key<Aes256Gcm>> {
         let mut hasher = Sha256::new();
         hasher.update(master_key.as_bytes());
         hasher.update(b"rustfs-kms-local");
@@ -797,6 +928,11 @@ impl LocalKmsClient {
     }
 
     async fn save_new_master_key(&self, master_key: &MasterKeyInfo, key_material: &[u8]) -> Result<()> {
+        // Creates never take the per-key write lock (`NoClobber` publishing
+        // already linearizes them), so they join the export fence here. This
+        // must stay the only fence acquisition on the create path: the fence
+        // read lock is not reentrant while an export waits for the write side.
+        let _fence = Arc::clone(&self.export_fence).read_owned().await;
         let key_path = self.master_key_path(&master_key.key_id)?;
         let content = self.encode_master_key(master_key, key_material)?;
         let temp_path = key_path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
@@ -2404,6 +2540,34 @@ mod tests {
         assert!(!is_orphan_commit_temp_name("mykey.tmp-"));
     }
 
+    #[tokio::test]
+    async fn link_durably_publishes_no_clobber_and_is_content_idempotent() {
+        let temp_dir = TempDir::new().expect("temp dir");
+        let source = temp_dir.path().join("staged");
+        let dest = temp_dir.path().join("published");
+        fs::write(&source, b"staged-content").await.expect("write source");
+
+        durable_file::link_durably(source.clone(), dest.clone())
+            .await
+            .expect("first link must succeed");
+        assert_eq!(fs::read(&dest).await.expect("read dest"), b"staged-content");
+
+        // Re-entry with identical content is idempotent success — exactly the
+        // resumed-cutover case.
+        durable_file::link_durably(source.clone(), dest.clone())
+            .await
+            .expect("re-linking identical content must be idempotent");
+
+        // Existing content that differs is a hard failure, never a clobber.
+        let foreign = temp_dir.path().join("foreign");
+        fs::write(&foreign, b"different-content").await.expect("write foreign");
+        let error = durable_file::link_durably(foreign, dest.clone())
+            .await
+            .expect_err("differing content must not be clobbered");
+        assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists);
+        assert_eq!(fs::read(&dest).await.expect("dest unchanged"), b"staged-content");
+    }
+
     #[tokio::test]
     async fn durable_commit_fsyncs_every_write_path() {
         use durable_file::fsync_recorder;

crates/kms/src/backends/scripted_vault.rs

@@ -26,16 +26,16 @@ use std::sync::{Arc, Mutex};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};
 
-/// One canned HTTP response.
-pub(crate) struct ScriptedResponse {
-    status: u16,
-    body: String,
+/// One scripted connection outcome.
+pub(crate) enum ScriptedResponse {
+    Http { status: u16, body: String },
+    Close,
 }
 
 impl ScriptedResponse {
     /// A 200 response carrying `data` inside the standard Vault envelope.
     pub(crate) fn ok(data: serde_json::Value) -> Self {
-        Self {
+        Self::Http {
             status: 200,
             body: serde_json::json!({
                 "request_id": "scripted",
@@ -50,18 +50,23 @@ impl ScriptedResponse {
 
     /// An error response in Vault's `{"errors": [...]}` format.
     pub(crate) fn error(status: u16, message: &str) -> Self {
-        Self {
+        Self::Http {
             status,
             body: serde_json::json!({ "errors": [message] }).to_string(),
         }
     }
+
+    /// Close the connection after consuming a request without sending an HTTP response.
+    pub(crate) fn close() -> Self {
+        Self::Close
+    }
 }
 
 /// A scripted stand-in Vault listening on a loopback port.
 pub(crate) struct ScriptedVault {
     /// Base address (`http://127.0.0.1:port`) to point a Vault client at.
     pub(crate) address: String,
-    requests: Arc<Mutex<Vec<String>>>,
+    requests: Arc<Mutex<Vec<(String, String)>>>,
 }
 
 impl ScriptedVault {
@@ -81,24 +86,21 @@ impl ScriptedVault {
                 let Ok((mut stream, _)) = listener.accept().await else {
                     return;
                 };
-                let Some(request_line) = read_request(&mut stream).await else {
+                let Some(request) = read_request(&mut stream).await else {
                     continue;
                 };
-                recorded
-                    .lock()
-                    .expect("scripted vault request log poisoned")
-                    .push(request_line);
+                recorded.lock().expect("scripted vault request log poisoned").push(request);
                 let response = responses
                     .next()
                     .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
-                let payload = format!(
-                    "HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
-                    response.status,
-                    response.body.len(),
-                    response.body
-                );
-                let _ = stream.write_all(payload.as_bytes()).await;
-                let _ = stream.shutdown().await;
+                if let ScriptedResponse::Http { status, body } = response {
+                    let payload = format!(
+                        "HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+                        body.len(),
+                    );
+                    let _ = stream.write_all(payload.as_bytes()).await;
+                    let _ = stream.shutdown().await;
+                }
             }
         });
 
@@ -107,14 +109,32 @@ impl ScriptedVault {
 
     /// The `METHOD /path` lines of every request served so far, in order.
     pub(crate) fn requests(&self) -> Vec<String> {
-        self.requests.lock().expect("scripted vault request log poisoned").clone()
+        self.requests
+            .lock()
+            .expect("scripted vault request log poisoned")
+            .iter()
+            .map(|(line, _)| line.clone())
+            .collect()
+    }
+
+    /// The request bodies, in the same order as [`Self::requests`]; empty for
+    /// bodyless requests. Lets tests assert what a write actually persisted
+    /// (record contents, check-and-set options), not just that a write happened.
+    pub(crate) fn request_bodies(&self) -> Vec<String> {
+        self.requests
+            .lock()
+            .expect("scripted vault request log poisoned")
+            .iter()
+            .map(|(_, body)| body.clone())
+            .collect()
     }
 }
 
 /// Read one HTTP/1.1 request (head plus content-length body) and return its
-/// `METHOD /path` line. Draining the body before responding keeps the client
-/// from seeing a connection reset while it is still writing.
-async fn read_request(stream: &mut TcpStream) -> Option<String> {
+/// `METHOD /path` line together with the body. Draining the body before
+/// responding keeps the client from seeing a connection reset while it is
+/// still writing.
+async fn read_request(stream: &mut TcpStream) -> Option<(String, String)> {
     let mut buffer = Vec::new();
     let mut chunk = [0u8; 4096];
     let head_end = loop {
@@ -146,14 +166,17 @@ async fn read_request(stream: &mut TcpStream) -> Option<String> {
         })
         .next()
         .unwrap_or(0);
-    let mut remaining = content_length.saturating_sub(buffer.len() - head_end);
+    let mut body = buffer[head_end..].to_vec();
+    let mut remaining = content_length.saturating_sub(body.len());
     while remaining > 0 {
         let read = stream.read(&mut chunk).await.ok()?;
         if read == 0 {
             break;
         }
+        body.extend_from_slice(&chunk[..read]);
         remaining = remaining.saturating_sub(read);
     }
+    body.truncate(content_length);
 
-    Some(format!("{method} {path}"))
+    Some((format!("{method} {path}"), String::from_utf8_lossy(&body).into_owned()))
 }

crates/kms/src/backends/vault_credentials.rs

@@ -57,6 +57,42 @@ const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5);
 /// Default seconds between token file re-reads for [`TokenFileSource`].
 const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30;
 
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// Both gauges describe the one credential generation currently installed, so
+// they carry no labels: the Vault address, mount, auth path and token are all
+// off limits as label values, and there is exactly one generation to describe.
+// The renewal loop republishes them on a bounded cadence while it waits, so a
+// scrape landing between refresh cycles never reads a TTL frozen at the last
+// refresh, or a fail-closed state that flipped after it.
+// ---------------------------------------------------------------------------
+
+/// Gauge: seconds left before the active Vault token expires; `0` once it has.
+const METRIC_TOKEN_TTL_SECONDS: &str = "rustfs_kms_vault_token_ttl_seconds";
+/// Gauge: `1` while [`VaultCredentialProvider::current`] refuses to hand out
+/// the token because it is inside the fail-closed safety window, `0` otherwise.
+const METRIC_CREDENTIALS_FAIL_CLOSED: &str = "rustfs_kms_vault_credentials_fail_closed";
+
+/// How often the renewal loop republishes the credential gauges while waiting.
+/// Bounds how stale a scrape can be, without any additional Vault traffic.
+const CREDENTIAL_GAUGE_INTERVAL: Duration = Duration::from_secs(10);
+
+/// Register metric descriptions once per process.
+fn describe_credential_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_gauge!(
+            METRIC_TOKEN_TTL_SECONDS,
+            "Seconds remaining before the Vault token backing the KMS backend expires"
+        );
+        metrics::describe_gauge!(
+            METRIC_CREDENTIALS_FAIL_CLOSED,
+            "1 while the Vault credential provider refuses to serve its token because the token is inside the fail-closed safety window"
+        );
+    });
+}
+
 /// A crate-owned secret value, zeroized on drop and redacted in Debug output.
 #[derive(Clone, Zeroize, ZeroizeOnDrop)]
 pub(crate) struct SecretString(String);
@@ -627,6 +663,45 @@ impl VaultCredentialProvider {
         Ok(handle)
     }
 
+    /// Publish the credential gauges for the generation currently installed.
+    ///
+    /// The fail-closed gauge re-evaluates the very gate
+    /// [`VaultCredentialProvider::current`] applies, so what operators see and
+    /// what the request path does cannot drift apart.
+    fn record_credential_gauges(&self) {
+        let handle = self.current.load();
+        let now = Instant::now();
+        let fail_closed = match handle.expires_at() {
+            Some(expires_at) => {
+                metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
+                now + self.policy.safety_window >= expires_at
+            }
+            // A generation without an expiry has no remaining TTL to report
+            // and can never lapse, so it can never fail closed either.
+            None => false,
+        };
+        metrics::gauge!(METRIC_CREDENTIALS_FAIL_CLOSED).set(if fail_closed { 1.0 } else { 0.0 });
+    }
+
+    /// Wait until `deadline`, republishing the credential gauges on the
+    /// observation cadence. Reports `false` when cancellation cut the wait
+    /// short.
+    async fn wait_publishing_gauges(&self, deadline: Instant, cancel: &CancellationToken) -> bool {
+        loop {
+            self.record_credential_gauges();
+            let now = Instant::now();
+            if now >= deadline {
+                return true;
+            }
+            let slice = (deadline - now).min(CREDENTIAL_GAUGE_INTERVAL);
+            tokio::select! {
+                biased;
+                _ = cancel.cancelled() => return false,
+                _ = tokio::time::sleep(slice) => {}
+            }
+        }
+    }
+
     /// Refresh the credentials if generation `observed` is still current.
     ///
     /// Single-flight: concurrent callers serialize on the refresh lock, and a
@@ -711,17 +786,22 @@ impl fmt::Debug for VaultCredentialProvider {
 /// again immediately (the renewal point is already in the past), so the
 /// provider keeps trying to recover even after the fail-closed window has
 /// been reached.
+///
+/// Both waits run through [`VaultCredentialProvider::wait_publishing_gauges`],
+/// which is the only place the credential gauges are published: the loop is
+/// already the component that tracks token expiry, and doing it here keeps the
+/// request path free of any metric work.
 async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: CancellationToken) {
+    describe_credential_metrics();
     loop {
         let handle = provider.snapshot();
         let Some(renew_at) = handle.renew_at() else {
             // The current generation never expires; nothing left to schedule.
+            provider.record_credential_gauges();
             return;
         };
-        tokio::select! {
-            biased;
-            _ = cancel.cancelled() => return,
-            _ = tokio::time::sleep_until(renew_at) => {}
+        if !provider.wait_publishing_gauges(renew_at, &cancel).await {
+            return;
         }
 
         match provider.refresh(handle.generation, &cancel).await {
@@ -733,10 +813,9 @@ async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: Cancellati
                     error = %error,
                     "Vault credential refresh failed; retrying until the credentials recover"
                 );
-                tokio::select! {
-                    biased;
-                    _ = cancel.cancelled() => return,
-                    _ = tokio::time::sleep(provider.policy.retry_interval) => {}
+                let retry_at = Instant::now() + provider.policy.retry_interval;
+                if !provider.wait_publishing_gauges(retry_at, &cancel).await {
+                    return;
                 }
             }
         }
@@ -1354,4 +1433,125 @@ mod tests {
         assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}");
         assert!(rendered.contains("vault-token"), "the file path is not a secret");
     }
+
+    // -- Metric emission ----------------------------------------------------
+    //
+    // Each test installs a thread-local debugging recorder and drives a
+    // paused-clock current-thread runtime inside it, so the renewal loop's
+    // gauge publications land on exact virtual timestamps.
+
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Run `test` on a paused current-thread runtime under a debugging
+    /// recorder and return one snapshot of everything it emitted.
+    ///
+    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
+    /// the recorded state, so taking it per assertion would only show the
+    /// first assertion any data.
+    fn record_metrics<Out>(
+        test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
+    ) -> (Vec<MetricEntry>, Out) {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let out = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_time()
+                .start_paused(true)
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (snapshotter.snapshot().into_vec(), out)
+    }
+
+    /// Last value of a gauge, plus the labels it was published with.
+    fn gauge(snapshot: &[MetricEntry], name: &str) -> Option<(f64, Vec<String>)> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(value)) => Some((
+                    value.into_inner(),
+                    composite.key().labels().map(|label| label.key().to_string()).collect(),
+                )),
+                _ => None,
+            }
+        })
+    }
+
+    #[test]
+    fn renewal_loop_republishes_token_ttl_while_it_waits() {
+        let (snapshot, ()) = record_metrics(|| {
+            Box::pin(async {
+                let (provider, _state) = scripted_provider(
+                    Duration::from_secs(60),
+                    true,
+                    test_policy(Duration::from_secs(10), Duration::from_secs(5)),
+                )
+                .await;
+                let task = provider.spawn_renewal_task().expect("lease-bound tokens need renewal");
+
+                // Well inside the first half of the lease: nothing has been
+                // renewed yet, so only the observation cadence can have moved
+                // the gauge off its initial 60s.
+                tokio::time::sleep(Duration::from_secs(25)).await;
+                task.shutdown().await;
+            })
+        });
+
+        let (ttl, labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
+        assert!(
+            (ttl - 40.0).abs() < 1.0,
+            "expected ~40s left of a 60s lease at the last observation, got {ttl}"
+        );
+        assert!(labels.is_empty(), "credential gauges must carry no labels, got {labels:?}");
+        assert_eq!(
+            gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
+            Some(0.0),
+            "a token outside its safety window must not report fail-closed"
+        );
+    }
+
+    #[test]
+    fn fail_closed_gauge_tracks_the_gate_the_request_path_applies() {
+        let (snapshot, refused) = record_metrics(|| {
+            Box::pin(async {
+                let (provider, state) = scripted_provider(
+                    Duration::from_secs(60),
+                    true,
+                    test_policy(Duration::from_secs(10), Duration::from_secs(5)),
+                )
+                .await;
+                state.fail_renew.store(true, Ordering::SeqCst);
+                state.fail_login.store(true, Ordering::SeqCst);
+                let task = provider.spawn_renewal_task().expect("renewal task");
+
+                // 60s lease minus the 10s safety window: by 51s the provider
+                // is refusing the token, and the gauge must already say so.
+                tokio::time::sleep(Duration::from_secs(51)).await;
+                let refused = provider.current().is_err();
+                task.shutdown().await;
+                refused
+            })
+        });
+
+        assert!(refused, "a token inside the safety window must be refused");
+        assert_eq!(
+            gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
+            Some(1.0),
+            "the gauge must report the same fail-closed state the request path enforces"
+        );
+        let (ttl, _labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
+        assert!(
+            (ttl - 10.0).abs() < 1.0,
+            "expected the TTL gauge to track the lease down into its safety window, got {ttl}"
+        );
+    }
 }

crates/kms/src/backup/manifest.rs

@@ -355,8 +355,10 @@ struct ManifestProbe {
 ///
 /// The manifest is the authoritative description of one backup bundle: what
 /// was captured, under which snapshot generation, protected by which backup
-/// KEK, and which restore responsibility applies. Field order is part of the
-/// canonical digest form and is frozen for this format version.
+/// KEK, and which restore responsibility applies. The digest's canonical
+/// form is the manifest's JSON value with the digest hex emptied (see
+/// [`Self::compute_digest`]); decoders verify it against the raw stored
+/// bytes and never re-serialize parsed fields.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct BackupManifest {
@@ -415,8 +417,14 @@ impl BackupManifest {
     ///
     /// Fail-closed order: truncated or malformed input, then unknown format
     /// version, then a missing completeness marker, then schema decoding
-    /// (unknown fields, duplicate fields, missing fields), then semantic
-    /// validation including digest verification.
+    /// (unknown fields, duplicate fields, missing fields), then digest
+    /// verification against the raw input bytes, then semantic validation.
+    ///
+    /// The digest is verified against the bytes as stored — parsed typed
+    /// fields are never re-serialized for verification, so a field whose
+    /// string form does not round-trip byte-identically through its parsed
+    /// representation (timestamps in environment-dependent time zone
+    /// spellings, for example) cannot produce a spurious mismatch.
     pub fn decode(bytes: &[u8]) -> Result<Self, BackupError> {
         let probe: ManifestProbe = serde_json::from_slice(bytes).map_err(map_serde_error)?;
         if probe.format_version != Self::FORMAT_VERSION {
@@ -429,7 +437,9 @@ impl BackupManifest {
             return Err(BackupError::incomplete_bundle("manifest has no completeness marker"));
         }
         let manifest: Self = serde_json::from_slice(bytes).map_err(map_serde_error)?;
-        manifest.validate()?;
+        manifest.validate_pre_digest()?;
+        Self::verify_digest_in_bytes(bytes, &manifest.manifest_digest)?;
+        manifest.validate_content()?;
         Ok(manifest)
     }
 
@@ -449,23 +459,30 @@ impl BackupManifest {
         Ok(self)
     }
 
-    /// Compute the digest over the canonical manifest bytes.
+    /// Compute the digest over the canonical manifest form.
     ///
-    /// Canonical form: compact JSON serialization of this manifest with the
-    /// digest hex emptied. Field order is struct declaration order and is
-    /// frozen for format version 1, so the same manifest content always
-    /// hashes to the same value.
+    /// Canonical form: the JSON *value* of the manifest with the digest hex
+    /// emptied, object keys rebuilt in bytewise-sorted order at every level
+    /// (see [`canonicalize_value`]), then serialized compactly. The value
+    /// layer is what makes sealing and decoding agree byte-for-byte — a
+    /// decoder recovers the identical value from the raw stored bytes
+    /// without round-tripping any typed field through parse-and-reprint —
+    /// and the explicit key sort makes the bytes independent of
+    /// `serde_json`'s map implementation (`preserve_order` on or off).
     pub fn compute_digest(&self) -> Result<ContentDigest, BackupError> {
         let mut unsealed = self.clone();
         unsealed.manifest_digest = ContentDigest::placeholder(self.manifest_digest.algorithm);
-        let canonical = serde_json::to_vec(&unsealed)
+        let value = serde_json::to_value(&unsealed)
             .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?;
-        match self.manifest_digest.algorithm {
-            DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)),
-        }
+        Self::digest_of_canonical_value(value, self.manifest_digest.algorithm)
     }
 
-    /// Verify the sealed digest against the current manifest content.
+    /// Verify the sealed digest against the current in-memory content.
+    ///
+    /// This is the producer-side check (sealing and [`Self::encode`]).
+    /// Decoders must use the raw stored bytes instead (see [`Self::decode`]):
+    /// re-serializing parsed fields is not guaranteed to reproduce the
+    /// stored spelling byte-for-byte.
     pub fn verify_digest(&self) -> Result<(), BackupError> {
         if !self.manifest_digest.is_well_formed() {
             return Err(BackupError::corrupted("manifest digest is not a well-formed digest value"));
@@ -478,6 +495,33 @@ impl BackupManifest {
         Ok(())
     }
 
+    /// Verify a declared digest against raw manifest bytes, normalizing only
+    /// through the JSON value layer and emptying the digest slot in place.
+    fn verify_digest_in_bytes(bytes: &[u8], declared: &ContentDigest) -> Result<(), BackupError> {
+        if !declared.is_well_formed() {
+            return Err(BackupError::corrupted("manifest digest is not a well-formed digest value"));
+        }
+        let mut value: serde_json::Value = serde_json::from_slice(bytes).map_err(map_serde_error)?;
+        let Some(slot) = value.get_mut("manifest_digest").and_then(|digest| digest.get_mut("hex")) else {
+            return Err(BackupError::corrupted("manifest has no digest slot"));
+        };
+        *slot = serde_json::Value::String(String::new());
+        if Self::digest_of_canonical_value(value, declared.algorithm)? != *declared {
+            return Err(BackupError::corrupted(
+                "manifest digest mismatch: content does not match the sealed digest",
+            ));
+        }
+        Ok(())
+    }
+
+    fn digest_of_canonical_value(value: serde_json::Value, algorithm: DigestAlgorithm) -> Result<ContentDigest, BackupError> {
+        let canonical = serde_json::to_vec(&canonicalize_value(value))
+            .map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?;
+        match algorithm {
+            DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)),
+        }
+    }
+
     /// Look up a required artifact by kind, failing closed when absent.
     pub fn require_artifact(&self, kind: ArtifactKind) -> Result<&ArtifactDescriptor, BackupError> {
         self.artifacts
@@ -486,11 +530,21 @@ impl BackupManifest {
             .ok_or_else(|| BackupError::missing_artifact(artifact_kind_name(kind)))
     }
 
-    /// Validate the full manifest contract.
+    /// Validate the full manifest contract against the in-memory content.
     ///
-    /// This is decode-side validation and also guards [`Self::encode`], so a
-    /// producer cannot publish a manifest a decoder would reject.
+    /// This guards [`Self::encode`], so a producer cannot publish a manifest
+    /// a decoder would reject. [`Self::decode`] runs the same checks but
+    /// verifies the digest against the raw input bytes instead.
     pub fn validate(&self) -> Result<(), BackupError> {
+        self.validate_pre_digest()?;
+        self.verify_digest()?;
+        self.validate_content()
+    }
+
+    /// Checks that must run before any digest verification: an unknown
+    /// version or an unsealed bundle is reported as its own typed error, not
+    /// as a digest mismatch.
+    fn validate_pre_digest(&self) -> Result<(), BackupError> {
         if self.format_version != Self::FORMAT_VERSION {
             return Err(BackupError::UnknownVersion {
                 found: self.format_version,
@@ -500,7 +554,12 @@ impl BackupManifest {
         if self.completeness != CompletenessState::Complete {
             return Err(BackupError::incomplete_bundle("completeness marker records an in-progress bundle"));
         }
-        self.verify_digest()?;
+        Ok(())
+    }
+
+    /// Semantic validation of everything except version, completeness, and
+    /// digest integrity.
+    fn validate_content(&self) -> Result<(), BackupError> {
         require_non_empty("backup_id", &self.backup_id)?;
         require_non_empty("rustfs_version", &self.rustfs_version)?;
         require_non_empty("deployment_identity", &self.deployment_identity)?;
@@ -645,6 +704,29 @@ fn map_serde_error(error: serde_json::Error) -> BackupError {
     }
 }
 
+/// Rebuild a JSON value with object keys in bytewise-sorted order at every
+/// nesting level (array element order is preserved).
+///
+/// `serde_json`'s map keeps keys sorted by default but preserves insertion
+/// order when the `preserve_order` feature is unified into the build by any
+/// other crate. Digest bytes must not depend on that, so the ordering is
+/// imposed explicitly here instead of being inherited from the map type.
+fn canonicalize_value(value: serde_json::Value) -> serde_json::Value {
+    match value {
+        serde_json::Value::Object(map) => {
+            let mut entries: Vec<(String, serde_json::Value)> = map.into_iter().collect();
+            entries.sort_by(|a, b| a.0.cmp(&b.0));
+            let mut sorted = serde_json::Map::with_capacity(entries.len());
+            for (key, entry) in entries {
+                sorted.insert(key, canonicalize_value(entry));
+            }
+            serde_json::Value::Object(sorted)
+        }
+        serde_json::Value::Array(items) => serde_json::Value::Array(items.into_iter().map(canonicalize_value).collect()),
+        other => other,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -736,7 +818,7 @@ mod tests {
     /// canonical form and frozen. If serialization layout or field order
     /// changes, this value changes and the fixture test fails — which is the
     /// point: that is a format-version bump, not a patch.
-    const FIXTURE_DIGEST_HEX: &str = "01accb3e2bc51e12d17d1efc52ad6ab9441c50bec52c3fb29e0a9f92b725cdaa";
+    const FIXTURE_DIGEST_HEX: &str = "a8b104d61c358cd75cc9a7691d2f7d2d96f73c53653bef8940a49e96dbdf7775";
 
     fn fixture() -> String {
         FIXTURE.replace("SEALED_DIGEST_HEX", FIXTURE_DIGEST_HEX)
@@ -1010,12 +1092,39 @@ mod tests {
         let with_discovery = fixture().replace("\"completeness\"", "\"capability_discovery\": [1], \"completeness\"");
         expect_corrupted(BackupManifest::decode(with_discovery.as_bytes()), "reserved");
 
-        // Explicit null carries no data and is tolerated as absence; digest
-        // verification still passes because null slots are skipped on
-        // serialization.
+        // An explicit null slot decodes as absence at the schema layer, but
+        // sealed bundles never contain the key (`skip_serializing_if`), so
+        // inserting one after sealing is a byte-level modification and the
+        // raw-bytes digest check rejects it.
         let with_null = fixture().replace("\"completeness\"", "\"key_versions\": null, \"completeness\"");
-        let decoded = BackupManifest::decode(with_null.as_bytes()).expect("null reserved slot should decode");
-        assert_eq!(decoded.key_versions, None);
+        expect_corrupted(BackupManifest::decode(with_null.as_bytes()), "digest mismatch");
+    }
+
+    #[test]
+    fn digest_verification_survives_non_round_tripping_timestamp_spellings() {
+        // A legacy `created_at` spelling (no time zone annotation) parses via
+        // the compat fallback and re-serializes differently ("+00:00[UTC]"),
+        // and host-dependent zone spellings can do the same. Digest
+        // verification therefore operates on the raw stored bytes and must
+        // never re-serialize parsed fields.
+        let sealed = seal(local_manifest_unsealed());
+        let mut value = serde_json::to_value(&sealed).expect("manifest should convert to a value");
+        value["created_at"] = serde_json::Value::String("2026-07-30T00:00:00+00:00".to_string());
+        value["manifest_digest"]["hex"] = serde_json::Value::String(String::new());
+        let digest = BackupManifest::digest_of_canonical_value(value.clone(), DigestAlgorithm::Sha256)
+            .expect("canonical digest should compute");
+        value["manifest_digest"]["hex"] = serde_json::Value::String(digest.hex);
+        let bytes = serde_json::to_vec(&value).expect("manifest bytes");
+
+        let decoded = BackupManifest::decode(&bytes).expect("a non-round-tripping timestamp spelling must not break decoding");
+
+        // Precondition: the spelling really does not survive a typed
+        // round-trip — otherwise this test is vacuous.
+        let reserialized = serde_json::to_value(&decoded).expect("decoded manifest should convert to a value");
+        assert_ne!(reserialized["created_at"], value["created_at"]);
+        // Which is exactly why the producer-side (in-memory) digest check
+        // cannot be used on decoded manifests.
+        assert!(decoded.verify_digest().is_err());
     }
 
     #[test]

crates/kms/src/backup/mod.rs

@@ -12,13 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Backup/restore contract types for KMS state.
+//! Backup/restore contracts and backup production for KMS state.
 //!
-//! This module is contract-only: it defines the versioned backup manifest,
-//! the per-backend responsibility matrix, typed failure modes, and the
-//! restore dry-run report. Nothing here is wired into handlers or backends;
-//! backup export, restore orchestration, and the admin API build on these
-//! types in follow-up changes.
+//! The contract side defines the versioned backup manifest, the per-backend
+//! responsibility matrix, typed failure modes, and the restore dry-run
+//! report. [`local_export`] implements the producer side and
+//! [`local_restore`] the consumer side for the Local backend as
+//! crate-internal APIs; the admin API builds on these pieces in follow-up
+//! changes.
 //!
 //! # Bundle model
 //!
@@ -50,6 +51,8 @@
 mod capability;
 mod dry_run;
 mod error;
+pub mod local_export;
+pub mod local_restore;
 mod manifest;
 
 pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
@@ -57,6 +60,14 @@ pub use dry_run::{
     ExternalDependencyMismatch, RestoreBlocker, RestoreBlockerCode, RestoreConflict, RestoreConflictKind, RestoreDryRunReport,
 };
 pub use error::BackupError;
+pub use local_export::{
+    BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup,
+    read_local_bundle_manifest,
+};
+pub use local_restore::{
+    LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore,
+    restore_local_backup,
+};
 pub use manifest::{
     AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest,
     DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot,

crates/kms/src/cache.rs

@@ -16,11 +16,84 @@
 
 use crate::types::KeyMetadata;
 use moka::future::Cache;
+use moka::notification::RemovalCause;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::Duration;
 
+/// Default lifetime of a cached key metadata entry.
+const DEFAULT_METADATA_TTL: Duration = Duration::from_secs(300);
+
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// Lookup outcomes and removals are counted here because moka exposes neither
+// hit nor miss counts; the numbers below are the cache's own, not derived.
+// Label values are exclusively static strings (lookup result, removal cause) —
+// key identifiers and key metadata must never reach a metric label.
+// ---------------------------------------------------------------------------
+
+/// Counter: key metadata lookups, by `result` (`hit` or `miss`).
+const METRIC_CACHE_LOOKUPS_TOTAL: &str = "rustfs_kms_metadata_cache_lookups_total";
+/// Counter: entries dropped from the cache, by `cause` (`expired`, `size`,
+/// `explicit`, `replaced`); only `expired` and `size` are true evictions, the
+/// rest are invalidations driven by key lifecycle operations.
+const METRIC_CACHE_EVICTIONS_TOTAL: &str = "rustfs_kms_metadata_cache_evictions_total";
+/// Gauge: entries currently held by the cache. Republished whenever the entry
+/// set can have changed — every write path, plus the lookups that miss, since
+/// TTL expiry drops entries without any write taking place.
+const METRIC_CACHE_ENTRIES: &str = "rustfs_kms_metadata_cache_entries";
+
+/// Register metric descriptions once per process.
+fn describe_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_counter!(METRIC_CACHE_LOOKUPS_TOTAL, "Total KMS key metadata cache lookups, by hit or miss");
+        metrics::describe_counter!(
+            METRIC_CACHE_EVICTIONS_TOTAL,
+            "Total entries dropped from the KMS key metadata cache, by removal cause"
+        );
+        metrics::describe_gauge!(METRIC_CACHE_ENTRIES, "Entries currently held by the KMS key metadata cache");
+    });
+}
+
+fn removal_cause_label(cause: RemovalCause) -> &'static str {
+    match cause {
+        RemovalCause::Expired => "expired",
+        RemovalCause::Explicit => "explicit",
+        RemovalCause::Replaced => "replaced",
+        RemovalCause::Size => "size",
+    }
+}
+
+/// Cumulative cache counters. Shared with moka's eviction listener, which runs
+/// outside any `&self` borrow, hence the `Arc` and the atomics.
+#[derive(Debug, Default)]
+struct CacheCounters {
+    hits: AtomicU64,
+    misses: AtomicU64,
+    evictions: AtomicU64,
+}
+
+/// Snapshot of the KMS key metadata cache counters.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct KmsCacheStats {
+    /// Entries currently held. Eventually consistent: moka applies pending
+    /// maintenance lazily.
+    pub entries: u64,
+    /// Lookups served from the cache since process start.
+    pub hits: u64,
+    /// Lookups that fell through to the backend since process start.
+    pub misses: u64,
+    /// Entries dropped since process start, whatever the cause (expiry,
+    /// capacity, invalidation, replacement).
+    pub evictions: u64,
+}
+
 /// KMS cache for storing frequently accessed keys and metadata
 pub struct KmsCache {
     key_metadata_cache: Cache<String, KeyMetadata>,
+    counters: Arc<CacheCounters>,
 }
 
 impl KmsCache {
@@ -33,11 +106,26 @@ impl KmsCache {
     /// A new instance of `KmsCache`
     ///
     pub fn new(capacity: u64) -> Self {
+        Self::with_ttl(capacity, DEFAULT_METADATA_TTL)
+    }
+
+    /// Create a new KMS cache with an explicit metadata time-to-live
+    fn with_ttl(capacity: u64, metadata_ttl: Duration) -> Self {
+        describe_metrics();
+
+        let counters = Arc::new(CacheCounters::default());
+        let eviction_counters = Arc::clone(&counters);
+
         Self {
             key_metadata_cache: Cache::builder()
                 .max_capacity(capacity)
-                .time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
+                .time_to_live(metadata_ttl)
+                .eviction_listener(move |_key: Arc<String>, _metadata: KeyMetadata, cause: RemovalCause| {
+                    eviction_counters.evictions.fetch_add(1, Ordering::Relaxed);
+                    metrics::counter!(METRIC_CACHE_EVICTIONS_TOTAL, "cause" => removal_cause_label(cause)).increment(1);
+                })
                 .build(),
+            counters,
         }
     }
 
@@ -50,7 +138,26 @@ impl KmsCache {
     /// An `Option` containing the `KeyMetadata` if found, or `None` if not found
     ///
     pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
-        self.key_metadata_cache.get(key_id).await
+        let cached = self.key_metadata_cache.get(key_id).await;
+        self.record_lookup(cached.is_some());
+
+        if cached.is_none() {
+            // TTL expiry is the one way the entry set shrinks without a write,
+            // and a miss is where it surfaces. moka expires lazily: the miss is
+            // reported at once, but the removal that decrements `entry_count`
+            // and reaches the eviction listener lands in the maintenance pass
+            // moka runs opportunistically on reads, on its own interval. So
+            // this converges the gauge within a housekeeping interval rather
+            // than on the first miss — enough to stop a cache that only ever
+            // expires, with no put, remove or clear to follow, from reporting a
+            // population that is gone. Forcing `run_pending_tasks` would
+            // tighten that at the cost of taking moka's maintenance lock on
+            // every miss, for a gauge `entry_count` only approximates anyway.
+            // Hits, the hot path, are left alone.
+            self.record_entry_count();
+        }
+
+        cached
     }
 
     /// Put key metadata into cache
@@ -62,6 +169,7 @@ impl KmsCache {
     pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
         self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
         self.key_metadata_cache.run_pending_tasks().await;
+        self.record_entry_count();
     }
 
     /// Remove key metadata from cache
@@ -71,6 +179,11 @@ impl KmsCache {
     ///
     pub async fn remove_key_metadata(&mut self, key_id: &str) {
         self.key_metadata_cache.remove(key_id).await;
+
+        // Flush maintenance so the removal notification and the entry gauge
+        // describe the cache as it is once this call returns.
+        self.key_metadata_cache.run_pending_tasks().await;
+        self.record_entry_count();
     }
 
     /// Clear all cached entries
@@ -79,18 +192,35 @@ impl KmsCache {
 
         // Wait for invalidation to complete
         self.key_metadata_cache.run_pending_tasks().await;
+        self.record_entry_count();
     }
 
-    /// Get cache statistics (hit count, miss count)
+    /// Get cache statistics
     ///
     /// # Returns
-    /// A tuple containing total entries and total misses
+    /// A [`KmsCacheStats`] snapshot of entry count, hits, misses and evictions
     ///
-    pub fn stats(&self) -> (u64, u64) {
-        (
-            self.key_metadata_cache.entry_count(),
-            0u64, // moka doesn't provide miss count directly
-        )
+    pub fn stats(&self) -> KmsCacheStats {
+        KmsCacheStats {
+            entries: self.key_metadata_cache.entry_count(),
+            hits: self.counters.hits.load(Ordering::Relaxed),
+            misses: self.counters.misses.load(Ordering::Relaxed),
+            evictions: self.counters.evictions.load(Ordering::Relaxed),
+        }
+    }
+
+    fn record_lookup(&self, hit: bool) {
+        let (counter, result) = if hit {
+            (&self.counters.hits, "hit")
+        } else {
+            (&self.counters.misses, "miss")
+        };
+        counter.fetch_add(1, Ordering::Relaxed);
+        metrics::counter!(METRIC_CACHE_LOOKUPS_TOTAL, "result" => result).increment(1);
+    }
+
+    fn record_entry_count(&self) {
+        metrics::gauge!(METRIC_CACHE_ENTRIES).set(self.key_metadata_cache.entry_count() as f64);
     }
 }
 
@@ -99,37 +229,90 @@ mod tests {
     use super::*;
     use crate::types::{KeyState, KeyUsage};
     use jiff::Zoned;
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
     use std::time::Duration;
 
-    #[derive(Debug, Clone)]
-    struct CacheInfo {
-        key_metadata_count: u64,
-    }
-
-    impl CacheInfo {
-        fn total_entries(&self) -> u64 {
-            self.key_metadata_count
-        }
-    }
-
     impl KmsCache {
-        fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
-            Self {
-                key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
-            }
-        }
-
-        fn info_for_tests(&self) -> CacheInfo {
-            CacheInfo {
-                key_metadata_count: self.key_metadata_cache.entry_count(),
-            }
-        }
-
         fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
             self.key_metadata_cache.contains_key(key_id)
         }
     }
 
+    fn test_metadata(key_id: &str) -> KeyMetadata {
+        KeyMetadata {
+            key_id: key_id.to_string(),
+            key_state: KeyState::Enabled,
+            key_usage: KeyUsage::EncryptDecrypt,
+            description: None,
+            creation_date: Zoned::now(),
+            deletion_date: None,
+            origin: "KMS".to_string(),
+            key_manager: "CUSTOMER".to_string(),
+            tags: std::collections::HashMap::new(),
+        }
+    }
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Drive `test` on a current-thread runtime under a thread-local debugging
+    /// recorder and return its output plus one snapshot of everything emitted.
+    ///
+    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
+    /// the recorded state, so taking it per assertion would leave every
+    /// assertion after the first with nothing to look at.
+    fn record_metrics<F, Fut, T>(test: F) -> (T, Vec<MetricEntry>)
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = T>,
+    {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let output = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (output, snapshotter.snapshot().into_vec())
+    }
+
+    /// Sum of counters with `name` whose labels include every `(key, value)` pair.
+    fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let key = composite.key();
+                let matches = composite.kind() == MetricKind::Counter
+                    && key.name() == name
+                    && labels
+                        .iter()
+                        .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected));
+                match (matches, value) {
+                    (true, DebugValue::Counter(count)) => Some(*count),
+                    _ => None,
+                }
+            })
+            .sum()
+    }
+
+    /// Last value recorded for the unlabelled gauge `name`.
+    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(gauge)) => Some(**gauge),
+                _ => None,
+            }
+        })
+    }
+
     #[tokio::test]
     async fn test_cache_operations() {
         let mut cache = KmsCache::new(100);
@@ -154,19 +337,16 @@ mod tests {
         assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
 
         // Test cache info
-        let info = cache.info_for_tests();
-        assert_eq!(info.key_metadata_count, 1);
-        assert_eq!(info.total_entries(), 1);
+        assert_eq!(cache.stats().entries, 1);
 
         // Test cache clearing
         cache.clear().await;
-        let info_after_clear = cache.info_for_tests();
-        assert_eq!(info_after_clear.total_entries(), 0);
+        assert_eq!(cache.stats().entries, 0);
     }
 
     #[tokio::test]
     async fn test_cache_with_custom_ttl() {
-        let mut cache = KmsCache::with_ttl_for_tests(
+        let mut cache = KmsCache::with_ttl(
             100,
             Duration::from_millis(100), // Short TTL for testing
         );
@@ -217,4 +397,92 @@ mod tests {
 
         assert!(cache.contains_key_metadata_for_tests("contains-test"));
     }
+
+    #[test]
+    fn lookups_report_real_hit_and_miss_counts() {
+        let (stats, snapshot) = record_metrics(|| async {
+            let mut cache = KmsCache::new(100);
+
+            assert!(cache.get_key_metadata("absent").await.is_none());
+            cache.put_key_metadata("present", &test_metadata("present")).await;
+            assert!(cache.get_key_metadata("present").await.is_some());
+            assert!(cache.get_key_metadata("absent").await.is_none());
+
+            cache.stats()
+        });
+
+        assert_eq!(stats.hits, 1);
+        assert_eq!(stats.misses, 2);
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "hit")]), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "miss")]), 2);
+    }
+
+    #[test]
+    fn removals_report_their_cause_and_the_resulting_entry_count() {
+        let (stats, snapshot) = record_metrics(|| async {
+            let mut cache = KmsCache::new(100);
+
+            cache.put_key_metadata("key", &test_metadata("key")).await;
+            cache.put_key_metadata("key", &test_metadata("key")).await;
+            cache.remove_key_metadata("key").await;
+
+            cache.stats()
+        });
+
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "replaced")]), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "explicit")]), 1);
+        assert_eq!(stats.evictions, 2);
+        assert_eq!(stats.entries, 0);
+        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
+    }
+
+    #[test]
+    fn capacity_pressure_reports_size_evictions() {
+        let (stats, snapshot) = record_metrics(|| async {
+            let mut cache = KmsCache::with_ttl(1, DEFAULT_METADATA_TTL);
+
+            cache.put_key_metadata("first", &test_metadata("first")).await;
+            cache.put_key_metadata("second", &test_metadata("second")).await;
+
+            cache.stats()
+        });
+
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "size")]), 1);
+        assert_eq!(stats.evictions, 1);
+        assert_eq!(stats.entries, 1);
+        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(1.0));
+    }
+
+    /// Entries that expire are dropped outside every write path, so the gauge
+    /// has to be republished from the lookup that observes the expiry — nothing
+    /// else runs afterwards to correct it.
+    ///
+    /// moka's clock is internal and cannot be driven by tokio's paused time,
+    /// hence the one short real sleep. The explicit `run_pending_tasks` stands
+    /// in for the maintenance moka schedules by itself on reads, so the test
+    /// does not depend on moka's internal 300ms housekeeping interval.
+    #[test]
+    fn expiry_without_further_writes_converges_the_entry_gauge() {
+        let ttl = Duration::from_millis(100);
+
+        let (stats, snapshot) = record_metrics(move || async move {
+            let mut cache = KmsCache::with_ttl(100, ttl);
+
+            cache.put_key_metadata("expiring", &test_metadata("expiring")).await;
+            assert_eq!(cache.stats().entries, 1);
+
+            tokio::time::sleep(Duration::from_millis(150)).await;
+            cache.key_metadata_cache.run_pending_tasks().await;
+
+            // The lookup that observes the expiry is the last thing to touch
+            // the cache: no put, remove or clear follows it.
+            assert!(cache.get_key_metadata("expiring").await.is_none());
+
+            cache.stats()
+        });
+
+        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "expired")]), 1);
+        assert_eq!(stats.entries, 0);
+        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
+    }
 }

crates/kms/src/config.rs

@@ -130,6 +130,21 @@ pub enum KmsBackend {
     Static,
 }
 
+impl KmsBackend {
+    /// Stable identifier for logs, metrics, and audit records.
+    ///
+    /// External consumers key off these values, so treat them as a wire
+    /// contract rather than a rendering detail.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            KmsBackend::VaultKv2 => "vault-kv2",
+            KmsBackend::VaultTransit => "vault-transit",
+            KmsBackend::Local => "local",
+            KmsBackend::Static => "static",
+        }
+    }
+}
+
 /// Main KMS configuration
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct KmsConfig {

crates/kms/src/deletion_worker.rs

@@ -22,18 +22,123 @@
 //! every node of a deployment concurrently — a key is only ever removed while
 //! its (re-read) record is an expired pending deletion or a tombstone.
 
+use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
 use crate::backends::{ExpiredKeyRemoval, KmsBackend};
-use crate::types::{KeyStatus, ListKeysRequest};
+use crate::error::Result;
+use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
 use async_trait::async_trait;
 use jiff::Zoned;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};
 
 /// How often the worker looks for expired pending deletions.
 pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
 
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// The sweep already pages through the whole key set, so everything below is
+// derived from the pages it has in hand: observing the key lifecycle costs no
+// extra backend call. Each metric is an aggregate — the gauges carry no labels
+// at all and the counter only a static outcome — because a per-key label would
+// carry key identifiers into the metric stream and grow the series count with
+// the key set. "This key is overdue for rotation" is a threshold on the
+// aggregate age and belongs in an alerting rule, not in a label.
+// ---------------------------------------------------------------------------
+
+/// Gauge: keys scheduled for deletion whose deadline has not passed, as of the
+/// end of the last sweep that saw the whole key set.
+const METRIC_PENDING_DELETION_KEYS: &str = "rustfs_kms_pending_deletion_keys";
+/// Gauge: keys left tombstoned by an interrupted removal and still awaiting
+/// the sweep, as of the end of the last sweep that saw the whole key set.
+const METRIC_TOMBSTONE_KEYS: &str = "rustfs_kms_deletion_tombstone_keys";
+/// Gauge: seconds since the least recently rotated usable key was rotated
+/// (its creation time when it was never rotated); `0` when there are none.
+const METRIC_OLDEST_ROTATION_AGE_SECONDS: &str = "rustfs_kms_oldest_key_rotation_age_seconds";
+/// Counter: keys the sweep acted on, by `outcome` (`removed`, `blocked`,
+/// `skipped`, `failed`).
+const METRIC_SWEEP_KEYS_TOTAL: &str = "rustfs_kms_deletion_sweep_keys_total";
+
+/// Register metric descriptions once per process.
+fn describe_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_gauge!(
+            METRIC_PENDING_DELETION_KEYS,
+            "KMS keys scheduled for deletion whose deadline has not passed yet"
+        );
+        metrics::describe_gauge!(
+            METRIC_TOMBSTONE_KEYS,
+            "KMS keys left tombstoned by an interrupted removal, still awaiting the deletion sweep"
+        );
+        metrics::describe_gauge!(
+            METRIC_OLDEST_ROTATION_AGE_SECONDS,
+            "Seconds since the least recently rotated usable KMS key was last rotated, counting from creation for keys that were never rotated"
+        );
+        metrics::describe_counter!(METRIC_SWEEP_KEYS_TOTAL, "Total keys acted on by the KMS deletion sweep, by outcome");
+    });
+}
+
+/// Aggregate view of the key set, accumulated while the sweep pages through it.
+///
+/// Counts and one maximum age only: these feed label-less gauges, so no key
+/// identifier can reach the metric stream through them.
+#[derive(Debug, Default, Clone, Copy, PartialEq)]
+struct KeyCensus {
+    pending_deletion: usize,
+    tombstones: usize,
+    /// Longest time since last rotation across keys that are still usable,
+    /// counting from creation for keys that were never rotated. Keys on their
+    /// way out are excluded: they will never be rotated again, and would
+    /// otherwise pin the gauge high until the sweep finishes removing them.
+    oldest_rotation_age_seconds: f64,
+}
+
+impl KeyCensus {
+    fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
+        match key.status {
+            KeyStatus::PendingDeletion => self.pending_deletion += 1,
+            KeyStatus::Deleted => self.tombstones += 1,
+            KeyStatus::Active | KeyStatus::Disabled => {
+                let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
+                self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
+            }
+        }
+    }
+}
+
+/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
+/// timestamp persisted by a node running ahead) cannot produce a negative age.
+fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
+    (now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
+}
+
+/// Publish one sweep's counters, plus the lifecycle gauges when `census` covers
+/// the whole key set.
+fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
+    describe_metrics();
+    for (outcome, count) in [
+        ("removed", report.removed.len()),
+        ("blocked", report.blocked.len()),
+        ("skipped", report.skipped),
+        ("failed", report.failed),
+    ] {
+        // Emitted even at zero so every outcome series exists from the first
+        // sweep on and a rate over it is defined.
+        metrics::counter!(METRIC_SWEEP_KEYS_TOTAL, "outcome" => outcome).increment(count as u64);
+    }
+
+    // A sweep that could not finish listing saw only part of the key set;
+    // publishing its counts would understate every gauge, so the previous
+    // (complete) values are left standing instead.
+    let Some(census) = census else { return };
+    metrics::gauge!(METRIC_PENDING_DELETION_KEYS).set(census.pending_deletion as f64);
+    metrics::gauge!(METRIC_TOMBSTONE_KEYS).set(census.tombstones as f64);
+    metrics::gauge!(METRIC_OLDEST_ROTATION_AGE_SECONDS).set(census.oldest_rotation_age_seconds);
+}
+
 /// Reports configuration that still references a KMS key.
 ///
 /// Consulted before any material is destroyed; a non-empty result blocks the
@@ -68,6 +173,8 @@ pub(crate) struct DeletionWorker {
     default_key_id: Option<String>,
     reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
     interval: Duration,
+    backend_kind: &'static str,
+    audit_sink: Option<Arc<dyn KmsAuditSink>>,
 }
 
 impl DeletionWorker {
@@ -75,15 +182,24 @@ impl DeletionWorker {
         backend: Arc<dyn KmsBackend>,
         default_key_id: Option<String>,
         reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
+        backend_kind: &'static str,
     ) -> Self {
         Self {
             backend,
             default_key_id,
             reference_checker,
+            backend_kind,
+            audit_sink: None,
             interval: DEFAULT_SWEEP_INTERVAL,
         }
     }
 
+    /// Audit each irreversible key removal to `sink`.
+    pub(crate) fn with_audit_sink(mut self, sink: Option<Arc<dyn KmsAuditSink>>) -> Self {
+        self.audit_sink = sink;
+        self
+    }
+
     pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
         tokio::spawn(async move { self.run(cancel).await })
     }
@@ -114,10 +230,15 @@ impl DeletionWorker {
 
     /// Run one sweep at the given time. Exposed separately so tests can drive
     /// the expiry logic deterministically.
+    ///
+    /// Also feeds the lifecycle gauges: every key it lists is counted into a
+    /// [`KeyCensus`] on the way past, so the observability comes out of the
+    /// pages the sweep already had to fetch.
     pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
         let mut report = SweepReport::default();
+        let mut census = KeyCensus::default();
         let mut marker: Option<String> = None;
-        loop {
+        let listed_everything = loop {
             let request = ListKeysRequest {
                 limit: Some(100),
                 marker: marker.clone(),
@@ -129,54 +250,99 @@ impl DeletionWorker {
                 Err(error) => {
                     warn!(%error, "KMS deletion sweep could not list keys");
                     report.failed += 1;
-                    return report;
+                    break false;
                 }
             };
             for key in &response.keys {
-                if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
-                    self.process_key(&key.key_id, now, &mut report).await;
+                // Keys this sweep destroys are left out of the census: the
+                // gauges describe the key set as it stands once the sweep is
+                // done, not as it was when the page was listed.
+                let removed = if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
+                    self.process_key(&key.key_id, now, &mut report).await
+                } else {
+                    false
+                };
+                if !removed {
+                    census.observe(key, now);
                 }
             }
             if !response.truncated {
-                break;
+                break true;
             }
             match response.next_marker {
                 Some(next_marker) => marker = Some(next_marker),
-                None => break,
+                // Truncated without a marker: the rest of the key set is out
+                // of reach, so the census is incomplete.
+                None => break false,
             }
-        }
+        };
+        record_sweep(&report, listed_everything.then_some(census));
         report
     }
 
-    async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) {
+    /// Handle one key that is on its way out; reports whether it was removed.
+    async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
         // Never remove a key that live configuration still points at. The
         // default key check is built in; broader references (bucket
         // encryption settings, ...) come from the injected checker.
         if self.default_key_id.as_deref() == Some(key_id) {
             warn!(key_id, "expired KMS key is still the default key; refusing removal");
             report.blocked.push(key_id.to_string());
-            return;
+            return false;
         }
         if let Some(checker) = &self.reference_checker {
             let references = checker.references(key_id).await;
             if !references.is_empty() {
                 warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal");
                 report.blocked.push(key_id.to_string());
-                return;
+                return false;
             }
         }
 
         // The backend re-checks state and deadline under its own write
         // synchronization, so a cancellation racing this sweep wins there.
-        match self.backend.remove_expired_key(key_id, now).await {
-            Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()),
-            Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1,
+        let started = Instant::now();
+        let outcome = self.backend.remove_expired_key(key_id, now).await;
+        match &outcome {
+            Ok(ExpiredKeyRemoval::Removed) => {
+                report.removed.push(key_id.to_string());
+                self.audit_removal(key_id, started, &outcome);
+                true
+            }
+            Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => {
+                report.skipped += 1;
+                false
+            }
             Err(error) => {
                 warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep");
                 report.failed += 1;
+                self.audit_removal(key_id, started, &outcome);
+                false
             }
         }
     }
+
+    /// Record an attempted destruction of key material.
+    ///
+    /// Only attempts that reached the backend are audited: a key the sweep
+    /// declined to touch (not yet due, still referenced, state changed under
+    /// us) was never at risk, and recording it as a deletion event would
+    /// drown the real ones.
+    fn audit_removal(&self, key_id: &str, started: Instant, outcome: &Result<ExpiredKeyRemoval>) {
+        let Some(sink) = self.audit_sink.as_ref() else {
+            return;
+        };
+
+        // Removal runs on a background sweep, so there is no request
+        // principal to attribute it to.
+        let context = OperationContext::internal();
+        sink.emit(
+            KmsAuditRecord::new(KmsAuditOperation::DeleteKey, &context, self.backend_kind)
+                .with_key_id(Some(key_id))
+                .with_latency(started.elapsed())
+                .with_result(outcome),
+        );
+    }
 }
 
 #[cfg(test)]
@@ -216,7 +382,7 @@ mod tests {
     }
 
     fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker {
-        DeletionWorker::new(backend, None, None)
+        DeletionWorker::new(backend, None, None, "local")
     }
 
     fn after_window() -> Zoned {
@@ -307,7 +473,7 @@ mod tests {
         schedule(&backend, &key_id).await;
 
         // Blocked while it is the configured default key.
-        let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None);
+        let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None, "local");
         let report = as_default.sweep(&after_window()).await;
         assert_eq!(report.blocked, vec![key_id.clone()]);
         assert!(report.removed.is_empty());
@@ -317,6 +483,7 @@ mod tests {
             backend.clone(),
             None,
             Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))),
+            "local",
         );
         let report = with_references.sweep(&after_window()).await;
         assert_eq!(report.blocked, vec![key_id.clone()]);
@@ -327,11 +494,53 @@ mod tests {
             .expect("blocked key must still exist");
 
         // Removed once nothing references it anymore.
-        let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))));
+        let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))), "local");
         let report = unreferenced.sweep(&after_window()).await;
         assert_eq!(report.removed, vec![key_id.clone()]);
     }
 
+    #[tokio::test]
+    async fn only_attempted_removals_are_audited() {
+        #[derive(Default)]
+        struct CapturingSink {
+            records: std::sync::Mutex<Vec<KmsAuditRecord>>,
+        }
+
+        impl KmsAuditSink for CapturingSink {
+            fn emit(&self, record: KmsAuditRecord) {
+                self.records.lock().expect("sink lock should not be poisoned").push(record);
+            }
+        }
+
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let backend = local_backend(&temp_dir).await;
+        let key_id = create_key(&backend, "audited-removal").await;
+        schedule(&backend, &key_id).await;
+
+        let sink = Arc::new(CapturingSink::default());
+        let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
+
+        // A key that is not yet due was never at risk, so it produces no record.
+        worker.sweep(&Zoned::now()).await;
+        assert!(
+            sink.records.lock().expect("sink lock").is_empty(),
+            "a key the sweep declined to touch must not be audited as a deletion"
+        );
+
+        worker.sweep(&after_window()).await;
+
+        let records = sink.records.lock().expect("sink lock");
+        assert_eq!(records.len(), 1, "the removal should be audited exactly once");
+        let record = &records[0];
+        assert_eq!(record.operation, crate::audit::KmsAuditOperation::DeleteKey);
+        assert_eq!(record.event, rustfs_s3_types::EventName::KmsKeyDeleted);
+        assert_eq!(record.outcome, crate::audit::KmsAuditOutcome::Success);
+        assert_eq!(record.key_id.as_deref(), Some(key_id.as_str()));
+        assert_eq!(record.backend, "local");
+        // Background work has no request principal to attribute.
+        assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
+    }
+
     #[tokio::test]
     async fn deadline_survives_backend_restart_and_sweep_completes_it() {
         let temp_dir = tempfile::tempdir().expect("temp dir");
@@ -395,4 +604,156 @@ mod tests {
         cancel.cancel();
         task.await.expect("worker task must stop after cancellation");
     }
+
+    // -- Metric emission ----------------------------------------------------
+
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Run `test` on a current-thread runtime under a debugging recorder and
+    /// return one snapshot of everything it emitted.
+    ///
+    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
+    /// the recorded state, so taking it per assertion would only show the
+    /// first assertion any data.
+    fn record_metrics<Out>(
+        test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
+    ) -> (Vec<MetricEntry>, Out) {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let out = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (snapshotter.snapshot().into_vec(), out)
+    }
+
+    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(value)) => Some(value.into_inner()),
+                _ => None,
+            }
+        })
+    }
+
+    fn counter_value(snapshot: &[MetricEntry], name: &str, outcome: &str) -> u64 {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let matches = composite.kind() == MetricKind::Counter
+                    && composite.key().name() == name
+                    && composite
+                        .key()
+                        .labels()
+                        .any(|label| label.key() == "outcome" && label.value() == outcome);
+                match (matches, value) {
+                    (true, DebugValue::Counter(count)) => Some(*count),
+                    _ => None,
+                }
+            })
+            .sum()
+    }
+
+    fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
+        KeyInfo {
+            key_id: key_id.to_string(),
+            description: None,
+            algorithm: "AES-256".to_string(),
+            usage: KeyUsage::EncryptDecrypt,
+            status,
+            version: 1,
+            metadata: std::collections::HashMap::new(),
+            tags: std::collections::HashMap::new(),
+            created_at,
+            rotated_at,
+            created_by: None,
+        }
+    }
+
+    #[test]
+    fn census_ages_from_rotation_and_ignores_departing_keys() {
+        let now = Zoned::now();
+        let day = Duration::from_secs(86400);
+        let mut census = KeyCensus::default();
+
+        // Rotation beats creation as the age baseline...
+        census.observe(
+            &key_info("rotated", KeyStatus::Active, now.clone() - 30 * day, Some(now.clone() - 2 * day)),
+            &now,
+        );
+        // ...and a key that was never rotated ages from its creation.
+        census.observe(&key_info("never-rotated", KeyStatus::Disabled, now.clone() - 5 * day, None), &now);
+        // Keys on their way out only ever move the counts: they will not be
+        // rotated again, so their age must not drive the rotation gauge.
+        census.observe(&key_info("pending", KeyStatus::PendingDeletion, now.clone() - 400 * day, None), &now);
+        census.observe(&key_info("tombstone", KeyStatus::Deleted, now.clone() - 400 * day, None), &now);
+
+        assert_eq!(census.pending_deletion, 1);
+        assert_eq!(census.tombstones, 1);
+        let expected = 5.0 * 86400.0;
+        assert!(
+            (census.oldest_rotation_age_seconds - expected).abs() < 1.0,
+            "expected the never-rotated key to set the age, got {}",
+            census.oldest_rotation_age_seconds
+        );
+    }
+
+    #[test]
+    fn sweep_publishes_lifecycle_gauges_without_key_labels() {
+        let (snapshot, key_ids) = record_metrics(|| {
+            Box::pin(async {
+                let temp_dir = tempfile::tempdir().expect("temp dir");
+                let backend = local_backend(&temp_dir).await;
+                let live = create_key(&backend, "live-key").await;
+                let doomed = create_key(&backend, "doomed-key").await;
+                schedule(&backend, &doomed).await;
+
+                // Three days in, still inside the seven-day window: the sweep
+                // observes the scheduled key instead of removing it.
+                let report = worker(backend.clone())
+                    .sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
+                    .await;
+                assert_eq!(report.skipped, 1);
+                assert!(report.removed.is_empty());
+                vec![live, doomed]
+            })
+        });
+
+        assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(1.0));
+        assert_eq!(gauge_value(&snapshot, METRIC_TOMBSTONE_KEYS), Some(0.0));
+        let age = gauge_value(&snapshot, METRIC_OLDEST_ROTATION_AGE_SECONDS).expect("rotation age gauge must be published");
+        // Both keys were just created, and only the usable one counts, so the
+        // reported age is the sweep's own offset into the future.
+        assert!(
+            (age - 3.0 * 86400.0).abs() < 60.0,
+            "expected the sweep offset as the rotation age, got {age}"
+        );
+        assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "skipped"), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "removed"), 0);
+
+        for (composite, ..) in &snapshot {
+            for label in composite.key().labels() {
+                for key_id in &key_ids {
+                    assert!(
+                        !label.value().contains(key_id.as_str()),
+                        "metric {} leaked a key identifier through label {}",
+                        composite.key().name(),
+                        label.key()
+                    );
+                }
+            }
+        }
+    }
 }

crates/kms/src/lib.rs

@@ -65,6 +65,7 @@
 
 // Core modules
 pub mod api_types;
+pub mod audit;
 pub mod backends;
 pub mod backup;
 mod cache;
@@ -74,6 +75,7 @@ mod encryption;
 mod error;
 pub mod manager;
 mod policy;
+pub mod probe;
 pub mod service;
 pub mod service_manager;
 mod time_serde;
@@ -86,11 +88,14 @@ pub use api_types::{
     StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
     UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
+pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
+pub use cache::KmsCacheStats;
 pub use config::*;
 pub use deletion_worker::DeletionReferenceChecker;
 pub use encryption::is_data_key_envelope;
 pub use error::{KmsError, KmsUnavailableError, Result};
 pub use manager::KmsManager;
+pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
     KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,

crates/kms/src/manager.rs

@@ -14,16 +14,18 @@
 
 //! KMS manager for handling key operations and backend coordination
 
+use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
 use crate::backends::KmsBackend;
-use crate::cache::KmsCache;
+use crate::cache::{KmsCache, KmsCacheStats};
 use crate::config::KmsConfig;
 use crate::error::Result;
 use crate::types::{
     CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
     DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
-    GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse,
+    GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
 };
 use std::sync::Arc;
+use std::time::Instant;
 use tokio::sync::RwLock;
 
 /// KMS Manager coordinates operations between backends and caching
@@ -33,6 +35,8 @@ pub struct KmsManager {
     cache: Arc<RwLock<KmsCache>>,
     default_key_id: Option<String>,
     enable_cache: bool,
+    backend_kind: &'static str,
+    audit_sink: Option<Arc<dyn KmsAuditSink>>,
 }
 
 impl KmsManager {
@@ -44,16 +48,72 @@ impl KmsManager {
             cache,
             default_key_id: config.default_key_id,
             enable_cache: config.enable_cache,
+            backend_kind: config.backend.as_str(),
+            audit_sink: None,
         }
     }
 
+    /// Send an audit record for every management operation to `sink`.
+    ///
+    /// Without a sink the manager builds no records at all, so a deployment
+    /// that does not consume KMS audit records is unaffected.
+    pub fn with_audit_sink(mut self, sink: Arc<dyn KmsAuditSink>) -> Self {
+        self.audit_sink = Some(sink);
+        self
+    }
+
     /// Get the default key ID if configured
     pub fn get_default_key_id(&self) -> Option<&String> {
         self.default_key_id.as_ref()
     }
 
+    /// Emit an audit record for a completed management operation.
+    ///
+    /// Called after the operation resolved, so nothing here can change its
+    /// result; the record is built only when a sink is installed.
+    fn audit<T>(
+        &self,
+        operation: KmsAuditOperation,
+        context: &OperationContext,
+        key_id: Option<&str>,
+        started: Instant,
+        result: &Result<T>,
+    ) {
+        let Some(sink) = self.audit_sink.as_ref() else {
+            return;
+        };
+
+        sink.emit(
+            KmsAuditRecord::new(operation, context, self.backend_kind)
+                .with_key_id(key_id)
+                .with_latency(started.elapsed())
+                .with_result(result),
+        );
+    }
+
     /// Create a new master key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::create_key_with_context`].
     pub async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
+        self.create_key_with_context(request, &OperationContext::internal()).await
+    }
+
+    /// Create a new master key on behalf of `context`'s principal
+    pub async fn create_key_with_context(
+        &self,
+        request: CreateKeyRequest,
+        context: &OperationContext,
+    ) -> Result<CreateKeyResponse> {
+        let started = Instant::now();
+        let key_name = request.key_name.clone();
+        let result = self.create_key_inner(request).await;
+        let key_id = result.as_ref().ok().map(|r| r.key_id.as_str()).or(key_name.as_deref());
+        self.audit(KmsAuditOperation::CreateKey, context, key_id, started, &result);
+        result
+    }
+
+    async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
         let response = self.backend.create_key(request).await?;
 
         // Cache the key metadata if enabled
@@ -84,7 +144,27 @@ impl KmsManager {
     }
 
     /// Describe a key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::describe_key_with_context`].
     pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
+        self.describe_key_with_context(request, &OperationContext::internal()).await
+    }
+
+    /// Describe a key on behalf of `context`'s principal
+    pub async fn describe_key_with_context(
+        &self,
+        request: DescribeKeyRequest,
+        context: &OperationContext,
+    ) -> Result<DescribeKeyResponse> {
+        let started = Instant::now();
+        let key_id = request.key_id.clone();
+        let result = self.describe_key_inner(request).await;
+        self.audit(KmsAuditOperation::DescribeKey, context, Some(&key_id), started, &result);
+        result
+    }
+
+    async fn describe_key_inner(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
         // Check cache first if enabled
         if self.enable_cache {
             let cache = self.cache.read().await;
@@ -109,12 +189,24 @@ impl KmsManager {
     }
 
     /// List keys
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::list_keys_with_context`].
     pub async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
-        self.backend.list_keys(request).await
+        self.list_keys_with_context(request, &OperationContext::internal()).await
     }
 
-    /// Get cache statistics
-    pub async fn cache_stats(&self) -> Option<(u64, u64)> {
+    /// List keys on behalf of `context`'s principal
+    pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
+        let started = Instant::now();
+        let result = self.backend.list_keys(request).await;
+        // Listing spans keys, so the record carries no key id.
+        self.audit(KmsAuditOperation::ListKeys, context, None, started, &result);
+        result
+    }
+
+    /// Get cache statistics, or `None` when caching is disabled
+    pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
         if self.enable_cache {
             let cache = self.cache.read().await;
             Some(cache.stats())
@@ -133,7 +225,27 @@ impl KmsManager {
     }
 
     /// Delete a key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::delete_key_with_context`].
     pub async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
+        self.delete_key_with_context(request, &OperationContext::internal()).await
+    }
+
+    /// Delete a key on behalf of `context`'s principal
+    pub async fn delete_key_with_context(
+        &self,
+        request: DeleteKeyRequest,
+        context: &OperationContext,
+    ) -> Result<DeleteKeyResponse> {
+        let started = Instant::now();
+        let key_id = request.key_id.clone();
+        let result = self.delete_key_inner(request).await;
+        self.audit(KmsAuditOperation::ScheduleKeyDeletion, context, Some(&key_id), started, &result);
+        result
+    }
+
+    async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
         let response = self.backend.delete_key(request).await?;
 
         // Remove from cache if enabled and key is being deleted
@@ -146,7 +258,28 @@ impl KmsManager {
     }
 
     /// Cancel key deletion
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::cancel_key_deletion_with_context`].
     pub async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
+        self.cancel_key_deletion_with_context(request, &OperationContext::internal())
+            .await
+    }
+
+    /// Cancel key deletion on behalf of `context`'s principal
+    pub async fn cancel_key_deletion_with_context(
+        &self,
+        request: CancelKeyDeletionRequest,
+        context: &OperationContext,
+    ) -> Result<CancelKeyDeletionResponse> {
+        let started = Instant::now();
+        let key_id = request.key_id.clone();
+        let result = self.cancel_key_deletion_inner(request).await;
+        self.audit(KmsAuditOperation::CancelKeyDeletion, context, Some(&key_id), started, &result);
+        result
+    }
+
+    async fn cancel_key_deletion_inner(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
         let response = self.backend.cancel_key_deletion(request).await?;
 
         // Update cache if enabled
@@ -159,24 +292,60 @@ impl KmsManager {
     }
 
     /// Enable a disabled key
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::enable_key_with_context`].
     pub async fn enable_key(&self, key_id: &str) -> Result<()> {
-        self.backend.enable_key(key_id).await?;
-        self.invalidate_cached_metadata(key_id).await;
-        Ok(())
+        self.enable_key_with_context(key_id, &OperationContext::internal()).await
+    }
+
+    /// Enable a disabled key on behalf of `context`'s principal
+    pub async fn enable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
+        let started = Instant::now();
+        let result = self.backend.enable_key(key_id).await;
+        if result.is_ok() {
+            self.invalidate_cached_metadata(key_id).await;
+        }
+        self.audit(KmsAuditOperation::EnableKey, context, Some(key_id), started, &result);
+        result
     }
 
     /// Disable a key; existing data remains decryptable
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::disable_key_with_context`].
     pub async fn disable_key(&self, key_id: &str) -> Result<()> {
-        self.backend.disable_key(key_id).await?;
-        self.invalidate_cached_metadata(key_id).await;
-        Ok(())
+        self.disable_key_with_context(key_id, &OperationContext::internal()).await
+    }
+
+    /// Disable a key on behalf of `context`'s principal
+    pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
+        let started = Instant::now();
+        let result = self.backend.disable_key(key_id).await;
+        if result.is_ok() {
+            self.invalidate_cached_metadata(key_id).await;
+        }
+        self.audit(KmsAuditOperation::DisableKey, context, Some(key_id), started, &result);
+        result
     }
 
     /// Rotate a key to a new version
+    ///
+    /// Audited as an internal operation; callers serving an authenticated
+    /// request should use [`Self::rotate_key_with_context`].
     pub async fn rotate_key(&self, key_id: &str) -> Result<()> {
-        self.backend.rotate_key(key_id).await?;
-        self.invalidate_cached_metadata(key_id).await;
-        Ok(())
+        self.rotate_key_with_context(key_id, &OperationContext::internal()).await
+    }
+
+    /// Rotate a key on behalf of `context`'s principal
+    pub async fn rotate_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
+        let started = Instant::now();
+        let result = self.backend.rotate_key(key_id).await;
+        if result.is_ok() {
+            self.invalidate_cached_metadata(key_id).await;
+        }
+        self.audit(KmsAuditOperation::RotateKey, context, Some(key_id), started, &result);
+        result
     }
 
     /// Drop cached metadata after a state mutation so the next describe
@@ -208,11 +377,470 @@ impl KmsManager {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::audit::KmsAuditOutcome;
     use crate::backends::local::LocalKmsBackend;
-    use crate::types::{KeySpec, KeyState, KeyUsage};
+    use crate::error::KmsError;
+    use crate::types::{KeyMetadata, KeySpec, KeyState, KeyUsage};
+    use async_trait::async_trait;
+    use base64::Engine as _;
+    use jiff::Zoned;
     use std::collections::HashMap;
+    use std::sync::Mutex;
     use tempfile::tempdir;
 
+    /// Sink that keeps every record so tests can assert on the audit trail.
+    #[derive(Default)]
+    struct CapturingSink {
+        records: Mutex<Vec<KmsAuditRecord>>,
+    }
+
+    impl KmsAuditSink for CapturingSink {
+        fn emit(&self, record: KmsAuditRecord) {
+            self.records
+                .lock()
+                .expect("audit records lock should not be poisoned")
+                .push(record);
+        }
+    }
+
+    impl CapturingSink {
+        fn records(&self) -> Vec<KmsAuditRecord> {
+            self.records
+                .lock()
+                .expect("audit records lock should not be poisoned")
+                .clone()
+        }
+
+        fn take_one(&self) -> KmsAuditRecord {
+            let mut records = self.records.lock().expect("audit records lock should not be poisoned");
+            assert_eq!(records.len(), 1, "expected exactly one audit record, got {records:?}");
+            records.remove(0)
+        }
+    }
+
+    /// Backend whose management operations all succeed or all fail, so a
+    /// single test can drive every audited operation down both paths —
+    /// including operations no real backend supports on both.
+    struct ScriptedBackend {
+        failure: Option<KmsError>,
+    }
+
+    impl ScriptedBackend {
+        fn succeeding() -> Self {
+            Self { failure: None }
+        }
+
+        fn failing(failure: KmsError) -> Self {
+            Self { failure: Some(failure) }
+        }
+
+        fn check(&self) -> Result<()> {
+            match &self.failure {
+                Some(failure) => Err(failure.clone()),
+                None => Ok(()),
+            }
+        }
+
+        fn metadata(key_id: &str) -> KeyMetadata {
+            KeyMetadata {
+                key_id: key_id.to_string(),
+                key_state: KeyState::Enabled,
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: None,
+                creation_date: Zoned::now(),
+                deletion_date: None,
+                origin: "RUSTFS_KMS".to_string(),
+                key_manager: "RUSTFS".to_string(),
+                tags: HashMap::new(),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl KmsBackend for ScriptedBackend {
+        async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
+            self.check()?;
+            let key_id = request.key_name.unwrap_or_else(|| "scripted-key".to_string());
+            Ok(CreateKeyResponse {
+                key_metadata: Self::metadata(&key_id),
+                key_id,
+            })
+        }
+
+        async fn encrypt(&self, _request: EncryptRequest) -> Result<EncryptResponse> {
+            unimplemented!("data plane is not audited by the manager")
+        }
+
+        async fn decrypt(&self, _request: DecryptRequest) -> Result<DecryptResponse> {
+            unimplemented!("data plane is not audited by the manager")
+        }
+
+        async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
+            unimplemented!("data plane is not audited by the manager")
+        }
+
+        async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
+            self.check()?;
+            Ok(DescribeKeyResponse {
+                key_metadata: Self::metadata(&request.key_id),
+            })
+        }
+
+        async fn list_keys(&self, _request: ListKeysRequest) -> Result<ListKeysResponse> {
+            self.check()?;
+            Ok(ListKeysResponse {
+                keys: Vec::new(),
+                next_marker: None,
+                truncated: false,
+            })
+        }
+
+        async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
+            self.check()?;
+            Ok(DeleteKeyResponse {
+                key_metadata: Self::metadata(&request.key_id),
+                key_id: request.key_id,
+                deletion_date: None,
+            })
+        }
+
+        async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
+            self.check()?;
+            Ok(CancelKeyDeletionResponse {
+                key_metadata: Self::metadata(&request.key_id),
+                key_id: request.key_id,
+            })
+        }
+
+        async fn enable_key(&self, _key_id: &str) -> Result<()> {
+            self.check()
+        }
+
+        async fn disable_key(&self, _key_id: &str) -> Result<()> {
+            self.check()
+        }
+
+        async fn rotate_key(&self, _key_id: &str) -> Result<()> {
+            self.check()
+        }
+
+        async fn health_check(&self) -> Result<bool> {
+            Ok(true)
+        }
+    }
+
+    const AUDITED_KEY_ID: &str = "audited-key";
+
+    /// Prefix length used when checking that a record embedded no *part* of a
+    /// secret. Long enough that a collision with unrelated text is not a real
+    /// concern.
+    const FRAGMENT_LEN: usize = 24;
+
+    fn scripted_manager(backend: ScriptedBackend) -> (KmsManager, Arc<CapturingSink>) {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let sink = Arc::new(CapturingSink::default());
+        let manager = KmsManager::new(Arc::new(backend), config).with_audit_sink(sink.clone());
+        (manager, sink)
+    }
+
+    fn request_context() -> OperationContext {
+        OperationContext::new("arn:aws:iam::user/alice".to_string())
+            .with_source_ip("192.0.2.10".to_string())
+            .with_user_agent("rustfs-admin/1".to_string())
+            .with_context("requestID".to_string(), "req-42".to_string())
+    }
+
+    /// Drive one management operation and return the single record it emitted.
+    async fn run_operation(manager: &KmsManager, operation: KmsAuditOperation, context: &OperationContext) -> Result<()> {
+        match operation {
+            KmsAuditOperation::CreateKey => manager
+                .create_key_with_context(
+                    CreateKeyRequest {
+                        key_name: Some(AUDITED_KEY_ID.to_string()),
+                        ..Default::default()
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::DescribeKey => manager
+                .describe_key_with_context(
+                    DescribeKeyRequest {
+                        key_id: AUDITED_KEY_ID.to_string(),
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::ListKeys => manager
+                .list_keys_with_context(ListKeysRequest::default(), context)
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::ScheduleKeyDeletion => manager
+                .delete_key_with_context(
+                    DeleteKeyRequest {
+                        key_id: AUDITED_KEY_ID.to_string(),
+                        pending_window_in_days: None,
+                        force_immediate: None,
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::CancelKeyDeletion => manager
+                .cancel_key_deletion_with_context(
+                    CancelKeyDeletionRequest {
+                        key_id: AUDITED_KEY_ID.to_string(),
+                    },
+                    context,
+                )
+                .await
+                .map(|_| ()),
+            KmsAuditOperation::EnableKey => manager.enable_key_with_context(AUDITED_KEY_ID, context).await,
+            KmsAuditOperation::DisableKey => manager.disable_key_with_context(AUDITED_KEY_ID, context).await,
+            KmsAuditOperation::RotateKey => manager.rotate_key_with_context(AUDITED_KEY_ID, context).await,
+            // Physical removal happens on the background sweep, not here.
+            KmsAuditOperation::DeleteKey => unreachable!("removal is audited by the deletion worker"),
+        }
+    }
+
+    /// Every management operation the manager serves, in audit terms.
+    const AUDITED_OPERATIONS: [KmsAuditOperation; 8] = [
+        KmsAuditOperation::CreateKey,
+        KmsAuditOperation::DescribeKey,
+        KmsAuditOperation::ListKeys,
+        KmsAuditOperation::ScheduleKeyDeletion,
+        KmsAuditOperation::CancelKeyDeletion,
+        KmsAuditOperation::EnableKey,
+        KmsAuditOperation::DisableKey,
+        KmsAuditOperation::RotateKey,
+    ];
+
+    #[tokio::test]
+    async fn every_management_operation_emits_a_complete_success_record() {
+        for operation in AUDITED_OPERATIONS {
+            let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
+            let context = request_context();
+
+            let outer = Instant::now();
+            run_operation(&manager, operation, &context)
+                .await
+                .unwrap_or_else(|error| panic!("{} should succeed: {error}", operation.as_str()));
+            let outer_elapsed = outer.elapsed();
+
+            let record = sink.take_one();
+            assert_eq!(record.operation, operation);
+            assert_eq!(record.event, operation.event_name());
+            assert_eq!(record.outcome, KmsAuditOutcome::Success);
+            assert_eq!(record.error_class, None);
+            assert_eq!(record.operation_id, context.operation_id);
+            assert_eq!(record.principal, "arn:aws:iam::user/alice");
+            assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
+            assert_eq!(record.user_agent.as_deref(), Some("rustfs-admin/1"));
+            assert_eq!(record.backend, "local");
+            assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-42"));
+            assert!(
+                record.latency <= outer_elapsed,
+                "{} reported a latency larger than the call it measured",
+                operation.as_str()
+            );
+
+            // Listing spans keys; every other operation names the key it touched.
+            if operation == KmsAuditOperation::ListKeys {
+                assert_eq!(record.key_id, None);
+            } else {
+                assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn every_management_operation_emits_a_failure_record() {
+        for operation in AUDITED_OPERATIONS {
+            let (manager, sink) = scripted_manager(ScriptedBackend::failing(KmsError::access_denied("denied by policy")));
+            let context = request_context();
+
+            let error = run_operation(&manager, operation, &context)
+                .await
+                .expect_err("scripted backend should reject the operation");
+            assert!(matches!(error, KmsError::AccessDenied { .. }));
+
+            let record = sink.take_one();
+            assert_eq!(record.operation, operation);
+            assert_eq!(record.event, operation.event_name());
+            assert_eq!(record.outcome, KmsAuditOutcome::Failure);
+            assert_eq!(record.error_class, Some("access_denied"));
+            assert_eq!(record.operation_id, context.operation_id);
+            assert_eq!(record.principal, "arn:aws:iam::user/alice");
+            assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
+
+            // A denied create still has to name the key the caller asked for,
+            // otherwise the record cannot answer "what were they after".
+            if operation != KmsAuditOperation::ListKeys {
+                assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn operations_are_unaffected_when_no_sink_is_installed() {
+        // The audit trail is optional; without a sink the manager must behave
+        // exactly as it did before records existed.
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let manager = KmsManager::new(Arc::new(ScriptedBackend::succeeding()), config);
+
+        for operation in AUDITED_OPERATIONS {
+            run_operation(&manager, operation, &request_context())
+                .await
+                .unwrap_or_else(|error| panic!("{} should succeed without a sink: {error}", operation.as_str()));
+        }
+    }
+
+    #[tokio::test]
+    async fn context_free_calls_are_attributed_to_the_internal_principal() {
+        // Callers that have no authenticated identity must still be
+        // distinguishable from an identity we failed to record.
+        let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
+
+        manager
+            .describe_key(DescribeKeyRequest {
+                key_id: AUDITED_KEY_ID.to_string(),
+            })
+            .await
+            .expect("describe should succeed");
+
+        let record = sink.take_one();
+        assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
+        assert_eq!(record.source_ip, None);
+        assert_eq!(record.user_agent, None);
+    }
+
+    #[tokio::test]
+    async fn unsupported_lifecycle_operations_are_audited_with_their_own_class() {
+        // The local backend has no version history, so rotation is a capability
+        // gap rather than a policy denial; the audit trail must say so.
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
+        let sink = Arc::new(CapturingSink::default());
+        let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
+
+        let key_id = manager
+            .create_key_with_context(
+                CreateKeyRequest {
+                    key_name: Some("rotate-me".to_string()),
+                    ..Default::default()
+                },
+                &request_context(),
+            )
+            .await
+            .expect("create should succeed")
+            .key_id;
+
+        manager
+            .rotate_key_with_context(&key_id, &request_context())
+            .await
+            .expect_err("local rotation must be rejected");
+
+        let records = sink.records();
+        let rotate = records.last().expect("rotation should be audited");
+        assert_eq!(rotate.operation, KmsAuditOperation::RotateKey);
+        assert_eq!(rotate.outcome, KmsAuditOutcome::Failure);
+        assert_eq!(rotate.error_class, Some("unsupported_capability"));
+        assert_eq!(rotate.key_id.as_deref(), Some(key_id.as_str()));
+    }
+
+    /// Negative assertion: no audit record may reproduce key material. Driven
+    /// against the real local backend so the assertion covers whatever the
+    /// backend actually hands back, not a hand-written stand-in.
+    #[tokio::test]
+    async fn audit_records_never_reproduce_key_material() {
+        let temp_dir = tempdir().expect("Failed to create temp dir");
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
+        let sink = Arc::new(CapturingSink::default());
+        let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
+
+        let grant_token = "grant-token-cec4d4b5a1";
+        let context = request_context();
+
+        let key_id = manager
+            .create_key_with_context(
+                CreateKeyRequest {
+                    key_name: Some("material-key".to_string()),
+                    ..Default::default()
+                },
+                &context,
+            )
+            .await
+            .expect("create should succeed")
+            .key_id;
+
+        // Produce real key material, then keep driving the management plane so
+        // any record built afterwards is covered by the assertions below.
+        let data_key = manager
+            .generate_data_key(GenerateDataKeyRequest {
+                key_id: key_id.clone(),
+                key_spec: KeySpec::Aes256,
+                encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
+            })
+            .await
+            .expect("data key generation should succeed");
+        let decrypted = manager
+            .decrypt(DecryptRequest {
+                ciphertext: data_key.ciphertext_blob.clone(),
+                encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
+                grant_tokens: vec![grant_token.to_string()],
+            })
+            .await
+            .expect("decrypt should succeed");
+
+        manager
+            .describe_key_with_context(DescribeKeyRequest { key_id: key_id.clone() }, &context)
+            .await
+            .expect("describe should succeed");
+        manager
+            .list_keys_with_context(ListKeysRequest::default(), &context)
+            .await
+            .expect("list should succeed");
+        manager
+            .disable_key_with_context(&key_id, &context)
+            .await
+            .expect("disable should succeed");
+        manager
+            .enable_key_with_context(&key_id, &context)
+            .await
+            .expect("enable should succeed");
+
+        let base64 = base64::engine::general_purpose::STANDARD;
+        let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
+        let mut forbidden = vec![grant_token.to_string()];
+        forbidden.extend(encodings(&data_key.plaintext_key));
+        forbidden.extend(encodings(&decrypted.plaintext));
+        forbidden.extend(encodings(&data_key.ciphertext_blob));
+        // Fragments catch a record that embedded only part of a blob.
+        let fragments: Vec<String> = forbidden
+            .iter()
+            .filter(|secret| secret.len() > FRAGMENT_LEN)
+            .map(|secret| secret[..FRAGMENT_LEN].to_string())
+            .collect();
+        forbidden.extend(fragments);
+
+        let records = sink.records();
+        assert!(!records.is_empty(), "management operations should have been audited");
+        for record in &records {
+            let rendered = format!("{record:?}");
+            for secret in &forbidden {
+                assert!(
+                    !rendered.contains(secret.as_str()),
+                    "audit record leaked key material or a grant token: {rendered}"
+                );
+            }
+        }
+    }
+
     #[tokio::test]
     async fn test_manager_operations() {
         let temp_dir = tempdir().expect("Failed to create temp dir");
@@ -254,9 +882,12 @@ mod tests {
         let describe_response = manager.describe_key(describe_request).await.expect("Failed to describe key");
         assert_eq!(describe_response.key_metadata.key_id, create_response.key_id);
 
-        // Test cache stats
-        let stats = manager.cache_stats().await;
-        assert!(stats.is_some());
+        // Creating the key populated the cache, so the describe above was
+        // served from it rather than from the backend.
+        let stats = manager.cache_stats().await.expect("cache is enabled");
+        assert_eq!(stats.entries, 1);
+        assert_eq!(stats.hits, 1);
+        assert_eq!(stats.misses, 0);
 
         // Test health check
         let health = manager.health_check().await.expect("Health check failed");

crates/kms/src/probe.rs

@@ -0,0 +1,894 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Background synthetic probe for the configured KMS backend.
+//!
+//! A health check that only reports "the backend answered" cannot distinguish a
+//! usable KMS from one that is reachable but no longer returns material that
+//! round-trips. Each probe round therefore generates a data key under a
+//! dedicated key, decrypts the returned ciphertext, and compares the plaintext,
+//! so a backend that answers without being usable becomes visible before object
+//! traffic depends on it.
+//!
+//! The probe key uses a fixed reserved id ([`PROBE_KEY_ID`]) and is created only
+//! when missing, so every node of a deployment converges on the same key and a
+//! create that loses the race against another node is a success, not an error.
+//!
+//! Backends that cannot host such a key — single-key read-only backends, or
+//! backends without data key generation — are reported as
+//! [`ProbeResult::Unsupported`] exactly once and then left alone: an
+//! unsupported backend is a configuration fact, not an outage, and must never
+//! raise failure metrics or alerts.
+//!
+//! Results are published two ways: as a lock-free [`ProbeStatus`] snapshot that
+//! readiness reporting can read without touching the backend, and as
+//! `rustfs_kms_probe_*` metrics. As everywhere else in this crate, metric labels
+//! carry only static outcome classes — never key identifiers, key material, or
+//! ciphertext.
+
+use crate::backends::KmsBackend;
+use crate::error::KmsError;
+use crate::types::{CreateKeyRequest, DecryptRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyUsage};
+use arc_swap::ArcSwap;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use subtle::ConstantTimeEq;
+use tokio::time::Instant;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, info, warn};
+use zeroize::Zeroize;
+
+/// Reserved id of the key every probe round runs through.
+///
+/// Fixed rather than per-node or per-process: all nodes must converge on one
+/// key instead of littering the backend with probe keys, and the reserved
+/// `rustfs-internal-` prefix marks it as RustFS-owned for operators reviewing
+/// the key list.
+pub const PROBE_KEY_ID: &str = "rustfs-internal-kms-probe";
+
+/// Description stored on the probe key when this node creates it.
+const PROBE_KEY_DESCRIPTION: &str = "RustFS internal KMS health probe key; created and used only by the synthetic probe";
+
+/// Default interval between probe rounds.
+pub const DEFAULT_PROBE_INTERVAL: Duration = Duration::from_secs(60);
+
+/// Floor for the configured interval. Probing faster buys no additional signal
+/// and multiplies backend load by the size of the deployment.
+pub const MIN_PROBE_INTERVAL: Duration = Duration::from_secs(5);
+
+/// Probe interval in whole seconds; `0` disables the probe entirely.
+pub const ENV_KMS_PROBE_INTERVAL_SECS: &str = "RUSTFS_KMS_PROBE_INTERVAL_SECS";
+
+/// Result of the most recent probe round.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProbeResult {
+    /// No round has completed yet for this service version.
+    Pending,
+    /// The full generate/decrypt/compare cycle succeeded.
+    Success,
+    /// The backend cannot host the probe key; no further rounds run and no
+    /// failures are recorded.
+    Unsupported,
+    /// The round failed with the given classification.
+    Failure(ProbeFailureKind),
+}
+
+impl ProbeResult {
+    /// Static metric label for this result.
+    fn as_label(self) -> &'static str {
+        match self {
+            ProbeResult::Pending => "pending",
+            ProbeResult::Success => "success",
+            ProbeResult::Unsupported => "unsupported",
+            ProbeResult::Failure(_) => "failure",
+        }
+    }
+}
+
+/// Stage of the round trip that failed, used as a static metric label.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProbeFailureKind {
+    /// The dedicated probe key could not be described or created.
+    KeyProvisioning,
+    /// Data key generation failed.
+    Generate,
+    /// The freshly generated ciphertext could not be decrypted.
+    Decrypt,
+    /// Decryption succeeded but returned material that differs from what was
+    /// generated — the backend answers without round-tripping.
+    Mismatch,
+}
+
+impl ProbeFailureKind {
+    /// Static metric label for this failure class.
+    fn as_label(self) -> &'static str {
+        match self {
+            ProbeFailureKind::KeyProvisioning => "key_provisioning",
+            ProbeFailureKind::Generate => "generate",
+            ProbeFailureKind::Decrypt => "decrypt",
+            ProbeFailureKind::Mismatch => "mismatch",
+        }
+    }
+}
+
+/// Snapshot of what the probe has observed so far.
+///
+/// Consumed by readiness reporting, which owns the staleness and consecutive
+/// failure thresholds; the probe itself only records what happened.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ProbeStatus {
+    /// Result of the most recent completed round.
+    pub result: ProbeResult,
+    /// When the most recent round completed.
+    pub last_round_at: Option<Instant>,
+    /// When the most recent successful round completed.
+    pub last_success_at: Option<Instant>,
+    /// Wall-clock timestamp of the most recent success, in Unix seconds, for
+    /// display and for the exported gauge. Use [`ProbeStatus::last_success_age`]
+    /// for staleness decisions: it is monotonic and immune to clock steps.
+    pub last_success_unix_secs: Option<i64>,
+    /// Failed rounds since the last success; reset by any non-failure result.
+    pub consecutive_failures: u32,
+}
+
+impl ProbeStatus {
+    /// Initial snapshot published before the first round completes.
+    fn pending() -> Self {
+        Self {
+            result: ProbeResult::Pending,
+            last_round_at: None,
+            last_success_at: None,
+            last_success_unix_secs: None,
+            consecutive_failures: 0,
+        }
+    }
+
+    /// How long ago the most recent round completed.
+    pub fn last_round_age(&self) -> Option<Duration> {
+        self.last_round_at.map(|at| at.elapsed())
+    }
+
+    /// How long ago the most recent successful round completed.
+    pub fn last_success_age(&self) -> Option<Duration> {
+        self.last_success_at.map(|at| at.elapsed())
+    }
+}
+
+/// Lock-free publication slot for [`ProbeStatus`].
+///
+/// Written by the single worker task and read by anyone; readers never block
+/// the probe and the probe never blocks a readiness handler.
+struct ProbeState {
+    status: ArcSwap<ProbeStatus>,
+}
+
+impl ProbeState {
+    fn new() -> Self {
+        Self {
+            status: ArcSwap::from_pointee(ProbeStatus::pending()),
+        }
+    }
+
+    fn status(&self) -> Arc<ProbeStatus> {
+        self.status.load_full()
+    }
+
+    /// Fold one round result into the published snapshot.
+    ///
+    /// Safe as a plain load-then-store because the owning worker task is the
+    /// only writer.
+    fn publish(&self, result: ProbeResult) {
+        let previous = self.status.load();
+        let now = Instant::now();
+        let (last_success_at, last_success_unix_secs) = match result {
+            ProbeResult::Success => (Some(now), Some(jiff::Timestamp::now().as_second())),
+            _ => (previous.last_success_at, previous.last_success_unix_secs),
+        };
+        let consecutive_failures = match result {
+            ProbeResult::Failure(_) => previous.consecutive_failures.saturating_add(1),
+            _ => 0,
+        };
+        self.status.store(Arc::new(ProbeStatus {
+            result,
+            last_round_at: Some(now),
+            last_success_at,
+            last_success_unix_secs,
+            consecutive_failures,
+        }));
+        record_state(result, last_success_unix_secs, consecutive_failures);
+    }
+}
+
+/// Owner of one service version's probe worker.
+///
+/// Mirrors the deletion worker's ownership model: cancelling stops the loop,
+/// and dropping the handle — a service version replaced without an explicit
+/// stop — cancels as a safety net.
+pub struct ProbeHandle {
+    state: Arc<ProbeState>,
+    cancel: CancellationToken,
+    task: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
+}
+
+impl ProbeHandle {
+    /// Latest published snapshot.
+    pub fn status(&self) -> Arc<ProbeStatus> {
+        self.state.status()
+    }
+
+    /// Stop the worker. The task observes the cancelled token on its next poll.
+    pub(crate) fn shutdown(&self) {
+        self.cancel.cancel();
+        if let Ok(mut task) = self.task.lock() {
+            drop(task.take());
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn is_cancelled(&self) -> bool {
+        self.cancel.is_cancelled()
+    }
+}
+
+impl Drop for ProbeHandle {
+    fn drop(&mut self) {
+        self.cancel.cancel();
+    }
+}
+
+/// Start the probe for a service version.
+///
+/// Returns `None` when the backend advertises no data key round trip, or when
+/// the operator disabled the probe; both cases leave no background task behind.
+pub(crate) fn spawn_probe_worker(backend: Arc<dyn KmsBackend>) -> Option<ProbeHandle> {
+    let capabilities = backend.capabilities();
+    if !capabilities.generate_data_key || !capabilities.decrypt {
+        return None;
+    }
+    let interval = configured_interval()?;
+    let state = Arc::new(ProbeState::new());
+    let cancel = CancellationToken::new();
+    let task = ProbeWorker::new(backend, state.clone(), interval).spawn(cancel.clone());
+    Some(ProbeHandle {
+        state,
+        cancel,
+        task: std::sync::Mutex::new(Some(task)),
+    })
+}
+
+/// Probe interval from the environment, or `None` when the probe is disabled.
+fn configured_interval() -> Option<Duration> {
+    parse_interval(std::env::var(ENV_KMS_PROBE_INTERVAL_SECS).ok().as_deref())
+}
+
+/// Interpret the configured interval: unset or unparsable falls back to the
+/// default, `0` disables the probe, and anything below [`MIN_PROBE_INTERVAL`]
+/// is raised to it so a misconfigured value cannot turn the probe into a load
+/// generator against the backend.
+fn parse_interval(value: Option<&str>) -> Option<Duration> {
+    let Some(value) = value else {
+        return Some(DEFAULT_PROBE_INTERVAL);
+    };
+    let Ok(seconds) = value.trim().parse::<u64>() else {
+        warn!(
+            variable = ENV_KMS_PROBE_INTERVAL_SECS,
+            "ignoring unparsable KMS probe interval; falling back to the default"
+        );
+        return Some(DEFAULT_PROBE_INTERVAL);
+    };
+    if seconds == 0 {
+        return None;
+    }
+    Some(Duration::from_secs(seconds).max(MIN_PROBE_INTERVAL))
+}
+
+/// Encryption context bound to every probe data key.
+///
+/// Static and free of per-node values, so material generated by one node stays
+/// decryptable by every other node running the same probe.
+fn probe_encryption_context() -> HashMap<String, String> {
+    HashMap::from([("rustfs-purpose".to_string(), "kms-probe".to_string())])
+}
+
+/// Outcome of making sure the dedicated probe key exists.
+enum KeyProvisioning {
+    Ready,
+    Unsupported,
+    Failed,
+}
+
+pub(crate) struct ProbeWorker {
+    backend: Arc<dyn KmsBackend>,
+    state: Arc<ProbeState>,
+    interval: Duration,
+}
+
+impl ProbeWorker {
+    fn new(backend: Arc<dyn KmsBackend>, state: Arc<ProbeState>, interval: Duration) -> Self {
+        Self {
+            backend,
+            state,
+            interval,
+        }
+    }
+
+    fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
+        tokio::spawn(async move { self.run(cancel).await })
+    }
+
+    async fn run(self, cancel: CancellationToken) {
+        describe_metrics();
+        let mut ticker = tokio::time::interval(self.interval);
+        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+        loop {
+            tokio::select! {
+                _ = cancel.cancelled() => {
+                    debug!("KMS probe worker stopped");
+                    return;
+                }
+                _ = ticker.tick() => {}
+            }
+            let started = Instant::now();
+            let result = self.round().await;
+            record_round(result, started.elapsed());
+            self.state.publish(result);
+            if result == ProbeResult::Unsupported {
+                // A capability gap does not change while a service version
+                // lives, so retrying it forever would only produce noise.
+                info!("KMS backend cannot host the synthetic probe key; probe disabled for this service version");
+                return;
+            }
+        }
+    }
+
+    /// Run one round. Exposed separately so tests can drive the round trip
+    /// deterministically without the interval loop.
+    pub(crate) async fn round(&self) -> ProbeResult {
+        let capabilities = self.backend.capabilities();
+        if !capabilities.generate_data_key || !capabilities.decrypt {
+            return ProbeResult::Unsupported;
+        }
+        match self.ensure_probe_key().await {
+            KeyProvisioning::Ready => {}
+            KeyProvisioning::Unsupported => return ProbeResult::Unsupported,
+            KeyProvisioning::Failed => return ProbeResult::Failure(ProbeFailureKind::KeyProvisioning),
+        }
+
+        let mut generated = match self
+            .backend
+            .generate_data_key(GenerateDataKeyRequest {
+                key_id: PROBE_KEY_ID.to_string(),
+                key_spec: KeySpec::Aes256,
+                encryption_context: probe_encryption_context(),
+            })
+            .await
+        {
+            Ok(generated) => generated,
+            Err(error) => {
+                warn!(%error, "KMS probe could not generate a data key");
+                return ProbeResult::Failure(ProbeFailureKind::Generate);
+            }
+        };
+
+        let decrypted = self
+            .backend
+            .decrypt(DecryptRequest {
+                ciphertext: generated.ciphertext_blob.clone(),
+                encryption_context: probe_encryption_context(),
+                grant_tokens: Vec::new(),
+            })
+            .await;
+
+        let result = match decrypted {
+            Ok(mut response) => {
+                let round_trips = bool::from(response.plaintext.ct_eq(&generated.plaintext_key));
+                response.plaintext.zeroize();
+                if round_trips {
+                    debug!("KMS probe round trip succeeded");
+                    ProbeResult::Success
+                } else {
+                    // The backend answered both calls but the material does not
+                    // survive the round trip: report it as loudly as an outage.
+                    warn!("KMS probe decrypted material does not match the generated data key");
+                    ProbeResult::Failure(ProbeFailureKind::Mismatch)
+                }
+            }
+            Err(error) => {
+                warn!(%error, "KMS probe could not decrypt its generated data key");
+                ProbeResult::Failure(ProbeFailureKind::Decrypt)
+            }
+        };
+        generated.plaintext_key.zeroize();
+        result
+    }
+
+    /// Create the probe key if it is missing.
+    ///
+    /// Idempotent by construction: an existing key is used as is, and a create
+    /// that loses the race against another node reports the key as ready
+    /// instead of failing the round.
+    async fn ensure_probe_key(&self) -> KeyProvisioning {
+        match self
+            .backend
+            .describe_key(DescribeKeyRequest {
+                key_id: PROBE_KEY_ID.to_string(),
+            })
+            .await
+        {
+            Ok(_) => return KeyProvisioning::Ready,
+            Err(KmsError::KeyNotFound { .. }) => {}
+            Err(error) if is_capability_gap(&error) => return KeyProvisioning::Unsupported,
+            Err(error) => {
+                warn!(%error, "KMS probe could not describe its probe key");
+                return KeyProvisioning::Failed;
+            }
+        }
+
+        match self
+            .backend
+            .create_key(CreateKeyRequest {
+                key_name: Some(PROBE_KEY_ID.to_string()),
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: Some(PROBE_KEY_DESCRIPTION.to_string()),
+                ..Default::default()
+            })
+            .await
+        {
+            Ok(_) => {
+                info!(key_id = PROBE_KEY_ID, "created the KMS synthetic probe key");
+                KeyProvisioning::Ready
+            }
+            // Another node created the key between the describe and the create.
+            Err(KmsError::KeyAlreadyExists { .. }) => KeyProvisioning::Ready,
+            Err(error) if is_capability_gap(&error) => KeyProvisioning::Unsupported,
+            Err(error) => {
+                warn!(%error, "KMS probe could not create its probe key");
+                KeyProvisioning::Failed
+            }
+        }
+    }
+}
+
+/// Whether an error means "this backend will never host a probe key".
+///
+/// Read-only single-key backends reject key creation outright, and backends
+/// without the operation report an unsupported capability. Both are properties
+/// of the configuration rather than symptoms of an outage, so they stop the
+/// probe instead of counting as failures.
+fn is_capability_gap(error: &KmsError) -> bool {
+    matches!(error, KmsError::UnsupportedCapability { .. } | KmsError::InvalidOperation { .. })
+}
+
+// ---------------------------------------------------------------------------
+// Metrics
+//
+// Label values are exclusively static outcome classes; the probe key id is a
+// compile-time constant and still never becomes a label, so the metric shape
+// stays independent of how many keys or nodes exist.
+// ---------------------------------------------------------------------------
+
+/// Counter: probe rounds completed, by `result`.
+const METRIC_PROBE_ROUNDS_TOTAL: &str = "rustfs_kms_probe_rounds_total";
+/// Counter: failed probe rounds, by `failure_kind`.
+const METRIC_PROBE_FAILURES_TOTAL: &str = "rustfs_kms_probe_failures_total";
+/// Histogram: wall-clock duration of one probe round, in seconds, by `result`.
+const METRIC_PROBE_DURATION_SECONDS: &str = "rustfs_kms_probe_duration_seconds";
+/// Gauge: Unix timestamp of the most recent successful round.
+const METRIC_PROBE_LAST_SUCCESS_TIMESTAMP: &str = "rustfs_kms_probe_last_success_timestamp_seconds";
+/// Gauge: failed rounds since the last success.
+const METRIC_PROBE_CONSECUTIVE_FAILURES: &str = "rustfs_kms_probe_consecutive_failures";
+
+/// Register metric descriptions once per process.
+fn describe_metrics() {
+    static DESCRIBE: std::sync::Once = std::sync::Once::new();
+    DESCRIBE.call_once(|| {
+        metrics::describe_counter!(METRIC_PROBE_ROUNDS_TOTAL, "Total synthetic KMS probe rounds completed, by result");
+        metrics::describe_counter!(
+            METRIC_PROBE_FAILURES_TOTAL,
+            "Total failed synthetic KMS probe rounds, by the round-trip stage that failed"
+        );
+        metrics::describe_histogram!(
+            METRIC_PROBE_DURATION_SECONDS,
+            "Wall-clock duration of one synthetic KMS probe round, in seconds"
+        );
+        metrics::describe_gauge!(
+            METRIC_PROBE_LAST_SUCCESS_TIMESTAMP,
+            "Unix timestamp of the most recent successful synthetic KMS probe round"
+        );
+        metrics::describe_gauge!(
+            METRIC_PROBE_CONSECUTIVE_FAILURES,
+            "Synthetic KMS probe rounds that have failed since the last success"
+        );
+    });
+}
+
+/// Record one completed round.
+///
+/// An unsupported backend is deliberately counted as its own result and never
+/// as a failure, so alerts on the failure counter stay silent for deployments
+/// whose backend cannot host the probe.
+fn record_round(result: ProbeResult, elapsed: Duration) {
+    metrics::counter!(METRIC_PROBE_ROUNDS_TOTAL, "result" => result.as_label()).increment(1);
+    if let ProbeResult::Failure(kind) = result {
+        metrics::counter!(METRIC_PROBE_FAILURES_TOTAL, "failure_kind" => kind.as_label()).increment(1);
+    }
+    metrics::histogram!(METRIC_PROBE_DURATION_SECONDS, "result" => result.as_label()).record(elapsed.as_secs_f64());
+}
+
+/// Mirror the published snapshot into the gauges.
+fn record_state(result: ProbeResult, last_success_unix_secs: Option<i64>, consecutive_failures: u32) {
+    if result == ProbeResult::Success
+        && let Some(timestamp) = last_success_unix_secs
+    {
+        // Only ever moved forward by a success, so a failing probe leaves the
+        // gauge at the last known good time and its age keeps growing.
+        metrics::gauge!(METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).set(timestamp as f64);
+    }
+    metrics::gauge!(METRIC_PROBE_CONSECUTIVE_FAILURES).set(f64::from(consecutive_failures));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::backends::local::LocalKmsBackend;
+    use crate::backends::static_kms::StaticKmsBackend;
+    use crate::config::KmsConfig;
+    use crate::types::{
+        CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyResponse, DecryptResponse, DeleteKeyRequest,
+        DeleteKeyResponse, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyResponse, KeyMetadata, KeyState,
+        ListKeysRequest, ListKeysResponse,
+    };
+    use async_trait::async_trait;
+    use base64::Engine as _;
+    use metrics_util::MetricKind;
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+    use std::future::Future;
+
+    async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<dyn KmsBackend> {
+        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
+        Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build"))
+    }
+
+    async fn static_backend() -> Arc<dyn KmsBackend> {
+        let config =
+            KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
+        Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
+    }
+
+    fn worker(backend: Arc<dyn KmsBackend>) -> ProbeWorker {
+        ProbeWorker::new(backend, Arc::new(ProbeState::new()), DEFAULT_PROBE_INTERVAL)
+    }
+
+    /// Backend that answers every probe call but corrupts the round trip in a
+    /// configurable way, so each failure classification can be provoked without
+    /// a real backend outage.
+    struct BrokenBackend {
+        fault: Fault,
+    }
+
+    #[derive(Clone, Copy)]
+    enum Fault {
+        Generate,
+        Decrypt,
+        /// Decrypt succeeds but returns material other than the generated key.
+        Mismatch,
+    }
+
+    impl BrokenBackend {
+        fn arc(fault: Fault) -> Arc<dyn KmsBackend> {
+            Arc::new(Self { fault })
+        }
+
+        fn probe_key_metadata() -> KeyMetadata {
+            KeyMetadata {
+                key_id: PROBE_KEY_ID.to_string(),
+                key_state: KeyState::Enabled,
+                key_usage: KeyUsage::EncryptDecrypt,
+                description: None,
+                creation_date: jiff::Zoned::now(),
+                deletion_date: None,
+                origin: "test".to_string(),
+                key_manager: "test".to_string(),
+                tags: HashMap::new(),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl KmsBackend for BrokenBackend {
+        async fn create_key(&self, _request: CreateKeyRequest) -> crate::error::Result<CreateKeyResponse> {
+            unimplemented!("the probe key always exists on this backend")
+        }
+
+        async fn encrypt(&self, _request: EncryptRequest) -> crate::error::Result<EncryptResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn decrypt(&self, _request: DecryptRequest) -> crate::error::Result<DecryptResponse> {
+            match self.fault {
+                Fault::Decrypt => Err(KmsError::backend_error("decrypt unavailable")),
+                Fault::Mismatch => Ok(DecryptResponse {
+                    plaintext: vec![0xffu8; 32],
+                    key_id: PROBE_KEY_ID.to_string(),
+                    encryption_algorithm: None,
+                }),
+                Fault::Generate => unreachable!("generation fails before decrypt is reached"),
+            }
+        }
+
+        async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> crate::error::Result<GenerateDataKeyResponse> {
+            match self.fault {
+                Fault::Generate => Err(KmsError::backend_error("data key generation unavailable")),
+                _ => Ok(GenerateDataKeyResponse {
+                    key_id: PROBE_KEY_ID.to_string(),
+                    plaintext_key: vec![0x11u8; 32],
+                    ciphertext_blob: vec![0x22u8; 48],
+                }),
+            }
+        }
+
+        async fn describe_key(&self, _request: DescribeKeyRequest) -> crate::error::Result<DescribeKeyResponse> {
+            Ok(DescribeKeyResponse {
+                key_metadata: Self::probe_key_metadata(),
+            })
+        }
+
+        async fn list_keys(&self, _request: ListKeysRequest) -> crate::error::Result<ListKeysResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn delete_key(&self, _request: DeleteKeyRequest) -> crate::error::Result<DeleteKeyResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn cancel_key_deletion(
+            &self,
+            _request: CancelKeyDeletionRequest,
+        ) -> crate::error::Result<CancelKeyDeletionResponse> {
+            unimplemented!("not exercised by probe tests")
+        }
+
+        async fn health_check(&self) -> crate::error::Result<bool> {
+            Ok(true)
+        }
+    }
+
+    #[tokio::test]
+    async fn round_creates_the_probe_key_once_and_round_trips_it() {
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let backend = local_backend(&temp_dir).await;
+        let worker = worker(backend.clone());
+
+        assert_eq!(worker.round().await, ProbeResult::Success);
+        let created = backend
+            .describe_key(DescribeKeyRequest {
+                key_id: PROBE_KEY_ID.to_string(),
+            })
+            .await
+            .expect("the first round must create the probe key");
+        assert_eq!(created.key_metadata.key_id, PROBE_KEY_ID);
+
+        // A second round reuses the key instead of failing on the existing one,
+        // which is also what a second node in the deployment would do.
+        assert_eq!(worker.round().await, ProbeResult::Success);
+        assert_eq!(
+            backend
+                .list_keys(ListKeysRequest {
+                    limit: Some(100),
+                    marker: None,
+                    usage_filter: None,
+                    status_filter: None,
+                })
+                .await
+                .expect("list keys")
+                .keys
+                .iter()
+                .filter(|key| key.key_id == PROBE_KEY_ID)
+                .count(),
+            1,
+            "probe key creation must be idempotent"
+        );
+    }
+
+    #[test]
+    fn read_only_backend_is_reported_unsupported_without_failures() {
+        let (snapshot, result) = record_metrics(|| {
+            Box::pin(async {
+                let worker = worker(static_backend().await);
+                let result = worker.round().await;
+                record_round(result, Duration::from_millis(1));
+                result
+            })
+        });
+
+        assert_eq!(result, ProbeResult::Unsupported);
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "unsupported")]), 1);
+        assert_eq!(
+            counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[]),
+            0,
+            "an unsupported backend must never raise probe failures"
+        );
+    }
+
+    #[tokio::test]
+    async fn each_round_trip_stage_gets_its_own_failure_classification() {
+        for (fault, expected) in [
+            (Fault::Generate, ProbeFailureKind::Generate),
+            (Fault::Decrypt, ProbeFailureKind::Decrypt),
+            (Fault::Mismatch, ProbeFailureKind::Mismatch),
+        ] {
+            assert_eq!(worker(BrokenBackend::arc(fault)).round().await, ProbeResult::Failure(expected));
+        }
+    }
+
+    #[test]
+    fn metrics_and_status_record_success_then_failure() {
+        let (snapshot, status) = record_metrics(|| {
+            Box::pin(async {
+                let temp_dir = tempfile::tempdir().expect("temp dir");
+                let state = Arc::new(ProbeState::new());
+                let healthy = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL);
+                let round = healthy.round().await;
+                record_round(round, Duration::from_millis(10));
+                state.publish(round);
+
+                let broken = ProbeWorker::new(BrokenBackend::arc(Fault::Mismatch), state.clone(), DEFAULT_PROBE_INTERVAL);
+                for _ in 0..2 {
+                    let round = broken.round().await;
+                    record_round(round, Duration::from_millis(10));
+                    state.publish(round);
+                }
+                state.status()
+            })
+        });
+
+        assert_eq!(status.result, ProbeResult::Failure(ProbeFailureKind::Mismatch));
+        assert_eq!(status.consecutive_failures, 2);
+        assert!(
+            status.last_success_at.is_some() && status.last_success_unix_secs.is_some(),
+            "a failing probe must keep reporting when it last succeeded"
+        );
+
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "success")]), 1);
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "failure")]), 2);
+        assert_eq!(counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[("failure_kind", "mismatch")]), 2);
+        assert_eq!(histogram_values(&snapshot, METRIC_PROBE_DURATION_SECONDS, &[]).len(), 3);
+        assert!(
+            gauge_value(&snapshot, METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).is_some_and(|timestamp| timestamp > 0.0),
+            "the last success gauge must carry a wall-clock timestamp"
+        );
+        assert_eq!(gauge_value(&snapshot, METRIC_PROBE_CONSECUTIVE_FAILURES), Some(2.0));
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn worker_publishes_status_and_stops_on_cancel() {
+        let temp_dir = tempfile::tempdir().expect("temp dir");
+        let state = Arc::new(ProbeState::new());
+        assert_eq!(state.status().result, ProbeResult::Pending);
+
+        let cancel = CancellationToken::new();
+        let task = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
+
+        // The paused clock auto-advances through the worker's interval ticks.
+        let mut observed = None;
+        for _ in 0..100 {
+            tokio::time::sleep(Duration::from_secs(1)).await;
+            if state.status().result != ProbeResult::Pending {
+                observed = Some(state.status());
+                break;
+            }
+        }
+        let observed = observed.expect("the worker loop must publish a round result");
+        assert_eq!(observed.result, ProbeResult::Success);
+        assert_eq!(observed.consecutive_failures, 0);
+        assert!(observed.last_success_at.is_some());
+
+        cancel.cancel();
+        task.await.expect("worker task must stop after cancellation");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn unsupported_backend_stops_the_worker_without_cancellation() {
+        let state = Arc::new(ProbeState::new());
+        let cancel = CancellationToken::new();
+        let task = ProbeWorker::new(static_backend().await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
+
+        task.await.expect("worker must exit on its own for an unsupported backend");
+        assert_eq!(state.status().result, ProbeResult::Unsupported);
+        assert!(!cancel.is_cancelled());
+    }
+
+    #[test]
+    fn interval_configuration_defaults_clamps_and_disables() {
+        assert_eq!(parse_interval(None), Some(DEFAULT_PROBE_INTERVAL));
+        assert_eq!(parse_interval(Some("not-a-number")), Some(DEFAULT_PROBE_INTERVAL));
+        assert_eq!(parse_interval(Some(" 120 ")), Some(Duration::from_secs(120)));
+        assert_eq!(parse_interval(Some("1")), Some(MIN_PROBE_INTERVAL));
+        assert_eq!(parse_interval(Some("0")), None, "zero must disable the probe");
+    }
+
+    // -- Metric assertion helpers, mirroring the policy engine's tests --------
+
+    type MetricEntry = (
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        DebugValue,
+    );
+
+    /// Run `test` on a paused current-thread runtime under a debugging recorder
+    /// and return one snapshot of everything it emitted.
+    fn record_metrics<Out>(test: impl FnOnce() -> std::pin::Pin<Box<dyn Future<Output = Out>>>) -> (Vec<MetricEntry>, Out) {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let out = metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_time()
+                .start_paused(true)
+                .build()
+                .expect("current-thread runtime must build");
+            runtime.block_on(test())
+        });
+        (snapshotter.snapshot().into_vec(), out)
+    }
+
+    fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
+        labels
+            .iter()
+            .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected))
+    }
+
+    fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let matches = composite.kind() == MetricKind::Counter
+                    && composite.key().name() == name
+                    && labels_match(composite.key(), labels);
+                match (matches, value) {
+                    (true, DebugValue::Counter(count)) => Some(*count),
+                    _ => None,
+                }
+            })
+            .sum()
+    }
+
+    fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
+        snapshot
+            .iter()
+            .filter_map(|(composite, _unit, _description, value)| {
+                let matches = composite.kind() == MetricKind::Histogram
+                    && composite.key().name() == name
+                    && labels_match(composite.key(), labels);
+                match (matches, value) {
+                    (true, DebugValue::Histogram(values)) => Some(values),
+                    _ => None,
+                }
+            })
+            .flatten()
+            .map(|value| value.into_inner())
+            .collect()
+    }
+
+    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
+        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
+            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
+            match (matches, value) {
+                (true, DebugValue::Gauge(gauge)) => Some(gauge.into_inner()),
+                _ => None,
+            }
+        })
+    }
+}

crates/kms/src/service.rs

@@ -14,6 +14,7 @@
 
 //! Object encryption service for S3-compatible encryption
 
+use crate::cache::KmsCacheStats;
 use crate::encryption::ciphers::{create_cipher, generate_iv};
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
@@ -112,6 +113,23 @@ impl ObjectEncryptionService {
         self.kms_manager.create_key(request).await
     }
 
+    /// Create a new master key on behalf of `context`'s principal
+    ///
+    /// # Arguments
+    /// * `request` - CreateKeyRequest with key parameters
+    /// * `context` - Identity and correlation data recorded in the audit trail
+    ///
+    /// # Returns
+    /// CreateKeyResponse with created key details
+    ///
+    pub async fn create_key_with_context(
+        &self,
+        request: CreateKeyRequest,
+        context: &OperationContext,
+    ) -> Result<CreateKeyResponse> {
+        self.kms_manager.create_key_with_context(request, context).await
+    }
+
     /// Describe a master key (delegates to KMS manager)
     ///
     /// # Arguments
@@ -124,6 +142,23 @@ impl ObjectEncryptionService {
         self.kms_manager.describe_key(request).await
     }
 
+    /// Describe a master key on behalf of `context`'s principal
+    ///
+    /// # Arguments
+    /// * `request` - DescribeKeyRequest with key ID
+    /// * `context` - Identity and correlation data recorded in the audit trail
+    ///
+    /// # Returns
+    /// DescribeKeyResponse with key metadata
+    ///
+    pub async fn describe_key_with_context(
+        &self,
+        request: DescribeKeyRequest,
+        context: &OperationContext,
+    ) -> Result<DescribeKeyResponse> {
+        self.kms_manager.describe_key_with_context(request, context).await
+    }
+
     /// List master keys (delegates to KMS manager)
     ///
     /// # Arguments
@@ -136,6 +171,19 @@ impl ObjectEncryptionService {
         self.kms_manager.list_keys(request).await
     }
 
+    /// List master keys on behalf of `context`'s principal
+    ///
+    /// # Arguments
+    /// * `request` - ListKeysRequest with listing parameters
+    /// * `context` - Identity and correlation data recorded in the audit trail
+    ///
+    /// # Returns
+    /// ListKeysResponse with list of keys
+    ///
+    pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
+        self.kms_manager.list_keys_with_context(request, context).await
+    }
+
     /// Generate a data encryption key (delegates to KMS manager)
     ///
     /// # Arguments
@@ -160,9 +208,9 @@ impl ObjectEncryptionService {
     /// Get cache statistics
     ///
     /// # Returns
-    /// Option with (hits, misses) if caching is enabled
+    /// A [`KmsCacheStats`] snapshot if caching is enabled, `None` otherwise
     ///
-    pub async fn cache_stats(&self) -> Option<(u64, u64)> {
+    pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
         self.kms_manager.cache_stats().await
     }
 

crates/kms/src/service_manager.rs

@@ -14,12 +14,14 @@
 
 //! KMS service manager for dynamic configuration and runtime management
 
+use crate::audit::KmsAuditSink;
 use crate::backends::vault_credentials::CredentialTaskHandle;
 use crate::backends::{KmsBackend, local::LocalKmsBackend};
 use crate::config::{BackendConfig, KmsConfig};
 use crate::deletion_worker::{DeletionReferenceChecker, DeletionWorker};
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
+use crate::probe::{ProbeHandle, ProbeStatus, spawn_probe_worker};
 use crate::service::ObjectEncryptionService;
 use arc_swap::ArcSwap;
 use sha2::{Digest, Sha256};
@@ -120,13 +122,23 @@ struct ServiceVersion {
     /// Background deletion worker owned by this service version, if the
     /// backend supports deletion scheduling
     deletion_worker: Option<Arc<DeletionWorkerHandle>>,
+    /// Background synthetic probe owned by this service version, if the
+    /// backend can round-trip a data key and the probe is enabled
+    probe_worker: Option<Arc<ProbeHandle>>,
 }
 
 impl ServiceVersion {
-    fn shutdown_deletion_worker(&self) {
+    /// Stop the background workers this version owns.
+    ///
+    /// The credential renewal task is recycled separately because its shutdown
+    /// is asynchronous.
+    fn shutdown_background_workers(&self) {
         if let Some(worker) = &self.deletion_worker {
             worker.shutdown();
         }
+        if let Some(probe) = &self.probe_worker {
+            probe.shutdown();
+        }
     }
 }
 
@@ -171,6 +183,8 @@ pub struct KmsServiceManager {
     lifecycle_mutex: Arc<Mutex<()>>,
     /// External reference checker consulted before expired keys are removed
     deletion_reference_checker: std::sync::RwLock<Option<Arc<dyn DeletionReferenceChecker>>>,
+    /// External sink receiving audit records for management operations
+    audit_sink: std::sync::RwLock<Option<Arc<dyn KmsAuditSink>>>,
 }
 
 impl KmsServiceManager {
@@ -185,9 +199,26 @@ impl KmsServiceManager {
             version_counter: Arc::new(AtomicU64::new(0)),
             lifecycle_mutex: Arc::new(Mutex::new(())),
             deletion_reference_checker: std::sync::RwLock::new(None),
+            audit_sink: std::sync::RwLock::new(None),
         }
     }
 
+    /// Install the sink that receives an audit record for every KMS
+    /// management operation. Takes effect for services built by the next
+    /// start or reconfigure.
+    ///
+    /// Without a sink no records are built, so KMS operations are unaffected
+    /// when auditing is not configured.
+    pub fn set_audit_sink(&self, sink: Arc<dyn KmsAuditSink>) {
+        if let Ok(mut slot) = self.audit_sink.write() {
+            *slot = Some(sink);
+        }
+    }
+
+    fn audit_sink(&self) -> Option<Arc<dyn KmsAuditSink>> {
+        self.audit_sink.read().ok().and_then(|slot| slot.clone())
+    }
+
     /// Install the reference checker consulted before the deletion worker
     /// removes an expired key. Takes effect for workers spawned by the next
     /// start or reconfigure.
@@ -379,7 +410,7 @@ impl KmsServiceManager {
         // Note: Existing Arc references will keep the service alive until operations complete
         let state = self.state.load_full();
         if let Some(current) = state.current_service.as_ref() {
-            current.shutdown_deletion_worker();
+            current.shutdown_background_workers();
         }
         self.state.store(Arc::new(RuntimeState {
             config: state.config.clone(),
@@ -514,6 +545,19 @@ impl KmsServiceManager {
         self.state.load().current_service.as_ref().map(|sv| sv.version)
     }
 
+    /// Latest synthetic probe snapshot of the running service version.
+    ///
+    /// `None` when KMS is not running, when the backend cannot round-trip a
+    /// data key, or when the probe is disabled. The read is lock-free and
+    /// performs no backend I/O, so a health endpoint can consult it without
+    /// turning an external KMS hiccup into probe pressure of its own; the
+    /// staleness and consecutive-failure thresholds belong to the caller.
+    pub fn probe_status(&self) -> Option<Arc<ProbeStatus>> {
+        let state = self.state.load();
+        let service_version = state.current_service.as_ref()?;
+        Some(service_version.probe_worker.as_ref()?.status())
+    }
+
     /// Health check for the KMS service
     pub async fn health_check(&self) -> Result<bool> {
         let checked_state = self.state.load_full();
@@ -585,7 +629,11 @@ impl KmsServiceManager {
         };
 
         // Create KMS manager
-        let kms_manager = Arc::new(KmsManager::new(backend, config.clone()));
+        let mut kms_manager = KmsManager::new(backend, config.clone());
+        if let Some(sink) = self.audit_sink() {
+            kms_manager = kms_manager.with_audit_sink(sink);
+        }
+        let kms_manager = Arc::new(kms_manager);
 
         // Create encryption service
         let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
@@ -596,6 +644,7 @@ impl KmsServiceManager {
             manager: kms_manager,
             credential_task,
             deletion_worker: None,
+            probe_worker: None,
         })
     }
 
@@ -609,9 +658,13 @@ impl KmsServiceManager {
 
     fn publish_running(&self, config: KmsConfig, mut service_version: ServiceVersion) {
         if let Some(previous) = self.state.load().current_service.as_ref() {
-            previous.shutdown_deletion_worker();
+            previous.shutdown_background_workers();
         }
         service_version.deletion_worker = self.spawn_deletion_worker(&config, &service_version);
+        // Started at publish time for the same reason as the deletion worker:
+        // a start or reconfigure candidate that never becomes current must not
+        // leak a running task or publish probe results nobody asked for.
+        service_version.probe_worker = spawn_probe_worker(service_version.manager.backend()).map(Arc::new);
         self.state.store(Arc::new(RuntimeState {
             config: Some(config),
             status: KmsServiceStatus::Running,
@@ -629,7 +682,13 @@ impl KmsServiceManager {
             return None;
         }
         let cancel = CancellationToken::new();
-        let worker = DeletionWorker::new(backend, config.default_key_id.clone(), self.deletion_reference_checker());
+        let worker = DeletionWorker::new(
+            backend,
+            config.default_key_id.clone(),
+            self.deletion_reference_checker(),
+            config.backend.as_str(),
+        )
+        .with_audit_sink(self.audit_sink());
         let task = worker.spawn(cancel.clone());
         Some(Arc::new(DeletionWorkerHandle {
             cancel,
@@ -980,6 +1039,49 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn probe_worker_follows_the_service_lifecycle() {
+        use tempfile::TempDir;
+
+        let key_dir = TempDir::new().expect("create local KMS directory");
+        let mut config = KmsConfig::local(key_dir.path().to_path_buf());
+        config.allow_insecure_dev_defaults = true;
+        let manager = KmsServiceManager::new();
+        manager.configure(config).await.expect("configure local KMS");
+        manager.start().await.expect("start local KMS");
+
+        let first_probe = manager
+            .state
+            .load()
+            .current_service
+            .as_ref()
+            .expect("running service")
+            .probe_worker
+            .clone()
+            .expect("a backend with a data key round trip must run a probe");
+        assert!(!first_probe.is_cancelled());
+        assert!(manager.probe_status().is_some(), "a running service must publish probe status");
+
+        // Replacing the service version replaces (and cancels) its probe.
+        manager.restart().await.expect("restart");
+        assert!(first_probe.is_cancelled(), "replaced version's probe must be cancelled");
+        let second_probe = manager
+            .state
+            .load()
+            .current_service
+            .as_ref()
+            .expect("running service")
+            .probe_worker
+            .clone()
+            .expect("restarted service must run a fresh probe");
+        assert!(!second_probe.is_cancelled());
+
+        // Stopping the service stops its probe and withdraws the snapshot.
+        manager.stop().await.expect("stop");
+        assert!(second_probe.is_cancelled(), "stop must cancel the probe worker");
+        assert!(manager.probe_status().is_none(), "a stopped service must publish no status");
+    }
+
     #[tokio::test]
     async fn reconfigure_allows_safe_local_runtime_settings_only() {
         use tempfile::TempDir;

crates/kms/src/types.rs

@@ -484,6 +484,20 @@ impl OperationContext {
         }
     }
 
+    /// Principal recorded for work the server starts on its own behalf.
+    ///
+    /// Namespaced so it can never collide with an access key or an IAM ARN,
+    /// which keeps "no human did this" distinguishable from "we lost track of
+    /// who did this" in the audit trail.
+    pub const INTERNAL_PRINCIPAL: &'static str = "rustfs:internal";
+
+    /// Context for an operation with no authenticated caller, such as
+    /// background maintenance or a call made while serving a data-plane
+    /// request that carries its own audit entry.
+    pub fn internal() -> Self {
+        Self::new(Self::INTERNAL_PRINCIPAL.to_string())
+    }
+
     /// Add additional context
     ///
     /// # Arguments

crates/kms/tests/vault_fault_injection.rs

@@ -14,8 +14,8 @@
 
 //! Fault-injection matrix for the Vault backend operation policy.
 //!
-//! Offline cases run against locally injected transport faults (a closed
-//! port, a listener that never responds) — deterministic, no external
+//! Offline cases run against locally injected transport faults (a listener
+//! that never responds) — deterministic, no external
 //! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault
 //! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`).
 //!
@@ -33,9 +33,11 @@ use std::time::Duration;
 
 use metrics_util::MetricKind;
 use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-use rustfs_kms::backends::KmsClient;
-use rustfs_kms::backends::vault::VaultKmsClient;
-use rustfs_kms::{KmsConfig, KmsError, VaultAuthMethod, VaultConfig};
+use rustfs_kms::backends::KmsBackend as KmsBackendTrait;
+use rustfs_kms::backends::vault::VaultKmsBackend;
+use rustfs_kms::{
+    BackendConfig, DescribeKeyRequest, KmsBackend as KmsBackendKind, KmsConfig, KmsError, VaultAuthMethod, VaultConfig,
+};
 
 const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total";
 const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total";
@@ -54,14 +56,23 @@ fn vault_config(address: &str, token: &str) -> VaultConfig {
     }
 }
 
-fn kms_config(attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig {
+fn kms_config(vault_config: VaultConfig, attempt_timeout: Duration, retry_attempts: u32) -> KmsConfig {
     KmsConfig {
+        backend: KmsBackendKind::VaultKv2,
+        backend_config: BackendConfig::VaultKv2(Box::new(vault_config)),
+        allow_insecure_dev_defaults: true,
         timeout: attempt_timeout,
         retry_attempts,
         ..KmsConfig::default()
     }
 }
 
+fn describe_key_request(key_id: &str) -> DescribeKeyRequest {
+    DescribeKeyRequest {
+        key_id: key_id.to_string(),
+    }
+}
+
 type MetricEntry = (
     metrics_util::CompositeKey,
     Option<metrics::Unit>,
@@ -107,45 +118,6 @@ fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)])
         .sum()
 }
 
-/// Connection refused: connection-class failures are retried up to the
-/// configured budget, then surface as a backend error.
-#[test]
-fn connection_refused_is_retried_within_budget() {
-    // Reserve a loopback port and release it so nothing is listening there.
-    let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve a loopback port");
-    let address = format!("http://{}", listener.local_addr().expect("reserved port addr"));
-    drop(listener);
-
-    let snapshot = record_metrics(|| {
-        Box::pin(async move {
-            let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_secs(2), 2))
-                .await
-                .expect("client construction performs no network calls");
-            let error = client
-                .describe_key("fault-injection-refused", None)
-                .await
-                .expect_err("a refused connection must fail the operation");
-            assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
-        })
-    });
-
-    assert_eq!(
-        counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]),
-        2,
-        "both budgeted attempts must observe the refused connection"
-    );
-    assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1);
-    // The static-token login records its own success; the Vault read must not.
-    assert_eq!(
-        counter_value(
-            &snapshot,
-            OPERATIONS_TOTAL,
-            &[("operation", "vault_kv2_read_key"), ("outcome", "success")]
-        ),
-        0
-    );
-}
-
 /// Stalled connection: a server that accepts but never responds is cut off by
 /// the per-attempt timeout (either the policy timer or the equally sized HTTP
 /// client timeout, whichever fires first) instead of hanging forever.
@@ -166,11 +138,10 @@ fn stalled_connection_is_cut_off_by_the_attempt_timeout() {
                 }
             });
 
-            let client = VaultKmsClient::new(vault_config(&address, "unused"), &kms_config(Duration::from_millis(250), 1))
+            let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_millis(250), 1))
                 .await
                 .expect("client construction performs no network calls");
-            let error = client
-                .describe_key("fault-injection-stalled", None)
+            let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled"))
                 .await
                 .expect_err("a stalled request must be cut off by the attempt timeout");
             assert!(
@@ -201,11 +172,10 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() {
     let snapshot = record_metrics(|| {
         Box::pin(async {
             let config = vault_config(&real_vault_address(), "fault-injection-invalid-token");
-            let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3))
+            let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3))
                 .await
                 .expect("client construction performs no network calls");
-            let error = client
-                .describe_key("fault-injection-forbidden", None)
+            let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden"))
                 .await
                 .expect_err("an invalid token must be rejected");
             assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
@@ -235,11 +205,10 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() {
     let snapshot = record_metrics(|| {
         Box::pin(async move {
             let config = vault_config(&real_vault_address(), &token);
-            let client = VaultKmsClient::new(config, &kms_config(Duration::from_secs(5), 3))
+            let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3))
                 .await
                 .expect("client construction performs no network calls");
-            let error = client
-                .describe_key("fault-injection-definitely-missing", None)
+            let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-definitely-missing"))
                 .await
                 .expect_err("a missing key must resolve to key-not-found");
             assert!(matches!(error, KmsError::KeyNotFound { .. }), "got {error:?}");

crates/notify/src/config_manager.rs

@@ -19,7 +19,7 @@ use crate::{
     resolve_notify_object_store_handle,
     rule_engine::NotifyRuleEngine,
     runtime_facade::NotifyRuntimeFacade,
-    with_notify_server_config_read_lock, with_notify_server_config_write_lock,
+    with_notify_server_config_read_lock,
 };
 use rustfs_config::notify::{
     NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
@@ -41,12 +41,32 @@ enum NotifyConfigStoreError {
     StorageNotAvailable,
     Read(String),
     Save(String),
+    Converge { persisted: bool, error: String },
+}
+
+fn notification_convergence_error(persisted: bool, error: impl std::fmt::Display) -> NotificationError {
+    let durable_state = if persisted { "persisted" } else { "unchanged" };
+    NotificationError::Configuration(format!(
+        "configuration was {durable_state} but notification runtime convergence failed: {error}"
+    ))
+}
+
+async fn supervise_config_update<T>(
+    mutation: impl std::future::Future<Output = Result<T, NotifyConfigStoreError>> + Send + 'static,
+) -> Result<T, NotifyConfigStoreError>
+where
+    T: Send + 'static,
+{
+    tokio::spawn(mutation).await.map_err(|error| {
+        let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
+        NotifyConfigStoreError::Lock(format!("notify config update task {outcome}"))
+    })?
 }
 
 async fn update_server_config<F>(
-    modifier: F,
+    mut modifier: F,
     lifecycle: NotifyLifecycleCoordinator,
-) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError>
+) -> Result<Option<(crate::lifecycle::NotificationLifecycleTransition, bool)>, NotifyConfigStoreError>
 where
     F: FnMut(&mut Config) -> bool + Send + 'static,
 {
@@ -54,51 +74,31 @@ where
         return Err(NotifyConfigStoreError::StorageNotAvailable);
     };
 
-    let store_for_read = store.clone();
-    let store_for_save = store.clone();
-    with_notify_server_config_write_lock(store, move || {
-        read_modify_write(
-            modifier,
-            move || async move {
-                crate::read_notify_server_config_without_migrate_no_lock(store_for_read)
-                    .await
-                    .map_err(NotifyConfigStoreError::Read)
-            },
-            move |config| async move {
-                crate::save_notify_server_config_no_lock(store_for_save, &config)
-                    .await
-                    .map_err(NotifyConfigStoreError::Save)
-            },
-            move |config| lifecycle.update_config(config),
-        )
+    supervise_config_update(async move {
+        let snapshot = crate::read_notify_server_config_snapshot(store.clone())
+            .await
+            .map_err(NotifyConfigStoreError::Read)?;
+        let mut config = snapshot.config.clone();
+        if !modifier(&mut config) {
+            return Ok(None);
+        }
+
+        let persisted = crate::save_notify_server_config_snapshot(store.clone(), &config, &snapshot)
+            .await
+            .map_err(NotifyConfigStoreError::Save)?;
+        drop(snapshot);
+
+        let read_store = store.clone();
+        with_notify_server_config_read_lock(store, move || async move {
+            let latest = crate::read_existing_notify_server_config_no_lock(read_store)
+                .await
+                .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?;
+            Ok::<_, NotifyConfigStoreError>(Some((lifecycle.update_config(latest), persisted)))
+        })
+        .await
+        .map_err(|error| NotifyConfigStoreError::Converge { persisted, error })?
     })
     .await
-    .map_err(NotifyConfigStoreError::Lock)?
-}
-
-async fn read_modify_write<F, R, RFut, S, SFut, P, T>(
-    mut modifier: F,
-    read: R,
-    save: S,
-    publish: P,
-) -> Result<Option<T>, NotifyConfigStoreError>
-where
-    F: FnMut(&mut Config) -> bool,
-    R: FnOnce() -> RFut,
-    RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>,
-    S: FnOnce(Config) -> SFut,
-    SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>,
-    P: FnOnce(Config) -> T,
-{
-    let mut new_config = read().await?;
-
-    if !modifier(&mut new_config) {
-        return Ok(None);
-    }
-
-    save(new_config.clone()).await?;
-
-    Ok(Some(publish(new_config)))
 }
 
 pub(crate) fn notify_configuration_hint() -> String {
@@ -345,16 +345,18 @@ impl NotifyConfigManager {
         if self.lifecycle.state() == NotificationRuntimeState::Terminated {
             return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
         }
-        let Some(transition) = update_server_config(modifier, self.lifecycle.clone())
-            .await
-            .map_err(|err| match err {
-                NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
-                NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
-                    "Failed to save target configuration: server storage not initialized".to_string(),
-                ),
-                NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
-                NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
-            })?
+        let Some((transition, persisted)) =
+            update_server_config(modifier, self.lifecycle.clone())
+                .await
+                .map_err(|err| match err {
+                    NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
+                    NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
+                        "Failed to save target configuration: server storage not initialized".to_string(),
+                    ),
+                    NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
+                    NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
+                    NotifyConfigStoreError::Converge { persisted, error } => notification_convergence_error(persisted, error),
+                })?
         else {
             debug!(
                 event = EVENT_NOTIFY_CONFIG_UPDATE,
@@ -367,23 +369,53 @@ impl NotifyConfigManager {
             return Ok(());
         };
 
+        let result = if persisted { "updated" } else { "unchanged" };
         info!(
             event = EVENT_NOTIFY_CONFIG_UPDATE,
             component = LOG_COMPONENT_NOTIFY,
             subsystem = LOG_SUBSYSTEM_CONFIG,
             action = "reload_if_changed",
-            result = "updated",
+            result,
             "notify config update"
         );
-        transition.wait().await
+        transition
+            .wait()
+            .await
+            .map_err(|err| notification_convergence_error(persisted, err))
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem};
+    use super::{
+        NotifyConfigManager, NotifyConfigStoreError, notification_convergence_error, runtime_target_id_for_subsystem,
+        supervise_config_update,
+    };
     use crate::rules::RulesMap;
     use crate::{NotificationError, NotificationRuntimeState};
+
+    #[tokio::test]
+    async fn supervised_config_update_survives_waiter_cancellation() {
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            supervise_config_update(async move {
+                let _ = started_tx.send(());
+                let _ = release_rx.await;
+                let _ = completed_tx.send(());
+                Ok::<_, NotifyConfigStoreError>(())
+            })
+            .await
+        });
+
+        started_rx.await.expect("config update should start");
+        waiter.abort();
+        release_tx.send(()).expect("config update should be released");
+        let completed = tokio::time::timeout(std::time::Duration::from_secs(30), completed_rx).await;
+        assert!(matches!(completed, Ok(Ok(()))), "detached config update should complete");
+    }
     use crate::{
         integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
         runtime_facade::NotifyRuntimeFacade,
@@ -392,14 +424,11 @@ mod tests {
         NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
         NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
     };
-    use rustfs_config::server_config::{Config, KVS};
+    use rustfs_config::server_config::Config;
     use rustfs_s3_types::EventName;
     use rustfs_targets::ReplayWorkerManager;
     use rustfs_targets::arn::TargetID;
-    use std::sync::{
-        Arc,
-        atomic::{AtomicBool, Ordering},
-    };
+    use std::sync::Arc;
     use tokio::sync::{RwLock, Semaphore};
 
     fn build_manager() -> NotifyConfigManager {
@@ -463,38 +492,6 @@ mod tests {
         assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
     }
 
-    #[tokio::test]
-    async fn read_modify_write_publishes_only_after_save() {
-        let saved = Arc::new(AtomicBool::new(false));
-        let saved_by_writer = saved.clone();
-        let observed_by_publisher = saved.clone();
-
-        read_modify_write(
-            |config| {
-                config
-                    .0
-                    .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
-                    .or_default()
-                    .insert("primary".to_string(), KVS::default());
-                true
-            },
-            || async { Ok::<_, NotifyConfigStoreError>(Config::default()) },
-            move |_config| async move {
-                saved_by_writer.store(true, Ordering::Release);
-                Ok::<_, NotifyConfigStoreError>(())
-            },
-            move |_config| {
-                assert!(
-                    observed_by_publisher.load(Ordering::Acquire),
-                    "publication must observe the completed save"
-                );
-            },
-        )
-        .await
-        .expect("read-modify-write should succeed")
-        .expect("changed config should publish");
-    }
-
     #[test]
     fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
         let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
@@ -550,4 +547,16 @@ mod tests {
         assert_eq!(target_id.id, "audittrail");
         assert_eq!(target_id.name, "postgres");
     }
+
+    #[test]
+    fn convergence_error_reports_durable_write_state() {
+        for (persisted, expected) in [(true, "configuration was persisted"), (false, "configuration was unchanged")] {
+            let error = notification_convergence_error(persisted, "injected failure");
+            let NotificationError::Configuration(message) = error else {
+                panic!("convergence error should be a configuration error");
+            };
+            assert!(message.contains(expected), "unexpected convergence error: {message}");
+            assert!(message.contains("injected failure"));
+        }
+    }
 }

crates/notify/src/lib.rs

@@ -57,7 +57,6 @@ pub use services::NotifyServices;
 pub use status_view::NotifyStatusView;
 pub use storage_api::NotifyStore;
 pub(crate) use storage_api::crate_boundary::{
-    read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
-    resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
-    with_notify_server_config_write_lock,
+    read_existing_notify_server_config_no_lock, read_notify_server_config_snapshot, resolve_notify_object_store_handle,
+    save_notify_server_config_snapshot, with_notify_server_config_read_lock,
 };

crates/notify/src/rules/mod.rs

@@ -22,6 +22,8 @@ mod pattern_rules_test;
 #[cfg(test)]
 mod pattern_test;
 mod rules_map;
+#[cfg(test)]
+mod rules_map_test;
 mod subscriber_index;
 mod subscriber_snapshot;
 mod target_id_set;

crates/notify/src/rules/rules_map_test.rs

@@ -0,0 +1,107 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Regressions for rule matching against the KMS event namespace.
+//!
+//! KMS events (backlog#1583) are appended to `EventName` for the audit sink,
+//! but they must stay invisible to bucket notification rules: a subscriber
+//! asking for `s3:ObjectCreated:*` — or for everything — must never receive
+//! key management activity.
+
+use super::RulesMap;
+use rustfs_s3_types::EventName;
+use rustfs_targets::arn::TargetID;
+
+const KMS_EVENTS: &[EventName] = &[
+    EventName::KmsKeyCreated,
+    EventName::KmsKeyRotated,
+    EventName::KmsKeyEnabled,
+    EventName::KmsKeyDisabled,
+    EventName::KmsKeyDeletionScheduled,
+    EventName::KmsKeyDeletionCancelled,
+    EventName::KmsKeyDeleted,
+    EventName::KmsKeyAccessed,
+    EventName::KmsServiceConfigured,
+    EventName::KmsServiceStarted,
+    EventName::KmsServiceStopped,
+];
+
+fn test_target() -> TargetID {
+    TargetID::new("primary".to_string(), "webhook".to_string())
+}
+
+fn rules_for(events: &[EventName]) -> RulesMap {
+    let mut rules = RulesMap::new();
+    rules.add_rule_config(events, String::new(), test_target());
+    rules
+}
+
+#[test]
+fn s3_wildcard_subscriptions_do_not_match_kms_events() {
+    let rules = rules_for(&[
+        EventName::ObjectCreatedAll,
+        EventName::ObjectAccessedAll,
+        EventName::ObjectRemovedAll,
+        EventName::ObjectTaggingAll,
+        EventName::ObjectReplicationAll,
+        EventName::ObjectRestoreAll,
+        EventName::ObjectTransitionAll,
+        EventName::LifecycleExpirationAll,
+        EventName::ObjectScannerAll,
+    ]);
+
+    for event in KMS_EVENTS {
+        assert!(!rules.has_subscriber(event), "{event} must not have an S3 wildcard subscriber");
+        assert!(
+            rules.match_rules(*event, "any/object").is_empty(),
+            "{event} must not match any S3 wildcard rule"
+        );
+    }
+}
+
+#[test]
+fn everything_subscription_does_not_match_kms_events() {
+    let rules = rules_for(&[EventName::Everything]);
+
+    // Sanity: the catch-all really is subscribed to the S3 surface.
+    assert!(rules.has_subscriber(&EventName::ObjectCreatedPut));
+
+    for event in KMS_EVENTS {
+        assert!(!rules.has_subscriber(event), "{event} must stay outside the s3 catch-all");
+        assert!(
+            rules.match_rules(*event, "any/object").is_empty(),
+            "{event} must not match the s3 catch-all rule"
+        );
+    }
+}
+
+#[test]
+fn kms_subscription_does_not_leak_into_s3_matching() {
+    // The inverse direction: even an explicit KMS subscription must not widen
+    // the mask so that S3 events start matching a KMS-only rule.
+    let rules = rules_for(KMS_EVENTS);
+
+    for event in [
+        EventName::ObjectCreatedPut,
+        EventName::ObjectRemovedDelete,
+        EventName::ObjectAccessedGet,
+        EventName::BucketCreated,
+    ] {
+        assert!(!rules.has_subscriber(&event), "{event} must not match a KMS-only rule");
+        assert!(
+            rules.match_rules(event, "any/object").is_empty(),
+            "{event} must not match a KMS-only rule"
+        );
+    }
+}

crates/notify/src/storage_api.rs

@@ -15,11 +15,10 @@
 use std::sync::Arc;
 
 use rustfs_ecstore::api::config::com::{
-    read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock,
     read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
-    save_server_config_no_lock as save_notify_server_config_to_backend_no_lock,
+    read_server_config_snapshot as read_notify_server_config_snapshot_from_backend,
+    save_server_config_snapshot as save_notify_server_config_snapshot_to_backend,
     with_server_config_read_lock as with_notify_server_config_read_lock_from_backend,
-    with_server_config_write_lock as with_notify_server_config_write_lock_from_backend,
 };
 use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
 pub use rustfs_ecstore::api::storage::ECStore as NotifyStore;
@@ -28,10 +27,10 @@ pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
     resolve_notify_object_store_handle_from_backend()
 }
 
-pub(crate) async fn read_notify_server_config_without_migrate_no_lock(
-    store: Arc<NotifyStore>,
-) -> Result<rustfs_config::server_config::Config, String> {
-    read_notify_config_without_migrate_from_backend_no_lock(store)
+pub(crate) type NotifyServerConfigSnapshot = rustfs_ecstore::api::config::com::ServerConfigSnapshot;
+
+pub(crate) async fn read_notify_server_config_snapshot(store: Arc<NotifyStore>) -> Result<NotifyServerConfigSnapshot, String> {
+    read_notify_server_config_snapshot_from_backend(store)
         .await
         .map_err(|err| err.to_string())
 }
@@ -44,22 +43,12 @@ pub(crate) async fn read_existing_notify_server_config_no_lock(
         .map_err(|err| err.to_string())
 }
 
-pub(crate) async fn save_notify_server_config_no_lock(
+pub(crate) async fn save_notify_server_config_snapshot(
     store: Arc<NotifyStore>,
     config: &rustfs_config::server_config::Config,
-) -> Result<(), String> {
-    save_notify_server_config_to_backend_no_lock(store, config)
-        .await
-        .map_err(|err| err.to_string())
-}
-
-pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
-where
-    F: FnOnce() -> Fut + Send + 'static,
-    Fut: std::future::Future<Output = T> + Send + 'static,
-    T: Send + 'static,
-{
-    with_notify_server_config_write_lock_from_backend(store, operation)
+    snapshot: &NotifyServerConfigSnapshot,
+) -> Result<bool, String> {
+    save_notify_server_config_snapshot_to_backend(store, config, snapshot)
         .await
         .map_err(|err| err.to_string())
 }
@@ -77,8 +66,7 @@ where
 
 pub(crate) mod crate_boundary {
     pub(crate) use super::{
-        read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
-        resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
-        with_notify_server_config_write_lock,
+        read_existing_notify_server_config_no_lock, read_notify_server_config_snapshot, resolve_notify_object_store_handle,
+        save_notify_server_config_snapshot, with_notify_server_config_read_lock,
     };
 }

crates/policy/src/policy.rs

@@ -72,4 +72,10 @@ pub enum Error {
 
     #[error("invalid resource, type: '{0}', pattern: '{1}'")]
     InvalidResource(String, String),
+
+    #[error("KMS resources require a statement whose actions are all KMS actions")]
+    KmsResourceWithNonKmsAction,
+
+    #[error("bucket policies do not support KMS actions or resources")]
+    KmsUnsupportedInBucketPolicy,
 }

crates/policy/src/policy/action.rs

@@ -728,6 +728,8 @@ pub enum KmsAction {
     ListKeysAction,
     #[strum(serialize = "kms:DescribeKey")]
     DescribeKeyAction,
+    #[strum(serialize = "kms:Decrypt")]
+    DecryptAction,
 }
 
 #[cfg(test)]
@@ -764,6 +766,7 @@ mod tests {
             ("kms:RotateKey", KmsAction::RotateKeyAction),
             ("kms:ListKeys", KmsAction::ListKeysAction),
             ("kms:DescribeKey", KmsAction::DescribeKeyAction),
+            ("kms:Decrypt", KmsAction::DecryptAction),
         ] {
             let action = Action::try_from(raw).expect("Should parse KMS action");
             assert_eq!(action, Action::KmsAction(expected));

crates/policy/src/policy/opa.rs

@@ -30,6 +30,10 @@ impl Args {
     }
 }
 
+pub fn is_configured() -> bool {
+    env::var_os(ENV_POLICY_PLUGIN_OPA_URL).is_some_and(|url| !url.is_empty())
+}
+
 #[derive(Debug, Clone)]
 pub struct AuthZPlugin {
     client: OpaHttpClient,
@@ -88,7 +92,12 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
         .build()
         .map_err(OpaConfigError::Connection)?;
 
-    match client.post(&config.url).send().await {
+    let mut request = client.post(&config.url);
+    if !config.auth_token.is_empty() {
+        request = request.header("Authorization", format!("Bearer {}", config.auth_token));
+    }
+
+    match request.send().await {
         Ok(resp) => {
             match resp.status() {
                 reqwest::StatusCode::OK => {

crates/policy/src/policy/policy.rs

@@ -1760,6 +1760,391 @@ mod test {
         );
     }
 
+    fn kms_args<'a>(
+        action: Action,
+        key_id: &'a str,
+        conditions: &'a HashMap<String, Vec<String>>,
+        claims: &'a HashMap<String, Value>,
+    ) -> Args<'a> {
+        Args {
+            account: "testuser",
+            groups: &None,
+            action,
+            bucket: "",
+            conditions,
+            is_owner: false,
+            object: key_id,
+            claims,
+            deny_only: false,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_kms_statement_with_key_resource_scopes_by_key() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:DisableKey"],
+      "Resource": ["arn:aws:kms:::key/key-a"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let action = Action::KmsAction(KmsAction::DisableKeyAction);
+
+        assert!(
+            policy.is_allowed(&kms_args(action, "key-a", &conditions, &claims)).await,
+            "the granted key must be allowed"
+        );
+        assert!(
+            !policy.is_allowed(&kms_args(action, "key-b", &conditions, &claims)).await,
+            "a key outside the granted resource must be denied"
+        );
+        assert!(
+            !policy
+                .is_allowed(&kms_args(Action::KmsAction(KmsAction::EnableKeyAction), "key-a", &conditions, &claims))
+                .await,
+            "an action outside the grant must stay denied even for the granted key"
+        );
+        assert!(
+            policy.is_allowed(&kms_args(action, "", &conditions, &claims)).await,
+            "call sites that do not pass a key resource keep the legacy match-every-key behaviour"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_kms_statement_with_wildcard_key_resource() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:GenerateDataKey"],
+      "Resource": ["arn:aws:kms:::key/app-*"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let action = Action::KmsAction(KmsAction::GenerateDataKeyAction);
+
+        assert!(
+            policy
+                .is_allowed(&kms_args(action, "app-primary", &conditions, &claims))
+                .await
+        );
+        assert!(
+            !policy
+                .is_allowed(&kms_args(action, "backup-primary", &conditions, &claims))
+                .await
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_kms_statement_without_resource_matches_every_key() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:*"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+
+        for key_id in ["", "key-a", "any-other-key"] {
+            assert!(
+                policy
+                    .is_allowed(&kms_args(Action::KmsAction(KmsAction::DisableKeyAction), key_id, &conditions, &claims))
+                    .await,
+                "resource-less KMS statement must keep matching every key (key_id: {key_id:?})"
+            );
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_kms_deny_with_wildcard_resource_overrides_allow() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:*"],
+      "Resource": ["arn:aws:kms:::key/key-a"]
+    },
+    {
+      "Effect": "Deny",
+      "Action": ["kms:DisableKey"],
+      "Resource": ["arn:aws:kms:::key/*"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+
+        assert!(
+            !policy
+                .is_allowed(&kms_args(Action::KmsAction(KmsAction::DisableKeyAction), "key-a", &conditions, &claims))
+                .await,
+            "a wildcard Deny must override the narrower Allow"
+        );
+        assert!(
+            policy
+                .is_allowed(&kms_args(Action::KmsAction(KmsAction::RotateKeyAction), "key-a", &conditions, &claims))
+                .await,
+            "actions outside the Deny keep the Allow"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_kms_statement_with_not_resource_excludes_keys() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:DescribeKey"],
+      "NotResource": ["arn:aws:kms:::key/prod-*"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let action = Action::KmsAction(KmsAction::DescribeKeyAction);
+
+        assert!(policy.is_allowed(&kms_args(action, "dev-key", &conditions, &claims)).await);
+        assert!(!policy.is_allowed(&kms_args(action, "prod-key", &conditions, &claims)).await);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_kms_statement_with_s3_resource_is_treated_as_unscoped() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        // Statements combining KMS actions with S3 resources predate KMS resource
+        // support and were always evaluated as if unscoped. Pin that they still
+        // match every key (a warning is logged during evaluation).
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:DisableKey"],
+      "Resource": ["arn:aws:s3:::somebucket/*"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let action = Action::KmsAction(KmsAction::DisableKeyAction);
+
+        for key_id in ["", "key-a"] {
+            assert!(
+                policy.is_allowed(&kms_args(action, key_id, &conditions, &claims)).await,
+                "malformed KMS statement with S3 resources must keep matching every key (key_id: {key_id:?})"
+            );
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_kms_alias_resource_parses_but_matches_no_key() -> Result<()> {
+        use crate::policy::action::{Action, KmsAction};
+
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:DescribeKey"],
+      "Resource": ["arn:aws:kms:::alias/app-alias"]
+    }
+  ]
+}"#,
+        )?;
+        let conditions = HashMap::new();
+        let claims = HashMap::new();
+        let action = Action::KmsAction(KmsAction::DescribeKeyAction);
+
+        assert!(
+            !policy.is_allowed(&kms_args(action, "app-alias", &conditions, &claims)).await,
+            "alias patterns are parse-only until alias resolution lands and must not match key requests"
+        );
+        assert!(
+            policy.is_allowed(&kms_args(action, "", &conditions, &claims)).await,
+            "call sites without a key resource keep the legacy match-every-key behaviour"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_kms_resource_policy_round_trips() {
+        let policy = Policy::parse_config(
+            br#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["kms:GenerateDataKey", "kms:Decrypt"],
+      "Resource": ["arn:aws:kms:::key/app-*", "arn:aws:kms:::alias/app-alias"]
+    }
+  ]
+}"#,
+        )
+        .expect("KMS resource policy should parse");
+
+        let json = serde_json::to_string(&policy).expect("policy should serialize");
+        let round_trip = Policy::parse_config(json.as_bytes()).expect("serialized KMS resource policy should re-parse");
+        assert_eq!(round_trip.statements[0].resources, policy.statements[0].resources);
+        assert_eq!(round_trip.statements[0].actions, policy.statements[0].actions);
+
+        let value: serde_json::Value = serde_json::from_str(&json).expect("JSON valid");
+        let resources: Vec<_> = value["Statement"][0]["Resource"]
+            .as_array()
+            .expect("Resource should serialize as an array")
+            .iter()
+            .map(|resource| resource.as_str().expect("Resource entries should be strings"))
+            .collect();
+        assert_eq!(resources, vec!["arn:aws:kms:::key/app-*", "arn:aws:kms:::alias/app-alias"]);
+    }
+
+    #[test]
+    fn test_kms_resource_with_non_kms_action_is_invalid() {
+        for action in ["s3:GetObject", "admin:ServerInfo", "sts:AssumeRole"] {
+            let data = format!(
+                r#"{{
+  "Version": "2012-10-17",
+  "Statement": [
+    {{
+      "Effect": "Allow",
+      "Action": ["{action}"],
+      "Resource": ["arn:aws:kms:::key/key-a"]
+    }}
+  ]
+}}"#
+            );
+
+            let result = Policy::parse_config(data.as_bytes());
+            assert!(
+                matches!(result.as_ref().unwrap_err(), Error::PolicyError(IamError::KmsResourceWithNonKmsAction)),
+                "{action} with a KMS resource should fail with KmsResourceWithNonKmsAction, got: {result:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_bucket_policy_with_kms_statement_is_invalid() {
+        for statement in [
+            r#"{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["kms:GenerateDataKey"],"Resource":["arn:aws:s3:::bucket/*"]}"#,
+            r#"{"Effect":"Allow","Principal":{"AWS":"*"},"Action":["s3:GetObject"],"Resource":["arn:aws:kms:::key/key-a"]}"#,
+            r#"{"Effect":"Allow","Principal":{"AWS":"*"},"NotAction":["kms:*"],"Resource":["arn:aws:s3:::bucket/*"]}"#,
+        ] {
+            let data = format!(r#"{{"Version":"2012-10-17","Statement":[{statement}]}}"#);
+            let policy: BucketPolicy =
+                serde_json::from_str(&data).expect("bucket policy with KMS content should still deserialize");
+            let result = policy.is_valid();
+            assert!(
+                matches!(result.as_ref().unwrap_err(), Error::PolicyError(IamError::KmsUnsupportedInBucketPolicy)),
+                "bucket policy statement {statement} should fail with KmsUnsupportedInBucketPolicy, got: {result:?}"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_stored_bucket_policy_with_kms_statement_loads_and_is_ignored() -> Result<()> {
+        // Policies stored before KMS statements were rejected at validation must keep
+        // deserializing, and their KMS statements must not affect bucket traffic.
+        let bucket_policy: BucketPolicy = serde_json::from_str(
+            r#"{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {"AWS": "*"},
+      "Action": ["kms:GenerateDataKey"],
+      "Resource": ["arn:aws:s3:::bucket/*"]
+    },
+    {
+      "Effect": "Allow",
+      "Principal": {"AWS": "*"},
+      "Action": ["s3:GetObject"],
+      "Resource": ["arn:aws:s3:::bucket/*"]
+    }
+  ]
+}"#,
+        )?;
+
+        let conditions = HashMap::new();
+        let args = BucketPolicyArgs {
+            account: "testuser",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::GetObjectAction),
+            bucket: "bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "a.txt",
+        };
+        assert!(
+            bucket_policy.is_allowed(&args).await,
+            "the S3 statement must keep working alongside an ignored KMS statement"
+        );
+
+        let put_args = BucketPolicyArgs {
+            account: "testuser",
+            groups: &None,
+            action: Action::S3Action(crate::policy::action::S3Action::PutObjectAction),
+            bucket: "bucket",
+            conditions: &conditions,
+            is_owner: false,
+            object: "a.txt",
+        };
+        assert!(
+            !bucket_policy.is_allowed(&put_args).await,
+            "the ignored KMS statement must not grant anything"
+        );
+
+        Ok(())
+    }
+
     #[test]
     fn test_mixed_action_families_are_invalid_even_with_resource() {
         let data = r#"

crates/policy/src/policy/resource.rs

@@ -40,7 +40,7 @@ impl Serialize for ResourceSet {
         for resource in &self.0 {
             let resource_str = match resource {
                 Resource::S3(value) => format!("{}{}", Resource::S3_PREFIX, value),
-                Resource::Kms(value) => value.clone(),
+                Resource::Kms(value) => format!("{}{}", Resource::KMS_PREFIX, value),
             };
             seq.serialize_element(&resource_str)?;
         }
@@ -171,6 +171,20 @@ pub enum Resource {
 
 impl Resource {
     pub const S3_PREFIX: &'static str = "arn:aws:s3:::";
+    /// KMS ARNs use the same empty-account form as [`Self::S3_PREFIX`]; the suffix is
+    /// `key/<key_id>` (wildcards allowed in the id), `alias/<name>`, or a bare `*`.
+    pub const KMS_PREFIX: &'static str = "arn:aws:kms:::";
+    /// Resource-type segment for key ids; request-side KMS resource strings are
+    /// `key/<key_id>` so they line up with these patterns.
+    pub const KMS_KEY_SEGMENT: &'static str = "key/";
+    /// Resource-type segment reserved for key aliases. Alias patterns parse and
+    /// validate, but requests are always evaluated against `key/<key_id>` strings,
+    /// so an alias pattern matches nothing until alias resolution lands.
+    pub const KMS_ALIAS_SEGMENT: &'static str = "alias/";
+
+    pub fn is_kms(&self) -> bool {
+        matches!(self, Resource::Kms(_))
+    }
 
     pub async fn is_match(&self, resource: &str, conditions: &HashMap<String, Vec<String>>) -> bool {
         self.is_match_with_resolver(resource, conditions, None).await
@@ -228,10 +242,13 @@ impl Resource {
 impl TryFrom<&str> for Resource {
     type Error = Error;
     fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
-        let Some(value) = value.strip_prefix(Self::S3_PREFIX) else {
+        let resource = if let Some(suffix) = value.strip_prefix(Self::S3_PREFIX) {
+            Resource::S3(suffix.into())
+        } else if let Some(suffix) = value.strip_prefix(Self::KMS_PREFIX) {
+            Resource::Kms(suffix.into())
+        } else {
             return Err(IamError::InvalidResource("unknown".into(), value.into()).into());
         };
-        let resource = Resource::S3(value.into());
 
         resource.is_valid()?;
         Ok(resource)
@@ -248,13 +265,17 @@ impl Validator for Resource {
                 }
             }
             Self::Kms(pattern) => {
-                if pattern.is_empty()
+                // A bare `*` matches every key resource; anything else must carry a
+                // resource-type segment. Key ids never contain separators (the KMS
+                // backends reject them), while alias names may nest ("alias/aws/s3").
+                let well_formed = pattern == "*"
                     || pattern
-                        .char_indices()
-                        .find(|&(_, c)| c == '/' || c == '\\' || c == '.')
-                        .map(|(i, _)| i)
-                        .is_some()
-                {
+                        .strip_prefix(Self::KMS_KEY_SEGMENT)
+                        .is_some_and(|id| !id.is_empty() && !id.contains('/') && !id.contains('\\'))
+                    || pattern
+                        .strip_prefix(Self::KMS_ALIAS_SEGMENT)
+                        .is_some_and(|name| !name.is_empty() && !name.contains('\\'));
+                if !well_formed {
                     return Err(IamError::InvalidResource("kms".into(), pattern.into()).into());
                 }
             }
@@ -270,7 +291,7 @@ impl Serialize for Resource {
     {
         match self {
             Resource::S3(s) => serializer.serialize_str(&format!("{}{}", Self::S3_PREFIX, s)),
-            Resource::Kms(s) => serializer.serialize_str(s),
+            Resource::Kms(s) => serializer.serialize_str(&format!("{}{}", Self::KMS_PREFIX, s)),
         }
     }
 }
@@ -308,9 +329,71 @@ mod tests {
     #[test_case("arn:aws:s3:::mybucket","mybucket/myobject" => false; "15")]
     #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/../victim-bucket/evil.txt" => false; "16")]
     #[test_case("arn:aws:s3:::attacker-bucket/*","attacker-bucket/safe/../../victim-bucket/evil.txt" => false; "17")]
+    #[test_case("arn:aws:kms:::key/mykey","key/mykey" => true; "kms exact key")]
+    #[test_case("arn:aws:kms:::key/mykey","key/otherkey" => false; "kms wrong key")]
+    #[test_case("arn:aws:kms:::key/*","key/mykey" => true; "kms key wildcard")]
+    #[test_case("arn:aws:kms:::key/app-*","key/app-primary" => true; "kms key prefix wildcard")]
+    #[test_case("arn:aws:kms:::key/app-*","key/backup-primary" => false; "kms key prefix mismatch")]
+    #[test_case("arn:aws:kms:::key/mykey?","key/mykey1" => true; "kms key question mark")]
+    #[test_case("arn:aws:kms:::*","key/mykey" => true; "kms bare star")]
+    #[test_case("arn:aws:kms:::key/mykey","key/mykey/../otherkey" => false; "kms traversal cleaned")]
+    #[test_case("arn:aws:kms:::alias/myalias","key/myalias" => false; "kms alias never matches key")]
+    #[test_case("arn:aws:kms:::alias/*","key/mykey" => false; "kms alias wildcard never matches key")]
+    #[test_case("arn:aws:kms:::key/mykey","mykey" => false; "kms bare id lacks key segment")]
     fn test_resource_is_match(resource: &str, object: &str) -> bool {
         let resource: Resource = resource.try_into().unwrap();
 
         pollster::block_on(resource.is_match(object, &HashMap::new()))
     }
+
+    #[test_case("arn:aws:kms:::key/mykey" => true; "key id parses")]
+    #[test_case("arn:aws:kms:::key/*" => true; "key wildcard parses")]
+    #[test_case("arn:aws:kms:::key/app-key.v2" => true; "key id with dot parses")]
+    #[test_case("arn:aws:kms:::alias/myalias" => true; "alias parses")]
+    #[test_case("arn:aws:kms:::alias/aws/s3" => true; "nested alias parses")]
+    #[test_case("arn:aws:kms:::*" => true; "bare star parses")]
+    #[test_case("arn:aws:kms:::" => false; "empty suffix rejected")]
+    #[test_case("arn:aws:kms:::key/" => false; "empty key id rejected")]
+    #[test_case("arn:aws:kms:::alias/" => false; "empty alias name rejected")]
+    #[test_case("arn:aws:kms:::mykey" => false; "missing resource type segment rejected")]
+    #[test_case("arn:aws:kms:::key/a/b" => false; "separator in key id rejected")]
+    #[test_case("arn:aws:kms:::key/a\\b" => false; "backslash in key id rejected")]
+    #[test_case("arn:aws:kms:us-east-1:123456789012:key/mykey" => false; "region and account form rejected")]
+    fn test_kms_resource_parse(resource: &str) -> bool {
+        Resource::try_from(resource).is_ok()
+    }
+
+    #[test]
+    fn test_kms_resource_serialization_round_trip() {
+        for raw in [
+            "arn:aws:kms:::key/mykey",
+            "arn:aws:kms:::key/*",
+            "arn:aws:kms:::alias/myalias",
+            "arn:aws:kms:::*",
+        ] {
+            let resource = Resource::try_from(raw).expect("KMS resource should parse");
+            assert!(resource.is_kms());
+
+            let json = serde_json::to_string(&resource).expect("KMS resource should serialize");
+            assert_eq!(json, format!("\"{raw}\""), "serialization must write back the full ARN");
+
+            let round_trip: Resource = serde_json::from_str(&json).expect("serialized KMS resource should deserialize");
+            assert_eq!(round_trip, resource);
+        }
+    }
+
+    #[test]
+    fn test_kms_resource_set_serialization_round_trip() {
+        use crate::policy::resource::ResourceSet;
+
+        let set: ResourceSet =
+            serde_json::from_str(r#"["arn:aws:kms:::key/app-*","arn:aws:kms:::alias/myalias"]"#).expect("set should parse");
+        assert_eq!(set.len(), 2);
+
+        let json = serde_json::to_string(&set).expect("set should serialize");
+        assert_eq!(json, r#"["arn:aws:kms:::key/app-*","arn:aws:kms:::alias/myalias"]"#);
+
+        let round_trip: ResourceSet = serde_json::from_str(&json).expect("serialized set should deserialize");
+        assert_eq!(round_trip, set);
+    }
 }

crates/policy/src/policy/statement.rs

@@ -16,6 +16,7 @@ use super::{
     ActionSet, Args, BucketPolicyArgs, Effect, Error as IamError, Functions, ID, Principal, ResourceSet, Validator,
     action::{Action, S3Action},
     function::key_name::{KeyName, S3KeyName},
+    resource::Resource,
     variables::{VariableContext, VariableResolver},
 };
 use crate::error::{Error, Result};
@@ -173,13 +174,79 @@ impl Statement {
         Some(ActionFamily::Mixed)
     }
 
+    /// Resource scope check for KMS statements, which match `arn:aws:kms:::key/<key_id>`
+    /// patterns instead of the bucket/object path grammar used by S3 statements.
+    ///
+    /// Call-site contract (wired up by the admin/SSE authorization paths): the requested
+    /// key identifier (pre-alias-resolution) travels in `args.object` with `args.bucket`
+    /// left empty. An empty `args.object` means the caller did not scope the request to a
+    /// key, which preserves the legacy match-every-key behaviour.
+    async fn kms_key_scope_matches(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool {
+        let kms_resources: Vec<&Resource> = self.resources.iter().filter(|resource| resource.is_kms()).collect();
+        let kms_not_resources: Vec<&Resource> = self.not_resources.iter().filter(|resource| resource.is_kms()).collect();
+
+        if kms_resources.len() != self.resources.len() || kms_not_resources.len() != self.not_resources.len() {
+            // Statements combining KMS actions with S3 resources predate KMS resource
+            // support and were always evaluated as if unscoped; keep that behaviour
+            // but surface it, since the S3 patterns never constrain key access.
+            tracing::warn!(
+                sid = %self.sid.0,
+                "KMS statement carries non-KMS resources; they are ignored and the statement matches every key"
+            );
+        }
+
+        if kms_resources.is_empty() && kms_not_resources.is_empty() {
+            // No KMS resources: the statement scopes by action only (legacy form).
+            return true;
+        }
+
+        if args.object.is_empty() {
+            // Call sites that do not pass a key resource keep the pre-resource-scoping
+            // behaviour where any key matches.
+            return true;
+        }
+
+        let requested = format!("{}{}", Resource::KMS_KEY_SEGMENT, args.object);
+
+        if !kms_resources.is_empty() {
+            let mut matched = false;
+            for resource in kms_resources {
+                if resource
+                    .is_match_with_resolver(&requested, args.conditions, Some(resolver))
+                    .await
+                {
+                    matched = true;
+                    break;
+                }
+            }
+            if !matched {
+                return false;
+            }
+        }
+
+        for resource in kms_not_resources {
+            if resource
+                .is_match_with_resolver(&requested, args.conditions, Some(resolver))
+                .await
+            {
+                return false;
+            }
+        }
+
+        true
+    }
+
     /// Returns true when this statement would reach `conditions.evaluate_with_resolver` in
-    /// [`Statement::is_allowed`] (including the KMS shortcut path). Does not evaluate conditions.
+    /// [`Statement::is_allowed`] (including the KMS resource path). Does not evaluate conditions.
     pub(crate) async fn request_reaches_condition_eval(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool {
         if (!self.actions.is_match(&args.action) && !self.actions.is_empty()) || self.not_actions.is_match(&args.action) {
             return false;
         }
 
+        if self.is_kms() {
+            return self.kms_key_scope_matches(args, resolver).await;
+        }
+
         let resource = build_resource(
             &args.action,
             args.bucket,
@@ -187,10 +254,6 @@ impl Statement {
             self.conditions.references_key_name(&KeyName::S3(S3KeyName::S3Prefix)),
         );
 
-        if self.is_kms() && (resource == "/" || self.resources.is_empty()) {
-            return true;
-        }
-
         if self.resources.is_empty() && self.not_resources.is_empty() && !self.is_admin() && !self.is_sts() {
             return false;
         }
@@ -273,6 +336,19 @@ impl Validator for Statement {
             return Err(IamError::BothResourceAndNotResource.into());
         }
 
+        // KMS resources only make sense on pure-KMS statements. The reverse
+        // combination (KMS actions with S3 resources) predates KMS resources,
+        // may already be stored, and stays loadable; evaluation treats it as
+        // unscoped and warns.
+        let has_kms_resource = self
+            .resources
+            .iter()
+            .chain(self.not_resources.iter())
+            .any(|resource| resource.is_kms());
+        if has_kms_resource && !matches!(action_family, Some(ActionFamily::Kms)) {
+            return Err(IamError::KmsResourceWithNonKmsAction.into());
+        }
+
         self.actions.is_valid()?;
         self.not_actions.is_valid()?;
         self.resources.is_valid()?;
@@ -323,6 +399,18 @@ pub struct BPStatement {
 impl BPStatement {
     /// Returns true when this statement would reach `conditions.evaluate` in [`BPStatement::is_allowed`].
     pub(crate) async fn request_reaches_condition_eval(&self, args: &BucketPolicyArgs<'_>) -> bool {
+        if !self.actions.is_empty() && self.actions.iter().all(|action| matches!(action, Action::KmsAction(_))) {
+            // Bucket policies cannot grant or deny KMS access; such statements are
+            // rejected at validation but may exist in policies stored before that
+            // check. Skip them so they never influence bucket traffic. Statements
+            // mixing KMS with S3 actions keep evaluating their S3 actions as before.
+            tracing::warn!(
+                sid = %self.sid.0,
+                "ignoring bucket policy statement with KMS actions during evaluation"
+            );
+            return false;
+        }
+
         if !self.principal.is_match(args.account) {
             return false;
         }
@@ -379,6 +467,24 @@ impl Validator for BPStatement {
             return Err(IamError::BothActionAndNotAction.into());
         }
 
+        // Bucket policies govern S3 access; KMS grants belong in identity policies.
+        // Rejected here (PutBucketPolicy) only: deserialization stays permissive so
+        // stored policies from before this check keep loading, and evaluation skips
+        // pure-KMS statements with a warning.
+        let has_kms_action = self
+            .actions
+            .iter()
+            .chain(self.not_actions.iter())
+            .any(|action| matches!(action, Action::KmsAction(_)));
+        let has_kms_resource = self
+            .resources
+            .iter()
+            .chain(self.not_resources.iter())
+            .any(|resource| resource.is_kms());
+        if has_kms_action || has_kms_resource {
+            return Err(IamError::KmsUnsupportedInBucketPolicy.into());
+        }
+
         if self.resources.is_empty() && self.not_resources.is_empty() {
             return Err(IamError::NonResource.into());
         }

crates/policy/tests/policy_eval_proptest.rs

@@ -26,6 +26,12 @@
 //!       policy is also allowed by the widened policy;
 //!   (c) an empty policy denies every non-owner request (default deny).
 //!
+//! Properties (d)-(g) extend the same invariants to KMS key resources
+//! (`arn:aws:kms:::key/<key_id>`, backlog#1582): Deny-first over key scopes,
+//! wildcard/resource-less supersets implying concrete key grants, exact key
+//! scoping without cross-key leaks, and the legacy resource-less
+//! match-every-key compatibility pin.
+//!
 //! Pure evaluation: no IO, no global state, parallel-safe. Statements are built
 //! from JSON exactly like production policies arriving via PutPolicy. Generated
 //! bucket/key/action pools avoid wildcard metacharacters so resource patterns
@@ -47,10 +53,23 @@ const OBJECT_ACTIONS: &[&str] = &[
     "s3:PutObjectTagging",
 ];
 
+/// Key-scoped KMS actions safe to pair with an `arn:aws:kms:::key/<id>` resource.
+const KMS_KEY_ACTIONS: &[&str] = &[
+    "kms:GenerateDataKey",
+    "kms:Decrypt",
+    "kms:DisableKey",
+    "kms:RotateKey",
+    "kms:DescribeKey",
+];
+
 fn statement_json(effect: &str, action: &str, resource: &str) -> String {
     format!(r#"{{"Effect":"{effect}","Action":["{action}"],"Resource":["{resource}"]}}"#)
 }
 
+fn resourceless_statement_json(effect: &str, action: &str) -> String {
+    format!(r#"{{"Effect":"{effect}","Action":["{action}"]}}"#)
+}
+
 fn policy_from_statements(statements: &[String]) -> Policy {
     let json = format!(r#"{{"Version":"2012-10-17","Statement":[{}]}}"#, statements.join(","));
     serde_json::from_str(&json).expect("generated policy JSON should parse")
@@ -88,6 +107,22 @@ fn action_strategy() -> impl Strategy<Value = &'static str> {
     proptest::sample::select(OBJECT_ACTIONS)
 }
 
+/// Strategy: a KMS key id without wildcard metacharacters or separators.
+fn key_id_strategy() -> impl Strategy<Value = String> {
+    "[a-z][a-z0-9-]{2,11}"
+}
+
+/// Strategy: one action name from the key-scoped KMS action pool.
+fn kms_action_strategy() -> impl Strategy<Value = &'static str> {
+    proptest::sample::select(KMS_KEY_ACTIONS)
+}
+
+/// KMS evaluation contract: the requested key id travels in `args.object` with an
+/// empty bucket (see `Statement::kms_key_scope_matches`).
+fn is_allowed_for_key(policy: &Policy, action: &str, key_id: &str) -> bool {
+    is_allowed(policy, action, "", key_id)
+}
+
 proptest! {
     /// (a) Deny anywhere wins: a Deny statement matching the request denies it,
     /// no matter how many broad Allow statements surround it or at which index
@@ -205,4 +240,122 @@ proptest! {
             "Policy::default() must deny {action} on {bucket}/{key}"
         );
     }
+
+    /// (d) KMS Deny anywhere wins: a Deny scoped to the exact key (or `key/*`)
+    /// denies the request no matter how many broad KMS Allow statements
+    /// (resource-less or `key/*`-scoped) surround it.
+    #[test]
+    fn kms_explicit_deny_anywhere_denies(
+        key_id in key_id_strategy(),
+        action in kms_action_strategy(),
+        allow_count in 0usize..4,
+        deny_pos_seed in 0usize..16,
+        broad in proptest::bool::ANY,
+        wildcard_deny in proptest::bool::ANY,
+    ) {
+        let mut statements: Vec<String> = (0..allow_count)
+            .map(|_| {
+                if broad {
+                    resourceless_statement_json("Allow", "kms:*")
+                } else {
+                    statement_json("Allow", "kms:*", "arn:aws:kms:::key/*")
+                }
+            })
+            .collect();
+
+        let deny_resource = if wildcard_deny {
+            "arn:aws:kms:::key/*".to_string()
+        } else {
+            format!("arn:aws:kms:::key/{key_id}")
+        };
+        let deny = statement_json("Deny", action, &deny_resource);
+        let deny_pos = deny_pos_seed % (statements.len() + 1);
+        statements.insert(deny_pos, deny);
+
+        let policy = policy_from_statements(&statements);
+
+        if allow_count > 0 {
+            let mut allows_only = statements.clone();
+            allows_only.remove(deny_pos);
+            let allow_policy = policy_from_statements(&allows_only);
+            prop_assert!(
+                is_allowed_for_key(&allow_policy, action, &key_id),
+                "sanity: the KMS Allow statements alone should permit {action} on key {key_id}"
+            );
+        }
+
+        prop_assert!(
+            !is_allowed_for_key(&policy, action, &key_id),
+            "explicit KMS Deny at index {deny_pos} of {} statements must deny {action} on key {key_id}",
+            statements.len()
+        );
+    }
+
+    /// (e) KMS wildcard superset implies the concrete key grant: whatever an
+    /// exact `key/<id>` Allow permits is also permitted by `key/*`, by a bare
+    /// `arn:aws:kms:::*`, and by the legacy resource-less statement form.
+    #[test]
+    fn kms_wildcard_superset_implies_concrete_match(
+        key_id in key_id_strategy(),
+        action in kms_action_strategy(),
+    ) {
+        let narrow = policy_from_statements(&[statement_json(
+            "Allow",
+            action,
+            &format!("arn:aws:kms:::key/{key_id}"),
+        )]);
+        let widened = policy_from_statements(&[statement_json("Allow", "kms:*", "arn:aws:kms:::key/*")]);
+        let star = policy_from_statements(&[statement_json("Allow", "kms:*", "arn:aws:kms:::*")]);
+        let resourceless = policy_from_statements(&[resourceless_statement_json("Allow", "kms:*")]);
+
+        prop_assert!(is_allowed_for_key(&narrow, action, &key_id), "narrow KMS policy must allow its own grant");
+        prop_assert!(is_allowed_for_key(&widened, action, &key_id), "kms:* on key/* must imply the concrete grant");
+        prop_assert!(is_allowed_for_key(&star, action, &key_id), "kms:* on arn:aws:kms:::* must imply the concrete grant");
+        prop_assert!(
+            is_allowed_for_key(&resourceless, action, &key_id),
+            "the legacy resource-less KMS statement must imply the concrete grant"
+        );
+    }
+
+    /// (f) Key scoping is exact: an Allow on `key/<a>` never leaks to a
+    /// different key id, while the compatibility contract keeps unscoped
+    /// requests (no key id passed) matching.
+    #[test]
+    fn kms_key_scope_does_not_leak_across_keys(
+        key_a in key_id_strategy(),
+        key_b in key_id_strategy(),
+        action in kms_action_strategy(),
+    ) {
+        prop_assume!(key_a != key_b);
+
+        let policy = policy_from_statements(&[statement_json(
+            "Allow",
+            action,
+            &format!("arn:aws:kms:::key/{key_a}"),
+        )]);
+
+        prop_assert!(is_allowed_for_key(&policy, action, &key_a), "the granted key must be allowed");
+        prop_assert!(
+            !is_allowed_for_key(&policy, action, &key_b),
+            "an Allow scoped to key {key_a} must not leak to key {key_b}"
+        );
+        prop_assert!(
+            is_allowed_for_key(&policy, action, ""),
+            "call sites that pass no key resource keep the legacy match-every-key behaviour"
+        );
+    }
+
+    /// (g) Legacy compatibility pin: resource-less KMS statements match every
+    /// generated key id, exactly as before KMS resources existed.
+    #[test]
+    fn kms_resourceless_statement_matches_every_key(
+        key_id in key_id_strategy(),
+        action in kms_action_strategy(),
+    ) {
+        let policy = policy_from_statements(&[resourceless_statement_json("Allow", action)]);
+        prop_assert!(
+            is_allowed_for_key(&policy, action, &key_id),
+            "resource-less KMS statement must keep matching {action} on key {key_id}"
+        );
+    }
 }

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1171,6 +1171,12 @@ pub struct SignalServiceResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
     #[prost(uint32, tag = "3")]
     pub protocol_version: u32,
+    /// Redacted fingerprint of the KMS configuration this node is running.
+    /// Only set for the "kms" dynamic config subsystem, and left unset when the
+    /// responder has no KMS configuration. Peers built before that subsystem
+    /// reject the signal outright rather than answering without this field.
+    #[prost(string, optional, tag = "4")]
+    pub config_fingerprint: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct ScannerActivityRequest {

crates/protos/src/node.proto

@@ -817,6 +817,11 @@ message SignalServiceResponse {
   bool success = 1;
   optional string error_info = 2;
   uint32 protocol_version = 3;
+  // Redacted fingerprint of the KMS configuration this node is running.
+  // Only set for the "kms" dynamic config subsystem, and left unset when the
+  // responder has no KMS configuration. Peers built before that subsystem
+  // reject the signal outright rather than answering without this field.
+  optional string config_fingerprint = 4;
 }
 
 message ScannerActivityRequest {

crates/s3-types/src/event_name.rs

@@ -86,6 +86,25 @@ pub enum EventName {
     ObjectRemovedAbortMultipartUpload,
     ObjectCreatedCreateMultipartUpload,
     ObjectRemovedDeleteObjects,
+
+    // KMS management-plane events, covering both key operations and the
+    // service-control endpoints that change which backend holds the keys. They
+    // travel to the audit sink only and are never produced by the bucket
+    // notification path, so no compound `s3:` event expands to them. New
+    // variants must keep being appended here: the discriminant of every
+    // preceding variant is a `mask()` bit position, and inserting in the middle
+    // would silently renumber existing bits.
+    KmsKeyCreated,
+    KmsKeyRotated,
+    KmsKeyEnabled,
+    KmsKeyDisabled,
+    KmsKeyDeletionScheduled,
+    KmsKeyDeletionCancelled,
+    KmsKeyDeleted,
+    KmsKeyAccessed,
+    KmsServiceConfigured,
+    KmsServiceStarted,
+    KmsServiceStopped,
 }
 
 // Single event type sequential array for Everything.expand()
@@ -186,6 +205,20 @@ impl EventName {
             "s3:Scanner:LargeVersions" => Ok(EventName::ScannerLargeVersions),
             "s3:Scanner:BigPrefix" => Ok(EventName::ScannerBigPrefix),
             "s3:Scanner:*" => Ok(EventName::ObjectScannerAll),
+            // KMS events use their own namespace so a `s3:` wildcard in a bucket
+            // notification config can never select them. They still round-trip
+            // because audit entries are persisted and replayed by the store targets.
+            "kms:Key:Created" => Ok(EventName::KmsKeyCreated),
+            "kms:Key:Rotated" => Ok(EventName::KmsKeyRotated),
+            "kms:Key:Enabled" => Ok(EventName::KmsKeyEnabled),
+            "kms:Key:Disabled" => Ok(EventName::KmsKeyDisabled),
+            "kms:Key:DeletionScheduled" => Ok(EventName::KmsKeyDeletionScheduled),
+            "kms:Key:DeletionCancelled" => Ok(EventName::KmsKeyDeletionCancelled),
+            "kms:Key:Deleted" => Ok(EventName::KmsKeyDeleted),
+            "kms:Key:Accessed" => Ok(EventName::KmsKeyAccessed),
+            "kms:Service:Configured" => Ok(EventName::KmsServiceConfigured),
+            "kms:Service:Started" => Ok(EventName::KmsServiceStarted),
+            "kms:Service:Stopped" => Ok(EventName::KmsServiceStopped),
             // `Everything` has no string representation (`as_str` yields ""), so it
             // cannot be parsed back from a string. Every other variant round-trips.
             _ => Err(ParseEventNameError(s.to_string())),
@@ -251,6 +284,17 @@ impl EventName {
             EventName::ObjectRemovedAbortMultipartUpload => "s3:ObjectRemoved:AbortMultipartUpload",
             EventName::ObjectCreatedCreateMultipartUpload => "s3:ObjectCreated:CreateMultipartUpload",
             EventName::ObjectRemovedDeleteObjects => "s3:ObjectRemoved:DeleteObjects",
+            EventName::KmsKeyCreated => "kms:Key:Created",
+            EventName::KmsKeyRotated => "kms:Key:Rotated",
+            EventName::KmsKeyEnabled => "kms:Key:Enabled",
+            EventName::KmsKeyDisabled => "kms:Key:Disabled",
+            EventName::KmsKeyDeletionScheduled => "kms:Key:DeletionScheduled",
+            EventName::KmsKeyDeletionCancelled => "kms:Key:DeletionCancelled",
+            EventName::KmsKeyDeleted => "kms:Key:Deleted",
+            EventName::KmsKeyAccessed => "kms:Key:Accessed",
+            EventName::KmsServiceConfigured => "kms:Service:Configured",
+            EventName::KmsServiceStarted => "kms:Service:Started",
+            EventName::KmsServiceStopped => "kms:Service:Stopped",
         }
     }
 
@@ -357,6 +401,28 @@ impl EventName {
                 | EventName::ObjectRemovedDeleteObjects
         )
     }
+
+    /// Returns `true` for KMS management-plane events.
+    ///
+    /// These are audit-only: they are never emitted through the bucket
+    /// notification pipeline, and no `s3:` event selector expands to them.
+    #[inline]
+    pub fn is_kms(&self) -> bool {
+        matches!(
+            self,
+            EventName::KmsKeyCreated
+                | EventName::KmsKeyRotated
+                | EventName::KmsKeyEnabled
+                | EventName::KmsKeyDisabled
+                | EventName::KmsKeyDeletionScheduled
+                | EventName::KmsKeyDeletionCancelled
+                | EventName::KmsKeyDeleted
+                | EventName::KmsKeyAccessed
+                | EventName::KmsServiceConfigured
+                | EventName::KmsServiceStarted
+                | EventName::KmsServiceStopped
+        )
+    }
 }
 
 /// Returns the S3 notification event schema version for a given event.
@@ -570,6 +636,32 @@ mod tests {
         EventName::ObjectRemovedAbortMultipartUpload,
         EventName::ObjectCreatedCreateMultipartUpload,
         EventName::ObjectRemovedDeleteObjects,
+        EventName::KmsKeyCreated,
+        EventName::KmsKeyRotated,
+        EventName::KmsKeyEnabled,
+        EventName::KmsKeyDisabled,
+        EventName::KmsKeyDeletionScheduled,
+        EventName::KmsKeyDeletionCancelled,
+        EventName::KmsKeyDeleted,
+        EventName::KmsKeyAccessed,
+        EventName::KmsServiceConfigured,
+        EventName::KmsServiceStarted,
+        EventName::KmsServiceStopped,
+    ];
+
+    /// Every KMS management-plane event.
+    const KMS_EVENT_NAMES: &[EventName] = &[
+        EventName::KmsKeyCreated,
+        EventName::KmsKeyRotated,
+        EventName::KmsKeyEnabled,
+        EventName::KmsKeyDisabled,
+        EventName::KmsKeyDeletionScheduled,
+        EventName::KmsKeyDeletionCancelled,
+        EventName::KmsKeyDeleted,
+        EventName::KmsKeyAccessed,
+        EventName::KmsServiceConfigured,
+        EventName::KmsServiceStarted,
+        EventName::KmsServiceStopped,
     ];
 
     /// Regression for backlog#965: `mask()` used to recurse forever for the
@@ -682,6 +774,54 @@ mod tests {
         }
     }
 
+    /// KMS events are audit-plane only. No `s3:` selector — including the
+    /// catch-all `Everything` and every compound "All" type — may share a bit
+    /// with them, otherwise a bucket notification rule would silently start
+    /// matching KMS activity.
+    #[test]
+    fn test_kms_event_masks_are_disjoint_from_every_s3_selector() {
+        let s3_selectors: Vec<EventName> = ALL_EVENT_NAMES.iter().copied().filter(|ev| !ev.is_kms()).collect();
+
+        let mut seen = 0u64;
+        for kms in KMS_EVENT_NAMES {
+            let mask = kms.mask();
+            assert_ne!(mask, 0, "KMS event {kms} must have a non-zero mask");
+            assert_eq!(seen & mask, 0, "KMS event {kms} mask overlaps another KMS event");
+            seen |= mask;
+
+            for s3 in &s3_selectors {
+                assert_eq!(s3.mask() & mask, 0, "KMS event {kms} mask collides with S3 selector {s3}");
+            }
+        }
+    }
+
+    /// KMS event names must live in their own namespace so that neither a
+    /// `s3:` prefix filter nor an `s3:...:*` wildcard can select them.
+    #[test]
+    fn test_kms_event_names_are_outside_the_s3_namespace() {
+        for ev in KMS_EVENT_NAMES {
+            assert!(ev.is_kms(), "{ev} should be classified as a KMS event");
+            assert!(ev.as_str().starts_with("kms:"), "unexpected KMS event name {:?}", ev.as_str());
+            assert_eq!(EventName::parse(ev.as_str()).as_ref(), Ok(ev), "KMS event {ev} must round-trip");
+            assert_eq!(ev.expand(), vec![*ev], "KMS event {ev} must expand to itself only");
+        }
+
+        for ev in ALL_EVENT_NAMES.iter().filter(|ev| !ev.is_kms()) {
+            assert!(!ev.as_str().starts_with("kms:"), "{ev} must not claim the KMS namespace");
+        }
+    }
+
+    /// `mask()` shifts by `discriminant - 1`, so the enum may hold at most 64
+    /// mask-bearing variants. Appending past that overflows the shift instead
+    /// of failing loudly, so keep the budget check next to the variants.
+    #[test]
+    fn test_mask_bit_budget_is_not_exhausted() {
+        for ev in ALL_EVENT_NAMES {
+            let value = *ev as u32;
+            assert!(value <= 64, "{ev} has discriminant {value}; mask() only has 64 bits to hand out");
+        }
+    }
+
     /// `Everything` must cover every sequential single-type bit.
     #[test]
     fn test_everything_mask_covers_all_single_types() {

crates/scanner/Cargo.toml

@@ -88,6 +88,7 @@ rmp-serde = { workspace = true }
 hmac = { workspace = true }
 sha2 = { workspace = true }
 rustfs-filemeta = { workspace = true }
+rustfs-lock.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
 rustfs-ecstore = { workspace = true }
 rustfs-storage-api = { workspace = true }

crates/scanner/src/remote_scanner.rs

@@ -12,17 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(test)]
+use crate::RUSTFS_META_BUCKET;
 use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
 use crate::scanner_io::{
-    DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, current_cache_root_or_prepare,
-    scanner_cache_lock_resource, scanner_cache_lock_timeout, scanner_set_disk_inventory,
+    DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, cache_root_entry_info,
+    current_cache_root_or_prepare, scanner_set_disk_inventory,
 };
 use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
-use crate::storage_api::scan::NamespaceLocking as _;
 use crate::{
     DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource,
-    DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, EcstoreError, RUSTFS_META_BUCKET, ScannerDiskExt as _, ScannerError,
-    ScannerObjectIO, StorageError, is_reserved_or_invalid_bucket, read_config, resolve_scanner_object_store_handle,
+    DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, EcstoreError, ScannerDiskExt as _, ScannerError, ScannerObjectIO,
+    StorageError, is_reserved_or_invalid_bucket, read_config, resolve_scanner_object_store_handle,
 };
 use hmac::{Hmac, KeyInit, Mac};
 use rustfs_common::heal_channel::HealScanMode;
@@ -63,6 +64,7 @@ const NS_SCANNER_DISK_HEALTH_TIMEOUT: Duration = Duration::from_secs(5);
 const NS_SCANNER_VALIDATED_CYCLE_TTL: Duration = Duration::from_secs(1);
 const NS_SCANNER_MAX_REPLAY_SESSIONS: usize = 65_536;
 const NS_SCANNER_MAX_ERROR_CHARS: usize = 4096;
+const NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX: &str = "retry_bucket:";
 const NS_SCANNER_FRAME_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-frame-v3";
 
 pub const NS_SCANNER_MAX_REQUEST_BODY_SIZE: usize = 16 * 1024;
@@ -317,6 +319,13 @@ impl RemoteScannerServerError {
         }
     }
 
+    fn retry_bucket(message: impl Into<String>) -> Self {
+        Self {
+            scope: RemoteScannerErrorScope::Bucket,
+            error: ScannerError::Other(format!("{}{}", NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX, message.into())),
+        }
+    }
+
     fn into_frame(self) -> RemoteScannerErrorFrame {
         RemoteScannerErrorFrame {
             scope: self.scope,
@@ -336,6 +345,7 @@ struct RemoteScannerStreamError {
     error: StorageError,
     progress_fully_reported: bool,
     retire_worker: bool,
+    retry_bucket: bool,
 }
 
 impl RemoteScannerStreamError {
@@ -344,6 +354,7 @@ impl RemoteScannerStreamError {
             error,
             progress_fully_reported: false,
             retire_worker: true,
+            retry_bucket: false,
         }
     }
 
@@ -352,6 +363,7 @@ impl RemoteScannerStreamError {
             error,
             progress_fully_reported: true,
             retire_worker: true,
+            retry_bucket: false,
         }
     }
 
@@ -360,6 +372,16 @@ impl RemoteScannerStreamError {
             error,
             progress_fully_reported: true,
             retire_worker: false,
+            retry_bucket: false,
+        }
+    }
+
+    fn retry_bucket(error: StorageError) -> Self {
+        Self {
+            error,
+            progress_fully_reported: true,
+            retire_worker: false,
+            retry_bucket: true,
         }
     }
 
@@ -951,16 +973,14 @@ async fn scan_and_persist_local_bucket(
             ))
         })?;
     let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
-    let lock_resource = scanner_cache_lock_resource(&cache_name);
-    let ns_lock = set
-        .new_ns_lock(RUSTFS_META_BUCKET, &lock_resource)
-        .await
-        .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache lock creation failed: {err}")))?;
-    let guard = ns_lock
-        .get_write_lock_quiet(scanner_cache_lock_timeout())
+    let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
         .await
         .map_err(|err| {
-            RemoteScannerServerError::worker(format!("remote namespace scanner cache lock acquisition failed: {err}"))
+            if err.is_contention() {
+                RemoteScannerServerError::retry_bucket(format!("remote namespace scanner cache lock contention: {err}"))
+            } else {
+                RemoteScannerServerError::worker(format!("remote namespace scanner cache lock acquisition failed: {err}"))
+            }
         })?;
     let mut cache = DataUsageCache::default();
     let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
@@ -1216,7 +1236,9 @@ fn finish_remote_scanner_stream(
             if !error.progress_fully_reported {
                 budget.cancel_after_unreported_remote_progress();
             }
-            let failure = if error.retire_worker {
+            let failure = if error.retry_bucket {
+                RemoteScannerFailure::retry_bucket(error.error)
+            } else if error.retire_worker {
                 RemoteScannerFailure::transport(error.error)
             } else {
                 RemoteScannerFailure::bucket(error.error)
@@ -1374,9 +1396,16 @@ where
                 return Ok(RemoteScannerOutcome::CycleAhead(required_cycle));
             }
             RemoteScannerFrameResult::Error(error_frame) => {
+                let retry_bucket = error_frame.message.starts_with(NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX);
+                let message = error_frame
+                    .message
+                    .strip_prefix(NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX)
+                    .map(str::trim_start)
+                    .unwrap_or(error_frame.message.as_str());
                 let error =
-                    StorageError::other(format!("remote namespace scanner failed: {}", limit_error_message(error_frame.message)));
+                    StorageError::other(format!("remote namespace scanner failed: {}", limit_error_message(message.to_string())));
                 return Err(match error_frame.scope {
+                    RemoteScannerErrorScope::Bucket if retry_bucket => RemoteScannerStreamError::retry_bucket(error),
                     RemoteScannerErrorScope::Bucket => RemoteScannerStreamError::bucket(error),
                     RemoteScannerErrorScope::Worker => RemoteScannerStreamError::reconciled(error),
                 });
@@ -2491,6 +2520,42 @@ mod tests {
         assert!(!budget.budget_elapsed());
     }
 
+    #[tokio::test]
+    async fn retry_bucket_error_terminal_frame_requeues_bucket_work() {
+        let request_id = Uuid::new_v4();
+        let writer_auth = FrameAuthenticator::for_test(request_id);
+        let reader_auth = FrameAuthenticator::for_test(request_id);
+        let (mut writer, reader) = tokio::io::duplex(4096);
+        tokio::spawn(async move {
+            let mut sequence = 0;
+            write_frame(
+                &mut writer,
+                &writer_auth,
+                &mut sequence,
+                &RemoteScannerFrame::terminal(
+                    RemoteScannerProgress::default(),
+                    RemoteScannerFrameResult::Error(RemoteScannerErrorFrame {
+                        scope: RemoteScannerErrorScope::Bucket,
+                        message: format!("{NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX} cache lock contention"),
+                    }),
+                ),
+            )
+            .await
+            .expect("retry-bucket error terminal frame should write");
+        });
+
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default());
+        let stream_result =
+            consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth)
+                .await;
+        let error = finish_remote_scanner_stream(stream_result, budget.as_ref()).expect_err("retry-bucket frame must fail");
+
+        assert!(error.retry_bucket_work());
+        assert!(!error.retire_worker());
+        assert!(error.to_string().contains("cache lock contention"));
+    }
+
     #[tokio::test]
     async fn worker_error_terminal_frame_retires_worker_without_cancelling_reported_budget() {
         let request_id = Uuid::new_v4();

crates/scanner/src/scanner.rs

@@ -949,7 +949,7 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
                 }
                 SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
                     return Err(format!(
-                        "scanner activity peer {host} cannot acknowledge distributed dirty usage with protocol {}",
+                        "scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
                         SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
                     ));
                 }
@@ -7114,9 +7114,17 @@ mod tests {
                 ..scanner_node_activity("epoch-a", 7, 3)
             },
         )]);
+        let previous = BTreeMap::from([(
+            "node-2".to_string(),
+            ScannerNodeActivity {
+                protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
+                ..scanner_node_activity("epoch-a", 7, 3)
+            },
+        )]);
         let current = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
 
         assert_ne!(scanner_activity_snapshot_digest(&legacy), scanner_activity_snapshot_digest(&current));
+        assert_ne!(scanner_activity_snapshot_digest(&previous), scanner_activity_snapshot_digest(&current));
     }
 
     #[test]

crates/scanner/src/scanner_folder.rs

@@ -75,7 +75,7 @@ const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500;
 const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
 const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
 const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
-const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60);
+const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
 const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
 const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
 const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
@@ -100,6 +100,21 @@ static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
 static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
 static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
 
+#[cfg(test)]
+type ListPathRawTimeoutSnapshot = (bool, Option<Duration>, Option<Duration>);
+
+fn scanner_abandoned_child_list_options() -> ListPathRawOptions {
+    // A complete heal walk scales with bucket size and may legitimately take
+    // longer than a fixed wall-clock budget. Keep the total duration unbounded;
+    // Retain the scanner's per-read stall budget and keep cancellation controlled
+    // by the scanner cycle token.
+    ListPathRawOptions {
+        skip_walkdir_total_timeout: true,
+        walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT),
+        ..Default::default()
+    }
+}
+
 pub fn data_usage_update_dir_cycles() -> u32 {
     rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
 }
@@ -1281,6 +1296,8 @@ pub struct FolderScanner {
     skip_heal: Arc<std::sync::atomic::AtomicBool>,
     local_disk: Arc<Disk>,
     pending_heals_changed: bool,
+    #[cfg(test)]
+    list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
 }
 
 impl FolderScanner {
@@ -2410,75 +2427,79 @@ impl FolderScanner {
                 let bucket_clone = bucket.clone();
                 let prefix_clone = prefix.clone();
                 let child_ctx_clone = child_ctx.clone();
+                #[cfg(test)]
+                let list_path_raw_options_observer = self.list_path_raw_options_observer.clone();
 
                 tokio::spawn(async move {
-                    if let Err(e) = list_path_raw(
-                        child_ctx_clone.clone(),
-                        ListPathRawOptions {
-                            disks,
-                            bucket: bucket_clone.clone(),
-                            path: prefix_clone.clone(),
-                            recursive: true,
-                            report_not_found: true,
-                            min_disks: disks_quorum,
-                            walkdir_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
-                            walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
-                            agreed: Some(Box::new(move |entry: MetaCacheEntry| {
-                                let entry_name = entry.name.clone();
-                                let agreed_tx = agreed_tx.clone();
-                                Box::pin(async move {
-                                    if let Err(e) = agreed_tx.send(entry_name).await {
-                                        error!(
-                                            target: "rustfs::scanner::folder",
-                                            event = EVENT_SCANNER_FOLDER_STATE,
-                                            component = LOG_COMPONENT_SCANNER,
-                                            subsystem = LOG_SUBSYSTEM_FOLDER,
-                                            entry = %entry.name,
-                                            state = "list_path_agreed_send_failed",
-                                            error = %e,
-                                            "Scanner list_path_raw agreed callback failed"
-                                        );
-                                    }
-                                })
-                            })),
-                            partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
-                                let partial_tx = partial_tx.clone();
-                                Box::pin(async move {
-                                    if let Err(e) = partial_tx.send(entries).await {
-                                        error!(
-                                            target: "rustfs::scanner::folder",
-                                            event = EVENT_SCANNER_FOLDER_STATE,
-                                            component = LOG_COMPONENT_SCANNER,
-                                            subsystem = LOG_SUBSYSTEM_FOLDER,
-                                            state = "list_path_partial_send_failed",
-                                            error = %e,
-                                            "Scanner list_path_raw partial callback failed"
-                                        );
-                                    }
-                                })
-                            })),
-                            finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
-                                let finished_tx = finished_tx.clone();
-                                let errs_clone = errs.to_vec();
-                                Box::pin(async move {
-                                    if let Err(e) = finished_tx.send(errs_clone).await {
-                                        error!(
-                                            target: "rustfs::scanner::folder",
-                                            event = EVENT_SCANNER_FOLDER_STATE,
-                                            component = LOG_COMPONENT_SCANNER,
-                                            subsystem = LOG_SUBSYSTEM_FOLDER,
-                                            state = "list_path_finished_send_failed",
-                                            error = %e,
-                                            "Scanner list_path_raw finished callback failed"
-                                        );
-                                    }
-                                })
-                            })),
-                            ..Default::default()
-                        },
-                    )
-                    .await
-                    {
+                    let options = ListPathRawOptions {
+                        disks,
+                        bucket: bucket_clone.clone(),
+                        path: prefix_clone.clone(),
+                        recursive: true,
+                        report_not_found: true,
+                        min_disks: disks_quorum,
+                        agreed: Some(Box::new(move |entry: MetaCacheEntry| {
+                            let entry_name = entry.name.clone();
+                            let agreed_tx = agreed_tx.clone();
+                            Box::pin(async move {
+                                if let Err(e) = agreed_tx.send(entry_name).await {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        entry = %entry.name,
+                                        state = "list_path_agreed_send_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw agreed callback failed"
+                                    );
+                                }
+                            })
+                        })),
+                        partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
+                            let partial_tx = partial_tx.clone();
+                            Box::pin(async move {
+                                if let Err(e) = partial_tx.send(entries).await {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        state = "list_path_partial_send_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw partial callback failed"
+                                    );
+                                }
+                            })
+                        })),
+                        finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
+                            let finished_tx = finished_tx.clone();
+                            let errs_clone = errs.to_vec();
+                            Box::pin(async move {
+                                if let Err(e) = finished_tx.send(errs_clone).await {
+                                    error!(
+                                        target: "rustfs::scanner::folder",
+                                        event = EVENT_SCANNER_FOLDER_STATE,
+                                        component = LOG_COMPONENT_SCANNER,
+                                        subsystem = LOG_SUBSYSTEM_FOLDER,
+                                        state = "list_path_finished_send_failed",
+                                        error = %e,
+                                        "Scanner list_path_raw finished callback failed"
+                                    );
+                                }
+                            })
+                        })),
+                        ..scanner_abandoned_child_list_options()
+                    };
+                    #[cfg(test)]
+                    if let Some(observer) = list_path_raw_options_observer {
+                        let _ = observer.send((
+                            options.skip_walkdir_total_timeout,
+                            options.walkdir_timeout,
+                            options.walkdir_stall_timeout,
+                        ));
+                    }
+                    if let Err(e) = list_path_raw(child_ctx_clone.clone(), options).await {
                         if is_missing_path_disk_error(&e) {
                             debug!(
                                 target: "rustfs::scanner::folder",
@@ -2820,6 +2841,8 @@ pub async fn scan_data_folder(
         skip_heal,
         local_disk,
         pending_heals_changed: false,
+        #[cfg(test)]
+        list_path_raw_options_observer: None,
     };
 
     // Check if context is cancelled
@@ -3026,6 +3049,7 @@ mod tests {
             skip_heal: Arc::new(AtomicBool::new(false)),
             local_disk: disk,
             pending_heals_changed: false,
+            list_path_raw_options_observer: None,
         };
 
         (scanner, temp_dir)
@@ -4210,6 +4234,8 @@ mod tests {
         scanner.heal_object_select = 1;
         scanner.disks = disks;
         scanner.disks_quorum = 2;
+        let (options_tx, mut options_rx) = mpsc::unbounded_channel();
+        scanner.list_path_raw_options_observer = Some(options_tx);
         scanner.old_cache.replace(
             &format!("{bucket}/{object}"),
             bucket,
@@ -4231,6 +4257,11 @@ mod tests {
             .expect("scan_folder should not hang after list_path_raw finishes")
             .expect("scan_folder should finish successfully");
 
+        let observed_options = tokio::time::timeout(Duration::from_secs(1), options_rx.recv())
+            .await
+            .expect("abandoned-child listing options should be observed promptly")
+            .expect("abandoned-child listing options channel should remain open");
+        assert_eq!(observed_options, (true, None, Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT)));
         let root = scanner
             .new_cache
             .checked_flatten(bucket)

crates/scanner/src/scanner_io.rs

@@ -30,6 +30,7 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e
 use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
 use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
 use rustfs_filemeta::FileMeta;
+use rustfs_lock::{LockError, NamespaceLockGuard};
 use rustfs_utils::path::path_join_buf;
 use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
 use sha2::{Digest as _, Sha256};
@@ -944,14 +945,85 @@ fn checked_bucket_usage_info(entry: &DataUsageEntry) -> Option<BucketUsageInfo>
     Some(usage)
 }
 
-pub(crate) fn scanner_cache_lock_resource(cache_name: &str) -> String {
-    path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, SCANNER_CACHE_LOCK_SUFFIX])
+pub(crate) fn scanner_cache_lock_resource(cache_name: &str, source: DataUsageCacheSource) -> String {
+    let lock_name = format!("{SCANNER_CACHE_LOCK_SUFFIX}.pool-{}.set-{}", source.pool_index, source.set_index);
+    path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, &lock_name])
 }
 
 pub(crate) fn scanner_cache_lock_timeout() -> Duration {
     Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
 }
 
+#[derive(Debug)]
+pub(crate) struct ScannerCacheLockGuards {
+    scoped: NamespaceLockGuard,
+}
+
+impl ScannerCacheLockGuards {
+    pub(crate) fn is_lock_lost(&self) -> bool {
+        self.scoped.is_lock_lost()
+    }
+}
+
+#[derive(Debug)]
+pub(crate) enum ScannerCacheLockError {
+    Create { resource: String, source: Error },
+    Acquire { resource: String, source: LockError },
+}
+
+impl ScannerCacheLockError {
+    pub(crate) fn state(&self) -> &'static str {
+        match self {
+            Self::Create { .. } => "lock_create_failed",
+            Self::Acquire { .. } => "lock_acquire_failed",
+        }
+    }
+
+    pub(crate) fn is_contention(&self) -> bool {
+        matches!(
+            self,
+            Self::Acquire {
+                source: LockError::Timeout { .. } | LockError::AlreadyLocked { .. },
+                ..
+            }
+        )
+    }
+}
+
+impl std::fmt::Display for ScannerCacheLockError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Create { resource, source } => write!(formatter, "create scanner cache lock {resource}: {source}"),
+            Self::Acquire { resource, source } => write!(formatter, "acquire scanner cache lock {resource}: {source}"),
+        }
+    }
+}
+
+pub(crate) async fn acquire_scanner_cache_locks(
+    store: &SetDisks,
+    cache_name: &str,
+    source: DataUsageCacheSource,
+) -> std::result::Result<ScannerCacheLockGuards, ScannerCacheLockError> {
+    let timeout = scanner_cache_lock_timeout();
+    let scoped_resource = scanner_cache_lock_resource(cache_name, source);
+    let scoped_lock = store
+        .new_ns_lock(RUSTFS_META_BUCKET, &scoped_resource)
+        .await
+        .map_err(|source| ScannerCacheLockError::Create {
+            resource: scoped_resource.clone(),
+            source,
+        })?;
+    let scoped = scoped_lock
+        .get_write_lock_quiet(timeout)
+        .await
+        .map_err(|source| ScannerCacheLockError::Acquire {
+            resource: scoped_resource,
+            source,
+        })?;
+
+    Ok(ScannerCacheLockGuards { scoped })
+}
+
 async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)
 where
     F: Future,
@@ -1697,6 +1769,19 @@ mod publish_gate_tests {
         assert!(!cache_snapshot_is_current(&cache, "photos", source, 11, 0, second));
     }
 
+    #[test]
+    fn scanner_cache_lock_resource_is_scoped_to_cache_source() {
+        let cache_name = "photos/.usage-cache.bin";
+        let first_source = DataUsageCacheSource::new(0, 1);
+        let same_source = DataUsageCacheSource::new(0, 1);
+        let other_source = DataUsageCacheSource::new(1, 0);
+
+        let first = scanner_cache_lock_resource(cache_name, first_source);
+        assert_eq!(first, scanner_cache_lock_resource(cache_name, same_source));
+        assert_ne!(first, scanner_cache_lock_resource(cache_name, other_source));
+        assert!(first.ends_with(".scanner-cycle.lock.pool-0.set-1"));
+    }
+
     #[test]
     fn count_budget_serializes_set_and_disk_work() {
         assert_eq!(scanner_budgeted_concurrency_limit(8, true), 1);
@@ -1797,24 +1882,7 @@ async fn persist_and_publish_cache_snapshot(
     cache_cycle_floor: &AtomicU64,
 ) -> Option<SystemTime> {
     let source = cache_snapshot.info.source?;
-    let lock_resource = scanner_cache_lock_resource(DATA_USAGE_CACHE_NAME);
-    let ns_lock = match store.new_ns_lock(RUSTFS_META_BUCKET, &lock_resource).await {
-        Ok(lock) => lock,
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner::io",
-                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_IO,
-                cache_name = DATA_USAGE_CACHE_NAME,
-                state = "lock_create_failed",
-                error = %err,
-                "Scanner cache snapshot lock creation failed"
-            );
-            return None;
-        }
-    };
-    let guard = match ns_lock.get_write_lock_quiet(scanner_cache_lock_timeout()).await {
+    let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
         Ok(guard) => guard,
         Err(err) => {
             error!(
@@ -1823,7 +1891,7 @@ async fn persist_and_publish_cache_snapshot(
                 component = LOG_COMPONENT_SCANNER,
                 subsystem = LOG_SUBSYSTEM_IO,
                 cache_name = DATA_USAGE_CACHE_NAME,
-                state = "lock_acquire_failed",
+                state = err.state(),
                 error = %err,
                 "Scanner cache snapshot lock acquisition failed"
             );
@@ -3080,32 +3148,35 @@ impl ScannerIOCache for SetDisks {
                         None
                     };
 
-                    // Lock order: scanner leader fence -> per-bucket cache lock ->
-                    // cache object read/write. The cache lock stays outermost for
-                    // local and rolling-upgrade workers so leader failover cannot
-                    // execute the same bucket concurrently.
-                    let lock_resource = scanner_cache_lock_resource(&cache_name);
-                    let ns_lock = match store_clone_clone.new_ns_lock(RUSTFS_META_BUCKET, &lock_resource).await {
-                        Ok(lock) => lock,
-                        Err(e) => {
-                            record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
-                            error!(
-                                target: "rustfs::scanner::io",
-                                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                                component = LOG_COMPONENT_SCANNER,
-                                subsystem = LOG_SUBSYSTEM_IO,
-                                bucket = %bucket.name,
-                                cache_name = %cache_name,
-                                state = "lock_create_failed",
-                                error = %e,
-                                "Scanner bucket cache lock creation failed"
-                            );
-                            continue;
-                        }
-                    };
-                    let cache_guard = match ns_lock.get_write_lock_quiet(scanner_cache_lock_timeout()).await {
+                    // Lock order: scanner leader fence -> set-scoped per-bucket cache lock ->
+                    // cache object read/write.
+                    let cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
                         Ok(guard) => guard,
                         Err(e) => {
+                            if e.is_contention() {
+                                if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
+                                    increment_disk_bucket_scans_queued(
+                                        &queued_disk_bucket_scans_clone,
+                                        &pool_label_clone,
+                                        &set_label_clone,
+                                    );
+                                } else {
+                                    record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
+                                }
+                                debug!(
+                                    target: "rustfs::scanner::io",
+                                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                                    component = LOG_COMPONENT_SCANNER,
+                                    subsystem = LOG_SUBSYSTEM_IO,
+                                    bucket = %bucket.name,
+                                    cache_name = %cache_name,
+                                    state = "lock_contention_requeued",
+                                    error = %e,
+                                    "Scanner bucket cache lock contention requeued bucket work"
+                                );
+                                break;
+                            }
+
                             record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
                             error!(
                                 target: "rustfs::scanner::io",
@@ -3114,7 +3185,7 @@ impl ScannerIOCache for SetDisks {
                                 subsystem = LOG_SUBSYSTEM_IO,
                                 bucket = %bucket.name,
                                 cache_name = %cache_name,
-                                state = "lock_acquire_failed",
+                                state = e.state(),
                                 error = %e,
                                 "Scanner bucket cache lock acquisition failed"
                             );
@@ -3894,6 +3965,52 @@ mod tests {
         (temp_dir, store)
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn scanner_cache_locks_block_same_source_workers() {
+        let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+        let set = &store.pools[0].disk_set[0];
+        let source = DataUsageCacheSource::new(0, 0);
+        let cache_name = "photos/.usage-cache.bin";
+
+        let guards = acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
+            .await
+            .expect("scanner cache locks should be acquired");
+        let scoped_lock = set
+            .new_ns_lock(RUSTFS_META_BUCKET, &scanner_cache_lock_resource(cache_name, source))
+            .await
+            .expect("scoped scanner cache lock should be created");
+        let scoped_err = scoped_lock
+            .get_write_lock_quiet(Duration::from_millis(100))
+            .await
+            .expect_err("same-source workers must be blocked while scanner cache lock is held");
+        assert!(matches!(scoped_err, LockError::Timeout { .. } | LockError::AlreadyLocked { .. }));
+
+        drop(guards);
+        acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
+            .await
+            .expect("scanner cache locks should be released when guards drop");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn scanner_cache_locks_allow_cross_source_workers() {
+        let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+        let first_set = &store.pools[0].disk_set[0];
+        let second_set = &store.pools[1].disk_set[0];
+        let cache_name = "photos/.usage-cache.bin";
+
+        let first = acquire_scanner_cache_locks(first_set.as_ref(), cache_name, DataUsageCacheSource::new(0, 0))
+            .await
+            .expect("first source scanner cache locks should be acquired");
+        let second = acquire_scanner_cache_locks(second_set.as_ref(), cache_name, DataUsageCacheSource::new(1, 0))
+            .await
+            .expect("different source scanner cache locks should not contend");
+
+        assert!(!first.is_lock_lost());
+        assert!(!second.is_lock_lost());
+    }
+
     #[tokio::test]
     async fn data_usage_publish_fails_when_receiver_is_closed() {
         let (updates, receiver) = mpsc::channel(1);

crates/storage-api/src/lib.rs

@@ -28,8 +28,8 @@ pub const NS_SCANNER_SESSION_SEQUENCE_QUERY: &str = "ns_scanner_session_sequence
 pub const NS_SCANNER_PROTOCOL_VERSION_QUERY: &str = "ns_scanner_protocol";
 pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3;
 pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0;
-pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 4;
-pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 5;
+pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
+pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 6;
 
 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 #[serde(deny_unknown_fields)]

deploy/observability/README.md

@@ -9,3 +9,5 @@ Import `grafana/rustfs-node-observability.json` into Grafana and select a Promet
 The dashboard uses the RustFS `server` metric label introduced with the node-local observability updates. `server` represents the RustFS node identity and is preferred for RustFS node comparisons. Prometheus `instance` still identifies the scrape target and remains useful for scrape/debugging views, but dashboards that compare RustFS nodes should group and filter by `server`.
 
 During a rolling upgrade, older nodes may still emit metrics without the `server` label. Complete the rollout before using this dashboard for node-by-node comparisons.
+
+`grafana/rustfs-kms-observability.json` covers the KMS backend operation metrics emitted at the operation-policy choke point. Unlike the node dashboard, KMS metrics do not carry the `server` label — use `job`/`instance` or promoted OTel resource attributes to split by node. Matching Prometheus alert rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`, and the alert response procedures are documented in `docs/operations/kms-observability-runbook.md`.

deploy/observability/grafana/rustfs-kms-observability.json

@@ -0,0 +1,793 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "KMS backend operation metrics emitted at the operation-policy choke point (crates/kms/src/policy.rs). All label values are static enum strings; key identifiers, key material, and tokens never appear in labels. These metrics do not carry the RustFS `server` label — use your scrape topology (job/instance or promoted OTel resource attributes) to split by node. Alert response procedures: docs/operations/kms-observability-runbook.md.",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Terminal outcomes of KMS backend operations. `fatal` means a non-retryable failure ended the operation on first observation; `budget_exhausted` and `deadline_exceeded` mean retries ran out; `cancelled` is normal during shutdown.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (outcome) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))",
+          "legendFormat": "{{outcome}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Backend Operation Rate by Outcome",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Operation names are static per-call-site identifiers (e.g. vault_kv2_read_key_version, vault_transit_encrypt, vault_login). `op_class` distinguishes read_idempotent, mutating_non_idempotent, and auth operations.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (operation, op_class) (rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval]))",
+          "legendFormat": "{{operation}} ({{op_class}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Backend Operation Rate by Operation",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Share of operations that terminated in fatal, budget_exhausted, or deadline_exceeded. The cancelled outcome is plotted separately because shutdown windows legitimately spike it. The ratio is meaningless at near-zero traffic — read it together with the operation rate panels.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome!~\"success|cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)",
+          "legendFormat": "non-success (excl. cancelled)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(rustfs_kms_backend_operations_total{outcome=\"cancelled\",operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum(rate(rustfs_kms_backend_operations_total{operation=~\"$operation\"}[$__rate_interval])), 1e-9)",
+          "legendFormat": "cancelled",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Non-Success Outcome Ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Per-attempt failures by retry classification. retryable_conn covers connection-level failures, retryable_status covers retryable backend status codes, attempt_timeout covers attempts cut off by the per-attempt timeout, and fatal covers non-retryable failures (auth, permission, malformed request). Sustained fatal traffic is always actionable.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total{operation=~\"$operation\"}[$__rate_interval]))",
+          "legendFormat": "{{error_class}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Attempt Failure Rate by Error Class",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Wall-clock duration of whole operations, including retries and backoff sleeps. A rising p99 with a flat p50 usually means a slow retry tail (backend degradation), not a uniform slowdown.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.50, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))",
+          "legendFormat": "p50",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))",
+          "legendFormat": "p99",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Operation Duration p50 / p99",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "p99 duration split by operation. Auth operations (vault_login, vault_token_renew) and mutating writes are expected to sit higher than idempotent reads.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket{operation=~\"$operation\"}[$__rate_interval])))",
+          "legendFormat": "{{operation}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Operation Duration p99 by Operation",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Attempts one operation used before completing. Healthy operations average close to 1. A rising average or p99 means the retry policy is absorbing backend failures — read together with the attempt-failure panel to see the failure class.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 12,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 4,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (operation) (rate(rustfs_kms_backend_operation_attempts_sum{operation=~\"$operation\"}[$__rate_interval])) / clamp_min(sum by (operation) (rate(rustfs_kms_backend_operation_attempts_count{operation=~\"$operation\"}[$__rate_interval])), 1e-9)",
+          "legendFormat": "{{operation}} avg",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum by (le) (rate(rustfs_kms_backend_operation_attempts_bucket{operation=~\"$operation\"}[$__rate_interval])))",
+          "legendFormat": "p99 (all operations)",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Operation Attempts Distribution",
+      "type": "timeseries"
+    },
+    {
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "id": 8,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "Placeholder for KMS metrics planned by sibling changes of rustfs/backlog#1584 that have **not landed yet**. Do not add panels for these names until the emitting code is merged, or the panels will render empty and mask real gaps.\n\n- TODO: key-cache effectiveness — hit/miss counters, entry gauge, eviction counter (replaces the former hardcoded-zero miss stat).\n- TODO: key lifecycle — pending-deletion/tombstone gauges from the deletion worker sweep, aggregate rotation-age gauge.\n- TODO: Vault credentials — token TTL remaining and fail-closed state gauges.\n- TODO: synthetic backend probe — probe outcome/failure-class metrics feeding the readiness cache.\n\nWhen a family lands, replace one bullet with a real panel and keep this list in sync with docs/operations/kms-observability-runbook.md (Coverage gaps).",
+        "mode": "markdown"
+      },
+      "title": "Planned Panels (TODO — metrics not landed yet)",
+      "type": "text"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "rustfs",
+    "observability",
+    "kms"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Prometheus",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${datasource}"
+        },
+        "definition": "label_values(rustfs_kms_backend_operations_total, operation)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "KMS operation",
+        "multi": true,
+        "name": "operation",
+        "options": [],
+        "query": "label_values(rustfs_kms_backend_operations_total, operation)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "RustFS KMS Observability",
+  "uid": "rustfs-kms-observability",
+  "version": 1,
+  "weekStart": ""
+}

docs/architecture/compat-cleanup-register.md

@@ -17,7 +17,7 @@ for later deletion.
 - `rustfs-5416-kubernetes-alias-dns` Kubernetes endpoint identity fallback: deployments created before explicit local endpoint identity may use resolvable aliases that do not match the Pod hostname. An implicit auto-mode zero match retains legacy DNS locality with a bounded deadline, while ambiguous matches and invalid explicit anchors still fail closed. Remove the fallback after every supported direct-upgrade chart and deployment manifest provides a canonical RUSTFS_LOCAL_ENDPOINT_HOST for domain-based distributed topologies.
 - `rustfs-5416-zero-retry-delay` startup retry-delay validation: releases before bounded topology convergence accept RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY values of 0 or 0ms. New servers replace those values with the safe nonzero default so a direct upgrade neither fails startup nor enters a busy loop. Reject zero after the minimum supported direct-upgrade release validates or rewrites this setting before rollout.
 - `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`.
-- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
+- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state, but predates set-scoped scanner cache locks. Current protocol v6 additionally fences scanner cache lock-domain changes, so distributed scanner cycles publish usage only after every peer reports protocol v6 state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while protocol-v5 peers are treated as previous-version peers that cannot safely participate in the new cache lock domain. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, remove the protocol-v4 activity codec after every supported peer implements protocol v5, and remove protocol-v5 previous-version rejection after every supported peer implements protocol v6; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
 - `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
 - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
 - `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC.

docs/operations/hotpath-warp-abba-runbook.md

@@ -0,0 +1,277 @@
+# Hotpath warp ABBA validation runbook
+
+This runbook describes how to collect formal Linux or production-cluster
+evidence for hotpath performance changes. Use it when a short local A/B smoke
+run is too noisy to decide whether a regression is real.
+
+The ABBA runner executes each workload and drive-sync cell as:
+
+```text
+A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline
+```
+
+`B1` and `B2` are compared with `A1` to measure the candidate delta. `A2` is
+also compared with `A1` to measure baseline drift. Treat a candidate regression
+as actionable only when the `A2` drift is passing or materially smaller than
+the `B1` and `B2` delta for the same workload.
+
+## Scope
+
+Use this runbook for hotpath profiling and performance validation of RustFS
+object I/O changes, especially when CPU, memory allocation, lock/channel wait
+time, request throughput, or tail latency is the review question.
+
+The script validates the same workload matrix as the hotpath warp A/B gate:
+
+| Workload | mode | size |
+| --- | --- | --- |
+| `put-4kib` | put | 4KiB |
+| `put-4mib` | put | 4MiB |
+| `get-4kib` | get | 4KiB |
+| `get-4mib` | get | 4MiB |
+| `get-10mib` | get | 10MiB |
+| `mixed-256k` | mixed | 256KiB |
+
+Each workload runs with `RUSTFS_DRIVE_SYNC_ENABLE=true` and
+`RUSTFS_DRIVE_SYNC_ENABLE=false`, so a full ABBA pass produces 48 measurement
+cells: 6 workloads x 2 drive-sync modes x 4 ABBA legs.
+
+## Prerequisites
+
+Run the formal pass on Linux, not on a laptop smoke environment.
+
+Required tools on the bench host:
+
+- `bash`, `curl`, `git`, and core GNU userland.
+- `warp` on `PATH`, or pass `--warp-bin`.
+- Two RustFS Linux binaries: one baseline and one candidate.
+- Enough isolated disks or directories for the local runner, or an externally
+  managed RustFS cluster for production-like validation.
+- Stable host telemetry collection such as `pidstat`, `mpstat`, `iostat`,
+  `sar`, `perf`, `heaptrack`, or the platform's equivalent observability stack.
+
+Cluster-mode requirements:
+
+- A deploy hook that can replace the RustFS binary on every node.
+- The hook must apply `RUSTFS_DRIVE_SYNC_ENABLE` for the current ABBA leg.
+- The hook must restart RustFS and return only after the rollout command has
+  been accepted. The ABBA script performs the HTTP readiness wait.
+- The benchmark client should run outside the RustFS nodes when possible.
+- Do not run against a production data set unless the workload bucket and test
+  credentials are isolated and approved for destructive benchmark traffic.
+
+## Build the binaries
+
+Build the baseline from the comparison commit, usually `origin/main` or the
+previous accepted release:
+
+```bash
+git fetch origin main
+git switch --detach origin/main
+cargo build --release -p rustfs --bins
+cp target/release/rustfs /tmp/rustfs-baseline
+```
+
+Build the candidate from the PR commit:
+
+```bash
+git switch <candidate-branch>
+cargo build --release -p rustfs --bins
+cp target/release/rustfs /tmp/rustfs-candidate
+```
+
+For cross-compiled cluster binaries, keep both outputs on the bench host and
+make the deploy hook copy the selected binary to the cluster. The ABBA runner
+passes the selected binary path through `HOTPATH_ABBA_BINARY`.
+
+## Local Linux runner
+
+Use local mode for a dedicated Linux runner with disposable data paths. This is
+not a substitute for a production-like cluster, but it is useful before spending
+cluster time.
+
+```bash
+scripts/run_hotpath_warp_abba.sh \
+  --baseline-bin /tmp/rustfs-baseline \
+  --candidate-bin /tmp/rustfs-candidate \
+  --address 127.0.0.1:9000 \
+  --data-root /var/tmp/rustfs-hotpath-abba \
+  --disks 4 \
+  --duration 120s \
+  --rounds 3 \
+  --cooldown 30 \
+  --concurrency 16 \
+  --out-dir target/hotpath-abba/linux-local
+```
+
+The script starts and stops RustFS for each ABBA leg. The data root is
+throwaway and should not contain important data.
+
+## Production-like cluster runner
+
+Use external mode when RustFS lifecycle is managed by ansible, systemd, a
+cluster scheduler, or a dedicated deployment harness. In this mode the ABBA
+script does not start RustFS directly; it calls `--deploy-hook` before each leg
+and then waits for `http://<endpoint><health-path>`.
+
+The deploy hook receives:
+
+| Environment variable | Value |
+| --- | --- |
+| `HOTPATH_ABBA_LEG` | `A1`, `B1`, `B2`, or `A2` |
+| `HOTPATH_ABBA_PHASE` | `baseline` or `candidate` |
+| `HOTPATH_ABBA_BINARY` | selected baseline or candidate binary path |
+| `HOTPATH_ABBA_DRIVE_SYNC` | `true` or `false` |
+
+Example ansible-shaped command:
+
+```bash
+scripts/run_hotpath_warp_abba.sh \
+  --baseline-bin /srv/rustfs-binaries/rustfs-baseline \
+  --candidate-bin /srv/rustfs-binaries/rustfs-candidate \
+  --endpoint rustfs-bench.example.internal:9000 \
+  --deploy-hook '
+    set -euo pipefail
+    cd /srv/rustfs-ansible
+    cp "${HOTPATH_ABBA_BINARY:?}" roles/rustfs/files/rustfs
+    export RUSTFS_DRIVE_SYNC_ENABLE="${HOTPATH_ABBA_DRIVE_SYNC:?}"
+    ansible-playbook -f 4 -l bench rustfs-manage.yml --tags stop
+    ansible-playbook -f 4 -l bench rustfs-manage.yml --tags config
+    ansible-playbook -f 4 -l bench rustfs-manage.yml --tags binary-copy
+    ansible-playbook -f 4 -l bench rustfs-manage.yml --tags start
+  ' \
+  --duration 180s \
+  --rounds 5 \
+  --cooldown 45 \
+  --concurrency 32 \
+  --out-dir target/hotpath-abba/cluster-pr-XXXX
+```
+
+For formal evidence, prefer `--rounds 5` or higher when the cluster budget
+allows it. The script enforces `--rounds >= 3`.
+
+## CPU and memory evidence
+
+ABBA warp output answers whether the candidate changed throughput or latency.
+Collect host telemetry at the same time to explain why.
+
+Recommended minimum:
+
+```bash
+mkdir -p target/hotpath-abba/cluster-pr-XXXX/telemetry
+
+pidstat -durh 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/pidstat.txt &
+PIDSTAT_PID=$!
+
+mpstat 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/mpstat.txt &
+MPSTAT_PID=$!
+
+iostat -xz 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/iostat.txt &
+IOSTAT_PID=$!
+```
+
+Stop the collectors after the ABBA script exits:
+
+```bash
+kill "$PIDSTAT_PID" "$MPSTAT_PID" "$IOSTAT_PID"
+```
+
+For deeper CPU attribution, run `perf record` around one representative
+workload after the ABBA gate identifies a candidate regression or improvement:
+
+```bash
+perf record -F 99 -g -- sleep 180
+perf report --stdio > target/hotpath-abba/cluster-pr-XXXX/telemetry/perf-report.txt
+```
+
+For allocation profiling, build the candidate with:
+
+```bash
+cargo build --release -p rustfs --bins --features hotpath-alloc
+```
+
+Then run the same ABBA command with that binary. Compare allocation-heavy
+function sections only within the same build mode. Do not compare
+`hotpath-alloc` binaries directly with default release binaries for throughput
+acceptance, because allocation instrumentation intentionally changes what is
+measured.
+
+For CPU hotpath sections emitted by hotpath, build with:
+
+```bash
+cargo build --release -p rustfs --bins --features hotpath-cpu
+```
+
+Use the CPU-enabled report to explain hotspots after the default or plain
+`hotpath` ABBA gate shows a real effect.
+
+## Output layout
+
+The ABBA runner writes:
+
+```text
+<out-dir>/
+  manifest.env
+  abba_schedule.csv
+  candidate_gate.md
+  baseline_drift_gate.md
+  summary.md
+  <workload>/<sync>/<leg>/median_summary.csv
+  <workload>/<sync>/<leg>/baseline_compare.csv
+```
+
+Attach or link at least these files in the issue or PR:
+
+- `summary.md`
+- `candidate_gate.md`
+- `baseline_drift_gate.md`
+- `abba_schedule.csv`
+- every `median_summary.csv` and `baseline_compare.csv` for a failed or
+  borderline workload
+- host telemetry files used to explain CPU, memory, or disk saturation
+
+## Interpretation
+
+Use this decision table:
+
+| Candidate gate | A2 drift gate | Interpretation |
+| --- | --- | --- |
+| PASS | PASS | Candidate is acceptable for the measured matrix. |
+| WARN | PASS | Candidate has a small measurable signal; inspect telemetry and decide if it is expected. |
+| FAIL | PASS | Candidate likely regressed the affected workload; investigate before merge. |
+| FAIL | FAIL on the same workload | Environment drift is high; rerun on a quieter runner or increase duration and rounds. |
+| PASS | FAIL | Candidate did not exceed the budget, but the rig was unstable; avoid using the numbers as proof of improvement. |
+
+When `B1` and `B2` disagree, treat the result as inconclusive even if the gate
+passes. Increase duration, rounds, cooldown, or runner isolation before drawing
+a conclusion.
+
+## AI execution checklist
+
+When delegating the run to an AI agent or an automation runner, provide these
+inputs explicitly:
+
+- repository checkout and candidate branch or commit;
+- baseline commit or binary path;
+- candidate binary path;
+- runner type: local Linux or external cluster;
+- endpoint, access key, secret key source, and region;
+- deploy hook path or exact command for cluster mode;
+- output directory;
+- required duration, rounds, cooldown, concurrency, and fail/warn budgets;
+- where to upload artifacts after the run.
+
+The AI agent should execute this sequence:
+
+1. Confirm `uname -a`, RustFS commits, binary SHA256 sums, `warp --version`,
+   CPU model, memory size, disk layout, and whether the run is local or cluster.
+2. Run `scripts/run_hotpath_warp_abba.sh --dry-run` with the final arguments.
+3. Run the real ABBA command with `--rounds >= 3`.
+4. Preserve the full output directory without editing generated CSV files.
+5. Read `summary.md`, `candidate_gate.md`, and `baseline_drift_gate.md`.
+6. Summarize only measured facts: candidate deltas, baseline drift, CPU or
+   memory saturation, and any failed workloads.
+7. Post the summary and artifact location to the tracking issue or PR.
+
+Do not report a performance win or loss when the baseline drift gate failed on
+the same workload and no rerun was collected.

docs/operations/kms-backend-security.md

@@ -2,7 +2,7 @@
 
 RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.
 
-For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md).
+For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md).
 
 ## Backend comparison
 
@@ -69,6 +69,77 @@ Decryption loads exactly the version recorded in the envelope and fails closed w
 
 Do not rotate any key until **every** RustFS node in the cluster runs a build that understands the `master_key_version` envelope field. Older binaries ignore the field and always decrypt with the current material: harmless while nothing has been rotated, but after a rotation they will fail to decrypt every object wrapped by an earlier key version. Complete the rolling upgrade of the entire cluster first, then rotate.
 
+This is the sharpest instance of a broader class of constraints; the rest are collected in [Mixed-version clusters during a rolling upgrade](#mixed-version-clusters-during-a-rolling-upgrade).
+
+## Mixed-version clusters during a rolling upgrade
+
+During a rolling upgrade the cluster runs two RustFS builds at once. That window matters more for KMS than for most subsystems, because KMS state is shared three ways: **Vault** holds the key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and — for configuration — can disagree for as long as the operator leaves them running.
+
+This section states only what is true of the current implementation. It is written for the KV2 and Transit backends; the Local backend is unsupported for multi-node deployments regardless of version (see the [deployment support matrix](#deployment-support-matrix)).
+
+### Persisted formats are backward compatible in both directions
+
+Nothing in this list requires a coordinated format cutover. The compatibility is deliberate and is covered by decode tests.
+
+- **DEK envelopes.** `DataKeyEnvelope::master_key_version` is optional and omitted when absent, so envelopes written by non-rotating backends stay byte-identical to the historical seven-field JSON shape. An upgraded node reading a pre-versioning envelope resolves `None` to the key's recorded baseline version, or — for a key that was never rotated, and so has no baseline — to the current version, which is exactly the pre-versioning behavior.
+- **KV2 key records.** `baseline_version` is read with a serde default, so records written by older builds deserialize unchanged, and `None` correctly means "never rotated".
+- **Transit metadata records.** Metadata persisted in KV v2 by either build decodes on the other.
+
+The one-way hazard is the rotation constraint above: an older binary reading a *new* envelope silently ignores the version field and decrypts with the current material.
+
+### Guarantees that hold only once every node is upgraded
+
+These are properties of the upgraded code, so a single node left behind removes them for the whole cluster.
+
+- **Check-and-set lifecycle writes.** Upgraded builds write every KV2 lifecycle mutation — create, enable, disable, tag metadata, schedule deletion, cancel deletion — as a versioned read followed by a check-and-set write, retrying on conflict by re-reading and re-validating the state gate (rustfs/rustfs#5518). Transit metadata writes got the same treatment (rustfs/rustfs#5520). Builds older than those write blind. A blind write from an old node can overwrite a check-and-set commit from an upgraded node without any conflict being reported, which is precisely the lost update the change was made to eliminate.
+- **`baseline_version` survives a write-back.** The KV2 key record does not deny unknown fields, so an old build reads a new record without error — and drops `baseline_version` when it writes that record back for any reason. A key that loses its baseline resolves pre-versioning envelopes to the current version again, which after a rotation means the wrong master key material. Any lifecycle operation issued to an old node is enough to trigger this.
+- **Version-record awareness.** Rotation stores each historical version under `{prefix}/{key_id}/versions/{N}` as a create-only record (check-and-set of 0), so two nodes racing the same version number produce exactly one creator; the loser adopts the persisted, never-current material or fails without touching the current pointer. Old builds have no concept of that sub-path: they never read or write it, and their key listing reports the KV2 directory entry (`my-key/`) as though it were a key, because the directory filter only exists in upgraded builds.
+
+### Windows in which nodes can legitimately disagree
+
+Even with every node on the same build, some state is process-local. These windows are bounded by design, except the last one.
+
+| What can diverge | Bound | Mechanism |
+| --- | --- | --- |
+| Transit key lifecycle state used by the `encrypt` and `generate_data_key` gates | ≤ 300 s (`METADATA_CACHE_TTL`) | Each node caches Transit metadata in process, TTL- and capacity-bounded, with targeted invalidation when a data-path call reports the key is gone server-side. A disable or schedule-deletion performed on one node is enforced on the others within one TTL at the latest, sooner if they hit that signal. |
+| `describe_key` output | ≤ 300 s | The manager-level key metadata cache. This is a reporting cache; the KV2 state gates do not read it. |
+| KV2 key lifecycle state | None | The KV2 backend re-reads the key record from Vault for every lifecycle and data-key operation, so a committed disable is effective on every upgraded node immediately. |
+| Active KMS configuration | Until the remaining nodes are restarted | See below. |
+
+Builds older than rustfs/rustfs#5520 held the Transit metadata cache with no TTL and no capacity bound. On such a node the divergence window is not 300 seconds but "until the process restarts": it can keep encrypting under a key that another node disabled, indefinitely.
+
+### Configuration is persisted cluster-wide but applied per node
+
+`POST /rustfs/admin/v3/kms/reconfigure` currently does two things: it switches the KMS service **on the node that handled the request**, and it persists the new configuration to cluster storage at `config/kms_config.json`. It does not notify peers. Every other node keeps running the configuration it started with until it is restarted, at which point it loads the persisted configuration during startup.
+
+The practical consequences today:
+
+- The configuration-split window has no upper bound other than the operator restarting the remaining nodes. Convergence is a restart, not a timeout.
+- During the window both configurations are live. If the reconfiguration changed backends, or changed the Vault mount or key prefix, different nodes write new key material to different places, and a key created through one node is invisible to the others.
+- `kms status` reflects the node that answered the request, so a single successful status response is not evidence that the cluster is consistent. Query every node.
+
+Treat `reconfigure` as the first step of a cluster-wide operation, not as the operation itself.
+
+### Recommended rolling upgrade order
+
+Follow the node-at-a-time procedure in the [multi-node restart runbook](rolling-restart.md); this adds the KMS-specific sequencing around it.
+
+1. **Freeze KMS administrative traffic** for the duration: no key creation, enable, disable, tagging, schedule-deletion, cancel-deletion, rotation, or reconfiguration. Object read and write traffic continues normally.
+2. **Upgrade one node at a time**, waiting for each to report ready before starting the next.
+3. **Verify no node is left behind** before unfreezing. A single old node is enough to reintroduce blind writes and to strip `baseline_version` on its next lifecycle write.
+4. **Resume administrative traffic.**
+5. **Only then perform the first rotation of any key.** Once the whole cluster understands `master_key_version`, rotation is safe; before that it is not.
+6. **If the KMS configuration was changed at any point**, restart the nodes that did not handle the request so they reload the persisted configuration, and confirm each one reports the intended backend.
+
+### Do not do these during a mixed-version window
+
+- **Rotate any key.** This is the hard constraint stated above; a rotation is unrecoverable for objects an old node must read.
+- **Issue any KV2 lifecycle write to an old node.** Its blind write can clobber a concurrent check-and-set commit and will drop `baseline_version` from the record.
+- **Create the same key ID from two nodes.** The create path is create-only on upgraded builds, but an old node's blind write does not honor that: the later writer's material wins and every DEK already wrapped with the earlier material becomes permanently unwrappable.
+- **Assume a disable or schedule-deletion took effect cluster-wide.** Old Transit nodes cache lifecycle state without expiry; confirm per node, or restart the old nodes, before treating a key as no longer in use.
+- **Reconfigure the KMS backend and consider it done.** The change applies to one node and is persisted; the rest need a restart.
+- **Delete or prune version records** under `{prefix}/{key_id}/versions/*` for any reason. This is never safe, mixed-version or not; see [Retention and destruction preconditions](#retention-and-destruction-preconditions).
+
 ## Choosing between Vault KV2 and Vault Transit
 
 Use **Vault Transit** (`VaultTransit`) when key material must be cryptographically isolated from anyone holding storage-level read access: Transit keeps key-encryption keys inside Vault and only ever returns ciphertext, and supports server-side key versioning/rotation.

docs/operations/kms-cryptographic-compliance.md

@@ -0,0 +1,148 @@
+# Cryptographic compliance positioning
+
+This document records where RustFS stands on cryptographic module validation, what may and may not be said about it in external material, and what each possible route to a stronger position would actually cost. It exists so that the question is answered once, from the code, instead of being re-litigated from assumptions about crate names and feature flags.
+
+For where master key material lives per backend and how rotation retention works, see [KMS backend security properties](kms-backend-security.md).
+
+## Status: not FIPS 140-3 validated
+
+**RustFS is not FIPS 140-3 (or 140-2) validated, and no component it links is running as a validated cryptographic module.** There is no CMVP certificate covering RustFS or the libraries it uses in the shipped configuration.
+
+This is a deliberate position, not an oversight. It is also not a statement about algorithm strength: the algorithms in use are standard, well-reviewed AEADs. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS has or currently pursues.
+
+### What the process actually links
+
+The table below is the audited inventory as of this document's writing. "Validated module" asks only whether the code performing the operation is a FIPS-validated cryptographic module; the answer is uniformly no.
+
+| Layer | Where | Implementation | Primitives | Validated module |
+| --- | --- | --- | --- | --- |
+| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange | No — this is the ordinary `aws-lc-rs` build, not the `aws-lc-fips-sys`-backed FIPS variant |
+| Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No |
+| DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No |
+| Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No |
+| Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
+| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS build | No |
+
+Two consequences follow directly from the table and are worth stating explicitly, because both are commonly assumed the other way:
+
+- **AWS-LC being present does not imply FIPS.** `aws-lc-rs` has a FIPS variant; the workspace does not enable it. Every `aws-lc-rs` dependency in the workspace is the default, non-FIPS build.
+- **The data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations. Swapping the TLS provider would not change that; see [route 1](#route-1-adopt-the-aws-lc-rs-fips-variant) for what would.
+
+## Terminology red lines for external material
+
+These rules apply to the README, CHANGELOG, release notes, marketing pages, sales decks, RFP responses, and security questionnaires. Claiming validation RustFS does not have is a false statement of fact with regulatory and contractual consequences, not a marketing overreach.
+
+### Never use
+
+- "FIPS validated", "FIPS certified", "FIPS 140-2/140-3 compliant", "FIPS compliant"
+- "FIPS mode", "runs in FIPS mode", "FIPS-enabled"
+- "NIST certified", "NIST approved", "CMVP certificate", any certificate number
+- "meets FIPS requirements", "satisfies FIPS", or any phrasing a reader would reasonably read as validation
+- The internal Cargo feature name `fips` as a product capability. It is a build-time algorithm selector (see below), and surfacing it as a feature name invites exactly the misreading this section exists to prevent.
+
+### Permitted, with the qualifier attached
+
+- **"FIPS-preferred algorithms"** — permitted only when accompanied, in the same paragraph or table cell, by an explicit non-validation statement. The defined meaning is: *the default algorithm selection is restricted to algorithms on the FIPS 140-3 approved list, implemented by software that has not been validated as a cryptographic module.*
+- Naming specific primitives factually ("AES-256-GCM", "ChaCha20-Poly1305", "PBKDF2-HMAC-SHA256") is always fine. Algorithm names carry no validation claim.
+
+Suggested boilerplate when the topic cannot be avoided:
+
+> RustFS encrypts object data with AES-256-GCM and supports ChaCha20-Poly1305. These are FIPS-approved algorithms, but the implementations are not FIPS 140-3 validated cryptographic modules and RustFS makes no FIPS validation claim.
+
+### Guard
+
+There is currently no FIPS-related wording anywhere in the repository's Markdown; that clean baseline is what a grep anchor test protects. Any future occurrence of the banned strings in shipped documentation should be treated as a defect and either removed or brought under the qualifier rule above.
+
+## The `rustfs-crypto` `fips` feature: what it actually does
+
+`crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way.
+
+| `fips` | Algorithm ID written | KDF | AEAD |
+| --- | --- | --- | --- |
+| enabled (default) | `ID::Pbkdf2AESGCM` (`0x02`) | PBKDF2-HMAC-SHA256, 8192 iterations | AES-256-GCM |
+| disabled | `ID::Argon2idAESGCM` (`0x00`) or `ID::Argon2idChaCHa20Poly1305` (`0x01`), chosen at runtime by CPU AES support | Argon2id (64 MiB, t=1, p=4) | AES-256-GCM or ChaCha20-Poly1305 |
+
+The selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src/encdec/stream_io.rs`; the algorithm identifiers and their KDF parameters live in `crates/crypto/src/encdec/id.rs`.
+
+Three properties matter for anyone reasoning about this feature:
+
+- **It affects writes only.** The decrypt path in `crates/crypto/src/encdec/id.rs` accepts all three identifiers unconditionally, and every ciphertext carries its identifier byte. Toggling the feature therefore never orphans existing data in either direction.
+- **It does not select a different implementation.** Both branches call RustCrypto. There is no validated module on either side of the switch, so the feature cannot move RustFS toward or away from validation.
+- **It is a trade-off, not an upgrade.** The FIPS-preferred branch uses PBKDF2-HMAC-SHA256 at 8192 iterations, a work factor well below current password-hashing guidance, whereas the non-FIPS branch uses memory-hard Argon2id. Against an attacker who has obtained an encrypted config or IAM blob and is attacking the passphrase offline, the default branch is the weaker of the two. Enabling the feature buys approved-algorithm alignment, not more resistance.
+
+### Rename recommendation
+
+The name `fips` states a compliance property the feature does not provide, and `rustfs-crypto` is published, so the name is visible to downstream consumers. Recommended direction:
+
+1. Introduce `fips-preferred-algs` as the real feature name, carrying the current behavior.
+2. Redefine `fips = ["fips-preferred-algs"]` so existing consumers keep building, and mark it deprecated in the crate documentation with a pointer to this document.
+3. Drop the `fips` alias after one release cycle.
+4. While renaming, raise the PBKDF2 iteration count or document the trade-off above at the feature definition, so the choice is explicit rather than inherited.
+
+This is a naming and documentation change only; no ciphertext format changes, because the identifier bytes stay as they are.
+
+## Routes to a stronger position, and what each costs
+
+### Route 1: adopt the `aws-lc-rs` FIPS variant
+
+Switch the whole process to `aws-lc-rs`'s FIPS build (backed by `aws-lc-fips-sys`) so cryptographic operations run inside a validated module boundary.
+
+**Scope.** The TLS provider swap is the small part — one feature flag plus the provider install sites. The substantial work is the data path: every AEAD call in `crates/kms/src/encryption/ciphers.rs`, `crates/kms/src/encryption/dek.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` would have to be re-implemented against `aws-lc-rs` primitives. Anything the validated module does not expose has to be dropped or moved out of the boundary: Argon2id has no FIPS status, so the Local backend's KDF and the non-FIPS branch of `rustfs-crypto` would need a compatibility story (read-only support for existing records, PBKDF2 for new ones), and ChaCha20-Poly1305 would become non-approved for new writes.
+
+**Build and platform cost.** `aws-lc-fips-sys` builds a pinned, validated source release and needs CMake, a C toolchain, and Go at build time; it supports a narrower target set than the ordinary crate. The platform matrix cost of plain AWS-LC is already documented and non-hypothetical: rustfs/backlog#883 records that the static musl release build compiles AWS-LC's `getentropy` entropy backend, which aborts on Linux kernels older than 3.17 (the Synology class of device), and that upstream considers this by design with no plan to fix it. The FIPS variant constrains the buildable matrix strictly harder than that, and pins upgrades to whatever the certified source revision allows.
+
+**What it would and would not buy.** Linking the validated module makes the accurate claim "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated". A product-level claim additionally requires a documented module boundary, approved-mode enforcement, power-on self-tests, key zeroization, and entropy-source documentation, plus the operational procedures to keep them true across releases.
+
+**Verdict.** Heavy, and it re-opens a platform-support question that is already an open problem. Justified only by a concrete customer or regulatory commitment that names FIPS as a requirement.
+
+### Route 2: let an externally validated KMS carry key operations
+
+Keep RustFS as-is and place key management inside someone else's validated boundary: the Vault Transit backend against a Vault deployment whose seal/HSM is validated, or an equivalent managed KMS.
+
+**Scope.** Mostly already built. The Transit backend (`VaultTransit`) never lets key-encryption key material leave Vault; RustFS only ever holds Transit ciphertext. What remains is configuration guidance, a supported-deployment statement, and the operational documentation that says which parts of the system are covered.
+
+**What it buys.** Master key generation, wrapping, unwrapping, and rotation happen inside the external module. That is a real, defensible partial answer to "where do keys live and who validated that": it covers the key operations, which is often the part an auditor actually asks about.
+
+**What it does not buy.** The object data path is untouched. DEKs are used for bulk AEAD by RustCrypto inside the RustFS process, and TLS still runs the non-FIPS AWS-LC build. The honest formulation is "key management operations are performed by an externally validated module; the object data path is not validated".
+
+**Verdict.** The nearest partial step, with no code rewrite and no platform-matrix risk. This is the route to point customers at when the requirement is about key custody rather than about a certificate covering the storage layer.
+
+### Route 3: make no validation claim (current default)
+
+Document the position, hold the terminology line, and revisit only when a requirement with a name attached shows up.
+
+**Cost.** This document plus the grep guard. Nothing else.
+
+**Verdict.** The current decision. FIPS 140-3 validation is explicitly not a roadmap target, and adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI.
+
+## Algorithm disablement and migration policy
+
+Retiring an algorithm from a storage system is not a code change; it is a data migration with a code change at each end. This section fixes the sequence so that no future deprecation removes a decrypt path while data still depends on it.
+
+### Every persisted artifact is self-describing
+
+The precondition for safe migration already holds: nothing relies on a global "current algorithm" setting to be decodable.
+
+- `rustfs-crypto` blobs carry the `ID` byte (`crates/crypto/src/encdec/id.rs`) immediately after the salt.
+- KMS ciphers are selected from the recorded `EncryptionAlgorithm` (`crates/kms/src/types.rs`).
+- DEK envelopes record which master key version wrapped them in `DataKeyEnvelope::master_key_version` (`crates/kms/src/encryption/dek.rs`).
+
+So for any stored object it is decidable, from the object alone, which algorithm and which key version it needs.
+
+### Deprecation classes
+
+Retirement moves an algorithm through these states, never skipping one:
+
+1. **Write-disabled, read-supported.** New writes select a replacement; existing data decrypts unchanged. This is the only step that is cheap and reversible.
+2. **Read-deprecated.** Reads still work but are counted and warned on, so the remaining population is measurable.
+3. **Read-removed.** The decrypt path is deleted. Permitted only once the remaining population is provably zero.
+
+### Sequencing rules
+
+- Never advance to read-removed on the strength of an argument that data "should have been" migrated. Removal requires evidence that nothing references the algorithm, not an elapsed-time policy.
+- A change to default algorithm selection is a compatibility event: it changes what new nodes write, which matters in a mixed-version cluster. Record it in the release notes and in the relevant crate's feature documentation, and check it against the [mixed-version constraints](kms-backend-security.md#mixed-version-clusters-during-a-rolling-upgrade).
+- Roll out write-disablement before the corresponding read change, and let the cluster fully converge in between. A build that cannot read what a peer is still writing is the failure mode to avoid.
+
+### Known gap
+
+Step 3 is currently unreachable for object data. There is no object rewrap or re-encryption capability, so there is no supported way to migrate already-written objects off an algorithm or off a master key version — the same gap that forces the rotation retention rule in [KMS backend security properties](kms-backend-security.md#retention-and-destruction-preconditions). Until a rewrap capability exists, treat every algorithm that has ever been written as permanently read-required, and confine deprecation to step 1.

docs/operations/kms-observability-runbook.md

@@ -0,0 +1,130 @@
+# KMS observability runbook
+
+This runbook covers the KMS backend operation metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md).
+
+## Metric reference
+
+All four metrics are emitted at the single operation-policy choke point (`crates/kms/src/policy.rs`) that every instrumented KMS backend call flows through. Label values are exclusively static enum strings — key identifiers, key material, ciphertext, paths, and tokens never appear in metric labels, and any change that would add such a label is a regression.
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `rustfs_kms_backend_operations_total` | counter | `operation`, `op_class`, `outcome` | Operations executed under the operation policy, counted once per terminal outcome |
+| `rustfs_kms_backend_attempt_failures_total` | counter | `operation`, `error_class` | Individual failed attempts, including attempts the retry policy later absorbed |
+| `rustfs_kms_backend_operation_duration_seconds` | histogram | `operation`, `outcome` | Wall-clock duration of a whole operation, including retries and backoff sleeps |
+| `rustfs_kms_backend_operation_attempts` | histogram | `operation`, `outcome` | Number of attempts one operation used before completing |
+
+Label values:
+
+- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `cancelled` (shutdown or caller cancellation).
+- `op_class`: `read_idempotent` (safe to retry), `mutating_non_idempotent` (never replayed — a retryable failure terminates after a single attempt because the server may have processed the request), `auth` (login and token renewal).
+- `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version).
+- `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`.
+
+Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. These metrics do not carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`).
+
+Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of KMS series on a cluster using those backends is expected, not an outage.
+
+## Dashboard
+
+Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` (Prometheus data source) and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed there.
+
+The dashboard's "Planned Panels (TODO)" text panel lists metric families designed under rustfs/backlog#1584 but not yet landed (key-cache effectiveness, lifecycle gauges, token TTL, synthetic probe). Do not add panels or alerts for those names until the emitting code is merged; keep that panel and the [Coverage gaps](#coverage-gaps-and-planned-metrics) section below in sync as they land.
+
+## Alert rules
+
+The rules live in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. The docker-compose Prometheus loads `/etc/prometheus/rules/*.yml`, so the `.yml` extension is load-bearing. Validate edits with `promtool check rules rustfs-kms-alerts.yml`.
+
+Every threshold in that file is a conservative default chosen without a production baseline; see [Threshold calibration](#threshold-calibration) before treating a firing alert as an SLO breach or a quiet one as health.
+
+## Alert response procedures
+
+### KmsBackendFatalErrors
+
+Meaning: attempts are failing with `error_class="fatal"` — failures the policy never retries. Each one is a KMS backend call that failed permanently (authentication, permissions, malformed request, or a missing key/version), so callers are seeing errors right now. This is the highest-signal KMS alert: fatal failures do not appear as background noise in a healthy system.
+
+Investigation:
+
+1. Break the rate down by operation: `sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m]))`.
+2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs.
+3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the minimal policy in [KMS backend security properties](kms-backend-security.md) (a policy that drifted or was re-scoped produces 403s that classify as fatal), and check the Vault audit log for the corresponding denied requests.
+4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the rotation retention preconditions in [KMS backend security properties](kms-backend-security.md) and verify nobody destroyed version records under the key subtree.
+5. Confirm blast radius with the outcome view: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="fatal"}[5m]))`.
+
+Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operation Rate by Outcome" dashboard panels; Vault server audit and server logs; S3-level 5xx on encrypted buckets.
+
+### KmsBackendHighErrorRate
+
+Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, or `deadline_exceeded`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page.
+
+Investigation:
+
+1. Break the failures down by outcome: `sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))`.
+2. If `fatal` dominates, follow [KmsBackendFatalErrors](#kmsbackendfatalerrors).
+3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted) — the backend is unavailable or too slow for longer than the retry policy can bridge.
+4. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured.
+
+Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts in this runbook.
+
+### KmsBackendP99LatencyHigh
+
+Meaning: the p99 wall-clock duration of KMS operations is sustained above 2s. The histogram includes retries and backoff sleeps, so a high p99 with a healthy p50 usually means a slow retry tail (a subset of calls failing and being retried), not a uniform slowdown.
+
+Investigation:
+
+1. Compare p50 and p99 on the "Operation Duration p50 / p99" panel. Flat p50 with elevated p99 points at retries; both elevated points at the backend or the network path being uniformly slow.
+2. Split by operation with `histogram_quantile(0.99, sum by (le, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])))` to see whether one backend call or all of them regressed.
+3. Check the attempts histogram: an average meaningfully above 1 confirms the latency is retry-driven; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the failure classes.
+4. If latency is not retry-driven, check the network path to Vault (TLS handshakes, DNS, proxies) and Vault's own telemetry (storage backend latency, load).
+5. Remember that this latency sits inside S3 request latency for encrypted objects: sustained p99 near the operation deadline will start converting into `deadline_exceeded` outcomes.
+
+Related signals: the "Operation Duration p99 by Operation" and "Operation Attempts Distribution" panels; `KMS backend attempt failed with a retryable error; backing off before retry` warnings (fields: `operation`, `attempt`, `error_class`, `backoff`).
+
+### KmsBackendAttemptFailureSpike
+
+Meaning: individual attempts are failing at a sustained rate across all error classes. The retry policy may still be absorbing these — operations can keep succeeding while this alert fires — but the system is burning retry budget and running degraded, and a small further degradation will surface to callers.
+
+Investigation:
+
+1. Break the rate down by class: `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total[5m]))`.
+2. `retryable_conn`: network-level failures — check connectivity, TLS, DNS, and whether Vault is down or restarting.
+3. `retryable_status`: the backend answered with a retryable error — check Vault health and seal status (a sealed Vault returns 503, which lands here), and Vault-side rate limiting.
+4. `attempt_timeout`: attempts are being cut off by the per-attempt timeout — either the backend is slow (correlate with [KmsBackendP99LatencyHigh](#kmsbackendp99latencyhigh)) or the configured attempt timeout is too tight for the deployment's network path.
+5. `fatal`: follow [KmsBackendFatalErrors](#kmsbackendfatalerrors).
+6. Grep RustFS logs for `KMS backend attempt failed with a retryable error; backing off before retry` — the structured fields (`operation`, `attempt`, `error_class`, `backoff`) identify which call sites are cycling.
+
+Related signals: the "Attempt Failure Rate by Error Class" panel; the attempts histogram average rising above 1.
+
+### KmsBackendRetryBudgetExhausted
+
+Meaning: operations are terminating as `budget_exhausted` or `deadline_exceeded` — every individual failure was retryable, but the backend stayed unhealthy for longer than the retry policy could bridge, so callers received hard failures.
+
+Investigation:
+
+1. Identify the failing operations: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m]))`.
+2. Establish how long the underlying failure has persisted from the attempt-failure rate history; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the class-specific diagnosis.
+3. Note the by-design case: `mutating_non_idempotent` operations (e.g. `vault_kv2_cas_write_key`, `vault_transit_create_key`) are never replayed, so a single retryable failure terminates them as `budget_exhausted` after one attempt. A spike confined to mutating operations means write-path failures, not an exhausted retry loop.
+4. `deadline_exceeded` clustering with duration p99 near the operation deadline means the budget is being spent on slow attempts rather than fast failures — treat as a latency problem first.
+5. Confirm client impact and, if the backend outage is confirmed external (Vault down), coordinate recovery there; RustFS will resume without intervention once the backend recovers.
+
+Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings in RustFS logs; Vault availability monitoring.
+
+## Threshold calibration
+
+Every numeric threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).
+
+## Coverage gaps and planned metrics
+
+The following families are designed under rustfs/backlog#1584 but land in separate changes; they are intentionally absent from the dashboard and alert rules, and referencing their names before the emitting code merges would produce permanently-empty panels and never-firing alerts:
+
+- Key-cache effectiveness: hit/miss counters, entry gauge, eviction counter (replacing the former hardcoded-zero miss statistic).
+- Key lifecycle: pending-deletion/tombstone gauges from the deletion-worker sweep and an aggregate rotation-age gauge.
+- Vault credentials: token TTL remaining and fail-closed state gauges (today only the log warnings quoted above exist).
+- Synthetic backend probe: probe outcome and failure-class metrics feeding the readiness cache.
+
+When one of these lands, add its panels, extend the alert rules, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel, and update this section.
+
+## Related documents
+
+- [KMS backend security properties](kms-backend-security.md) — backend trust boundaries, minimal Vault policies, rotation retention preconditions.
+- [Vault KMS authentication runbook](vault-kms-authentication.md) — credential sources, refresh behavior, and the fail-closed window.
+- `deploy/observability/README.md` — dashboard import notes for all RustFS dashboards.

docs/testing/e2e-suite-inventory.md

@@ -9,7 +9,7 @@
 > (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`.
 > 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147
 > repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke
-> lane and 27 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
+> lane and 28 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
 > nightly lane (`.github/workflows/e2e-replication-nightly.yml`).
 > Note: counts exclude `#[ignore]`d tests (nextest lists them separately).
 > The SSE-S3 replication contract is ignored under backlog#1291 until its
@@ -17,11 +17,11 @@
 
 | module | tests | PR smoke |
 |---|---|---|
-| admin_auth_test | 3 | ✅ |
+| admin_auth_test | 4 | ✅ |
 | admin_iam_crud_test | 2 | ✅ |
 | admin_pools_test | 1 | ✅ |
 | admin_timeout_regression_test | 1 |  |
-| anonymous_access_test | 3 | ✅ |
+| anonymous_access_test | 4 | ✅ |
 | api_rate_limit_test | 3 |  |
 | archive_download_integrity_test | 13 |  |
 | bucket_logging_test | 3 |  |
@@ -29,7 +29,7 @@
 | checksum_upload_test | 7 |  |
 | cluster_concurrency_test | 2 |  |
 | cluster_multidrive_pool_test | 2 |  |
-| common | 10 |  |
+| common | 12 |  |
 | compression_test | 1 |  |
 | connection_cap_test | 2 |  |
 | console_smoke_test | 1 | ✅ |
@@ -51,7 +51,9 @@
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
 | heal_erasure_disk_rebuild_test | 3 |  |
-| kms | 40 |  |
+| inline_fast_path_cluster_test | 14 |  |
+| internode_rpc_signature_e2e_test | 5 |  |
+| kms | 41 |  |
 | leading_slash_key_test | 2 | ✅ |
 | list_buckets_double_slash_test | 3 | ✅ |
 | list_object_versions_metadata_extension_test | 1 |  |
@@ -72,7 +74,7 @@
 | protocols | 16 |  |
 | quota_test | 14 |  |
 | reliability_disk_fault_test | 3 |  |
-| reliant | 10 | 5 ✅ |
+| reliant | 24 | 18 ✅ |
 | replication_extension_test | 48 | 20 ✅ +28 🌙 |
 | security_boundary_test | 4 |  |
 | ssec_copy_test | 2 | ✅ |
@@ -81,10 +83,11 @@
 | special_chars_test | 14 | ✅ |
 | stale_multipart_cleanup_cluster_test | 1 |  |
 | storage_class_capability_test | 4 | ✅ |
+| sts_query_compat_test | 3 | ✅ |
 | tls_gen | 3 |  |
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
 `notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
 
-**Total listed: 486 tests across 67 modules · PR smoke subset: 129 tests / 32 modules** (30 full modules + 5 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-07-26.
+**Total listed: 527 tests across 70 modules · PR smoke subset: 147 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 28 tests** · generated 2026-07-30.

rustfs/src/admin/auth.rs

@@ -46,6 +46,20 @@ impl<'a> AdminResourceScope<'a> {
     pub fn bucket_object(bucket: &'a str, object: &'a str) -> Self {
         Self { bucket, object }
     }
+
+    /// Scope an admin request to a single KMS key.
+    ///
+    /// The policy crate carries the requested key identifier in the object slot
+    /// with an empty bucket (see `Statement::kms_key_scope_matches`); `Args` has
+    /// no dedicated field for it. An empty `key_id` means the caller could not
+    /// name a target key, which keeps the pre-resource-scoping behaviour where a
+    /// KMS statement matches every key.
+    pub fn kms_key(key_id: &'a str) -> Self {
+        Self {
+            bucket: "",
+            object: key_id,
+        }
+    }
 }
 
 pub async fn validate_admin_request(
@@ -174,6 +188,34 @@ pub async fn validate_admin_request_with_bucket_object(
     evaluate_admin_actions(iam_store, &ctx, &actions, resource.bucket, resource.object).await
 }
 
+/// Admin gate for KMS endpoints that act on one key.
+///
+/// `key_id` is the identifier as requested (before any alias resolution), so a
+/// policy scoped to `arn:aws:kms:::key/<id>` only authorizes that key. Endpoints
+/// without a target key pass `""` and stay unscoped, which is also what a
+/// malformed request resolves to: the request is rejected on its own parse error
+/// right after the gate, so no key is ever touched under an unscoped decision.
+pub async fn validate_admin_request_with_kms_key(
+    headers: &HeaderMap,
+    cred: &Credentials,
+    is_owner: bool,
+    deny_only: bool,
+    actions: Vec<Action>,
+    remote_addr: Option<std::net::SocketAddr>,
+    key_id: &str,
+) -> S3Result<()> {
+    validate_admin_request_with_bucket_object(
+        headers,
+        cred,
+        is_owner,
+        deny_only,
+        actions,
+        remote_addr,
+        AdminResourceScope::kms_key(key_id),
+    )
+    .await
+}
+
 /// Unified authentication request handler for both UI and CLI
 ///
 /// This function provides a single entry point for authentication,
@@ -248,12 +290,13 @@ mod tests {
     use rustfs_iam::manager::IamCache;
     use rustfs_iam::store::{GroupInfo, MappedPolicy, UserType};
     use rustfs_policy::auth::UserIdentity;
-    use rustfs_policy::policy::PolicyDoc;
-    use rustfs_policy::policy::action::AdminAction;
+    use rustfs_policy::policy::action::{AdminAction, KmsAction};
+    use rustfs_policy::policy::{Policy, PolicyDoc};
     use serde::Serialize;
     use serde::de::DeserializeOwned;
     use std::collections::HashMap;
     use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
+    use time::OffsetDateTime;
 
     /// A `Store` whose methods are never invoked by these tests: the gate only
     /// reaches the persistence layer for non-owner principals, and the
@@ -383,14 +426,14 @@ mod tests {
         }
     }
 
-    /// Build an `IamSys` backed by an empty in-memory cache. The cache is
-    /// constructed directly (all fields are public, mirroring the iam crate's
-    /// own `build_test_iam_cache`) so no cluster-backed `ObjectStore` or disk
-    /// load is required. Readiness state is irrelevant: the gate calls
-    /// `is_allowed` directly, which does not gate on `is_ready`.
-    fn iam_sys_with_empty_store() -> Arc<IamSys<EmptyStore>> {
+    /// An `IamCache` over an empty in-memory snapshot. It is constructed
+    /// directly (all fields are public, mirroring the iam crate's own
+    /// `build_test_iam_cache`) so no cluster-backed `ObjectStore` or disk load is
+    /// required. Readiness state is irrelevant: the gate calls `is_allowed`
+    /// directly, which does not gate on `is_ready`.
+    fn iam_cache_with_empty_store() -> IamCache<EmptyStore> {
         let (send_chan, _rx) = tokio::sync::mpsc::channel::<i64>(1);
-        let cache = IamCache {
+        IamCache {
             cache: Cache::default(),
             api: EmptyStore,
             state: Arc::new(AtomicU8::new(0)),
@@ -401,10 +444,47 @@ mod tests {
             sync_failures: AtomicU64::new(0),
             sync_successes: AtomicU64::new(0),
             last_sync_duration_millis: AtomicU64::new(0),
-        };
-        Arc::new(IamSys::new(Arc::new(cache)))
+        }
     }
 
+    fn iam_sys_with_empty_store() -> Arc<IamSys<EmptyStore>> {
+        Arc::new(IamSys::new(Arc::new(iam_cache_with_empty_store())))
+    }
+
+    /// Same in-memory `IamSys`, with one regular (non-owner, non-temp,
+    /// non-service) account carrying `document` as its only attached policy. All
+    /// three cache entries the evaluation path consults are pre-seeded, so no
+    /// `EmptyStore` method is ever reached.
+    fn iam_sys_with_policy(account: &str, document: &str) -> Arc<IamSys<EmptyStore>> {
+        let store = iam_cache_with_empty_store();
+        let now = OffsetDateTime::now_utc();
+        let cache = &store.cache;
+
+        cache.add_or_update_user(
+            account,
+            &UserIdentity::new(Credentials {
+                access_key: account.to_string(),
+                secret_key: "kms-operator-secret".to_string(),
+                status: "on".to_string(),
+                ..Default::default()
+            }),
+            now,
+        );
+        cache.add_or_update_policy_doc(
+            KMS_TEST_POLICY_NAME,
+            &PolicyDoc {
+                policy: Policy::parse_config(document.as_bytes()).expect("test policy should parse"),
+                ..Default::default()
+            },
+            now,
+        );
+        cache.add_or_update_user_policy(account, &MappedPolicy::new(KMS_TEST_POLICY_NAME), now);
+
+        Arc::new(IamSys::new(Arc::new(store)))
+    }
+
+    const KMS_TEST_POLICY_NAME: &str = "kms-key-a-only";
+
     fn ctx_for<'a>(headers: &'a HeaderMap, cred: &'a Credentials, is_owner: bool) -> AuthContext<'a> {
         AuthContext {
             headers,
@@ -471,6 +551,87 @@ mod tests {
         assert_access_denied(res);
     }
 
+    /// KMS scoping rides the object slot with an empty bucket, matching the
+    /// contract the policy crate evaluates KMS statements against.
+    #[test]
+    fn kms_scope_carries_the_key_id_in_the_object_slot() {
+        let scope = AdminResourceScope::kms_key("key-a");
+        assert_eq!(scope.bucket, "");
+        assert_eq!(scope.object, "key-a");
+
+        let unscoped = AdminResourceScope::kms_key("");
+        assert_eq!(unscoped.object, "", "an absent key id must stay unscoped");
+    }
+
+    const KMS_OPERATOR: &str = "kms-operator";
+
+    fn kms_disable_key_actions() -> Vec<Action> {
+        vec![Action::KmsAction(KmsAction::DisableKeyAction)]
+    }
+
+    async fn gate_for_key(iam: Arc<IamSys<EmptyStore>>, key_id: &str, is_owner: bool) -> S3Result<()> {
+        let headers = HeaderMap::new();
+        let cred = Credentials {
+            access_key: KMS_OPERATOR.to_string(),
+            secret_key: "kms-operator-secret".to_string(),
+            status: "on".to_string(),
+            ..Default::default()
+        };
+        let ctx = ctx_for(&headers, &cred, is_owner);
+        let scope = AdminResourceScope::kms_key(key_id);
+
+        evaluate_admin_actions(iam, &ctx, &kms_disable_key_actions(), scope.bucket, scope.object).await
+    }
+
+    /// End-to-end through the gate: a `kms:DisableKey` grant limited to `key/A`
+    /// authorizes key A and denies key B with `AccessDenied`. This is the seam the
+    /// per-handler tests cannot reach — it pins that the scope actually travels
+    /// into the policy verdict rather than being dropped on the way
+    /// (rustfs/backlog#1582).
+    #[tokio::test]
+    async fn kms_scoped_gate_denies_a_key_outside_the_policy_scope() {
+        let document = r#"{"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["kms:DisableKey"],"Resource":["arn:aws:kms:::key/key-a"]}
+        ]}"#;
+
+        let allowed = gate_for_key(iam_sys_with_policy(KMS_OPERATOR, document), "key-a", false).await;
+        assert!(allowed.is_ok(), "the key the policy names must pass the gate");
+
+        assert_access_denied(gate_for_key(iam_sys_with_policy(KMS_OPERATOR, document), "key-b", false).await);
+    }
+
+    /// Compatibility pins for the same gate: a resource-less KMS statement keeps
+    /// authorizing every key, and the owner short-circuit is unaffected by scoping.
+    #[tokio::test]
+    async fn kms_scoped_gate_keeps_unscoped_policies_and_owners_unrestricted() {
+        let unscoped = r#"{"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["kms:DisableKey"]}
+        ]}"#;
+        let scoped = r#"{"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["kms:DisableKey"],"Resource":["arn:aws:kms:::key/key-a"]}
+        ]}"#;
+
+        for key in ["key-a", "key-b"] {
+            let res = gate_for_key(iam_sys_with_policy(KMS_OPERATOR, unscoped), key, false).await;
+            assert!(res.is_ok(), "a resource-less KMS statement must keep authorizing {key}");
+
+            let res = gate_for_key(iam_sys_with_policy(KMS_OPERATOR, scoped), key, true).await;
+            assert!(res.is_ok(), "the owner short-circuit must still authorize {key}");
+        }
+    }
+
+    /// An endpoint with no target key passes an empty scope, which must behave
+    /// exactly like the unscoped gate it replaced.
+    #[tokio::test]
+    async fn kms_gate_without_a_target_key_matches_every_key() {
+        let scoped = r#"{"Version":"2012-10-17","Statement":[
+            {"Effect":"Allow","Action":["kms:DisableKey"],"Resource":["arn:aws:kms:::key/key-a"]}
+        ]}"#;
+
+        let res = gate_for_key(iam_sys_with_policy(KMS_OPERATOR, scoped), "", false).await;
+        assert!(res.is_ok(), "an absent key id must keep the pre-resource-scoping verdict");
+    }
+
     /// The multi-action loop authorizes as soon as one candidate action passes
     /// (owner short-circuits every action), and denies when none pass.
     #[tokio::test]