ci: correct the companion-workflow comments

The ci-docs-only header claimed the ruleset already requires "Quick Checks";
it does not, and this PR deliberately leaves it that way. Say what is true
today and why the job is mirrored ahead of the ruleset change.

Also move the byte-identical requirement onto ci.yml's quick-checks job, which
is the more likely edit site, instead of pointing at a comment that was not
there.

.config/make/tests.mak

@@ -28,8 +28,6 @@ script-tests: ## Run shell script tests
 	./scripts/test_entrypoint_credentials.sh
 	./scripts/test_internode_grpc_ab_bench.sh
 	./scripts/test_object_batch_bench_enhanced.sh
-	./scripts/test_hotpath_warp_ab_gate.sh
-	./scripts/test_hotpath_warp_abba.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh

.config/nextest.toml

@@ -300,6 +300,9 @@ path = "junit.xml"
 # negative-path siblings of each family stay in as regression guards.
 #   * rustfs#4843 — over-limit archive entry paths hard-reject the whole
 #     archive even under ignore-errors semantics.
+#   * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
+#     under parallel load (multi-node in-process clusters; natural home is
+#     ci-7's nightly cluster lane).
 [profile.e2e-full]
 default-filter = """
     package(e2e_test)
@@ -308,6 +311,7 @@ default-filter = """
     & !test(/^replication_extension_test::/)
     & !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
     & !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
+    & !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
 """
 fail-fast = false
 

.github/actions/setup/action.yml

@@ -25,13 +25,9 @@ inputs:
     required: false
     default: "rustfs-deps"
   cache-save-if:
-    description: >-
-      Whether to save the cache. The fail-safe default is 'false': a caller that
-      wants to populate a cache must opt in explicitly, so a forgotten input
-      costs a cold cache (minutes) rather than silently consuming the
-      repository-wide 10GB Actions cache quota and evicting other lanes.
+    description: "Condition for saving cache"
     required: false
-    default: "false"
+    default: "true"
   install-cross-tools:
     description: "Install cross-compilation tools"
     required: false
@@ -40,43 +36,28 @@ inputs:
     description: "Target architecture to add"
     required: false
     default: ""
-  install-build-packaging-tools:
-    description: >-
-      Install musl-tools/zip/unzip, needed for musl linking and release
-      packaging. Off for CI test lanes, which use none of them.
+  github-token:
+    description: "GitHub token for API access"
     required: false
-    default: "true"
-  install-test-tools:
-    description: >-
-      Install cargo-nextest and the rustfmt/clippy components. Off for release
-      and audit lanes, which run no tests and no lints.
-    required: false
-    default: "true"
+    default: ""
 
 runs:
   using: "composite"
   steps:
-    # protobuf-compiler is deliberately absent: the setup-protoc step below
-    # installs 34.1 into the tool cache and prepends it to PATH, so the apt
-    # build (older, and never version-matched) was shadowed on every run and
-    # simply never used.
     - name: Install system dependencies (Ubuntu)
       if: runner.os == 'Linux'
       shell: bash
       run: |
         sudo apt-get update
         sudo apt-get install -y \
+          musl-tools \
           build-essential \
           pkg-config \
           libssl-dev \
-          ripgrep
-
-    # musl-gcc is needed by the native musl release leg, and zip/unzip by the
-    # release packaging steps. No CI test lane touches any of them.
-    - name: Install packaging and cross-linking dependencies (Ubuntu)
-      if: runner.os == 'Linux' && inputs.install-build-packaging-tools == 'true'
-      shell: bash
-      run: sudo apt-get install -y musl-tools zip unzip
+          ripgrep \
+          unzip \
+          zip \
+          protobuf-compiler
 
     - name: Install protoc
       uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
@@ -94,7 +75,7 @@ runs:
       with:
         toolchain: ${{ inputs.rust-version }}
         targets: ${{ inputs.target }}
-        components: ${{ inputs.install-test-tools == 'true' && 'rustfmt, clippy' || '' }}
+        components: rustfmt, clippy
 
     - name: Install Zig
       if: inputs.install-cross-tools == 'true'
@@ -105,24 +86,12 @@ runs:
       uses: taiki-e/install-action@a21ae4029b089b9ddc45704028756f51ab8abe48 # cargo-zigbuild
 
     - name: Install cargo-nextest
-      if: inputs.install-test-tools == 'true'
       uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
 
     - name: Setup Rust cache
       uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
       with:
-        # false is rust-cache's own default. With true, cleanup.ts returns
-        # *before* pruning ~/.cargo/registry/src, and config.ts archives the
-        # whole registry — so every cache carried the unpacked source tree of
-        # every dependency, not just "a few extra crates".
-        #
-        # No coverage is lost: getPackages runs `cargo metadata --all-features`,
-        # a strict superset of any single lane's feature closure, and -sys crates
-        # are explicitly exempted from pruning (their src timestamps would
-        # otherwise trigger rebuilds). Anything pruned is re-unpacked from the
-        # .crate files still in registry/cache, whose mtimes crates.io
-        # normalises, so cargo fingerprints stay valid.
-        cache-all-crates: false
+        cache-all-crates: true
         cache-on-failure: true
         shared-key: ${{ inputs.cache-shared-key }}
         save-if: ${{ inputs.cache-save-if }}

.github/workflows/architecture-migration-rules.yml

@@ -37,7 +37,6 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -46,11 +45,8 @@ jobs:
     name: Architecture Migration Rules
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Install ripgrep
         run: |

.github/workflows/audit.yml

@@ -39,12 +39,7 @@ on:
       - 'scripts/security/check_preview_release_workflow.sh'
       - 'scripts/security/check_workflow_pins.sh'
   schedule:
-    # Daily, not weekly. This schedule exists to catch RustSec advisories
-    # published against an unchanged dependency tree; at weekly cadence a new
-    # advisory could sit unnoticed for seven days. The check list is unchanged —
-    # splitting it into a light daily advisories-only run and a weekly full run
-    # would create runs where sources/bans/licenses go unverified.
-    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
+    - cron: '0 3 * * 0' # Weekly on Sunday 03:00 UTC (staggered after the midnight ci/build crons)
   workflow_dispatch:
 
 permissions:
@@ -64,7 +59,6 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -80,32 +74,11 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
-      # cargo-deny compiles nothing, so the full setup composite (apt packages,
-      # protoc, flatc, nextest, rustfmt/clippy) was pure overhead here. It does
-      # still need a real cargo: `cargo deny check` runs `cargo metadata`, and
-      # Cargo.toml pins datafusion and s3s as git dependencies, which must be
-      # materialised into ~/.cargo/git — a cold clone is hundreds of MB, so the
-      # cache stays.
-      - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-
-      # Was relying on the composite's default, which used to be "true": every
-      # PR touching Cargo.toml/Cargo.lock saved a second, PR-scoped copy of this
-      # cache and pushed the main-scoped lanes out of the 10GB quota. The
-      # default is now "false", but state it explicitly — see
-      # scripts/security/check_cache_save_if.sh.
-      - name: Setup Rust cache
-        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
         with:
-          # Same reasoning as the setup composite: true archives every
-          # dependency's unpacked source tree.
-          cache-all-crates: false
-          cache-on-failure: true
-          shared-key: rustfs-cargo-deny
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: rustfs-cargo-deny
 
       - name: Install cargo-deny
         uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
@@ -123,28 +96,16 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Report unpinned GitHub Actions
         run: ./scripts/security/check_workflow_pins.sh --enforce
 
-      - name: Check setup cache-save-if is explicit
-        run: ./scripts/security/check_cache_save_if.sh
-
-      - name: Check every job declares a timeout
-        run: ./scripts/security/check_job_timeouts.sh
-
-      - name: Check checkouts clear their credentials
-        run: ./scripts/security/check_persist_credentials.sh
-
       - name: Check preview release workflow policy
         run: ./scripts/security/check_preview_release_workflow.sh
 
   dependency-review:
     name: Dependency Review
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     if: github.event_name == 'pull_request' && github.event.action != 'closed'
     permissions:
       contents: read
@@ -152,8 +113,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Dependency Review
         uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5
@@ -166,28 +125,3 @@ jobs:
           # conscious re-review of the license/provenance claim (backlog#1181).
           allow-dependencies-licenses: pkg:cargo/rustfs-uring@0.1.0
           comment-summary-in-pr: always
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    # dependency-review is deliberately excluded: it only runs on pull_request,
-    # so it can never contribute a failure to a scheduled run.
-    needs: [cargo-deny, workflow-pin-report]
-    # A scheduled cargo-deny failure usually means the dependency tree just
-    # matched a newly published advisory — the single most important signal this
-    # workflow produces, and until now it was only visible to whoever happened to
-    # open the Actions tab. Same ci-8 mechanism coverage.yml and
-    # e2e-replication-nightly.yml already use.
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build.yml

@@ -50,18 +50,12 @@ on:
       - "**/*.svg"
       - ".gitignore"
       - ".dockerignore"
-      - "flake.lock"
   schedule:
     - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
   workflow_dispatch:
     inputs:
       build_docker:
-        # Advisory only. docker.yml triggers on workflow_run and its job-level
-        # condition requires the triggering event to be a tag push, so a manual
-        # dispatch of this workflow never produces images regardless of this
-        # value. Kept because the summary step reports it; wiring it up would
-        # mean teaching docker.yml's version parser a second event shape.
-        description: "Build and push Docker images after binary build (ignored: dispatch runs never reach docker.yml)"
+        description: "Build and push Docker images after binary build"
         required: false
         default: true
         type: boolean
@@ -89,7 +83,6 @@ jobs:
   build-check:
     name: Build Strategy Check
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     outputs:
       should_build: ${{ steps.check.outputs.should_build }}
       build_type: ${{ steps.check.outputs.build_type }}
@@ -99,8 +92,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Determine build strategy
         id: check
@@ -173,7 +164,6 @@ jobs:
     name: Prepare Platform Matrix
     needs: build-check
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     outputs:
       matrix: ${{ steps.select.outputs.matrix }}
       selected: ${{ steps.select.outputs.selected }}
@@ -181,14 +171,10 @@ jobs:
       - name: Select target platforms
         id: select
         shell: bash
-        env:
-          # via env, not interpolation: a dispatch input is free-form text and
-          # would otherwise be pasted into the script for bash to evaluate.
-          RAW_PLATFORMS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}
         run: |
           set -euo pipefail
 
-          selected="$RAW_PLATFORMS"
+          selected="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}"
           selected="$(echo "${selected}" | tr -d '[:space:]')"
           if [[ -z "${selected}" ]]; then
             selected="all"
@@ -259,7 +245,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
-          persist-credentials: false
           fetch-depth: 0
 
       - name: Setup Rust environment
@@ -268,17 +253,9 @@ jobs:
           rust-version: stable
           target: ${{ matrix.target }}
           cache-shared-key: build-${{ matrix.target }}
-          # main only. A cache saved on refs/tags/X is scoped to that tag: no
-          # other tag, no main run and no PR can restore it, so every release
-          # cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
-          # six legs each) that nobody could read, evicting the hot lanes from
-          # the repo-wide 10GB quota. Tag builds still restore the main-scoped
-          # cache, since default-branch caches are readable from every ref.
-          # The one real cost: re-running a failed leg of the same tag no longer
-          # finds that tag's own warm cache and falls back to main's.
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }}
           install-cross-tools: ${{ matrix.cross }}
-          install-test-tools: 'false'
 
       - name: Download static console assets
         shell: bash
@@ -725,14 +702,9 @@ jobs:
     needs: [ build-check, build-rustfs ]
     if: always() && needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Build completion summary
         shell: bash
-        env:
-          # dispatch input via env: free-form text must not be pasted into the
-          # script for bash to evaluate.
-          INPUT_BUILD_DOCKER: ${{ github.event.inputs.build_docker }}
         run: |
           BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
           VERSION="${{ needs.build-check.outputs.version }}"
@@ -774,7 +746,7 @@ jobs:
           echo "🐳 Docker Images:"
           if [[ "$BUILD_TYPE" == "preview" ]]; then
             echo "⏭️  Preview tags do not publish Docker images"
-          elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
+          elif [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then
             echo "⏭️  Docker image build was skipped (binary only build)"
           elif [[ "$BUILD_STATUS" == "success" ]]; then
             echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
@@ -788,7 +760,6 @@ jobs:
     needs: [ build-check, build-rustfs ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     permissions:
       contents: write
     outputs:
@@ -798,7 +769,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
-          persist-credentials: false
           fetch-depth: 0
 
       - name: Create GitHub Release
@@ -849,15 +819,12 @@ jobs:
     needs: [ build-check, build-rustfs, create-release ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     permissions:
       contents: write
       actions: read
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Download all build artifacts
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -943,7 +910,6 @@ jobs:
     needs: [ build-check, publish-release ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     steps:
       - name: Update latest.json
         env:
@@ -1003,7 +969,6 @@ jobs:
     needs: [ build-check, create-release, upload-release-assets ]
     if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     permissions:
       contents: write
     steps:

.github/workflows/cache-warm.yml

@@ -1,265 +0,0 @@
-# Copyright 2026 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Sole writer of the Rust dependency caches that ci.yml restores.
-#
-# Why this is a separate workflow rather than steps inside ci.yml: ci.yml's
-# concurrency group cancels in-progress runs on main pushes, and merges land far
-# faster than its 70-minute pipeline. Measured over 15 consecutive main pushes:
-# 12 cancelled, 2 failed, 0 succeeded. A cancelled run never reaches
-# Swatinem/rust-cache's post step (cache-on-failure does not cover cancellation),
-# so the writer lanes were saving nothing and every PR paid a cold restore —
-# 11.8-20.9 minutes of "Setup Rust environment" against 0.7-3.4 warm.
-#
-# Splitting cache writing out of the test pipeline lets ci.yml keep cancelling
-# superseded runs (which is correct — nobody needs test results for a commit
-# that is already three merges behind) while the caches still get written.
-#
-# The group below deliberately does NOT cancel in progress; see the comment on
-# it for how that bounds concurrency and why it is scoped by event.
-#
-# Each job below owns exactly one shared-key and is the only place that sets
-# cache-save-if to anything but 'false' for it; every lane in ci.yml reads.
-# scripts/security/check_cache_save_if.sh keeps the declarations explicit.
-#
-# The builds are supersets of what the reading lanes compile, because a reader
-# restores only what the writer saved. Feature resolution matters here: a lane
-# built with e2e-test-hooks resolves dependency features differently, which
-# changes -Cmetadata, so the plain build does not cover it. See
-# rustfs/backlog#1600.
-
-name: Cache Warm
-
-on:
-  push:
-    branches: [ main ]
-    # Mirrors ci.yml's push paths-ignore: if a commit cannot change what ci.yml
-    # compiles, it cannot change what ci.yml needs restored either.
-    paths-ignore:
-      - "**.md"
-      - "docs/**"
-      - "deploy/**"
-      - "scripts/dev_*.sh"
-      - "scripts/probe.sh"
-      - "LICENSE*"
-      - ".gitignore"
-      - ".dockerignore"
-      - "README*"
-      - "**/*.png"
-      - "**/*.jpg"
-      - "**/*.svg"
-      - ".github/workflows/build.yml"
-      - ".github/workflows/docker.yml"
-      - ".github/workflows/audit.yml"
-  workflow_dispatch:
-    inputs:
-      emit_timings:
-        description: >-
-          Also emit cargo --timings for the ci-dev build and upload it. Used to
-          decide whether sccache is worth adopting (rustfs/backlog#1601 gate).
-        required: false
-        default: false
-        type: boolean
-
-permissions:
-  contents: read
-
-# Scoped by event. A push run and a dispatch run do not compete: GitHub keeps
-# one running plus one pending per group, so with a single shared group a
-# manually dispatched run was displaced as pending by the next merge and
-# cancelled — observed three times in a row, which made the --timings gate in
-# rustfs/backlog#1601 effectively impossible to trigger while main was busy.
-#
-# Still no cancel-in-progress: a burst of merges collapses into "current run
-# finishes, newest queued run follows" rather than a pile-up, which is what
-# bounds this workflow to one self-hosted runner per event type.
-#
-# The two paths can now overlap and race to save the same key. That is benign:
-# the loser finds the key already present and skips, and both builds produce the
-# same artifacts from the same commit.
-concurrency:
-  group: cache-warm-${{ github.event_name }}
-  cancel-in-progress: false
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
-  # e2e-tests, e2e-full.
-  warm-ci-dev:
-    name: Warm ci-dev
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'true'
-          install-build-packaging-tools: 'false'
-
-      # rustfs/backlog#1601 gate. sccache can only cache compilation units whose
-      # --emit includes link, so it covers workspace rlibs and nothing else:
-      # clippy is metadata-only, and the ~100 test binaries, the rustfs bin and
-      # every build script invoke the system linker. Before spending a bucket,
-      # credentials and a supply-chain boundary on it, measure how much of the
-      # build is actually rlib codegen.
-      #
-      # Read from the report: workspace lib codegen as a share of the build, and
-      # s3select-query's own rlib as a share. The plan adopts sccache only above
-      # 50% and 25% respectively; if linking dominates instead, the answer is
-      # mold/lld plus split-debuginfo, which is exactly the part sccache cannot
-      # touch. Off by default — this doubles the ci-dev build.
-      - name: Build ci-dev superset (with --timings)
-        if: inputs.emit_timings
-        env:
-          CARGO_BUILD_JOBS: "2"
-        run: cargo build --workspace --all-targets --timings
-
-      - name: Upload cargo timings report
-        if: inputs.emit_timings
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: cargo-timings-ci-dev
-          path: target/cargo-timings/
-          retention-days: 30
-          if-no-files-found: error
-
-      # --all-targets covers the test binaries nextest builds, including
-      # e2e_test, which test-and-lint's own run excludes. The second build adds
-      # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
-      # and that no lint lane enables.
-      - name: Build ci-dev superset
-        env:
-          # Same limit ci.yml puts on its nextest step: this builds the same
-          # ~100 workspace test binaries, and three concurrent links saturate the
-          # self-hosted runner's overlay I/O and can wedge Cargo (#5394).
-          CARGO_BUILD_JOBS: "2"
-        run: |
-          cargo build --workspace --all-targets
-          cargo build -p rustfs --bins --features e2e-test-hooks
-
-      # Runs before rust-cache's post step, so these are the sizes it is about
-      # to archive. Reported so the cache-all-crates decision stays evidence-led:
-      # registry/src is what that flag prunes, registry/cache is what the pruned
-      # sources are re-unpacked from. See rustfs/backlog#1600.
-      - name: Report cache input sizes
-        if: always()
-        run: |
-          # tee, not a plain redirect: sent only to $GITHUB_STEP_SUMMARY these
-          # numbers are readable in the UI but absent from the job log, and the
-          # REST API exposes the log, not the summary — which made the figures
-          # unreachable for exactly the scripted comparison they exist for.
-          sizes="$(du -sh ~/.cargo/registry/src ~/.cargo/registry/cache \
-                          ~/.cargo/registry/index ~/.cargo/git target 2>/dev/null || true)"
-          echo "cache-input-sizes-begin"
-          printf '%s\n' "$sizes"
-          echo "cache-input-sizes-end"
-          {
-            echo "### Cache input sizes (ci-dev)"
-            echo '```'
-            printf '%s\n' "$sizes"
-            echo '```'
-          } >> "$GITHUB_STEP_SUMMARY"
-
-  # Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
-  warm-ci-feat-rio:
-    name: Warm ci-feat-rio
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-feat-rio
-          cache-save-if: 'true'
-          install-build-packaging-tools: 'false'
-
-      - name: Build ci-feat-rio superset
-        run: |
-          cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
-          cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
-
-  # Readers: the swift and sftp legs of test-and-lint-protocols. Built in
-  # sequence rather than as `--features swift,sftp`, which is a combination no
-  # lane actually compiles; running both leaves the union in target/.
-  warm-ci-feat-proto:
-    name: Warm ci-feat-proto
-    runs-on: sm-standard-4
-    timeout-minutes: 90
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-feat-proto
-          cache-save-if: 'true'
-          install-build-packaging-tools: 'false'
-
-      - name: Build ci-feat-proto superset
-        run: |
-          cargo build -p rustfs -p rustfs-protocols --all-targets --features swift
-          cargo build -p rustfs -p rustfs-protocols --all-targets --features sftp
-
-  # Reader: uring-integration. Runs on ubuntu-latest to match it: rust-cache's
-  # key covers runner.os and arch but not the runner label or image, so a cache
-  # written on sm-standard-4 would be restored by the hosted runner as if it
-  # belonged to it.
-  warm-ci-uring:
-    name: Warm ci-uring
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: ci-uring
-          cache-save-if: 'true'
-          install-build-packaging-tools: 'false'
-
-      - name: Install build dependencies
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
-
-      - name: Build ci-uring superset
-        run: cargo build -p rustfs-ecstore --all-targets

.github/workflows/ci-docs-only.yml

@@ -23,8 +23,8 @@
 # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
 #
 # "Quick Checks" is mirrored here ahead of the ruleset change that will make it
-# required too (rustfs/backlog#1599). Until that change lands this job is
-# inert; mirroring it first is what lets the ruleset change happen without
+# required too (rustfs/backlog#1599). It is not required yet, so today this job
+# is inert; adding it first is what lets that ruleset change land without
 # stranding docs-only PRs on a check nobody reports.
 #
 # Keep the paths list below in sync with the pull_request paths-ignore list
@@ -53,7 +53,6 @@ on:
       - ".github/workflows/build.yml"
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
-      - "flake.lock"
 
 permissions:
   contents: read
@@ -79,8 +78,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Install ripgrep
         run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -114,12 +111,6 @@ jobs:
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
 
-      - name: Check CI paths stay in sync
-        run: ./scripts/check_ci_paths_sync.sh
-
-      - name: Check io_uring lane --lib precondition
-        run: ./scripts/check_uring_lane_lib_only.sh
-
   test-and-lint:
     name: Test and Lint
     runs-on: ubuntu-latest
@@ -127,8 +118,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       # Docs-only PRs skip the full code CI, but they are exactly where a
       # planning-type document could be slipped in (git add -f bypasses

.github/workflows/ci.yml

@@ -33,7 +33,6 @@ on:
       - ".github/workflows/build.yml"
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
-      - "flake.lock"
   pull_request:
     types: [ opened, synchronize, reopened, closed ]
     branches: [ main ]
@@ -55,7 +54,6 @@ on:
       - ".github/workflows/build.yml"
       - ".github/workflows/docker.yml"
       - ".github/workflows/audit.yml"
-      - "flake.lock"
   merge_group:
     types: [ checks_requested ]
   schedule:
@@ -83,7 +81,6 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -92,11 +89,8 @@ jobs:
     name: Typos
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
@@ -114,8 +108,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Install ripgrep
         run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -149,48 +141,24 @@ jobs:
       - name: Check no planning docs committed
         run: ./scripts/check_no_planning_docs.sh
 
-      - name: Check CI paths stay in sync
-        run: ./scripts/check_ci_paths_sync.sh
-
-      - name: Check io_uring lane --lib precondition
-        run: ./scripts/check_uring_lane_lib_only.sh
-
   test-and-lint:
     name: Test and Lint
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 90
-    # Both lines are required. Job-level `permissions` replaces the workflow
-    # block rather than merging with it, so declaring only `actions: write`
-    # would drop `contents: read` and break this job's checkout and the
-    # repo-token the setup action hands to setup-protoc.
-    permissions:
-      contents: read
-      actions: write
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          # This job's token can cancel runs and delete Actions caches. Checkout
-          # otherwise writes it into .git/config, where a PR's own build.rs or
-          # proc-macro could read it back out.
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          # Every lane in this workflow reads its cache and none writes it.
-          # cache-warm.yml is the sole writer for all four keys: this workflow
-          # cancels superseded runs on main, and a cancelled run never reaches
-          # rust-cache's post step, so writing from here saved nothing (12 of 15
-          # consecutive main-push runs were cancelled). See rustfs/backlog#1600.
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-test
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Prepare test evidence
         run: |
@@ -205,54 +173,43 @@ jobs:
       # Clippy runs before the test pass: lint failures are the most common
       # CI-only breakage and should surface in minutes, not after 20+ minutes
       # of tests.
-      # Sampled too: clippy is the natural control arm for any CARGO_BUILD_JOBS
-      # experiment, since --all-targets is check-only for workspace members and
-      # never links the ~100 test binaries the limit exists to throttle.
       - name: Run clippy lints
-        run: |
-          ./scripts/ci/resource_sampler.sh start clippy
-          trap './scripts/ci/resource_sampler.sh stop' EXIT
-          cargo clippy --all-targets -- -D warnings
+        run: cargo clippy --all-targets -- -D warnings
 
       - name: Run nextest tests
         env:
-          # #5394 mitigation, now under a measured experiment (backlog#1601).
-          #
-          # 2 was chosen when three concurrent workspace test links were believed
-          # to saturate the runner's overlay I/O and wedge Cargo until the 75m
-          # timeout. cgroup v2 readings from the sampler show the pod actually
-          # has 14 CPUs and 28GB (peak use 2.1GB), so 2 throttles compilation to
-          # a seventh of what is available and memory was never the constraint —
-          # the label name "sm-standard-4" had led everyone, including the
-          # original mitigation, to assume 4 cores.
-          #
-          # Raised to 3 on main pushes and manual dispatches; PRs keep 2 so the
-          # merge path is untouched while the experiment runs.
-          #
-          # Dispatch is included because push alone cannot supply the samples:
-          # this workflow cancels superseded runs on main, and only 4 of the last
-          # 20 push-triggered Test and Lint jobs reached a terminal state — at
-          # that rate ten samples would take roughly fifty merges. The
-          # concurrency group is scoped by event_name, so a dispatched run has
-          # its own group and is not cancelled by merge traffic, which makes the
-          # sample collectable on demand rather than by waiting.
-          #
-          # Baseline over 17 samples at 2:
-          # median nextest/clippy step ratio 1.95, spread 1.85-2.06. The gate-2
-          # criterion is that ratio dropping at least 10% (below ~1.76) with no
-          # 75m timeout and no run showing three consecutive samples of
-          # rustc/collect2/rust-lld in D state. If it does not, the conclusion is
-          # "this limit is not the bottleneck" — fix it back at 2 and record the
-          # experiment, which is a result, not a failure.
-          #
-          # Must stay step-level: rust-cache hashes CARGO/CC/CFLAGS/CXX/CMAKE/RUST
-          # prefixed variables from process.env into the cache key, so promoting
-          # this to job level would rotate every key on this lane.
-          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
+          # Three concurrent workspace test links saturate the self-hosted
+          # runner's overlay I/O and can wedge Cargo until the 75m timeout.
+          CARGO_BUILD_JOBS: "2"
         run: |
           mkdir -p artifacts/test-and-lint
-          ./scripts/ci/resource_sampler.sh start nextest
-          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          # Evidence sampler for issue #5394: the post-mortem pgrep below runs
+          # only after `timeout` has already TERM'd the whole cargo process
+          # group, so it cannot name a wedged process. Sample system and
+          # process state every 60s instead; the last samples before the
+          # timeout show what was stuck (rustc, linker, build script, memory
+          # pressure, ...). The log rides along in the existing artifact.
+          (
+            while true; do
+              {
+                echo "=== $(date --utc --iso-8601=seconds)"
+                echo "--- load";  cat /proc/loadavg
+                echo "--- psi";   grep -H . /proc/pressure/* 2>/dev/null || true
+                echo "--- mem";   free -m
+                echo "--- disk";  df -h / /home/runner 2>/dev/null || df -h /
+                echo "--- top-rss"
+                ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
+                echo "--- build/test processes"
+                ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
+                echo "--- d-state (uninterruptible IO)"
+                ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
+                echo
+              } >> artifacts/test-and-lint/sampler.log 2>&1 || true
+              sleep 60
+            done
+          ) &
+          sampler_pid=$!
+          trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
           set +e
           NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
             cargo nextest run --profile ci --all --exclude e2e_test \
@@ -324,48 +281,6 @@ jobs:
       - name: Run rebalance/decommission migration proofs
         run: ./scripts/check_migration_gate_count.sh
 
-      # Early stop. Once this job has failed the PR cannot merge, so the sibling
-      # lanes are burning runners on a result nobody can act on: on run
-      # 30674613104 three lanes had already failed while Test and Lint and the
-      # rio-v2 variant kept going past 70 minutes.
-      #
-      # Only this job may cancel. The lanes that are NOT required checks
-      # (protocols, ILM, e2e, s3-tests) must never hold that power: a flake in
-      # one of them would turn the required "Test and Lint" into `cancelled`,
-      # which blocks the merge. Today a maintainer can merge with sftp red, and
-      # that has to stay true.
-      #
-      # These steps run last so the `if: always()` artifact upload above still
-      # captures logs and diagnostics before the run goes away.
-      - name: Annotate early-stop reason
-        if: failure() && github.event_name == 'pull_request'
-        run: |
-          echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
-          echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
-          echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
-
-      # curl rather than `gh`: every existing `gh` call in this repo runs on
-      # ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
-      # ship no C toolchain, see the e2e job below), so `gh` is not known to
-      # exist here.
-      #
-      # Fork PRs are excluded explicitly instead of relying on the error path:
-      # their GITHUB_TOKEN is forced read-only and job-level permissions cannot
-      # raise it, so the call would always 403. Skipping keeps their logs clean.
-      - name: Cancel run on failure (same-repo PR only)
-        if: >-
-          failure() && github.event_name == 'pull_request'
-          && github.event.pull_request.head.repo.full_name == github.repository
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          curl -fsS -X POST \
-            -H "Authorization: Bearer ${GH_TOKEN}" \
-            -H "Accept: application/vnd.github+json" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" || true
-
   # Dedicated serial lane for the ILM / lifecycle integration tests. These tests
   # drive the object layer through process-global singletons (the GLOBAL_ENV
   # ECStore, the global tier-config manager, background-expiry workers) and bind
@@ -379,7 +294,6 @@ jobs:
   test-ilm-integration-serial:
     name: ILM Integration (serial)
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 45
     env:
@@ -387,16 +301,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-ilm-serial
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
       # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
@@ -419,7 +331,6 @@ jobs:
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 60
     env:
@@ -427,16 +338,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-feat-rio
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-test-rio-v2
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Run rio-v2 clippy lints
         run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
@@ -449,17 +358,10 @@ jobs:
   test-and-lint-protocols:
     name: "Test and Lint (${{ matrix.features.name }})"
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 60
     strategy:
-      # On a PR, one failing protocol leg is enough to know the PR is not ready,
-      # so stop the sibling leg instead of paying another ~40 minutes for it.
-      # Everywhere else (main pushes, the merge queue, the weekly schedule) keep
-      # the full signal: there we want to know whether swift AND sftp are broken,
-      # not just whichever failed first. This is the only part of the early-stop
-      # work that also covers fork PRs, since it needs no token.
-      fail-fast: ${{ github.event_name == 'pull_request' }}
+      fail-fast: false
       matrix:
         features:
           - name: swift
@@ -471,16 +373,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-feat-proto
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-test-${{ matrix.features.name }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Run clippy with ${{ matrix.features.name }}
         run: |
@@ -493,7 +393,6 @@ jobs:
   build-rustfs-debug-binary:
     name: Build RustFS Debug Binary
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     env:
@@ -501,16 +400,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-rustfs-debug-binary
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build debug binary
         run: cargo build -p rustfs --bins --features e2e-test-hooks
@@ -526,7 +423,6 @@ jobs:
   build-rustfs-debug-binary-rio-v2:
     name: Build RustFS Debug Binary (rio-v2)
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     runs-on: sm-standard-4
     timeout-minutes: 30
     env:
@@ -534,16 +430,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-feat-rio
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-rustfs-debug-binary-rio-v2
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build debug binary with rio-v2
         run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
@@ -565,7 +459,6 @@ jobs:
     # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
     # 30662728539) and kept the cancellation run in progress for minutes.
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    needs: [ quick-checks ]
     # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
     # a container, applies no seccomp filter that would block io_uring_setup — so
     # the probe succeeds and the tests exercise the real UringBackend/FdCache/
@@ -576,24 +469,17 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          # Keeps its own key rather than joining ci-dev. rust-cache's key is
-          # built from runner.os/arch plus rustc and lockfile fingerprints — it
-          # does NOT include the runner label or image. ubuntu-latest and
-          # sm-standard-4 are therefore indistinguishable to it, so sharing a key
-          # would let two different system images overwrite each other's
-          # artifacts, and would make a 2-core hosted runner unpack ci-dev's ~3GB
-          # instead of this lane's ~1.3GB. cache-warm.yml warms this key on
-          # ubuntu-latest for the same reason.
           cache-shared-key: ci-uring
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Install build dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
 
       # ext4 supports O_DIRECT; the runner's default TMPDIR may sit on tmpfs or
       # overlayfs, where open(O_DIRECT) returns EINVAL/EOPNOTSUPP and the native
@@ -620,17 +506,7 @@ jobs:
           RUSTFS_IO_URING_READ_ENABLE: "true"
           RUSTFS_URING_TESTS_MUST_RUN: "1"
           TMPDIR: /mnt/rustfs-odirect
-        # --lib narrows what gets compiled, not what gets run: every selected
-        # test lives in the lib target. The 7 integration binaries under
-        # crates/ecstore/tests/ each reported "running 0 tests" here, so they
-        # were compiled and linked for nothing.
-        #
-        # The `uring_` filter must stay exactly as it is. libtest matches on
-        # substring, so it also selects names containing `during_` — 6 of the 18
-        # selected tests are such incidental matches. Narrowing the filter to
-        # `io_uring` would silently drop them, which is a coverage change.
-        # scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
-        run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture
+        run: cargo test -p rustfs-ecstore uring_ -- --test-threads=1 --nocapture
 
   e2e-tests:
     name: End-to-End Tests
@@ -640,8 +516,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       # Full setup with dependency caching: the smoke-suite step below
       # compiles the e2e_test crate, which pulls in most of the workspace.
@@ -650,9 +524,9 @@ jobs:
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-e2e
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
@@ -726,16 +600,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          cache-shared-key: ci-e2e
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       # Download after the cache restore so the freshly built binary from the
       # build job always wins over anything restored into target/debug.
@@ -771,8 +643,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Clean up previous test run
         run: |
@@ -827,8 +697,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Download debug binary
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -901,8 +769,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Download debug binary
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7

.github/workflows/cla.yml

@@ -22,18 +22,11 @@ on:
   issue_comment:
     types: [created, edited]
 
-# Least privilege at the top, widened per job below. This workflow runs on
-# pull_request_target and issue_comment, so it holds full secrets on every fork
-# PR and on any comment anyone writes — the one place in this repository where a
-# compromised action would be handed a repo-write token. It does not check out
-# or execute PR code, so there is no pwn-request path today, but the blast
-# radius should not depend on that staying true.
-#
-# contents: write in particular was never used: the signature records are
-# written to rustfs/cla through the scoped app token created below, and nothing
-# here writes to this repository's contents.
 permissions:
-  contents: read
+  contents: write
+  pull-requests: write
+  issues: write
+  checks: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
@@ -43,26 +36,14 @@ jobs:
   cancel-closed-pr-runs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
-    # Echoes one line; the run exists only so the concurrency group cancels the
-    # in-flight run of a closed PR.
-    permissions: {}
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
 
   cla:
     if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }}
-    # checks: write reports the merge-queue check run; pull-requests and issues
-    # let cla-bot comment and label. contents stays read — see the note above.
-    permissions:
-      contents: read
-      checks: write
-      issues: write
-      pull-requests: write
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     steps:
       - name: Report CLA result for merge queue
         if: github.event_name == 'merge_group'

.github/workflows/coverage.yml

@@ -62,16 +62,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
           cache-shared-key: ci-coverage
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          install-build-packaging-tools: 'false'
 
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
@@ -118,8 +116,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/docker.yml

@@ -85,7 +85,6 @@ jobs:
        github.event.workflow_run.head_branch != 'main' &&
        !contains(github.event.workflow_run.head_branch, '-preview'))
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     outputs:
       should_build: ${{ steps.check.outputs.should_build }}
       should_push: ${{ steps.check.outputs.should_push }}
@@ -98,18 +97,11 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
-          persist-credentials: false
           # For workflow_run events, checkout the specific commit that triggered the workflow
           ref: ${{ github.event.workflow_run.head_sha || github.sha }}
 
       - name: Check build conditions
         id: check
-        env:
-          # dispatch inputs via env, not `${{ }}` interpolation: they are
-          # free-form strings and would otherwise be evaluated by bash.
-          INPUT_VERSION: ${{ github.event.inputs.version }}
-          INPUT_PUSH_IMAGES: ${{ github.event.inputs.push_images }}
-          INPUT_FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
         run: |
           should_build=false
           should_push=false
@@ -210,9 +202,9 @@ jobs:
 
           elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             # Manual trigger
-            input_version="$INPUT_VERSION"
+            input_version="${{ github.event.inputs.version }}"
             version="${input_version}"
-            should_push="$INPUT_PUSH_IMAGES"
+            should_push="${{ github.event.inputs.push_images }}"
             should_build=true
 
             # Get short SHA
@@ -220,7 +212,7 @@ jobs:
 
             echo "🎯 Manual Docker build triggered:"
             echo "   📋 Requested version: $input_version"
-            echo "   🔧 Force rebuild: $INPUT_FORCE_REBUILD"
+            echo "   🔧 Force rebuild: ${{ github.event.inputs.force_rebuild }}"
             echo "   🚀 Push images: $should_push"
 
             case "$input_version" in
@@ -306,8 +298,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Login to Docker Hub
         uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
@@ -343,28 +333,32 @@ jobs:
           CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
           VARIANT_SUFFIX="${{ matrix.suffix }}"
 
-          # Convert version format for Dockerfile compatibility. The former
-          # DOCKER_CHANNEL was "release" down every branch and was passed as a
-          # build-arg no Dockerfile declares, so it is gone.
+          # Convert version format for Dockerfile compatibility
           case "$VERSION" in
             "latest")
+              # For stable latest, use RELEASE=latest + release CHANNEL
               DOCKER_RELEASE="latest"
+              DOCKER_CHANNEL="release"
               ;;
             v*)
               # For versioned releases (v1.0.0), remove 'v' prefix for Dockerfile
               DOCKER_RELEASE="${VERSION#v}"
+              DOCKER_CHANNEL="release"
               ;;
             *)
               # For other versions, pass as-is
               DOCKER_RELEASE="${VERSION}"
+              DOCKER_CHANNEL="release"
               ;;
           esac
 
           echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT"
+          echo "docker_channel=$DOCKER_CHANNEL" >> "$GITHUB_OUTPUT"
 
           echo "🐳 Docker build parameters:"
           echo "  - Original version: $VERSION"
           echo "  - Docker RELEASE: $DOCKER_RELEASE"
+          echo "  - Docker CHANNEL: $DOCKER_CHANNEL"
 
           # Generate tags based on build type
           # Only support release and prerelease builds (no development builds)
@@ -418,24 +412,18 @@ jobs:
           push: ${{ needs.build-check.outputs.should_push == 'true' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          # No layer cache. This build compiles nothing — it downloads a
-          # release zip and runs apk/apt — so the cache could only save the
-          # minute or two those take, while creating a correctness problem: with
-          # RELEASE=latest the binary URL is resolved by curl *inside* a RUN
-          # layer, and the layer key does not include what that resolved to. A
-          # rebuild at the same RELEASE value (dispatch with version=latest, or
-          # a re-run of the same version) would hit the old layer and ship the
-          # previous release's binary. mode=max also consumed the same 10GB
-          # Actions cache quota the Rust lanes are fighting over.
-          #
-          # Only RELEASE is passed: it is the sole build-arg the Dockerfiles
-          # declare besides TARGETARCH. BUILDTIME, VERSION, BUILD_TYPE, REVISION
-          # and CHANNEL were never read by any stage (and BUILDTIME's $(date ...)
-          # was a literal here, not a shell substitution). BUILD_DATE and VCS_REF
-          # are declared by the Dockerfiles but deliberately left unset —
-          # supplying them would change the published image labels.
+          cache-from: |
+            type=gha,scope=docker-${{ matrix.variant }}
+          cache-to: |
+            type=gha,mode=max,scope=docker-${{ matrix.variant }}
           build-args: |
+            BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+            VERSION=${{ needs.build-check.outputs.version }}
+            BUILD_TYPE=${{ needs.build-check.outputs.build_type }}
+            REVISION=${{ github.sha }}
             RELEASE=${{ steps.meta.outputs.docker_release }}
+            CHANNEL=${{ steps.meta.outputs.docker_channel }}
+            BUILDKIT_INLINE_CACHE=1
           provenance: true
           sbom: true
           # Add retry mechanism by splitting the build process
@@ -451,7 +439,6 @@ jobs:
     needs: [ build-check, build-docker ]
     if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     permissions:
       contents: read
       security-events: write
@@ -506,7 +493,6 @@ jobs:
     needs: [ build-check, build-docker ]
     if: always() && needs.build-check.outputs.should_build == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Docker build completion summary
         run: |

.github/workflows/e2e-replication-nightly.yml

@@ -64,16 +64,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
           cache-shared-key: ci-e2e-repl
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          install-build-packaging-tools: 'false'
 
       # awscurl lets the STS dual-node test actually exercise its path. Without
       # it the test skips gracefully with a visible log line
@@ -126,8 +124,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/e2e-s3tests.yml

@@ -45,13 +45,6 @@
 # The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
 # via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: e2e-s3tests
 
 on:
@@ -142,8 +135,6 @@ jobs:
       TEST_MODE: ${{ matrix.test-mode }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       # Provision Python explicitly rather than trusting the runner image to
       # ship a working pip (ci-1: a bare python3 without pip is what broke the
@@ -363,8 +354,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/fuzz.yml

@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Fuzz
 
 on:
@@ -66,7 +59,6 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -87,14 +79,13 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: nightly
           cache-shared-key: fuzz-${{ hashFiles('fuzz/Cargo.lock') }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}
 
       - name: Install cargo-fuzz
@@ -154,8 +145,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -211,8 +200,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Download prebuilt fuzz binaries
         uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -260,8 +247,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/helm-package.yml

@@ -32,7 +32,6 @@ permissions:
 jobs:
   build-helm-package:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     if: |
       (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
       (
@@ -50,26 +49,16 @@ jobs:
     steps:
       - name: Checkout helm chart repo
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
-      # Both inputs reach the shell through env rather than `${{ }}`
-      # interpolation. A git ref name may contain `$(...)` — anything without a
-      # space is a legal tag — and interpolation pastes it into the script
-      # verbatim, where bash would run it. Reading "$RAW_INPUT" instead makes it
-      # data.
       - name: Normalize release version
         id: version
-        env:
-          RAW_INPUT: ${{ github.event.inputs.version }}
-          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
         run: |
           set -eux
 
           if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            RAW="$RAW_INPUT"
+            RAW="${{ github.event.inputs.version }}"
           else
-            RAW="$RAW_BRANCH"
+            RAW="${{ github.event.workflow_run.head_branch }}"
           fi
 
           case "$RAW" in
@@ -84,13 +73,10 @@ jobs:
           ./scripts/helm_chart_version.sh "$RAW_TAG"
 
       - name: Replace chart version and app version
-        env:
-          CHART_VERSION: ${{ steps.version.outputs.chart_version }}
-          APP_VERSION: ${{ steps.version.outputs.app_version }}
         run: |
           set -eux
-          sed -i -E "s/^version:.*/version: \"${CHART_VERSION}\"/" helm/rustfs/Chart.yaml
-          sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" helm/rustfs/Chart.yaml
+          sed -i -E 's/^version:.*/version: "${{ steps.version.outputs.chart_version }}"/' helm/rustfs/Chart.yaml
+          sed -i -E 's/^appVersion:.*/appVersion: "${{ steps.version.outputs.app_version }}"/' helm/rustfs/Chart.yaml
 
       - name: Set up Helm
         uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
@@ -115,7 +101,6 @@ jobs:
 
   publish-helm-package:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     needs: [ build-helm-package ]
     if: needs.build-helm-package.result == 'success'
 
@@ -123,8 +108,6 @@ jobs:
       - name: Checkout helm package repo
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
-          # persist-credentials-exempt: this checkout's token IS the push credential —
-          # the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
           repository: rustfs/helm
           token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
 
@@ -140,19 +123,11 @@ jobs:
       - name: Generate index
         run: helm repo index . --url https://charts.rustfs.com
 
-      # app_version is derived from the triggering tag name, and this job holds
-      # the cross-repository push token with rustfs/helm already checked out —
-      # the worst place in the repo to paste an attacker-influenced string into
-      # a shell line. Passed through env so bash treats it as data.
       - name: Push helm package and index file
-        env:
-          GIT_USERNAME: ${{ secrets.USERNAME }}
-          GIT_EMAIL: ${{ secrets.EMAIL_ADDRESS }}
-          APP_VERSION: ${{ needs.build-helm-package.outputs.app_version }}
         run: |
           set -eux
-          git config --global user.name "${GIT_USERNAME}"
-          git config --global user.email "${GIT_EMAIL}"
+          git config --global user.name "${{ secrets.USERNAME }}"
+          git config --global user.email "${{ secrets.EMAIL_ADDRESS }}"
           git add .
-          git commit -m "Update rustfs helm package with ${APP_VERSION}." || echo "No changes to commit"
+          git commit -m "Update rustfs helm package with ${{ needs.build-helm-package.outputs.app_version }}." || echo "No changes to commit"
           git push origin main

.github/workflows/issue-translator.yml

@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: "issue-translator"
 on:
   issue_comment:
@@ -33,7 +26,6 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - uses: usthe/issues-translate-action@b41f55ddc81d7d54bd542a4f289fe28ec081898e # v2.7
         with:

.github/workflows/minio-interop.yml

@@ -23,13 +23,6 @@
 # Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
 # unlike the self-hosted fleet, whose pods drift in Docker/pip availability
 # (see the infra note in e2e-s3tests.yml). Nightly + manual only.
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: minio-interop
 
 on:
@@ -58,14 +51,13 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
           rust-version: stable
           cache-shared-key: ci-minio-interop
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Generate real MinIO fixtures via Docker

.github/workflows/mint.yml

@@ -45,13 +45,6 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: mint
 
 on:
@@ -125,8 +118,6 @@ jobs:
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Enable buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -272,8 +263,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/nightly-gnu.yml

@@ -1,57 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Nightly GNU Build
-
-on:
-  schedule:
-    - cron: "0 0 * * *"
-      timezone: "Asia/Shanghai"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: nightly-gnu-build-main-${{ github.event_name }}
-  cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
-
-env:
-  CARGO_TERM_COLOR: always
-  RUST_BACKTRACE: 1
-
-jobs:
-  build:
-    name: Build x86_64 GNU
-    runs-on: sm-standard-2
-    timeout-minutes: 150
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-    steps:
-      - name: Checkout main branch
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-          ref: main
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          cache-shared-key: build-x86_64-unknown-linux-gnu
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
-          install-test-tools: 'false'
-
-      - name: Build RustFS
-        run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins

.github/workflows/nix-flake-update.yml

@@ -19,12 +19,9 @@ on:
   schedule:
     - cron: '0 5 * * 0' # Weekly on Sunday 05:00 UTC (staggered after the midnight ci/build crons)
 
-# GITHUB_TOKEN only needs to read the repository here: the branch push and the
-# pull request are both created by update-flake-lock using the
-# FLAKE_UPDATE_TOKEN PAT below, not by this token. Leaving write on it hands a
-# repo-write credential to an unattended weekly job that does not use it.
 permissions:
-  contents: read
+  contents: write
+  pull-requests: write
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -40,10 +37,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        # persist-credentials-exempt: update-flake-lock pushes the branch and opens
-        # the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
-        # than reusing .git/config, but that is unverified — exempt until a
-        # workflow_dispatch run confirms it (rustfs/backlog#1602).
 
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3

.github/workflows/nix.yml

@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Nix CI
 
 on:
@@ -53,7 +46,6 @@ jobs:
     name: Cancel Closed PR Runs
     if: github.event_name == 'pull_request' && github.event.action == 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - name: Explain cancellation run
         run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -71,8 +63,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Install Nix
         uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0

.github/workflows/performance-ab.yml

@@ -22,13 +22,6 @@
 # a deliberate correctness cost (e.g. the #4221 fsync durability fix) is
 # recorded but does not block (rustfs/backlog#935 correction 1).
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Performance A/B
 
 on:
@@ -99,8 +92,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
 
       - name: Setup Rust environment
         uses: ./.github/actions/setup
@@ -108,6 +99,7 @@ jobs:
           rust-version: stable
           cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build release rustfs
         run: cargo build --release --bin rustfs
@@ -150,7 +142,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
-          persist-credentials: false
           fetch-depth: 0 # baseline is built from origin/main
 
       - name: Setup Rust environment
@@ -159,6 +150,7 @@ jobs:
           rust-version: stable
           cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install warp
         run: |
@@ -170,15 +162,13 @@ jobs:
 
       - name: Decide exemption
         id: exempt
-        env:
-          INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
         run: |
           allow="false"
           if [[ "${{ github.event_name }}" == "pull_request" ]] \
             && ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then
             allow="true"
           fi
-          if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
+          if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then
             allow="true"
           fi
           echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
@@ -225,34 +215,8 @@ jobs:
           cp target/release/rustfs baseline-bin/rustfs
           echo "built=true" >> "$GITHUB_OUTPUT"
 
-      - name: Build baseline on cache miss (different candidate)
-        id: baseline_build
-        if: >-
-          steps.baseline_cache.outputs.cache-hit != 'true' &&
-          steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
-        run: |
-          set -euo pipefail
-          baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}"
-          baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}"
-          git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}"
-          cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target"
-          mkdir -p baseline-bin
-          cp "$baseline_target/release/rustfs" baseline-bin/rustfs
-          git worktree remove --force "$baseline_root"
-          echo "built=true" >> "$GITHUB_OUTPUT"
-
-      - name: Build candidate binary
-        id: candidate_build
-        if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
-        run: |
-          set -euo pipefail
-          cargo build --release --bin rustfs
-          mkdir -p candidate-bin
-          cp target/release/rustfs candidate-bin/rustfs
-          echo "built=true" >> "$GITHUB_OUTPUT"
-
       - name: Save self-healed baseline to cache
-        if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true'
+        if: steps.selfheal.outputs.built == 'true'
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
         with:
           path: baseline-bin/rustfs
@@ -260,58 +224,54 @@ jobs:
 
       - name: Run warp A/B and gate
         id: ab
-        env:
-          INPUT_DURATION: ${{ github.event.inputs.duration }}
         run: |
           set -euo pipefail
-          # The formal runner executes A1 baseline -> B1 candidate -> B2 candidate
-          # -> A2 baseline for each workload and drive-sync cell. It requires three
-          # rounds per leg to emit tail latency and error-rate evidence.
+          # Budget note: with perf-3's cached baseline the nightly does no source
+          # build on a cache hit, so the wall-clock is dominated by the short warp
+          # matrix — duration/rounds/cooldown are kept small to fit all 24 cells
+          # (6 workloads x 2 phases x 2 drive-sync) rather than dropping cells.
           # --health-timeout 180 outlasts the server's own 120s startup-readiness
           # budget, which the rig's previous 60s health poll undershot (the first
           # two nightly failures). perf-6 recalibrates these once the noise study
           # lands.
-          duration="${INPUT_DURATION:-12s}"
+          duration="${{ github.event.inputs.duration || '12s' }}"
           baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
           candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
           baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
           selfheal_built="${{ steps.selfheal.outputs.built }}"
-          baseline_built="${{ steps.baseline_build.outputs.built }}"
-          candidate_built="${{ steps.candidate_build.outputs.built }}"
 
-          args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha")
+          args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
 
-          if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then
+          if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" ]]; then
             chmod +x baseline-bin/rustfs
             base_bin="$PWD/baseline-bin/rustfs"
             args+=(--baseline-bin "$base_bin")
             if [[ "$baseline_hit" == "true" ]]; then
               base_src="actions-cache (rustfs-baseline-$baseline_sha)"
-            elif [[ "$selfheal_built" == "true" ]]; then
-              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
             else
-              base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
+              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
             fi
             if [[ "$candidate_sha" == "$baseline_sha" ]]; then
               # Nightly on main: the candidate is the same commit as the baseline,
               # so reuse the one binary for both phases and skip all builds.
-              args+=(--candidate-bin "$base_bin")
+              args+=(--candidate-bin "$base_bin" --skip-build)
               cand_src="same binary as baseline (same commit)"
-            elif [[ "$candidate_built" == "true" ]]; then
-              chmod +x candidate-bin/rustfs
-              args+=(--candidate-bin "$PWD/candidate-bin/rustfs")
-              cand_src="source build of the checked-out ref"
             else
-              echo "::error::candidate binary was not built" >&2
-              exit 2
+              cand_src="source build of the checked-out ref"
             fi
           else
-            echo "::error::baseline binary was not restored or built" >&2
-            exit 2
+            # Cache miss with candidate != baseline (opt-in PR gate only): fall
+            # back to the source double-build. With the post-#4806 LTO profile
+            # this will overrun the job budget and alert; rerun once the push
+            # cache build for origin/main has completed, or wait for perf-7's
+            # merge-base caching.
+            args+=(--baseline-ref origin/main)
+            base_src="source build of origin/main (cache miss)"
+            cand_src="source build of the checked-out ref"
           fi
 
-          echo "baseline binary: $base_src"
-          echo "candidate binary: $cand_src"
+          args+=(--provenance-note "baseline commit: $baseline_sha - $base_src")
+          args+=(--provenance-note "candidate commit: $candidate_sha - $cand_src")
 
           if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
             args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
@@ -319,7 +279,7 @@ jobs:
           # Do not let a gate FAIL abort the job here; capture status and surface
           # it after the PR comment is posted.
           set +e
-          bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
+          bash scripts/run_hotpath_warp_ab.sh "${args[@]}"
           echo "status=$?" >> "$GITHUB_OUTPUT"
           set -e
           # Locate the newest run dir + gate.md for the summary/comment/artifact
@@ -327,10 +287,10 @@ jobs:
           # holds server-logs/ for diagnosis.
           # Run dirs are UTC-timestamp names (no special chars); ls is safe here.
           # shellcheck disable=SC2012
-          run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)"
+          run_dir="$(ls -td target/hotpath-ab/*/ 2>/dev/null | head -n1 || true)"
           echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
           # shellcheck disable=SC2012
-          gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)"
+          gate_md="$(ls -t target/hotpath-ab/*/gate.md 2>/dev/null | head -n1 || true)"
           echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"
 
       - name: Upload A/B results
@@ -338,10 +298,10 @@ jobs:
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
         with:
           name: hotpath-warp-ab-${{ github.run_number }}
-          # Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
+          # Includes per-cell median_summary.csv / baseline_compare.csv, gate.md,
           # and server-logs/ (rustfs.log + startup env per phase) so a failed run
           # is diagnosable. Short retention: this is churny nightly debug data.
-          path: target/hotpath-abba/
+          path: target/hotpath-ab/
           if-no-files-found: warn
           retention-days: 14
 
@@ -425,8 +385,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/runner-hygiene.yml

@@ -1,81 +0,0 @@
-# Copyright 2026 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Asserts that the self-hosted runners are still ephemeral — one job per pod.
-#
-# This repository is public and its pull_request jobs run on those runners,
-# executing the PR's own build.rs, proc-macros and tests. The only thing keeping
-# that code from reaching a later job is that each ARC pod handles exactly one
-# job and is then destroyed. That guarantee lives in the ARC scale-set
-# configuration, outside this repository, where it can be changed without any PR
-# — so it is asserted here from the outside, against real run data, instead of
-# being assumed.
-#
-# Monthly rather than per-PR: the property changes only when someone
-# reconfigures the scale set, and the check costs a few dozen API calls.
-# See docs/ci/runners.md and rustfs/backlog#1602.
-
-name: Runner Hygiene
-
-on:
-  schedule:
-    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: runner-hygiene
-  cancel-in-progress: false
-
-jobs:
-  check-ephemerality:
-    name: Check runner ephemerality
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      # Exit 2 (inconclusive / broken) is deliberately not a pass: a window
-      # where every sm-* job was still queued would otherwise look identical to
-      # a clean bill of health.
-      - name: Assert one job per self-hosted runner
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: ./scripts/ci/check_runner_ephemerality.sh 40
-
-  alert-on-failure:
-    name: Alert on scheduled failure
-    needs: [check-ephemerality]
-    # Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
-    # scheduled runs file a tracking issue, manual dispatch stays quiet so
-    # debugging never produces a spurious alert.
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-      - name: Open or update failure-tracking issue
-        uses: ./.github/actions/schedule-failure-issue
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/schedule-failure-alert-drill.yml

@@ -24,13 +24,6 @@
 # The run itself is expected to end red (the forced failure); only the
 # alert-on-failure job result matters.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: Schedule Failure Alert Drill
 
 on:
@@ -63,8 +56,6 @@ jobs:
       issues: write
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
       - name: Open or update failure-tracking issue
         uses: ./.github/actions/schedule-failure-issue
         with:

.github/workflows/stale.yml

@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# DISABLED. This workflow is switched off in the repository's Actions settings
-# (state: disabled_manually) and does not run on any trigger, including its cron
-# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
-# reading this file, which has already misled at least one audit — hence this
-# banner. Re-enabling is a UI action; anyone doing so should first check that the
-# workflow still matches the current CI layout. See rustfs/backlog#1603.
-#
 name: "Mark stale issues"
 on:
   schedule:
@@ -27,7 +20,6 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
     steps:
       - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
         with:

.github/workflows/star-history.yml

@@ -15,7 +15,6 @@ concurrency:
 jobs:
   update:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
     steps:
       - uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0
         with:

AGENTS.md

@@ -21,22 +21,6 @@ If repo-level instructions conflict, follow the nearest file and keep behavior a
 - Avoid redundant file reads, repeated commands, and unnecessary exploratory work once enough context is available.
 - A good result is a minimal diff with clear assumptions, no over-engineering, and independent verification that survives Adversarial Validation (below).
 
-## Worktree and Disk Hygiene
-
-- Unless the requester explicitly says otherwise, treat every new implementation task as isolated work: fetch the latest `origin/main`, confirm the requested change is not already present there, and create a dedicated feature branch and worktree from that exact upstream commit before editing. Do not implement new work directly in the primary checkout or reuse a worktree from another task.
-- Check available disk space before creating the worktree or starting dependency downloads, builds, tests, coverage, or other artifact-heavy commands. For long-running or artifact-heavy work, re-check disk usage at natural phase boundaries and before broad validation; if remaining space may not safely accommodate the next command, stop and reclaim task-owned artifacts before continuing.
-- Keep cleanup scoped and safe: remove generated build/test/coverage artifacts and temporary files created by the task when they are no longer needed, and never delete another task's worktree or uncommitted files. Prefer shared dependency caches where supported instead of duplicating large artifacts across worktrees.
-- At handoff, report the disk-space checks, cleanup performed, and any retained worktree or artifacts with the reason they are still needed.
-
-## PR Lifecycle Monitoring
-
-- Creating or updating a PR is not the terminal state. Unless the requester explicitly limits the task to PR creation, monitor the PR through its terminal state: merged, closed, or explicitly handed off because progress requires user or maintainer action.
-- While the task is active, monitor CI/check runs, review decisions and unresolved threads, mergeability and conflicts, and unexpected head/base changes. Prefer event-driven or bounded waits provided by the current environment over frequent polling; report only state changes, actionable failures, or meaningful prolonged delays.
-- Investigate every failing check and review comment before changing code. Fix failures attributable to the task, run the verification required for the new diff, push the update, respond to or resolve the corresponding review threads, and resume monitoring. Do not weaken checks, dismiss valid feedback, or retry flaky failures merely to obtain a green result.
-- Treat opening, green CI, approval, and mergeability as intermediate states. Never merge without the required reviewer approval or explicit authority. If progress depends on credentials, infrastructure, a maintainer decision, or another external action, report the exact blocker and the evidence already collected.
-- If the current execution environment cannot remain active until the next PR event, use a supported automation, monitor, or thread wakeup when available and within scope. Otherwise leave an explicit handoff containing the PR, current state, next event to observe, and pending cleanup; do not imply that background monitoring exists when none is scheduled.
-- After observing a merge, verify the commits are preserved on the upstream base, ensure the worktree is clean, remove the dedicated worktree, prune stale worktree metadata, and delete the local task branch when it is no longer in use. For a closed or abandoned PR, preserve any unmerged work unless deletion was explicitly authorized. Do not delete remote branches unless explicitly requested or repository automation owns that cleanup.
-
 ## Autonomy and Approval Boundaries
 
 - Inquiry tasks (answer, explain, review, diagnose, plan): report findings; do not change files unless a fix is explicitly requested.

Cargo.lock

@@ -290,11 +290,11 @@ dependencies = [
 
 [[package]]
 name = "ar_archive_writer"
-version = "0.5.3"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
+checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
 dependencies = [
- "object 0.39.1",
+ "object 0.37.3",
 ]
 
 [[package]]
@@ -599,16 +599,6 @@ dependencies = [
  "syn 2.0.119",
 ]
 
-[[package]]
-name = "assert-json-diff"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
-dependencies = [
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "astral-tokio-tar"
 version = "0.6.4"
@@ -953,32 +943,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "aws-sdk-kms"
-version = "1.114.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
-dependencies = [
- "arc-swap",
- "aws-credential-types",
- "aws-runtime",
- "aws-smithy-async",
- "aws-smithy-http",
- "aws-smithy-json",
- "aws-smithy-observability",
- "aws-smithy-runtime",
- "aws-smithy-runtime-api",
- "aws-smithy-schema",
- "aws-smithy-types",
- "aws-types",
- "bytes",
- "fastrand",
- "http 0.2.12",
- "http 1.5.0",
- "regex-lite",
- "tracing",
-]
-
 [[package]]
 name = "aws-sdk-s3"
 version = "1.140.0"
@@ -1194,23 +1158,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
 dependencies = [
  "aws-smithy-async",
- "aws-smithy-protocol-test",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
- "bytes",
  "h2",
  "http 1.5.0",
- "http-body 1.1.0",
  "hyper",
  "hyper-rustls",
  "hyper-util",
- "indexmap 2.14.0",
  "pin-project-lite",
  "rustls",
  "rustls-native-certs",
  "rustls-pki-types",
- "serde",
- "serde_json",
  "tokio",
  "tokio-rustls",
  "tower",
@@ -1237,25 +1195,6 @@ dependencies = [
  "aws-smithy-runtime-api",
 ]
 
-[[package]]
-name = "aws-smithy-protocol-test"
-version = "0.64.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a"
-dependencies = [
- "assert-json-diff",
- "aws-smithy-runtime-api",
- "base64-simd",
- "cbor-diag",
- "ciborium",
- "http 0.2.12",
- "pretty_assertions",
- "regex-lite",
- "roxmltree",
- "serde_json",
- "thiserror 2.0.19",
-]
-
 [[package]]
 name = "aws-smithy-query"
 version = "0.62.0"
@@ -1589,7 +1528,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]
 
 [[package]]
@@ -1608,7 +1547,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]
 
 [[package]]
@@ -1659,7 +1598,7 @@ version = "3.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
 dependencies = [
- "darling 0.23.0",
+ "darling 0.20.11",
  "ident_case",
  "prettyplease",
  "proc-macro2",
@@ -1740,9 +1679,9 @@ dependencies = [
 
 [[package]]
 name = "bytesize"
-version = "2.6.0"
+version = "2.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493"
+checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
 
 [[package]]
 name = "bytestring"
@@ -1819,25 +1758,6 @@ dependencies = [
  "cipher 0.5.2",
 ]
 
-[[package]]
-name = "cbor-diag"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429"
-dependencies = [
- "bs58",
- "chrono",
- "data-encoding",
- "half",
- "nom 7.1.3",
- "num-bigint",
- "num-rational",
- "num-traits",
- "separator",
- "url",
- "uuid",
-]
-
 [[package]]
 name = "cc"
 version = "1.4.0"
@@ -1950,7 +1870,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.6",
+ "crypto-common 0.1.7",
  "inout 0.1.4",
 ]
 
@@ -1968,9 +1888,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.5"
+version = "4.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1978,9 +1898,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.5"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2408,7 +2328,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
  "rand_core 0.6.4",
  "subtle",
  "zeroize",
@@ -2433,11 +2353,11 @@ dependencies = [
 
 [[package]]
 name = "crypto-common"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
  "typenum",
 ]
 
@@ -3634,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer 0.10.4",
  "const-oid 0.9.6",
- "crypto-common 0.1.6",
+ "crypto-common 0.1.7",
  "subtle",
 ]
 
@@ -3894,7 +3814,7 @@ dependencies = [
  "crypto-bigint 0.5.5",
  "digest 0.10.7",
  "ff 0.13.1",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
  "group 0.13.0",
  "hkdf 0.12.4",
  "pem-rfc7468 0.7.0",
@@ -4339,9 +4259,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.9"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
  "typenum",
  "version_check",
@@ -4354,7 +4274,7 @@ version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
  "rustversion",
  "typenum",
 ]
@@ -5381,7 +5301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
  "block-padding 0.3.3",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]
 
 [[package]]
@@ -5915,7 +5835,8 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.49"
-source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
 dependencies = [
  "cc",
  "cty",
@@ -6324,7 +6245,8 @@ dependencies = [
 [[package]]
 name = "mimalloc"
 version = "0.1.52"
-source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
 dependencies = [
  "libmimalloc-sys",
 ]
@@ -6746,17 +6668,6 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "num-rational"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
-dependencies = [
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -6821,7 +6732,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.21.7",
  "chrono",
  "getrandom 0.2.17",
  "http 1.5.0",
@@ -7946,7 +7857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
  "heck",
- "itertools 0.14.0",
+ "itertools 0.13.0",
  "log",
  "multimap",
  "once_cell",
@@ -7966,7 +7877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
  "heck",
- "itertools 0.14.0",
+ "itertools 0.13.0",
  "log",
  "multimap",
  "petgraph 0.8.3",
@@ -7987,7 +7898,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
 dependencies = [
  "anyhow",
- "itertools 0.14.0",
+ "itertools 0.13.0",
  "proc-macro2",
  "quote",
  "syn 2.0.119",
@@ -8000,7 +7911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
 dependencies = [
  "anyhow",
- "itertools 0.14.0",
+ "itertools 0.13.0",
  "proc-macro2",
  "quote",
  "syn 2.0.119",
@@ -8720,15 +8631,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "roxmltree"
-version = "0.14.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
-dependencies = [
- "xmlparser",
-]
-
 [[package]]
 name = "rsa"
 version = "0.9.10"
@@ -8822,9 +8724,9 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.62.5"
+version = "0.62.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
+checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
 dependencies = [
  "aes 0.9.2",
  "aws-lc-rs",
@@ -9125,7 +9027,6 @@ dependencies = [
  "url",
  "urlencoding",
  "uuid",
- "zeroize",
  "zip",
  "zstd",
 ]
@@ -9610,16 +9511,10 @@ dependencies = [
  "arc-swap",
  "argon2",
  "async-trait",
- "aws-config",
- "aws-sdk-kms",
- "aws-smithy-http-client",
- "aws-smithy-runtime-api",
- "aws-smithy-types",
  "base64 0.23.0",
  "chacha20poly1305",
  "hex",
  "hotpath",
- "http 1.5.0",
  "insta",
  "jiff",
  "md-5 0.11.0",
@@ -9628,7 +9523,6 @@ dependencies = [
  "moka",
  "rand 0.10.2",
  "reqwest",
- "rustfs-s3-types",
  "rustfs-security-governance",
  "rustfs-utils",
  "rustify",
@@ -9814,7 +9708,6 @@ dependencies = [
  "hotpath",
  "jiff",
  "libc",
- "log",
  "metrics",
  "num_cpus",
  "nvml-wrapper",
@@ -9850,7 +9743,6 @@ dependencies = [
  "tracing-error",
  "tracing-opentelemetry",
  "tracing-subscriber",
- "url",
  "zstd",
 ]
 
@@ -9882,7 +9774,6 @@ dependencies = [
  "time",
  "tokio",
  "tracing",
- "tracing-subscriber",
 ]
 
 [[package]]
@@ -10136,7 +10027,6 @@ dependencies = [
  "rustfs-data-usage",
  "rustfs-ecstore",
  "rustfs-filemeta",
- "rustfs-lock",
  "rustfs-storage-api",
  "rustfs-utils",
  "s3s",
@@ -10717,7 +10607,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
  "base16ct 0.2.0",
  "der 0.7.10",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
  "pkcs8 0.10.2",
  "subtle",
  "zeroize",
@@ -10770,12 +10660,6 @@ dependencies = [
  "serde_core",
 ]
 
-[[package]]
-name = "separator"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5"
-
 [[package]]
 name = "seq-macro"
 version = "0.3.6"
@@ -11713,7 +11597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",

Cargo.toml

@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
 # Serialization and Data Formats
 apache-avro = "0.21.0"
 bytes = { version = "1.12.1" }
-bytesize = "2.6.0"
+bytesize = "2.4.2"
 byteorder = "1.5.0"
 flatbuffers = "25.12.19"
 form_urlencoded = "1.2.2"
@@ -227,7 +227,6 @@ atoi = "3.1.0"
 atomic_enum = "0.3.0"
 aws-config = { version = "1.10.1" }
 aws-credential-types = { version = "1.3.0" }
-aws-sdk-kms = { default-features = false, version = "1.114.0" }
 aws-sdk-s3 = { default-features = false, version = "1.140.0" }
 aws-sdk-sts = { default-features = false, version = "1.110.0" }
 aws-smithy-http-client = { default-features = false, version = "1.2.0" }
@@ -236,7 +235,7 @@ aws-smithy-types = { version = "1.6.1" }
 base64 = "0.23.0"
 base64-simd = "0.8.0"
 brotli = "8.0.4"
-clap = { version = "4.6.5" }
+clap = { version = "4.6.4" }
 const-str = { version = "1.1.0" }
 convert_case = "0.11.0"
 criterion = { version = "0.8" }
@@ -341,15 +340,14 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "10.0.1" }
 rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.62.5" }
+russh = { version = "0.62.4" }
 russh-sftp = "2.3.0"
 
 # WebDAV
 dav-server = "0.11.0"
 
 # Performance Analysis and Memory Profiling
-mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
-libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
+mimalloc = "0.1.52"
 hotpath = { version = "0.22.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

crates/config/src/constants/app.rs

@@ -230,19 +230,6 @@ pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
 /// Default value: false
 pub const DEFAULT_KMS_ENABLE: bool = false;
 
-/// Environment variable enabling per-key KMS authorization on the SSE-KMS data path.
-///
-/// When enabled, an SSE-KMS write additionally requires `kms:GenerateDataKey` and an
-/// SSE-KMS read additionally requires `kms:Decrypt` on the resolved key, evaluated as
-/// the requesting identity. SSE-S3 and SSE-C are unaffected.
-pub const ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY: &str = "RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY";
-
-/// Default per-key KMS authorization mode for the SSE-KMS data path.
-///
-/// Off for now so deployments whose identity policies only grant s3 actions keep
-/// working; the default flips to on in a later release.
-pub const DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY: bool = false;
-
 /// Environment variable for server KMS backend.
 pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
 

crates/e2e_test/src/admin_iam_crud_test.rs

@@ -26,16 +26,75 @@
 //! Later batches tracked on backlog#1154: config get/set, info, pools status,
 //! group lifecycle, import/export IAM.
 
-use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::{Client, Config};
+use http::header::{CONTENT_TYPE, HOST};
 use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
 use serial_test::serial;
 use std::error::Error;
 use tokio::time::{Duration, sleep};
 
 type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+type BoxError = Box<dyn Error + Send + Sync>;
+
+/// Signs and sends an admin HTTP request with the given credential, returning
+/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
+/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
+async fn admin_request(
+    base_url: &str,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<(StatusCode, String), BoxError> {
+    let url = format!("{base_url}{path_and_query}");
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let mut builder = http::Request::builder()
+        .method(method.clone())
+        .uri(uri)
+        .header(HOST, authority)
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if body.is_some() {
+        builder = builder.header(CONTENT_TYPE, "application/json");
+    }
+
+    let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
+    let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let mut request = local_http_client().request(reqwest_method, &url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    if let Some(body) = body {
+        request = request.body(body);
+    }
+    let response = request.send().await?;
+    let status = response.status();
+    let text = response.text().await.unwrap_or_default();
+    Ok((status, text))
+}
+
+/// Root-credential admin request that must succeed; returns the response body.
+async fn admin_ok(
+    env: &RustFSTestEnvironment,
+    method: http::Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> Result<String, BoxError> {
+    let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
+    if !status.is_success() {
+        return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
+    }
+    Ok(text)
+}
 
 fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
     let config = Config::builder()

crates/e2e_test/src/common.rs

@@ -24,12 +24,7 @@
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use http::header::{CONTENT_TYPE, HOST};
 use reqwest::Client as HttpClient;
-use reqwest::StatusCode;
-use rustfs_signer::constants::UNSIGNED_PAYLOAD;
-use rustfs_signer::sign_v4;
-use s3s::Body;
 use std::ffi::OsStr;
 use std::fs as stdfs;
 use std::path::{Path, PathBuf};
@@ -80,58 +75,6 @@ pub fn local_http_client() -> HttpClient {
         .expect("failed to build local reqwest client")
 }
 
-/// Signs and sends an admin HTTP request with the given credentials.
-pub(crate) async fn admin_request(
-    base_url: &str,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-    access_key: &str,
-    secret_key: &str,
-) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
-    let url = format!("{base_url}{path_and_query}");
-    let uri = url.parse::<http::Uri>()?;
-    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
-    let mut request = http::Request::builder()
-        .method(method.clone())
-        .uri(uri)
-        .header(HOST, authority)
-        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
-    if body.is_some() {
-        request = request.header(CONTENT_TYPE, "application/json");
-    }
-
-    let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
-    let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
-
-    let mut request = local_http_client().request(method, &url);
-    for (name, value) in signed.headers() {
-        request = request.header(name, value);
-    }
-    if let Some(body) = body {
-        request = request.body(body);
-    }
-    let response = request.send().await?;
-    let status = response.status();
-    let body = response.text().await?;
-    Ok((status, body))
-}
-
-/// Sends a root-credential admin request and returns its successful response body.
-pub(crate) async fn admin_ok(
-    env: &RustFSTestEnvironment,
-    method: http::Method,
-    path_and_query: &str,
-    body: Option<String>,
-) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
-    let (status, response_body) =
-        admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
-    if !status.is_success() {
-        return Err(format!("{method} {path_and_query} failed: {status} {response_body}").into());
-    }
-    Ok(response_body)
-}
-
 /// Resolve the RustFS binary relative to the workspace.
 pub fn rustfs_binary_path() -> PathBuf {
     rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())

crates/e2e_test/src/kms/configured_roundtrip_test.rs

@@ -126,10 +126,7 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
     assert_eq!(cancelled["success"], true);
     assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled");
 
-    // A default server refuses to skip the waiting window (rustfs/backlog#1585):
-    // immediate deletion is unrecoverable and takes every object encrypted under
-    // the key with it, so the endpoint must reject it rather than honour it.
-    let refused = kms_admin_request(
+    let removed = kms_admin_request(
         base_url,
         http::Method::DELETE,
         "/rustfs/admin/v3/kms/keys/delete",
@@ -143,49 +140,37 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
         access_key,
         secret_key,
     )
-    .await
-    .err()
-    .ok_or("immediate KMS key deletion must be refused on a default server")?;
-    assert!(
-        refused.to_string().contains("400 Bad Request"),
-        "refused immediate deletion must report a client error: {refused}"
-    );
-
-    // The refused request left the key alone, so the window-bounded path still
-    // has something to schedule.
-    let described = kms_admin_request(
-        base_url,
-        http::Method::GET,
-        &format!("/rustfs/admin/v3/kms/keys/{key_id}"),
-        None,
-        access_key,
-        secret_key,
-    )
     .await?;
-    let described: serde_json::Value = serde_json::from_str(&described)?;
-    assert_eq!(
-        described["key_metadata"]["key_state"], "Enabled",
-        "a refused immediate deletion must leave the key usable"
-    );
+    let removed: serde_json::Value = serde_json::from_str(&removed)?;
+    assert_eq!(removed["success"], true);
 
-    let rescheduled = kms_admin_request(
-        base_url,
-        http::Method::DELETE,
-        "/rustfs/admin/v3/kms/keys/delete",
-        Some(
-            &serde_json::json!({
-                "key_id": key_id,
-                "pending_window_in_days": 7
-            })
-            .to_string(),
-        ),
-        access_key,
-        secret_key,
-    )
-    .await?;
-    let rescheduled: serde_json::Value = serde_json::from_str(&rescheduled)?;
-    assert_eq!(rescheduled["success"], true);
-    assert!(rescheduled["deletion_date"].is_string());
+    let listed =
+        kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
+    let listed: serde_json::Value = serde_json::from_str(&listed)?;
+    assert_eq!(listed["success"], true);
+    let keys = listed["keys"]
+        .as_array()
+        .ok_or("list KMS keys response omitted keys after deletion")?;
+    if let Some(key) = keys.iter().find(|key| key["key_id"] == key_id) {
+        assert_eq!(key["status"], "PendingDeletion", "a retained force-deleted key must be pending deletion");
+        let removed = kms_admin_request(
+            base_url,
+            http::Method::DELETE,
+            "/rustfs/admin/v3/kms/keys/delete",
+            Some(
+                &serde_json::json!({
+                    "key_id": key_id,
+                    "force_immediate": true
+                })
+                .to_string(),
+            ),
+            access_key,
+            secret_key,
+        )
+        .await?;
+        let removed: serde_json::Value = serde_json::from_str(&removed)?;
+        assert_eq!(removed["success"], true);
+    }
 
     let listed =
         kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
@@ -194,11 +179,10 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
     let keys = listed["keys"]
         .as_array()
         .ok_or("final list KMS keys response omitted keys after deletion")?;
-    let key = keys
-        .iter()
-        .find(|key| key["key_id"] == key_id)
-        .ok_or("a key awaiting its deletion window must still be listed")?;
-    assert_eq!(key["status"], "PendingDeletion", "a scheduled key must be pending deletion");
+    assert!(
+        keys.iter().all(|key| key["key_id"] != key_id),
+        "force-deleted KMS key must no longer appear in list"
+    );
     Ok(())
 }
 

crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs

@@ -1,483 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Negative authorization matrix for per-key KMS access control.
-//!
-//! Every case here is an end-to-end denial that the pre-`kms` resource server
-//! allowed, so a regression that reopens one of them fails this file rather than
-//! only a unit test. The matrix varies one dimension at a time:
-//!
-//! - **wrong identity**: a caller holding S3 rights but no `kms` grant at all
-//! - **wrong key**: a caller scoped to key A naming key B
-//! - **wrong action**: a caller holding `kms:GenerateDataKey` but not `kms:Decrypt`
-//!   (and, on the admin plane, `kms:DisableKey` but not `kms:RotateKey`)
-//! - **wrong context**: an explicit `Deny` beating a wildcard `Allow`, and SSE-S3
-//!   staying exempt from `kms` authorization
-//!
-//! Each matrix opens with a positive control. Without it a denial proves nothing:
-//! an identity whose policy has not propagated yet is denied everything.
-
-use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
-use crate::common::{admin_ok, admin_request, init_logging};
-use aws_sdk_s3::Client;
-use aws_sdk_s3::config::{Config, Credentials, Region};
-use aws_sdk_s3::error::ProvideErrorMetadata;
-use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::ServerSideEncryption;
-use serial_test::serial;
-use std::time::Duration;
-use tracing::info;
-
-type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-
-const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
-const OTHER_KEY: &str = "kms-matrix-other-key";
-const BUCKET: &str = "kms-authz-matrix";
-const SECRET: &str = "kms-matrix-secret";
-const PAYLOAD: &[u8] = b"kms authorization matrix payload";
-
-/// How long an identity change may take to reach the request path.
-const IAM_PROPAGATION: Duration = Duration::from_secs(20);
-
-fn s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
-    let config = Config::builder()
-        .credentials_provider(Credentials::new(access_key, secret_key, None, None, "kms-authz-matrix"))
-        .region(Region::new("us-east-1"))
-        .endpoint_url(url)
-        .force_path_style(true)
-        .behavior_version_latest()
-        .build();
-    Client::from_conf(config)
-}
-
-/// Start a server whose SSE-KMS data path authorizes against the named key.
-///
-/// The enforcement switch defaults to off for compatibility, so it has to be set
-/// explicitly; without it every negative case below would silently pass as an allow.
-async fn start_enforcing_server(env: &mut LocalKMSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
-    create_key_with_specific_id(&env.kms_keys_dir, ALLOWED_KEY).await?;
-    create_key_with_specific_id(&env.kms_keys_dir, OTHER_KEY).await?;
-
-    let key_dir = env.kms_keys_dir.clone();
-    let args = vec![
-        "--kms-enable",
-        "--kms-backend",
-        "local",
-        "--kms-key-dir",
-        key_dir.as_str(),
-        "--kms-default-key-id",
-        ALLOWED_KEY,
-    ];
-
-    let mut envs = vec![("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")];
-    envs.extend_from_slice(extra_env);
-
-    env.base_env.start_rustfs_server_with_env(args, &envs).await?;
-    Ok(())
-}
-
-/// Create `user` with `policy_document` attached under a canned policy of the same name.
-async fn provision_user(env: &LocalKMSTestEnvironment, user: &str, policy_document: &str) -> TestResult {
-    admin_ok(
-        &env.base_env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/add-canned-policy?name={user}"),
-        Some(policy_document.to_string()),
-    )
-    .await?;
-    provision_user_with_policy(env, user, user).await
-}
-
-/// Create `user` and attach an existing policy (built-in or canned) by name.
-async fn provision_user_with_policy(env: &LocalKMSTestEnvironment, user: &str, policy_name: &str) -> TestResult {
-    admin_ok(
-        &env.base_env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
-        Some(serde_json::json!({ "secretKey": SECRET, "status": "enabled" }).to_string()),
-    )
-    .await?;
-    admin_ok(
-        &env.base_env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={user}&isGroup=false"),
-        None,
-    )
-    .await?;
-    Ok(())
-}
-
-/// The S3 half of every data-path policy below: full object access, no KMS grant.
-fn s3_full_access_statement() -> serde_json::Value {
-    serde_json::json!({
-        "Effect": "Allow",
-        "Action": ["s3:*"],
-        "Resource": ["arn:aws:s3:::*"]
-    })
-}
-
-fn policy_document(statements: Vec<serde_json::Value>) -> String {
-    serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
-}
-
-async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
-    client
-        .put_object()
-        .bucket(BUCKET)
-        .key(key)
-        .body(ByteStream::from_static(PAYLOAD))
-        .server_side_encryption(ServerSideEncryption::AwsKms)
-        .ssekms_key_id(kms_key_id)
-        .send()
-        .await
-        .map(|_| ())
-        .map_err(aws_sdk_s3::Error::from)
-}
-
-/// Assert the operation failed with `AccessDenied` rather than any other error.
-///
-/// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
-/// would hide both a leak of key state and an outage masquerading as a denial.
-fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
-    let error = result.expect_err(&format!("{what} must be denied"));
-    assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
-}
-
-/// Retry an SSE-KMS write until the identity's policy has reached the request path.
-async fn wait_for_sse_kms_write(client: &Client, key: &str, kms_key_id: &str) -> TestResult {
-    let deadline = tokio::time::Instant::now() + IAM_PROPAGATION;
-    loop {
-        match put_sse_kms(client, key, kms_key_id).await {
-            Ok(()) => return Ok(()),
-            Err(error) if tokio::time::Instant::now() >= deadline => {
-                return Err(format!("positive control never became authorized: {error:?}").into());
-            }
-            Err(_) => tokio::time::sleep(Duration::from_millis(500)).await,
-        }
-    }
-}
-
-/// Retry an admin call until it stops returning 403, i.e. the policy is live.
-async fn wait_for_admin_success(
-    env: &LocalKMSTestEnvironment,
-    user: &str,
-    method: http::Method,
-    path: &str,
-    body: Option<String>,
-) -> TestResult {
-    let deadline = tokio::time::Instant::now() + IAM_PROPAGATION;
-    loop {
-        let (status, response) = admin_request(&env.base_env.url, method.clone(), path, body.clone(), user, SECRET).await?;
-        if status.is_success() {
-            return Ok(());
-        }
-        if tokio::time::Instant::now() >= deadline {
-            return Err(format!("positive control never became authorized: {method} {path} -> {status} {response}").into());
-        }
-        tokio::time::sleep(Duration::from_millis(500)).await;
-    }
-}
-
-async fn assert_admin_denied(
-    env: &LocalKMSTestEnvironment,
-    user: &str,
-    method: http::Method,
-    path: &str,
-    body: Option<String>,
-    what: &str,
-) -> TestResult {
-    let (status, response) = admin_request(&env.base_env.url, method, path, body, user, SECRET).await?;
-    assert_eq!(status.as_u16(), 403, "{what} must be denied, got {status}: {response}");
-    assert!(response.contains("AccessDenied"), "{what} must carry AccessDenied: {response}");
-    Ok(())
-}
-
-fn disable_body(key_id: &str) -> String {
-    serde_json::json!({ "key_id": key_id }).to_string()
-}
-
-/// Data-path matrix: SSE-KMS writes and reads are authorized against the resolved key.
-#[tokio::test]
-#[serial]
-async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
-    init_logging();
-
-    let mut env = LocalKMSTestEnvironment::new().await?;
-    start_enforcing_server(&mut env, &[("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true")]).await?;
-    env.base_env.create_test_bucket(BUCKET).await?;
-
-    // Scoped to ALLOWED_KEY only.
-    provision_user(
-        &env,
-        "kmsmatrixscoped",
-        &policy_document(vec![
-            s3_full_access_statement(),
-            serde_json::json!({
-                "Effect": "Allow",
-                "Action": ["kms:GenerateDataKey", "kms:Decrypt"],
-                "Resource": [format!("arn:aws:kms:::key/{ALLOWED_KEY}")]
-            }),
-        ]),
-    )
-    .await?;
-    // S3 rights only: the identity shape that existed before per-key authorization.
-    provision_user(&env, "kmsmatrixs3only", &policy_document(vec![s3_full_access_statement()])).await?;
-    // May wrap a data key but may never unwrap one.
-    provision_user(
-        &env,
-        "kmsmatrixwriter",
-        &policy_document(vec![
-            s3_full_access_statement(),
-            serde_json::json!({
-                "Effect": "Allow",
-                "Action": ["kms:GenerateDataKey"],
-                "Resource": ["arn:aws:kms:::*"]
-            }),
-        ]),
-    )
-    .await?;
-    // Wildcard allow, explicit deny on one key.
-    provision_user(
-        &env,
-        "kmsmatrixdenied",
-        &policy_document(vec![
-            s3_full_access_statement(),
-            serde_json::json!({
-                "Effect": "Allow",
-                "Action": ["kms:*"],
-                "Resource": ["arn:aws:kms:::*"]
-            }),
-            serde_json::json!({
-                "Effect": "Deny",
-                "Action": ["kms:*"],
-                "Resource": [format!("arn:aws:kms:::key/{OTHER_KEY}")]
-            }),
-        ]),
-    )
-    .await?;
-
-    let scoped = s3_client(&env.base_env.url, "kmsmatrixscoped", SECRET);
-    let s3_only = s3_client(&env.base_env.url, "kmsmatrixs3only", SECRET);
-    let writer = s3_client(&env.base_env.url, "kmsmatrixwriter", SECRET);
-    let denied = s3_client(&env.base_env.url, "kmsmatrixdenied", SECRET);
-
-    // --- positive control -----------------------------------------------------
-    wait_for_sse_kms_write(&scoped, "scoped/allowed", ALLOWED_KEY).await?;
-    let read = scoped.get_object().bucket(BUCKET).key("scoped/allowed").send().await?;
-    assert_eq!(read.body.collect().await?.into_bytes().as_ref(), PAYLOAD);
-    info!("positive control: scoped identity may write and read under its own key");
-
-    // --- wrong key ------------------------------------------------------------
-    assert_access_denied(
-        put_sse_kms(&scoped, "scoped/other", OTHER_KEY).await,
-        "SSE-KMS write under a key outside the identity's scope",
-    );
-
-    // --- wrong identity -------------------------------------------------------
-    assert_access_denied(
-        put_sse_kms(&s3_only, "s3only/allowed", ALLOWED_KEY).await,
-        "SSE-KMS write by an identity holding no kms grant",
-    );
-    // The object the scoped identity wrote is readable by its owner only.
-    assert_access_denied(
-        s3_only
-            .get_object()
-            .bucket(BUCKET)
-            .key("scoped/allowed")
-            .send()
-            .await
-            .map(|_| ())
-            .map_err(aws_sdk_s3::Error::from),
-        "SSE-KMS read by an identity holding no kms grant",
-    );
-
-    // --- wrong action ---------------------------------------------------------
-    wait_for_sse_kms_write(&writer, "writer/allowed", ALLOWED_KEY).await?;
-    assert_access_denied(
-        writer
-            .get_object()
-            .bucket(BUCKET)
-            .key("writer/allowed")
-            .send()
-            .await
-            .map(|_| ())
-            .map_err(aws_sdk_s3::Error::from),
-        "SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
-    );
-
-    // --- wrong context: explicit Deny beats a wildcard Allow -------------------
-    wait_for_sse_kms_write(&denied, "denied/allowed", ALLOWED_KEY).await?;
-    assert_access_denied(
-        put_sse_kms(&denied, "denied/other", OTHER_KEY).await,
-        "SSE-KMS write under a key covered by an explicit Deny",
-    );
-
-    // --- wrong context: SSE-S3 is out of scope --------------------------------
-    // SSE-S3 wraps its data key with a server-owned key the caller never names, so
-    // it must stay reachable for an identity with no kms grant at all.
-    s3_only
-        .put_object()
-        .bucket(BUCKET)
-        .key("s3only/sse-s3")
-        .body(ByteStream::from_static(PAYLOAD))
-        .server_side_encryption(ServerSideEncryption::Aes256)
-        .send()
-        .await?;
-    let sse_s3_read = s3_only.get_object().bucket(BUCKET).key("s3only/sse-s3").send().await?;
-    assert_eq!(sse_s3_read.body.collect().await?.into_bytes().as_ref(), PAYLOAD);
-
-    // ... and so must an unencrypted object.
-    s3_only
-        .put_object()
-        .bucket(BUCKET)
-        .key("s3only/plain")
-        .body(ByteStream::from_static(PAYLOAD))
-        .send()
-        .await?;
-    s3_only.get_object().bucket(BUCKET).key("s3only/plain").send().await?;
-
-    Ok(())
-}
-
-/// Admin-plane matrix: KMS key endpoints are authorized against the key they name.
-///
-/// Runs without the SSE enforcement switch: admin scoping is unconditional, and
-/// leaving the switch off proves the two planes are independent.
-#[tokio::test]
-#[serial]
-async fn kms_admin_per_key_authorization_negative_matrix() -> TestResult {
-    init_logging();
-
-    let mut env = LocalKMSTestEnvironment::new().await?;
-    start_enforcing_server(&mut env, &[]).await?;
-
-    // Built-in role templates, attached by name.
-    provision_user_with_policy(&env, "kmsmatrixkeyadmin", "KMSKeyAdministrator").await?;
-    provision_user_with_policy(&env, "kmsmatrixauditor", "KMSAuditor").await?;
-    // A narrowed copy of the administrator template, scoped to one key.
-    provision_user(
-        &env,
-        "kmsmatrixscopedadmin",
-        &policy_document(vec![serde_json::json!({
-            "Effect": "Allow",
-            "Action": ["kms:DisableKey", "kms:EnableKey"],
-            "Resource": [format!("arn:aws:kms:::key/{ALLOWED_KEY}")]
-        })]),
-    )
-    .await?;
-
-    // --- positive control -----------------------------------------------------
-    wait_for_admin_success(
-        &env,
-        "kmsmatrixkeyadmin",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/disable",
-        Some(disable_body(OTHER_KEY)),
-    )
-    .await?;
-    admin_request(
-        &env.base_env.url,
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/enable",
-        Some(disable_body(OTHER_KEY)),
-        "kmsmatrixkeyadmin",
-        SECRET,
-    )
-    .await?;
-
-    // --- wrong action: the administrator template withholds service-wide powers -
-    assert_admin_denied(
-        &env,
-        "kmsmatrixkeyadmin",
-        http::Method::GET,
-        "/rustfs/admin/v3/kms/config",
-        None,
-        "KMSKeyAdministrator reading the KMS backend configuration (kms:Configure)",
-    )
-    .await?;
-    assert_admin_denied(
-        &env,
-        "kmsmatrixkeyadmin",
-        http::Method::GET,
-        "/rustfs/admin/v3/kms/backup",
-        None,
-        "KMSKeyAdministrator exporting a backup bundle (kms:Backup)",
-    )
-    .await?;
-    // Separation of duties: managing a key never implies using it.
-    assert_admin_denied(
-        &env,
-        "kmsmatrixkeyadmin",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/generate-data-key",
-        Some(serde_json::json!({ "key_id": ALLOWED_KEY }).to_string()),
-        "KMSKeyAdministrator generating a data key (kms:GenerateDataKey)",
-    )
-    .await?;
-
-    // --- wrong action: the auditor template is read-only ----------------------
-    wait_for_admin_success(&env, "kmsmatrixauditor", http::Method::GET, "/rustfs/admin/v3/kms/keys", None).await?;
-    assert_admin_denied(
-        &env,
-        "kmsmatrixauditor",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/disable",
-        Some(disable_body(ALLOWED_KEY)),
-        "KMSAuditor disabling a key (kms:DisableKey)",
-    )
-    .await?;
-
-    // --- wrong key ------------------------------------------------------------
-    wait_for_admin_success(
-        &env,
-        "kmsmatrixscopedadmin",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/disable",
-        Some(disable_body(ALLOWED_KEY)),
-    )
-    .await?;
-    assert_admin_denied(
-        &env,
-        "kmsmatrixscopedadmin",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/disable",
-        Some(disable_body(OTHER_KEY)),
-        "key-scoped administrator disabling a key outside its scope",
-    )
-    .await?;
-    // --- wrong action, same key ----------------------------------------------
-    assert_admin_denied(
-        &env,
-        "kmsmatrixscopedadmin",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/rotate",
-        Some(disable_body(ALLOWED_KEY)),
-        "key-scoped administrator rotating a key it may only enable and disable",
-    )
-    .await?;
-
-    // --- wrong identity -------------------------------------------------------
-    provision_user(&env, "kmsmatrixnokms", &policy_document(vec![s3_full_access_statement()])).await?;
-    assert_admin_denied(
-        &env,
-        "kmsmatrixnokms",
-        http::Method::POST,
-        "/rustfs/admin/v3/kms/keys/disable",
-        Some(disable_body(ALLOWED_KEY)),
-        "identity holding no kms grant disabling a key",
-    )
-    .await?;
-
-    Ok(())
-}

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -449,31 +449,29 @@ async fn test_vault_kms_key_crud(
 
     info!("✅ Delete verification: Key state correctly changed to: {}", key_state);
 
-    // Force Delete - a default server refuses to skip the waiting window
-    // (rustfs/backlog#1585): destroying the key material immediately would take
-    // every object encrypted under the key with it.
-    let force_delete_error = crate::common::execute_awscurl(
+    // Force Delete - Force immediate deletion for PendingDeletion key
+    let force_delete_response = crate::common::execute_awscurl(
         &format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
         "DELETE",
         None,
         access_key,
         secret_key,
     )
-    .await
-    .expect_err("Immediate KMS key deletion must be refused on a default server");
-    info!("✅ Force Delete: correctly refused for key {}: {}", key_id, force_delete_error);
+    .await?;
 
-    // The refused request must leave the key exactly as it was: still present,
-    // still pending deletion, still recoverable through cancel-deletion.
-    let describe_after_refusal =
-        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
-    let describe_after_refusal: serde_json::Value = serde_json::from_str(&describe_after_refusal)?;
-    assert_eq!(
-        describe_after_refusal["key_metadata"]["key_state"], "PendingDeletion",
-        "A refused immediate deletion must leave the key pending deletion"
-    );
+    // Parse and validate the force delete response
+    let force_delete_result: serde_json::Value = serde_json::from_str(&force_delete_response)?;
+    assert_eq!(force_delete_result["success"], true, "Force delete operation must return success=true");
+    info!("✅ Force Delete: Successfully force deleted key: {}", key_id);
 
-    info!("✅ Force Delete verification: Key survived the refused immediate deletion");
+    // Verify key no longer exists after force deletion (should return error)
+    let describe_force_deleted_result =
+        crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;
+
+    // After force deletion, key should not be found (GET should fail)
+    assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
+
+    info!("✅ Force Delete verification: Key was permanently deleted and is no longer accessible");
 
     info!("Vault KMS key CRUD operations completed successfully");
     Ok(())

crates/e2e_test/src/kms/mod.rs

@@ -53,6 +53,3 @@ mod copy_object_version_restore_sse_test;
 
 #[cfg(test)]
 mod configured_roundtrip_test;
-
-#[cfg(test)]
-mod kms_authorization_negative_matrix_test;

crates/e2e_test/src/reliant/lock.rs

@@ -15,7 +15,9 @@
 
 use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
 use rustfs_lock::client::{LockClient, local::LocalClient};
-use rustfs_lock::{GlobalLockManager, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey};
+use rustfs_lock::{
+    GlobalLockManager, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey,
+};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -33,11 +35,7 @@ struct FailingClient;
 #[async_trait::async_trait]
 impl rustfs_lock::LockClient for FailingClient {
     async fn acquire_lock(&self, _request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
-        // Match RemoteClient's transport-failure response so the coordinator can count this node toward quorum loss.
-        Ok(LockResponse::failure(
-            "Remote lock RPC failed: simulated gRPC node failure",
-            Duration::ZERO,
-        ))
+        Err(LockError::internal("simulated gRPC node failure"))
     }
 
     async fn release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {

crates/e2e_test/src/reliant/tiering.rs

@@ -382,6 +382,7 @@ struct ManualTransitionRunReport {
     skipped_delete_marker: u64,
     skipped_directory: u64,
     skipped_replication: u64,
+    skipped_already_transitioned: u64,
     skipped_already_in_flight: u64,
     skipped_queue_full: u64,
     skipped_queue_closed: u64,
@@ -407,6 +408,41 @@ fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionR
     }
 }
 
+fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) {
+    assert_completed_or_in_flight_partial(state, report, context);
+    if report.skipped_already_in_flight > 0 {
+        assert!(
+            report.scanned <= expected_objects,
+            "{context}: scanned more objects than the conflict scope contains: {report:#?}"
+        );
+        assert!(
+            report.eligible <= expected_objects,
+            "{context}: marked more objects eligible than the conflict scope contains: {report:#?}"
+        );
+        assert_eq!(
+            report.enqueued + report.skipped_already_in_flight,
+            report.eligible,
+            "{context}: partial in-flight accounting must cover every eligible object: {report:#?}"
+        );
+    } else {
+        assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}");
+        assert_eq!(
+            report.eligible + report.skipped_already_transitioned,
+            expected_objects,
+            "{context}: {report:#?}"
+        );
+        assert_eq!(
+            report.enqueued + report.skipped_already_in_flight,
+            expected_objects,
+            "{context}: {report:#?}"
+        );
+    }
+    assert_eq!(
+        report.transition_completed, report.enqueued,
+        "{context}: winner must wait for all queued transitions: {report:#?}"
+    );
+}
+
 #[derive(Debug, Deserialize)]
 struct ManualTransitionQueueSnapshot {
     queue_capacity: u64,
@@ -1240,15 +1276,8 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
 
     let mut hot = RustFSTestEnvironment::new().await?;
-    hot.start_rustfs_server_with_env(
-        vec![],
-        &[
-            ("RUSTFS_SCANNER_ENABLED", "false"),
-            ("RUSTFS_SCANNER_CYCLE", "3600"),
-            (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
-        ],
-    )
-    .await?;
+    hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
+        .await?;
     let hot_client = hot.create_s3_client();
     add_rustfs_tier(&hot, &cold).await?;
 
@@ -1318,21 +1347,20 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
     assert_eq!(conflict.cancel_endpoint, status_endpoint);
     assert!(!conflict.scope_key.is_empty());
 
-    manual_transition_job_cancel(&hot, cancel_endpoint).await?;
-
     let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
     assert_eq!(terminal.job_id, job_id);
-    assert_eq!(terminal.status, "cancelled", "terminal conflict winner response: {terminal:#?}");
     assert!(!terminal.report.dry_run);
     assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
     assert_eq!(terminal.report.prefix, accepted.report.prefix);
-    assert!(terminal.report.cancelled, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(terminal.report.scanned, 0, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(terminal.report.enqueued, 0, "terminal conflict winner response: {terminal:#?}");
-    assert_eq!(
-        terminal.report.transition_completed, 0,
-        "terminal conflict winner response: {terminal:#?}"
+    assert_conflict_winner_report(
+        &terminal.status,
+        &terminal.report,
+        MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
+        "terminal conflict winner response",
     );
+    assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}");
+    assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}");
     let after_remote_count = cold_tier_object_count(&cold_client).await?;
     assert!(after_remote_count >= before_remote_count);
     assert!(after_remote_count <= before_remote_count + MANUAL_ASYNC_CONFLICT_OBJECTS);

crates/e2e_test/src/replication_extension_test.rs

@@ -29,9 +29,8 @@ use aws_sdk_s3::types::{
 use aws_sdk_s3::{Client, Config};
 use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
 use bytes::Bytes;
-use flate2::read::GzDecoder;
 use futures::{Stream, StreamExt};
-use http::header::{CONTENT_ENCODING, CONTENT_TYPE, HOST};
+use http::header::{CONTENT_TYPE, HOST};
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
 use hyper::server::conn::http1;
@@ -39,10 +38,6 @@ use hyper::service::service_fn;
 use hyper::{Request, Response};
 use hyper_util::rt::TokioIo;
 use local_ip_address::local_ip;
-use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
-use opentelemetry_proto::tonic::common::v1::{KeyValue, any_value::Value as AnyValue};
-use opentelemetry_proto::tonic::metrics::v1::{Metric, metric, number_data_point};
-use prost::Message;
 use rcgen::{
     BasicConstraints, CertificateParams, CertifiedIssuer, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose,
     SanType, generate_simple_self_signed,
@@ -61,7 +56,6 @@ use sha2::{Digest, Sha256};
 use std::collections::BTreeMap;
 use std::convert::Infallible;
 use std::error::Error;
-use std::io::Read;
 use std::net::IpAddr;
 use std::path::Path;
 use std::process::Command;
@@ -70,13 +64,11 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use time::{Duration as TimeDuration, OffsetDateTime};
 use tokio::fs;
 use tokio::net::TcpListener;
-use tokio::sync::{Mutex, watch};
-use tokio::task::JoinHandle;
+use tokio::sync::watch;
 use tokio::task::JoinSet;
 use tokio::time::{Duration, sleep, timeout};
 
 type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
-type BacklogMetricPoints = Arc<Mutex<BTreeMap<String, BTreeMap<String, (u64, f64)>>>>;
 
 /// A replication source server validates the remote target endpoint, and the e2e
 /// target runs on loopback (127.0.0.1), which RustFS's SSRF egress guard rejects by
@@ -115,252 +107,6 @@ const REPL17_KMS_KEY_ID: &str = "repl17-local-key";
 const REPL17_SSEC_KEY: &str = "01234567890123456789012345678901";
 const REPLICATION_FAILED_EVENT: &str = "s3:Replication:OperationFailedReplication";
 const REPLICATION_EVENT_MAX_BUFFER_BYTES: usize = 1024 * 1024;
-const OTLP_METRICS_BODY_LIMIT: u64 = 4 * 1024 * 1024;
-const BUCKET_LABEL: &str = "bucket";
-const TOTAL_FAILED_COUNT_METRIC: &str = "rustfs_bucket_replication_total_failed_count";
-const CURRENT_BACKLOG_COUNT_METRIC: &str = "rustfs_bucket_replication_current_backlog_count";
-const CURRENT_BACKLOG_BYTES_METRIC: &str = "rustfs_bucket_replication_current_backlog_bytes";
-const MRF_PENDING_COUNT_METRIC: &str = "rustfs_bucket_replication_mrf_pending_count";
-const MRF_PENDING_BYTES_METRIC: &str = "rustfs_bucket_replication_mrf_pending_bytes";
-
-struct ReplicationBacklogMetricCollector {
-    endpoint: String,
-    values: BacklogMetricPoints,
-    task: JoinHandle<()>,
-}
-
-impl ReplicationBacklogMetricCollector {
-    async fn start() -> Result<Self, Box<dyn Error + Send + Sync>> {
-        let listener = TcpListener::bind("127.0.0.1:0").await?;
-        let endpoint = format!("http://{}/v1/metrics", listener.local_addr()?);
-        let values = Arc::new(Mutex::new(BTreeMap::new()));
-        let task_values = values.clone();
-        let task = tokio::spawn(async move {
-            loop {
-                let Ok((stream, _)) = listener.accept().await else {
-                    break;
-                };
-                let values = task_values.clone();
-                tokio::spawn(async move {
-                    let _ = http1::Builder::new()
-                        .serve_connection(
-                            TokioIo::new(stream),
-                            service_fn(move |request| handle_backlog_metric_export(request, values.clone())),
-                        )
-                        .await;
-                });
-            }
-        });
-
-        Ok(Self { endpoint, values, task })
-    }
-
-    fn root_endpoint(&self) -> &str {
-        self.endpoint.trim_end_matches("/v1/metrics")
-    }
-
-    async fn bucket_metric_value(&self, metric: &str, bucket: &str) -> f64 {
-        self.values
-            .lock()
-            .await
-            .get(metric)
-            .and_then(|buckets| buckets.get(bucket))
-            .map(|(_, value)| *value)
-            .unwrap_or_default()
-    }
-
-    async fn wait_for_bucket_metric(
-        &self,
-        metric: &str,
-        bucket: &str,
-        expected: impl Fn(f64) -> bool,
-        description: &str,
-    ) -> Result<f64, Box<dyn Error + Send + Sync>> {
-        let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
-        loop {
-            let value = self.bucket_metric_value(metric, bucket).await;
-            if expected(value) {
-                return Ok(value);
-            }
-            if tokio::time::Instant::now() >= deadline {
-                let snapshot = self.values.lock().await.clone();
-                return Err(format!("timed out waiting for {metric} on bucket {bucket} to satisfy {description}; last={value}, snapshot={snapshot:?}").into());
-            }
-            sleep(Duration::from_millis(200)).await;
-        }
-    }
-}
-
-impl Drop for ReplicationBacklogMetricCollector {
-    fn drop(&mut self) {
-        self.task.abort();
-    }
-}
-
-async fn handle_backlog_metric_export(
-    request: Request<Incoming>,
-    values: BacklogMetricPoints,
-) -> Result<Response<Full<Bytes>>, Infallible> {
-    if request.uri().path() != "/v1/metrics" {
-        return Ok(empty_http_response(StatusCode::NOT_FOUND));
-    }
-
-    let gzip = request
-        .headers()
-        .get(CONTENT_ENCODING)
-        .and_then(|value| value.to_str().ok())
-        .is_some_and(|value| value.eq_ignore_ascii_case("gzip"));
-    let Ok(collected) = request.into_body().collect().await else {
-        return Ok(empty_http_response(StatusCode::BAD_REQUEST));
-    };
-    let body = collected.to_bytes();
-    if body.len() as u64 > OTLP_METRICS_BODY_LIMIT {
-        return Ok(empty_http_response(StatusCode::PAYLOAD_TOO_LARGE));
-    }
-    let payload = if gzip {
-        let mut decoder = GzDecoder::new(body.as_ref());
-        let mut decoded = Vec::new();
-        if decoder
-            .by_ref()
-            .take(OTLP_METRICS_BODY_LIMIT + 1)
-            .read_to_end(&mut decoded)
-            .is_err()
-            || decoded.len() as u64 > OTLP_METRICS_BODY_LIMIT
-        {
-            return Ok(empty_http_response(StatusCode::BAD_REQUEST));
-        }
-        decoded
-    } else {
-        body.to_vec()
-    };
-
-    match ExportMetricsServiceRequest::decode(payload.as_slice()) {
-        Ok(export) => {
-            let mut values = values.lock().await;
-            record_backlog_metrics(&export, &mut values);
-            Ok(empty_http_response(StatusCode::OK))
-        }
-        Err(_) => Ok(empty_http_response(StatusCode::BAD_REQUEST)),
-    }
-}
-
-fn empty_http_response(status: StatusCode) -> Response<Full<Bytes>> {
-    Response::builder()
-        .status(status)
-        .body(Full::new(Bytes::new()))
-        .expect("static HTTP response is valid")
-}
-
-fn record_backlog_metrics(export: &ExportMetricsServiceRequest, values: &mut BTreeMap<String, BTreeMap<String, (u64, f64)>>) {
-    for resource_metrics in &export.resource_metrics {
-        for scope_metrics in &resource_metrics.scope_metrics {
-            for metric in &scope_metrics.metrics {
-                record_backlog_metric(metric, values);
-            }
-        }
-    }
-}
-
-fn record_backlog_metric(metric: &Metric, values: &mut BTreeMap<String, BTreeMap<String, (u64, f64)>>) {
-    if ![
-        TOTAL_FAILED_COUNT_METRIC,
-        CURRENT_BACKLOG_COUNT_METRIC,
-        CURRENT_BACKLOG_BYTES_METRIC,
-        MRF_PENDING_COUNT_METRIC,
-        MRF_PENDING_BYTES_METRIC,
-    ]
-    .contains(&metric.name.as_str())
-    {
-        return;
-    }
-
-    let points = match &metric.data {
-        Some(metric::Data::Gauge(gauge)) => gauge.data_points.as_slice(),
-        Some(metric::Data::Sum(sum)) => sum.data_points.as_slice(),
-        _ => return,
-    };
-    for point in points {
-        let Some(bucket) = attribute_string(&point.attributes, BUCKET_LABEL) else {
-            continue;
-        };
-        let Some(value) = number_point_value(point.value.as_ref()) else {
-            continue;
-        };
-        values
-            .entry(metric.name.clone())
-            .or_default()
-            .entry(bucket.to_string())
-            .and_modify(|current| {
-                if point.time_unix_nano >= current.0 {
-                    *current = (point.time_unix_nano, value);
-                }
-            })
-            .or_insert((point.time_unix_nano, value));
-    }
-}
-
-fn number_point_value(value: Option<&number_data_point::Value>) -> Option<f64> {
-    match value? {
-        number_data_point::Value::AsDouble(value) => Some(*value),
-        number_data_point::Value::AsInt(value) => Some(*value as f64),
-    }
-}
-
-fn attribute_string<'a>(attributes: &'a [KeyValue], wanted_key: &str) -> Option<&'a str> {
-    attributes.iter().find_map(|attribute| {
-        if attribute.key != wanted_key {
-            return None;
-        }
-        match attribute.value.as_ref()?.value.as_ref()? {
-            AnyValue::StringValue(value) => Some(value.as_str()),
-            _ => None,
-        }
-    })
-}
-
-struct SlowReplicationTargetGuard {
-    task: Option<JoinHandle<()>>,
-}
-
-impl SlowReplicationTargetGuard {
-    async fn bind(address: &str, response_delay: Duration) -> Result<Self, Box<dyn Error + Send + Sync>> {
-        let listener = TcpListener::bind(address).await?;
-        let task = tokio::spawn(async move {
-            loop {
-                let Ok((stream, _)) = listener.accept().await else {
-                    break;
-                };
-                tokio::spawn(async move {
-                    let _ = http1::Builder::new()
-                        .serve_connection(
-                            TokioIo::new(stream),
-                            service_fn(move |_request| async move {
-                                sleep(response_delay).await;
-                                Ok::<_, Infallible>(empty_http_response(StatusCode::SERVICE_UNAVAILABLE))
-                            }),
-                        )
-                        .await;
-                });
-            }
-        });
-        Ok(Self { task: Some(task) })
-    }
-
-    async fn stop(mut self) {
-        if let Some(task) = self.task.take() {
-            task.abort();
-            let _ = task.await;
-        }
-    }
-}
-
-impl Drop for SlowReplicationTargetGuard {
-    fn drop(&mut self) {
-        if let Some(task) = self.task.take() {
-            task.abort();
-        }
-    }
-}
 
 #[derive(Debug, Clone, serde::Deserialize)]
 struct ReplicationResetStatusResponse {
@@ -1507,10 +1253,6 @@ async fn build_sse_replication_pair(
             ("RUSTFS_KMS_KEY_DIR", source_kms_key_dir.as_str()),
             ("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID),
             ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
-            // Per-key KMS authorization is on so this contract is pinned in the
-            // configuration replication will eventually ship with: the replication
-            // worker carries no request identity and must stay exempt.
-            ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
         ]);
     }
     source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
@@ -1523,7 +1265,6 @@ async fn build_sse_replication_pair(
             ("RUSTFS_KMS_KEY_DIR", target_kms_key_dir.as_str()),
             ("RUSTFS_KMS_DEFAULT_KEY_ID", REPL17_KMS_KEY_ID),
             ("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
-            ("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
         ]);
     }
     target_env
@@ -3918,139 +3659,6 @@ async fn test_bucket_replication_recovers_after_target_outage() -> TestResult {
     Ok(())
 }
 
-/// backlog#1610 - black-box bucket replication backlog observability.
-///
-/// The source exports metrics through the same OTLP path production uses. A slow
-/// loopback target keeps replication workers occupied long enough for the metrics
-/// runtime to publish non-zero bucket backlog gauges; after the real target
-/// returns, replication must converge and the exported current/MRF pending gauges
-/// must settle back to zero even though the historical failed counter remains
-/// non-zero.
-#[tokio::test]
-#[serial]
-async fn test_bucket_replication_backlog_metrics_observe_outage_and_recovery() -> TestResult {
-    init_logging();
-
-    let collector = ReplicationBacklogMetricCollector::start().await?;
-    let mut source_env = RustFSTestEnvironment::new().await?;
-    let metric_root = collector.root_endpoint().to_string();
-    let metric_endpoint = collector.endpoint.clone();
-    let mut source_env_vars: Vec<(&str, &str)> = replication_fast_env().into_iter().collect();
-    source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
-    source_env_vars.extend_from_slice(FAST_SCANNER_ENV);
-    source_env_vars.extend_from_slice(&[
-        ("RUSTFS_OBS_ENDPOINT", metric_root.as_str()),
-        ("RUSTFS_OBS_METRIC_ENDPOINT", metric_endpoint.as_str()),
-        ("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "true"),
-        ("RUSTFS_OBS_TRACES_EXPORT_ENABLED", "false"),
-        ("RUSTFS_OBS_LOGS_EXPORT_ENABLED", "false"),
-        ("RUSTFS_OBS_METER_INTERVAL", "1"),
-        ("RUSTFS_OBS_USE_STDOUT", "false"),
-        ("RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC", "1"),
-    ]);
-    source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
-
-    let mut target_env = RustFSTestEnvironment::new().await?;
-    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
-
-    let source_bucket = "repl-backlog-metrics-src";
-    let target_bucket = "repl-backlog-metrics-dst";
-    let source_client = source_env.create_s3_client();
-    let target_client = target_env.create_s3_client();
-
-    source_client.create_bucket().bucket(source_bucket).send().await?;
-    target_client.create_bucket().bucket(target_bucket).send().await?;
-    enable_bucket_versioning(&source_env, source_bucket).await?;
-    enable_bucket_versioning(&target_env, target_bucket).await?;
-
-    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
-    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
-
-    source_client
-        .put_object()
-        .bucket(source_bucket)
-        .key("before-outage.txt")
-        .body(ByteStream::from_static(b"baseline written before backlog metrics outage"))
-        .send()
-        .await?;
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-
-    target_env.stop_server();
-    let slow_target = SlowReplicationTargetGuard::bind(&target_env.address, Duration::from_secs(5)).await?;
-
-    let outage_keys = ["metrics-outage-1.txt", "metrics-outage-2.txt", "metrics-outage-3.txt"];
-    for key in outage_keys {
-        source_client
-            .put_object()
-            .bucket(source_bucket)
-            .key(key)
-            .body(ByteStream::from(
-                format!("written while backlog metrics target was slow: {key}").into_bytes(),
-            ))
-            .send()
-            .await?;
-    }
-
-    let current_backlog = collector
-        .wait_for_bucket_metric(
-            CURRENT_BACKLOG_COUNT_METRIC,
-            source_bucket,
-            |value| value >= 1.0,
-            "be at least 1 during outage",
-        )
-        .await?;
-    let current_bytes = collector
-        .wait_for_bucket_metric(
-            CURRENT_BACKLOG_BYTES_METRIC,
-            source_bucket,
-            |value| value > 0.0,
-            "report bytes during outage",
-        )
-        .await?;
-    assert!(
-        current_bytes >= current_backlog,
-        "backlog bytes should be at least the object count while queued; count={current_backlog}, bytes={current_bytes}"
-    );
-    let failed_count = collector
-        .wait_for_bucket_metric(
-            TOTAL_FAILED_COUNT_METRIC,
-            source_bucket,
-            |value| value >= 1.0,
-            "record at least one failed replication attempt during outage",
-        )
-        .await?;
-
-    slow_target.stop().await;
-    target_env.restart_server_preserving_data(vec![], &[]).await?;
-
-    assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
-    for metric in [
-        CURRENT_BACKLOG_COUNT_METRIC,
-        CURRENT_BACKLOG_BYTES_METRIC,
-        MRF_PENDING_COUNT_METRIC,
-        MRF_PENDING_BYTES_METRIC,
-    ] {
-        collector
-            .wait_for_bucket_metric(metric, source_bucket, |value| value == 0.0, "settle back to zero after recovery")
-            .await?;
-    }
-    let final_failed_count = collector.bucket_metric_value(TOTAL_FAILED_COUNT_METRIC, source_bucket).await;
-    assert!(
-        final_failed_count >= failed_count,
-        "historical failed counter should remain non-zero after recovery while current backlog is zero; before={failed_count}, after={final_failed_count}"
-    );
-
-    let target_state = list_replication_state(&target_client, target_bucket).await?;
-    for key in ["before-outage.txt"].into_iter().chain(outage_keys) {
-        assert!(
-            target_state.iter().any(|entry| entry.key == key && !entry.delete_marker),
-            "target missing object {key} after backlog metrics recovery; state={target_state:?}"
-        );
-    }
-
-    Ok(())
-}
-
 /// backlog#1147 repl-5, scenario (b) — failure state survives a source restart
 /// (mirrors backlog#858 delete-decision re-derivation and #859 no-drop).
 ///

crates/e2e_test/src/sts_query_compat_test.rs

@@ -12,35 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_sts::config::retry::RetryConfig;
 use aws_sdk_sts::config::{Credentials, Region};
 use aws_sdk_sts::error::ProvideErrorMetadata;
 use aws_sdk_sts::operation::RequestId;
 use aws_sdk_sts::{Client, Config};
 use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
-use bytes::Bytes;
-use http::header::{AUTHORIZATION, CONTENT_TYPE};
-use http::{Request, Response};
-use http_body_util::{BodyExt, Full};
-use hyper::body::Incoming;
-use hyper::server::conn::http1;
-use hyper::service::service_fn;
-use hyper_util::rt::TokioIo;
-use serde_json::Value;
+use http::header::{CONTENT_TYPE, HOST};
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
 use serial_test::serial;
-use std::collections::BTreeSet;
-use std::convert::Infallible;
 use std::error::Error;
-use std::sync::Arc;
-use tokio::net::TcpListener;
-use tokio::sync::{Notify, mpsc};
-use tokio::task::{JoinHandle, JoinSet};
-use tokio::time::{Duration, timeout};
 
 type BoxError = Box<dyn Error + Send + Sync>;
 type TestResult = Result<(), BoxError>;
-const OPA_AUTH_TOKEN: &str = "sts-opa-token";
 
 fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
     let mut config = Config::builder()
@@ -62,14 +49,32 @@ fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Opti
 }
 
 async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
-    let body = admin_ok(
-        env,
-        http::Method::PUT,
-        "/rustfs/admin/v3/add-service-accounts",
-        Some(serde_json::json!({ "targetUser": env.access_key.clone() }).to_string()),
-    )
-    .await?;
-    let response: Value = serde_json::from_str(&body)?;
+    let path = "/rustfs/admin/v3/add-service-accounts";
+    let url = format!("{}{path}", env.url);
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
+    let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
+    let request = http::Request::builder()
+        .method(http::Method::PUT)
+        .uri(uri)
+        .header(HOST, authority)
+        .header(CONTENT_TYPE, "application/json")
+        .header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
+        .body(Body::empty())?;
+    let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
+    let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
+    let mut request = local_http_client().put(&url);
+    for (name, value) in signed.headers() {
+        request = request.header(name, value);
+    }
+    let response = request.body(body).send().await?;
+    let status = response.status();
+    let body = response.text().await?;
+    if !status.is_success() {
+        return Err(format!("create service account failed: {status} {body}").into());
+    }
+
+    let response: serde_json::Value = serde_json::from_str(&body)?;
     let access_key = response["credentials"]["accessKey"]
         .as_str()
         .ok_or("service account response should contain credentials.accessKey")?
@@ -81,237 +86,28 @@ async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(Str
     Ok((access_key, secret_key))
 }
 
-async fn create_user_with_policy(
-    env: &RustFSTestEnvironment,
-    user: &str,
-    secret: &str,
-    policy_name: &str,
-    statements: Value,
-) -> TestResult {
-    create_user(env, user, secret).await?;
-    admin_ok(
-        env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
-        Some(
-            serde_json::json!({
-                "Version": "2012-10-17",
-                "Statement": statements,
-            })
-            .to_string(),
-        ),
-    )
-    .await?;
-    admin_ok(
-        env,
-        http::Method::POST,
-        "/rustfs/admin/v3/idp/builtin/policy/attach",
-        Some(serde_json::json!({ "policies": [policy_name], "user": user }).to_string()),
-    )
-    .await?;
-    Ok(())
-}
-
-async fn create_user(env: &RustFSTestEnvironment, user: &str, secret: &str) -> TestResult {
-    admin_ok(
-        env,
-        http::Method::PUT,
-        &format!("/rustfs/admin/v3/add-user?accessKey={user}"),
-        Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
-    )
-    .await?;
-    Ok(())
-}
-
-async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
+async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
     let error = client
         .assume_role()
         .role_arn("arn:aws:iam::123456789012:role/test")
         .role_session_name("sts-query-compat-e2e")
         .send()
         .await
-        .expect_err("AssumeRole must be denied");
+        .expect_err("credential chaining must be denied");
     let service_error = error
         .as_service_error()
-        .ok_or_else(|| format!("{context} should deserialize as an STS service error: {error:?}"))?;
+        .ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
 
     assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
     assert_eq!(service_error.code(), Some("AccessDenied"));
     assert_eq!(service_error.message(), Some("Access Denied"));
     assert!(
         error.request_id().is_some_and(|request_id| !request_id.is_empty()),
-        "{context} should include a request ID"
+        "{credential_kind} denial should include a request ID"
     );
     Ok(())
 }
 
-async fn handle_opa_request(
-    request: Request<Incoming>,
-    requests: mpsc::UnboundedSender<Value>,
-    validation_started: mpsc::UnboundedSender<()>,
-    validation_mode: OpaValidationMode,
-    expected_authorization: Option<String>,
-) -> Result<Response<Full<Bytes>>, Infallible> {
-    if let Some(expected_authorization) = expected_authorization
-        && request.headers().get(AUTHORIZATION).and_then(|value| value.to_str().ok()) != Some(expected_authorization.as_str())
-    {
-        return Ok(Response::builder()
-            .status(401)
-            .body(Full::new(Bytes::new()))
-            .expect("static OPA unauthorized response must be valid"));
-    }
-
-    let body = match request.into_body().collect().await {
-        Ok(body) => body.to_bytes(),
-        Err(error) => {
-            return Ok(Response::builder()
-                .status(400)
-                .body(Full::new(Bytes::from(error.to_string())))
-                .expect("static OPA error response must be valid"));
-        }
-    };
-
-    let payload = if body.is_empty() {
-        None
-    } else {
-        match serde_json::from_slice::<Value>(&body) {
-            Ok(payload) => Some(payload),
-            Err(error) => {
-                return Ok(Response::builder()
-                    .status(400)
-                    .body(Full::new(Bytes::from(error.to_string())))
-                    .expect("static OPA error response must be valid"));
-            }
-        }
-    };
-    if payload.is_none() {
-        let _ = validation_started.send(());
-        if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
-            release.notified().await;
-            return Ok(Response::builder()
-                .status(503)
-                .body(Full::new(Bytes::new()))
-                .expect("static OPA unavailable response must be valid"));
-        }
-    }
-    let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
-        Some(Value::String(account)) if account == "opaallow" => payload
-            .as_ref()
-            .and_then(|value| value.pointer("/input/context/deny_only"))
-            .and_then(Value::as_bool)
-            .unwrap_or(false),
-        Some(Value::String(account)) if account == "opadeny" => false,
-        None => true,
-        _ => false,
-    };
-    if let Some(payload) = payload {
-        let _ = requests.send(payload);
-    }
-    let body =
-        serde_json::to_vec(&serde_json::json!({ "result": { "allow": allow } })).expect("static OPA response must serialize");
-    Ok(Response::builder()
-        .header(CONTENT_TYPE, "application/json")
-        .body(Full::new(Bytes::from(body)))
-        .expect("static OPA response must be valid"))
-}
-
-#[derive(Clone)]
-enum OpaValidationMode {
-    Ready,
-    DelayedUnavailable(Arc<Notify>),
-}
-
-struct OpaMock {
-    url: String,
-    requests: mpsc::UnboundedReceiver<Value>,
-    validation_started: mpsc::UnboundedReceiver<()>,
-    validation_release: Option<Arc<Notify>>,
-    task: JoinHandle<()>,
-}
-
-impl OpaMock {
-    async fn start() -> Result<Self, BoxError> {
-        Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
-    }
-
-    async fn start_delayed_unavailable() -> Result<Self, BoxError> {
-        let release = Arc::new(Notify::new());
-        Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
-    }
-
-    async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
-        let listener = TcpListener::bind("127.0.0.1:0").await?;
-        let url = format!("http://{}/v1/data/rustfs/authz/allow", listener.local_addr()?);
-        let (requests_tx, requests) = mpsc::unbounded_channel();
-        let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
-        let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
-        let validation_release = match &validation_mode {
-            OpaValidationMode::Ready => None,
-            OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
-        };
-        let task = tokio::spawn(async move {
-            let mut connections = JoinSet::new();
-            loop {
-                tokio::select! {
-                    accepted = listener.accept() => {
-                        let Ok((stream, _)) = accepted else { break };
-                        let requests = requests_tx.clone();
-                        let validation_started = validation_started_tx.clone();
-                        let validation_mode = validation_mode.clone();
-                        let expected_authorization = expected_authorization.clone();
-                        connections.spawn(async move {
-                            let handler = service_fn(move |request| {
-                                handle_opa_request(
-                                    request,
-                                    requests.clone(),
-                                    validation_started.clone(),
-                                    validation_mode.clone(),
-                                    expected_authorization.clone(),
-                                )
-                            });
-                            let _ = http1::Builder::new()
-                                .serve_connection(TokioIo::new(stream), handler)
-                                .await;
-                        });
-                    }
-                    _ = connections.join_next(), if !connections.is_empty() => {}
-                }
-            }
-        });
-        Ok(Self {
-            url,
-            requests,
-            validation_started,
-            validation_release,
-            task,
-        })
-    }
-
-    async fn next_request(&mut self) -> Result<Value, BoxError> {
-        timeout(Duration::from_secs(5), self.requests.recv())
-            .await?
-            .ok_or_else(|| "OPA request channel closed".into())
-    }
-
-    async fn wait_for_validation(&mut self) -> TestResult {
-        timeout(Duration::from_secs(5), self.validation_started.recv())
-            .await?
-            .ok_or_else(|| "OPA validation channel closed".into())
-    }
-
-    fn release_validation(&self) {
-        if let Some(release) = &self.validation_release {
-            release.notify_one();
-        }
-    }
-}
-
-impl Drop for OpaMock {
-    fn drop(&mut self) {
-        self.task.abort();
-    }
-}
-
 #[tokio::test]
 #[serial]
 async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
@@ -359,7 +155,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
         "signature rejection should include a request ID"
     );
 
-    assert_access_denied(
+    assert_chaining_denied(
         &sts_client(
             &env.url,
             temporary.access_key_id(),
@@ -371,187 +167,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
     .await?;
 
     let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
-    assert_access_denied(
-        &sts_client(&env.url, &service_access_key, &service_secret_key, None),
-        "service-account denial",
-    )
-    .await?;
-
-    let implicit_user = "stsimplicit";
-    let explicit_allow_user = "stsallow";
-    let explicit_deny_user = "stsdeny";
-    let policyless_user = "stspolicyless";
-    let secret = "stsAuthzSecret123";
-    create_user(&env, policyless_user, secret).await?;
-    assert_access_denied(&sts_client(&env.url, policyless_user, secret, None), "policyless user").await?;
-    create_user_with_policy(
-        &env,
-        implicit_user,
-        secret,
-        "sts-implicit-policy",
-        serde_json::json!([{
-            "Effect": "Allow",
-            "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["arn:aws:s3:::*"],
-        }]),
-    )
-    .await?;
-    create_user_with_policy(
-        &env,
-        explicit_allow_user,
-        secret,
-        "sts-allow-policy",
-        serde_json::json!([{
-            "Effect": "Allow",
-            "Action": ["sts:AssumeRole"],
-            "Resource": ["arn:aws:s3:::*"],
-        }]),
-    )
-    .await?;
-    create_user_with_policy(
-        &env,
-        explicit_deny_user,
-        secret,
-        "sts-deny-policy",
-        serde_json::json!([
-            {
-                "Effect": "Allow",
-                "Action": ["sts:AssumeRole"],
-                "Resource": ["arn:aws:s3:::*"],
-            },
-            {
-                "Effect": "Deny",
-                "Action": ["sts:AssumeRole"],
-                "Resource": ["arn:aws:s3:::*"],
-            }
-        ]),
-    )
-    .await?;
-
-    for user in [implicit_user, explicit_allow_user] {
-        let output = sts_client(&env.url, user, secret, None)
-            .assume_role()
-            .role_arn("arn:aws:iam::123456789012:role/test")
-            .role_session_name("sts-authz-e2e")
-            .send()
-            .await
-            .map_err(|error| format!("{user} should be allowed to call AssumeRole: {error:?}"))?;
-        let credentials = output
-            .credentials()
-            .ok_or_else(|| format!("{user} AssumeRole response should contain credentials"))?;
-        assert!(!credentials.access_key_id().is_empty());
-        assert!(!credentials.secret_access_key().is_empty());
-        assert!(!credentials.session_token().is_empty());
-    }
-    assert_access_denied(&sts_client(&env.url, explicit_deny_user, secret, None), "explicit sts:AssumeRole Deny").await?;
-
-    env.stop_server();
-    Ok(())
-}
-
-#[tokio::test]
-#[serial]
-async fn test_sts_assume_role_opa_contract() -> TestResult {
-    init_logging();
-
-    let mut opa = OpaMock::start().await?;
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_with_env(
-        vec![],
-        &[
-            ("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
-            ("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
-        ],
-    )
-    .await?;
-
-    let secret = "stsOpaSecret123";
-    create_user_with_policy(
-        &env,
-        "opaallow",
-        secret,
-        "sts-opa-local-deny-policy",
-        serde_json::json!([{
-            "Effect": "Deny",
-            "Action": ["sts:AssumeRole"],
-            "Resource": ["arn:aws:s3:::*"],
-        }]),
-    )
-    .await?;
-    create_user_with_policy(
-        &env,
-        "opadeny",
-        secret,
-        "sts-opa-local-allow-policy",
-        serde_json::json!([{
-            "Effect": "Allow",
-            "Action": ["sts:AssumeRole"],
-            "Resource": ["arn:aws:s3:::*"],
-        }]),
-    )
-    .await?;
-
-    sts_client(&env.url, "opaallow", secret, None)
-        .assume_role()
-        .role_arn("arn:aws:iam::123456789012:role/test")
-        .role_session_name("sts-opa-contract")
-        .send()
-        .await
-        .map_err(|error| format!("OPA allow should override the local explicit Deny: {error:?}"))?;
-    assert_access_denied(
-        &sts_client(&env.url, "opadeny", secret, None),
-        "OPA denial despite local sts:AssumeRole Allow",
-    )
-    .await?;
-
-    let mut accounts = BTreeSet::new();
-    for _ in 0..2 {
-        let request = opa.next_request().await?;
-        assert_eq!(request.pointer("/input/action").and_then(Value::as_str), Some("sts:AssumeRole"));
-        assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(true));
-        let account = request
-            .pointer("/input/identity/account")
-            .and_then(Value::as_str)
-            .ok_or("OPA input should include identity.account")?;
-        accounts.insert(account.to_owned());
-    }
-    assert_eq!(accounts, BTreeSet::from(["opaallow".to_owned(), "opadeny".to_owned()]));
-
-    env.stop_server();
-    Ok(())
-}
-
-#[tokio::test]
-#[serial]
-async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
-    init_logging();
-
-    let mut opa = OpaMock::start_delayed_unavailable().await?;
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
-        .await?;
-    opa.wait_for_validation().await?;
-
-    let user = "opaunavailable";
-    let secret = "stsOpaUnavailableSecret123";
-    create_user_with_policy(
-        &env,
-        user,
-        secret,
-        "sts-opa-unavailable-local-policy",
-        serde_json::json!([{
-            "Effect": "Allow",
-            "Action": ["s3:ListAllMyBuckets"],
-            "Resource": ["arn:aws:s3:::*"],
-        }]),
-    )
-    .await?;
-
-    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
-
-    opa.release_validation();
-    tokio::time::sleep(Duration::from_millis(200)).await;
-    assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
+    assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
 
     env.stop_server();
     Ok(())

crates/ecstore/src/api/mod.rs

@@ -172,11 +172,6 @@ pub mod bucket {
     }
 
     pub mod replication {
-        pub use crate::bucket::replication::replication_pool::{
-            DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
-            MrfBucketBacklogObservability, durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot,
-            mrf_backlog_observability_snapshot,
-        };
         pub use crate::bucket::replication::{
             BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
             MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
@@ -184,13 +179,13 @@ pub mod bucket {
             ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
             ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
             ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
-            ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
-            VersionPurgeStatusType, delete_replication_state_from_config, delete_replication_version_id,
-            get_global_replication_pool, get_global_replication_stats, init_background_replication, read_durable_mrf_backlog,
-            replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
-            resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
-            should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
-            unsupported_replication_config_field, validate_replication_config_target_arns, version_purge_status_to_filemeta,
+            ReplicationType, ResyncOpts, ResyncStatusType, TargetReplicationResyncStatus, VersionPurgeStatusType,
+            delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
+            get_global_replication_stats, init_background_replication, read_durable_mrf_backlog, replication_state_to_filemeta,
+            replication_status_to_filemeta, replication_statuses_map, replication_target_arns, resync_start_conflict_id,
+            should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
+            should_use_existing_delete_replication_source, validate_replication_config_target_arns,
+            version_purge_status_to_filemeta,
         };
     }
 
@@ -424,15 +419,15 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
-        PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-        ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
-        gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
-        node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
-        set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
-        tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
-        verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
-        verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
+        AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
+        PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
+        ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
+        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
+        sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
+        verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
+        verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
+        verify_tonic_rpc_signature_with_bootstrap,
     };
 }
 

crates/ecstore/src/bucket/replication/mod.rs

@@ -46,7 +46,7 @@ mod runtime_boundary;
 pub use datatypes::ResyncStatusType;
 pub use replication_config_boundary::{
     ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
-    should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
+    should_remove_replication_target, validate_replication_config_target_arns,
 };
 #[cfg(test)]
 pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
@@ -78,7 +78,7 @@ pub use replication_queue_boundary::{
 };
 pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
 pub use replication_scanner_bridge::ReplicationScannerBridge;
-pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
+pub use replication_state::ReplicationStats;
 pub use replication_stats_boundary::BucketStats;
 pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
 pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;

crates/ecstore/src/bucket/replication/replication_config_boundary.rs

@@ -14,5 +14,5 @@
 
 pub use rustfs_replication::{
     ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, replication_target_arns,
-    should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_target_arns,
+    should_remove_replication_target, validate_replication_config_target_arns,
 };

crates/ecstore/src/bucket/replication/replication_object_bridge.rs

@@ -14,11 +14,8 @@
 
 use std::{collections::HashMap, sync::Arc};
 
-use super::replication_error_boundary::Result;
 use super::replication_filemeta_boundary::{ReplicateDecision, ReplicationStatusType, ReplicationType};
-use super::replication_object_config::{
-    check_replicate_delete, check_replicate_delete_strict, get_must_replicate_options, must_replicate,
-};
+use super::replication_object_config::{check_replicate_delete, get_must_replicate_options, must_replicate};
 use super::replication_object_decision_boundary::MustReplicateOptions;
 use super::replication_pool::{schedule_replication, schedule_replication_delete};
 use super::replication_queue_boundary::DeletedObjectReplicationInfo;
@@ -53,16 +50,6 @@ impl ReplicationObjectBridge {
         check_replicate_delete(bucket, object, source, opts, get_error).await
     }
 
-    pub async fn check_delete_strict(
-        bucket: &str,
-        object: &ObjectToDelete,
-        source: &ObjectInfo,
-        opts: &ObjectOptions,
-        get_error: Option<String>,
-    ) -> Result<ReplicateDecision> {
-        check_replicate_delete_strict(bucket, object, source, opts, get_error).await
-    }
-
     pub async fn schedule_object<S: ReplicationStorage>(
         object: ObjectInfo,
         storage: Arc<S>,

crates/ecstore/src/bucket/replication/replication_object_config.rs

@@ -169,8 +169,9 @@ pub(crate) async fn check_replicate_delete(
     del_opts: &ObjectOptions,
     gerr: Option<String>,
 ) -> ReplicateDecision {
-    match check_replicate_delete_strict(bucket, dobj, oi, del_opts, gerr).await {
-        Ok(decision) => decision,
+    let rcfg = match get_replication_config(bucket).await {
+        Ok(Some(config)) => config,
+        Ok(None) => return ReplicateDecision::default(),
         Err(err) => {
             error!(
                 event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
@@ -181,30 +182,16 @@ pub(crate) async fn check_replicate_delete(
                 error = %err,
                 "Failed to look up replication config for delete replication"
             );
-            ReplicateDecision::default()
+            return ReplicateDecision::default();
         }
-    }
-}
-
-pub(crate) async fn check_replicate_delete_strict(
-    bucket: &str,
-    dobj: &ObjectToDelete,
-    oi: &ObjectInfo,
-    del_opts: &ObjectOptions,
-    gerr: Option<String>,
-) -> Result<ReplicateDecision> {
-    let rcfg = match get_replication_config(bucket).await {
-        Ok(Some(config)) => config,
-        Ok(None) => return Ok(ReplicateDecision::default()),
-        Err(err) => return Err(err),
     };
 
     if del_opts.replication_request {
-        return Ok(ReplicateDecision::default());
+        return ReplicateDecision::default();
     }
 
-    if !del_opts.versioned && !del_opts.version_suspended {
-        return Ok(ReplicateDecision::default());
+    if !del_opts.versioned {
+        return ReplicateDecision::default();
     }
 
     let replication_delete = object_to_delete_for_replication(dobj);
@@ -222,7 +209,7 @@ pub(crate) async fn check_replicate_delete_strict(
     let mut dsc = ReplicateDecision::new();
 
     if tgt_arns.is_empty() {
-        return Ok(dsc);
+        return dsc;
     }
 
     for tgt_arn in tgt_arns {
@@ -252,7 +239,7 @@ pub(crate) async fn check_replicate_delete_strict(
         dsc.set(tgt_dsc);
     }
 
-    Ok(dsc)
+    dsc
 }
 
 pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision {

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -1151,6 +1151,60 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
         dobj.delete_object.version_id
     };
 
+    let _rcfg = match get_replication_config(&bucket).await {
+        Ok(Some(config)) => config,
+        Ok(None) => {
+            debug!(
+                event = EVENT_REPLICATION_DELETE_SKIPPED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                bucket = %bucket,
+                reason = "replication_config_missing",
+                "Skipping replication delete because replication config is missing"
+            );
+            send_local_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: dobj.delete_object.object_name.clone(),
+                    version_id,
+                    delete_marker: dobj.delete_object.delete_marker,
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                ..Default::default()
+            });
+
+            return;
+        }
+        Err(err) => {
+            debug!(
+                event = EVENT_REPLICATION_DELETE_SKIPPED,
+                component = LOG_COMPONENT_ECSTORE,
+                subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                bucket = %bucket,
+                error = %err,
+                reason = "replication_config_lookup_failed",
+                "Skipping replication delete because replication config lookup failed"
+            );
+            send_local_event(EventArgs {
+                event_name: EventName::ObjectReplicationNotTracked.to_string(),
+                bucket_name: bucket.clone(),
+                object: ObjectInfo {
+                    bucket: bucket.clone(),
+                    name: dobj.delete_object.object_name.clone(),
+                    version_id,
+                    delete_marker: dobj.delete_object.delete_marker,
+                    ..Default::default()
+                },
+                user_agent: "Internal: [Replication]".to_string(),
+                ..Default::default()
+            });
+            return;
+        }
+    };
+
     if dobj.delete_object.delete_marker
         && let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id
     {

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -22,9 +22,9 @@ use super::replication_stats_boundary::{
     QueueCache, ReplicationMetricScope, SRMetricsSummary, XferStats,
 };
 use super::runtime_boundary as runtime_sources;
-use std::collections::{HashMap, hash_map::Entry};
+use std::collections::HashMap;
+use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
-use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
 use std::time::{Duration, SystemTime};
 use tokio::sync::{Mutex, RwLock};
 use tokio::time::interval;
@@ -143,7 +143,7 @@ pub struct ReplicationStats {
     // Active worker statistics
     pub workers: Arc<Mutex<ActiveWorkerStat>>,
     // Queue statistics cache
-    pub q_cache: Arc<StdMutex<QueueCache>>,
+    pub q_cache: Arc<Mutex<QueueCache>>,
     // Proxy statistics cache
     pub p_cache: Arc<Mutex<ProxyStatsCache>>,
     // MRF backlog statistics (simplified)
@@ -153,89 +153,12 @@ pub struct ReplicationStats {
     pub most_recent_stats: Arc<Mutex<HashMap<String, BucketStats>>>,
 }
 
-#[derive(Debug, Clone, Default, PartialEq, Eq)]
-pub struct RuntimeReplicationTargetBacklog {
-    pub bucket: String,
-    pub target_arn: String,
-    pub count: u64,
-    pub bytes: u64,
-}
-
-type TargetQueueKey = (String, String);
-type TargetQueueCache = HashMap<TargetQueueKey, InQueueMetric>;
-
-struct TargetQueueCacheSlot {
-    owner: Weak<StdMutex<QueueCache>>,
-    metrics: TargetQueueCache,
-}
-
-impl TargetQueueCacheSlot {
-    fn new(owner: &Arc<StdMutex<QueueCache>>) -> Self {
-        Self {
-            owner: Arc::downgrade(owner),
-            metrics: TargetQueueCache::default(),
-        }
-    }
-
-    fn belongs_to(&self, owner: &Arc<StdMutex<QueueCache>>) -> bool {
-        self.owner.upgrade().is_some_and(|current| Arc::ptr_eq(&current, owner))
-    }
-}
-
-// Keep runtime target counters outside ReplicationStats to preserve its public struct shape.
-static TARGET_QUEUE_CACHES: LazyLock<StdMutex<Vec<TargetQueueCacheSlot>>> = LazyLock::new(|| StdMutex::new(Vec::new()));
-
-fn i64_to_u64_floor_zero(value: i64) -> u64 {
-    u64::try_from(value.max(0)).unwrap_or(0)
-}
-
-fn normalized_target_arns(target_arns: &[String]) -> Vec<&str> {
-    let mut target_arns = target_arns
-        .iter()
-        .map(String::as_str)
-        .filter(|target_arn| !target_arn.is_empty())
-        .collect::<Vec<_>>();
-    target_arns.sort_unstable();
-    target_arns.dedup();
-    target_arns
-}
-
-fn target_queue_cache_snapshot(cache: &TargetQueueCache) -> Vec<RuntimeReplicationTargetBacklog> {
-    cache
-        .iter()
-        .filter_map(|((bucket, target_arn), metric)| {
-            let count = i64_to_u64_floor_zero(metric.curr.get_current_count());
-            let bytes = i64_to_u64_floor_zero(metric.curr.get_current_bytes());
-            (count > 0 || bytes > 0).then(|| RuntimeReplicationTargetBacklog {
-                bucket: bucket.clone(),
-                target_arn: target_arn.clone(),
-                count,
-                bytes,
-            })
-        })
-        .collect()
-}
-
-fn prune_stale_target_queue_caches(caches: &mut Vec<TargetQueueCacheSlot>) {
-    caches.retain(|slot| slot.owner.strong_count() > 0);
-}
-
-fn with_target_queue_caches<T>(f: impl FnOnce(&mut Vec<TargetQueueCacheSlot>) -> T) -> T {
-    match TARGET_QUEUE_CACHES.lock() {
-        Ok(mut caches) => f(&mut caches),
-        Err(poisoned) => {
-            let mut caches = poisoned.into_inner();
-            f(&mut caches)
-        }
-    }
-}
-
 impl ReplicationStats {
     pub fn new() -> Self {
         Self {
             sr_stats: Arc::new(SRStats::new()),
             workers: Arc::new(Mutex::new(ActiveWorkerStat::new())),
-            q_cache: Arc::new(StdMutex::new(QueueCache::new())),
+            q_cache: Arc::new(Mutex::new(QueueCache::new())),
             p_cache: Arc::new(Mutex::new(ProxyStatsCache::new())),
             mrf_stats: HashMap::new(),
             cache: Arc::new(RwLock::new(HashMap::new())),
@@ -275,9 +198,8 @@ impl ReplicationStats {
             let mut interval = interval(Duration::from_secs(2));
             loop {
                 interval.tick().await;
-                if let Ok(mut cache) = q_cache_clone.lock() {
-                    cache.update();
-                }
+                let mut cache = q_cache_clone.lock().await;
+                cache.update();
             }
         });
     }
@@ -469,13 +391,12 @@ impl ReplicationStats {
         drop(cache);
 
         {
-            if let Ok(q_cache) = self.q_cache.lock() {
-                for (bucket, queue_stats) in &q_cache.bucket_stats {
-                    let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
-                    bucket_stats.q_stat = queue_stats.snapshot();
-                    bucket_stats.mark_node_local_provider_available();
-                    bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
-                }
+            let q_cache = self.q_cache.lock().await;
+            for (bucket, queue_stats) in &q_cache.bucket_stats {
+                let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
+                bucket_stats.q_stat = queue_stats.snapshot();
+                bucket_stats.mark_node_local_provider_available();
+                bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
             }
         }
 
@@ -508,11 +429,8 @@ impl ReplicationStats {
         let boot_time = SystemTime::UNIX_EPOCH; // simplified implementation
         let uptime = SystemTime::now().duration_since(boot_time).unwrap_or_default().as_secs() as i64;
 
-        let queued = self
-            .q_cache
-            .lock()
-            .map(|q_cache| q_cache.get_site_stats())
-            .unwrap_or_default();
+        let q_cache = self.q_cache.lock().await;
+        let queued = q_cache.get_site_stats();
 
         let p_cache = self.p_cache.lock().await;
         let proxied = p_cache.get_site_stats();
@@ -715,9 +633,8 @@ impl ReplicationStats {
         drop(cache);
 
         {
-            if let Ok(q_cache) = self.q_cache.lock()
-                && let Some(queue_stats) = q_cache.bucket_stats.get(bucket)
-            {
+            let q_cache = self.q_cache.lock().await;
+            if let Some(queue_stats) = q_cache.bucket_stats.get(bucket) {
                 replication_stats.q_stat = queue_stats.snapshot();
             }
         }
@@ -748,79 +665,31 @@ impl ReplicationStats {
     }
 
     /// Increase queue statistics
-    pub fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
-        if let Ok(mut q_cache) = self.q_cache.lock() {
-            q_cache.inc(bucket, size);
-        }
+    pub async fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
+        let mut q_cache = self.q_cache.lock().await;
+        let stats = q_cache
+            .bucket_stats
+            .entry(bucket.to_string())
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
+
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
     }
 
     /// Decrease queue statistics
-    pub fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
-        if let Ok(mut q_cache) = self.q_cache.lock() {
-            q_cache.dec(bucket, size);
-        }
-    }
+    pub async fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
+        let mut q_cache = self.q_cache.lock().await;
+        let stats = q_cache
+            .bucket_stats
+            .entry(bucket.to_string())
+            .or_insert_with(InQueueMetric::default);
+        stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
 
-    pub(crate) fn inc_target_q(&self, bucket: &str, target_arns: &[String], size: i64) {
-        let target_arns = normalized_target_arns(target_arns);
-        if target_arns.is_empty() {
-            return;
-        }
-        with_target_queue_caches(|caches| {
-            prune_stale_target_queue_caches(caches);
-            let slot_index = match caches.iter().position(|slot| slot.belongs_to(&self.q_cache)) {
-                Some(index) => index,
-                None => {
-                    caches.push(TargetQueueCacheSlot::new(&self.q_cache));
-                    caches.len() - 1
-                }
-            };
-            let slot = &mut caches[slot_index];
-            let bucket = bucket.to_string();
-            for target_arn in target_arns {
-                let metric = match slot.metrics.entry((bucket.clone(), target_arn.to_string())) {
-                    Entry::Occupied(entry) => entry.into_mut(),
-                    Entry::Vacant(entry) => entry.insert(InQueueMetric::default()),
-                };
-                metric.curr.add_current(size, 1);
-            }
-        });
-    }
-
-    pub(crate) fn dec_target_q(&self, bucket: &str, target_arns: &[String], size: i64) {
-        let target_arns = normalized_target_arns(target_arns);
-        if target_arns.is_empty() {
-            return;
-        }
-        with_target_queue_caches(|caches| {
-            prune_stale_target_queue_caches(caches);
-            if let Some(slot) = caches.iter_mut().find(|slot| slot.belongs_to(&self.q_cache)) {
-                let bucket = bucket.to_string();
-                for target_arn in target_arns {
-                    if let Some(metric) = slot.metrics.get_mut(&(bucket.clone(), target_arn.to_string())) {
-                        metric.curr.subtract_current(size, 1);
-                    }
-                }
-                slot.metrics
-                    .retain(|_, metric| metric.curr.get_current_count() > 0 || metric.curr.get_current_bytes() > 0);
-            }
-        });
-    }
-
-    pub fn runtime_target_backlog_snapshot(&self) -> Vec<RuntimeReplicationTargetBacklog> {
-        let mut snapshot = with_target_queue_caches(|caches| {
-            caches
-                .iter()
-                .find(|slot| slot.belongs_to(&self.q_cache))
-                .map(|slot| target_queue_cache_snapshot(&slot.metrics))
-                .unwrap_or_default()
-        });
-        snapshot.sort_by(|left, right| {
-            left.bucket
-                .cmp(&right.bucket)
-                .then_with(|| left.target_arn.cmp(&right.target_arn))
-        });
-        snapshot
+        q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
+        q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
     }
 
     /// Increase proxy metrics
@@ -846,94 +715,6 @@ impl Default for ReplicationStats {
 mod tests {
     use super::*;
 
-    #[test]
-    fn runtime_target_backlog_snapshot_tracks_targets() {
-        let stats = ReplicationStats::new();
-        stats.inc_target_q(
-            "photos",
-            &[
-                "arn:rustfs:replication:target-b".to_string(),
-                "arn:rustfs:replication:target-a".to_string(),
-                "arn:rustfs:replication:target-a".to_string(),
-            ],
-            1024,
-        );
-
-        let snapshot = stats.runtime_target_backlog_snapshot();
-
-        assert_eq!(
-            snapshot,
-            vec![
-                RuntimeReplicationTargetBacklog {
-                    bucket: "photos".to_string(),
-                    target_arn: "arn:rustfs:replication:target-a".to_string(),
-                    count: 1,
-                    bytes: 1024,
-                },
-                RuntimeReplicationTargetBacklog {
-                    bucket: "photos".to_string(),
-                    target_arn: "arn:rustfs:replication:target-b".to_string(),
-                    count: 1,
-                    bytes: 1024,
-                },
-            ]
-        );
-    }
-
-    #[test]
-    fn runtime_target_backlog_ignores_empty_targets() {
-        let stats = ReplicationStats::new();
-        stats.inc_target_q("photos", &["".to_string()], 1024);
-
-        assert!(stats.runtime_target_backlog_snapshot().is_empty());
-    }
-
-    #[test]
-    fn runtime_target_backlog_is_scoped_to_stats_instance() {
-        let first = ReplicationStats::new();
-        let second = ReplicationStats::new();
-        first.inc_target_q("photos", &["arn:rustfs:replication:target-a".to_string()], 1024);
-
-        assert!(second.runtime_target_backlog_snapshot().is_empty());
-        assert_eq!(first.runtime_target_backlog_snapshot()[0].count, 1);
-    }
-
-    #[test]
-    fn runtime_target_backlog_decrements_with_saturation() {
-        let stats = ReplicationStats::new();
-        let target_arns = ["arn:rustfs:replication:target-a".to_string()];
-        stats.inc_target_q("photos", &target_arns, 1024);
-        stats.dec_target_q("photos", &target_arns, 2048);
-
-        assert!(stats.runtime_target_backlog_snapshot().is_empty());
-    }
-
-    #[test]
-    fn runtime_target_backlog_prunes_stale_sidecar_on_next_access() {
-        {
-            let stats = ReplicationStats::new();
-            stats.inc_target_q("photos", &["arn:rustfs:replication:target-a".to_string()], 1024);
-            assert!(
-                TARGET_QUEUE_CACHES
-                    .lock()
-                    .expect("target queue cache mutex")
-                    .iter()
-                    .any(|slot| slot.owner.strong_count() > 0)
-            );
-        }
-
-        let stats = ReplicationStats::new();
-        stats.inc_target_q("photos", &["arn:rustfs:replication:target-b".to_string()], 1024);
-
-        assert!(
-            TARGET_QUEUE_CACHES
-                .lock()
-                .expect("target queue cache mutex")
-                .iter()
-                .all(|slot| slot.owner.strong_count() > 0)
-        );
-    }
-
     #[tokio::test]
     async fn test_replication_stats_new() {
         let stats = ReplicationStats::new();
@@ -1020,14 +801,14 @@ mod tests {
     async fn latest_stats_include_queue_until_drained() {
         let stats = ReplicationStats::new();
 
-        stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object);
+        stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object).await;
         let queued = stats.get_latest_replication_stats("queued-bucket").await;
         assert!(queued.replication_stats.provider_available);
         assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
         assert_eq!(queued.replication_stats.q_stat.curr.bytes, 4096);
         assert_eq!(queued.replication_stats.queue_scope, ReplicationMetricScope::NodeLocal);
 
-        stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object);
+        stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object).await;
         let drained = stats.get_latest_replication_stats("queued-bucket").await;
         assert_eq!(drained.replication_stats.q_stat.curr.count, 0);
         assert_eq!(drained.replication_stats.q_stat.curr.bytes, 0);
@@ -1130,7 +911,7 @@ mod tests {
         for _ in 0..32 {
             let stats = Arc::clone(&stats);
             tasks.push(tokio::spawn(async move {
-                stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object);
+                stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object).await;
             }));
         }
         for task in tasks {

crates/ecstore/src/cluster/rpc/mod.rs

@@ -44,7 +44,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub(crate) use peer_rest_client::TierConfigReloadOutcome;
 pub use peer_rest_client::{
-    KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
     SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -45,9 +45,9 @@ use rustfs_protos::proto_gen::node_service::{
     HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
     LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
     LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
-    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
-    StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
-    TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
+    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
+    StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
+    TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
     tier_mutation_control_service_client::TierMutationControlServiceClient,
 };
 pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -71,12 +71,6 @@ use uuid::Uuid;
 
 pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
 pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
-/// Dynamic config subsystem for the cluster-persisted KMS configuration.
-///
-/// KMS configuration lives in its own cluster object rather than in the server
-/// config document, so it is not a `ServerConfig` subsystem; it only shares the
-/// reload signal transport.
-pub const KMS_SIGNAL_SUBSYSTEM: &str = "kms";
 const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
 const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
 const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
@@ -105,13 +99,8 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
 }
 
 fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
-    // The version stays pinned to DYNAMIC_CONFIG_PROTOCOL_VERSION rather than
-    // being bumped per subsystem: the comparison is shared, so raising it would
-    // retire peers that already converge scanner and heal config correctly.
-    // Subsystems added after a peer was built are rejected by that peer's own
-    // subsystem allow-list, which surfaces as an explicit failed signal.
     if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
-        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS | KMS_SIGNAL_SUBSYSTEM)
+        && matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
         && protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
     {
         return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
@@ -1511,22 +1500,6 @@ impl PeerRestClient {
     }
 
     pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
-        self.signal_service_checked(sig, sub_sys, dry_run).await.map(|_| ())
-    }
-
-    /// Report the KMS configuration fingerprint the peer is currently running.
-    ///
-    /// Sent as a dry-run reload signal so the peer answers without swapping its
-    /// own configuration. `None` means the peer has no KMS configuration. The
-    /// fingerprint is advisory and feeds cluster status reporting only, so the
-    /// response is not proof-signed.
-    pub async fn kms_config_fingerprint(&self) -> Result<Option<String>> {
-        self.signal_service_checked(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, true)
-            .await
-            .map(|response| response.config_fingerprint)
-    }
-
-    async fn signal_service_checked(&self, sig: u64, sub_sys: &str, dry_run: bool) -> Result<SignalServiceResponse> {
         self.finalize_result(
             async {
                 let mut client = self.get_client().await?;
@@ -1547,7 +1520,7 @@ impl PeerRestClient {
                     return Err(Error::other(""));
                 }
                 validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
-                Ok(response)
+                Ok(())
             }
             .await,
         )
@@ -2374,19 +2347,6 @@ mod tests {
             .expect("full refresh compatibility is guarded by its scanner preflight");
     }
 
-    #[test]
-    fn dynamic_kms_config_requires_versioned_peer_acknowledgement() {
-        let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, 0)
-            .expect_err("an unversioned peer must not claim KMS config convergence");
-        assert!(err.to_string().contains("does not support dynamic"));
-        validate_signal_service_protocol(
-            SERVICE_SIGNAL_RELOAD_DYNAMIC,
-            KMS_SIGNAL_SUBSYSTEM,
-            rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
-        )
-        .expect("a current peer should support dynamic KMS config");
-    }
-
     #[test]
     fn peer_rest_client_marks_network_like_errors() {
         assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));

crates/ecstore/src/disk/local.rs

@@ -5078,7 +5078,7 @@ impl LocalDisk {
         Ok((buf, mtime))
     }
 
-    #[hotpath::measure(impl_type = "LocalDisk")]
+    #[hotpath::measure]
     async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
         check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
 
@@ -5121,7 +5121,7 @@ impl LocalDisk {
         Ok((data, modtime))
     }
 
-    #[hotpath::measure(impl_type = "LocalDisk")]
+    #[hotpath::measure]
     async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
         // TODO: timeout support
         let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
@@ -6641,49 +6641,6 @@ impl LocalDisk {
         let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
         restore_delete_rollback_after_error(object_dir, &xl_path, Some(rollback_dir), volume, object, stage, err).await
     }
-
-    /// Execute every deferred data-dir deletion pending on `volume` right now,
-    /// even while snapshot leases are still held. Bucket deletion requires it:
-    /// a streaming reader defers the physical cleanup of an already-deleted
-    /// version, and a non-force `delete_volume` would otherwise fail closed
-    /// with `VolumeNotEmpty` on those remnants even though the bucket is
-    /// logically empty. The still-active readers keep their open descriptors;
-    /// only path-based reopens observe the removal.
-    async fn settle_pending_snapshot_deletes(&self, volume: &str) {
-        let pending: Vec<(SnapshotLeaseKey, DeleteOptions)> = {
-            let mut registry = self.snapshot_leases.lock().await;
-            registry
-                .entries
-                .iter_mut()
-                .filter(|(key, entry)| key.volume == volume && !entry.deleting && entry.pending_delete.is_some())
-                .map(|(key, entry)| {
-                    entry.deleting = true;
-                    (key.clone(), entry.pending_delete.clone().expect("filtered on Some"))
-                })
-                .collect()
-        };
-
-        for (key, opts) in pending {
-            let result = self.delete_unleased(&key.volume, &key.path, &opts).await;
-            let mut registry = self.snapshot_leases.lock().await;
-            match result {
-                Ok(()) => {
-                    registry.entries.remove(&key);
-                }
-                Err(err) => {
-                    if let Some(entry) = registry.entries.get_mut(&key) {
-                        entry.deleting = false;
-                    }
-                    warn!(
-                        volume = %key.volume,
-                        path = %key.path,
-                        error = %err,
-                        "failed to settle deferred data-dir deletion before volume removal"
-                    );
-                }
-            }
-        }
-    }
 }
 
 #[async_trait::async_trait]
@@ -9165,14 +9122,6 @@ impl DiskAPI for LocalDisk {
         let p = self.get_bucket_path(volume)?;
         let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await;
 
-        // A streaming reader's snapshot lease defers the physical cleanup of
-        // data dirs whose version delete already committed. Those remnants are
-        // logically deleted, so run the parked cleanups now instead of letting
-        // the non-force removal below fail closed on them (the s3-tests SSE-C
-        // teardown races exactly this way: DeleteObjects, then DeleteBucket
-        // while an abandoned GET body still pins the lease).
-        self.settle_pending_snapshot_deletes(volume).await;
-
         // Non-force removes empty directory remnants children-first with
         // non-recursive rmdir calls. A file that exists during the scan, or
         // appears before its parent is removed, fails closed with
@@ -16032,74 +15981,6 @@ mod test {
         assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
     }
 
-    #[tokio::test]
-    async fn delete_volume_settles_lease_deferred_cleanup() {
-        use tempfile::tempdir;
-
-        let root_dir = tempdir().expect("temp dir should be created");
-        let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
-        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
-        let volume = "snapshot-lease-bucket-delete";
-        let object = "multipart_enc";
-        let version_id = Uuid::new_v4();
-        let data_dir = Uuid::new_v4();
-        let rollback_dir = Uuid::new_v4();
-        let data_path = path_join_buf(&[object, &data_dir.to_string()]);
-        let part = path_join_buf(&[&data_path, "part.1"]);
-        ensure_test_volume(&disk, volume).await;
-        disk.write_all(volume, &part, Bytes::from_static(b"payload"))
-            .await
-            .expect("shard should be written");
-        let fi = test_file_info(object, version_id, Some(data_dir), None);
-        disk.write_all(volume, &path_join_buf(&[object, STORAGE_FORMAT_FILE]), test_meta(fi.clone()).into())
-            .await
-            .expect("metadata should be written");
-
-        // An abandoned streaming GET pins the data dir with a snapshot lease.
-        let snapshot = disk
-            .acquire_snapshot_lease(volume, &data_path)
-            .await
-            .expect("snapshot lease should be acquired");
-        disk.delete_version(
-            volume,
-            object,
-            fi.clone(),
-            false,
-            DeleteOptions {
-                old_data_dir: Some(rollback_dir),
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("version delete should commit metadata");
-        disk.delete(
-            volume,
-            &format!("{object}/{rollback_dir}"),
-            DeleteOptions {
-                recursive: true,
-                immediate: true,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("version delete should schedule physical cleanup");
-
-        // The bucket is logically empty; a non-force volume delete must settle
-        // the deferred data-dir cleanup instead of failing with VolumeNotEmpty.
-        disk.delete_volume(volume, false)
-            .await
-            .expect("bucket delete must not observe lease-deferred remnants");
-        assert!(matches!(
-            disk.read_all(volume, &part).await,
-            Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
-        ));
-
-        // The late lease release finds nothing pending and stays idempotent.
-        disk.release_snapshot_lease(volume, &data_path, snapshot)
-            .await
-            .expect("releasing the lease after bucket deletion should be a no-op");
-    }
-
     #[tokio::test]
     async fn version_delete_cleanup_intent_survives_local_disk_restart() {
         use tempfile::tempdir;

crates/ecstore/src/erasure/coding/bitrot.rs

@@ -103,7 +103,7 @@ where
     /// or `out` is larger than one shard. On error `out`'s contents are
     /// unspecified but never contain bytes that failed the hash check — the copy
     /// happens only after verification.
-    #[hotpath::measure(impl_type = "BitrotReader")]
+    #[hotpath::measure]
     pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
         let want = out.len();
         self.begin_read(want)?;
@@ -303,7 +303,7 @@ where
 
     /// Write a (hash+data) block. Returns the number of data bytes written.
     /// Returns an error if called after a short write or if data exceeds shard_size.
-    #[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
+    #[hotpath::measure(label = "BitrotWriter::write")]
     pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
         if buf.is_empty() {
             return Ok(0);

crates/ecstore/src/erasure/coding/decode.rs

@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
 where
     R: crate::erasure::coding::ShardSource,
 {
-    #[hotpath::measure(impl_type = "ParallelReader")]
+    #[hotpath::measure]
     pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
         // On the reconstruction-verifying GET path, read every live shard reader
         // in lockstep so all readers advance one block per stripe and stay
@@ -1505,7 +1505,7 @@ where
 }
 
 impl Erasure {
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub async fn decode<W, R>(
         &self,
         writer: &mut W,
@@ -1645,28 +1645,15 @@ impl Erasure {
             }
             Err(e) => {
                 record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
-                let reason = classify_io_error(&e);
-                if reason == GetObjectFailureReason::DownstreamClosed {
-                    debug!(
-                        block_offset,
-                        block_length,
-                        bytes_written = *written,
-                        stage = GET_STAGE_EMIT,
-                        reason = reason.as_str(),
-                        error = ?e,
-                        "Erasure decode stopped after downstream closed"
-                    );
-                } else {
-                    error!(
-                        block_offset,
-                        block_length,
-                        bytes_written = *written,
-                        stage = GET_STAGE_EMIT,
-                        reason = reason.as_str(),
-                        error = ?e,
-                        "Erasure decode failed to emit reconstructed data"
-                    );
-                }
+                error!(
+                    block_offset,
+                    block_length,
+                    bytes_written = *written,
+                    stage = GET_STAGE_EMIT,
+                    reason = classify_io_error(&e).as_str(),
+                    error = ?e,
+                    "Erasure decode failed to emit reconstructed data"
+                );
                 *ret_err = Some(e);
                 return StripeFlow::Stop;
             }
@@ -1958,7 +1945,7 @@ mod tests {
     use std::io::Cursor;
     use std::pin::Pin;
     use std::sync::{
-        Arc, Mutex,
+        Arc,
         atomic::{AtomicUsize, Ordering},
     };
     use std::task::{Context, Poll};
@@ -2133,59 +2120,6 @@ mod tests {
         }
     }
 
-    struct DownstreamClosedWriter;
-
-    impl AsyncWrite for DownstreamClosedWriter {
-        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
-            Poll::Ready(Err(crate::diagnostics::get::mark_get_object_downstream_closed(io::Error::new(
-                ErrorKind::BrokenPipe,
-                "injected downstream close",
-            ))))
-        }
-
-        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-
-        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-    }
-
-    #[derive(Clone, Default)]
-    struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
-
-    struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
-
-    impl CapturedLogs {
-        fn contents(&self) -> String {
-            String::from_utf8(self.0.lock().expect("captured logs mutex should not be poisoned").clone())
-                .expect("captured logs should be valid UTF-8")
-        }
-    }
-
-    impl std::io::Write for CapturedLogWriter {
-        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-            self.0
-                .lock()
-                .expect("captured logs mutex should not be poisoned")
-                .extend_from_slice(buf);
-            Ok(buf.len())
-        }
-
-        fn flush(&mut self) -> std::io::Result<()> {
-            Ok(())
-        }
-    }
-
-    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
-        type Writer = CapturedLogWriter;
-
-        fn make_writer(&'a self) -> Self::Writer {
-            CapturedLogWriter(Arc::clone(&self.0))
-        }
-    }
-
     #[test]
     fn parallel_reader_constructor_variants_preserve_read_cost_and_verification_flags() {
         let erasure = Erasure::new(2, 1, 64);
@@ -2281,47 +2215,6 @@ mod tests {
         assert_eq!(err.to_string(), "injected emit failure");
     }
 
-    #[tokio::test(flavor = "current_thread")]
-    async fn erasure_decode_logs_reconstructed_downstream_close_at_debug() {
-        let logs = CapturedLogs::default();
-        let subscriber = tracing_subscriber::fmt()
-            .with_max_level(tracing::Level::DEBUG)
-            .with_writer(logs.clone())
-            .with_ansi(false)
-            .without_time()
-            .finish();
-        let _guard = tracing::subscriber::set_default(subscriber);
-
-        let erasure = Erasure::new(2, 1, 64);
-        let data: Vec<u8> = (0..64).collect();
-        let shard_size = erasure.shard_size();
-        let encoded = erasure.encode_data(&data).expect("test data should encode");
-        let readers = vec![
-            None,
-            Some(BitrotReader::new(
-                Cursor::new(encoded[1].to_vec()),
-                shard_size,
-                HashAlgorithm::None,
-                false,
-            )),
-            Some(BitrotReader::new(
-                Cursor::new(encoded[2].to_vec()),
-                shard_size,
-                HashAlgorithm::None,
-                false,
-            )),
-        ];
-
-        let mut writer = DownstreamClosedWriter;
-        let (written, err) = erasure.decode(&mut writer, readers, 0, data.len(), data.len()).await;
-
-        assert_eq!(written, 0);
-        assert_eq!(err.expect("downstream close must still terminate the GET").kind(), ErrorKind::BrokenPipe);
-        let captured = logs.contents();
-        assert!(captured.contains("Erasure decode stopped after downstream closed"));
-        assert!(!captured.contains("Erasure decode failed to emit reconstructed data"));
-    }
-
     #[tokio::test]
     async fn test_erasure_decode_rejects_reader_count_and_range_overflow() {
         let erasure = Erasure::new(2, 1, 64);

crates/ecstore/src/erasure/coding/encode.rs

@@ -28,11 +28,8 @@ use std::vec;
 use tokio::io::AsyncRead;
 use tokio::runtime::RuntimeFlavor;
 use tokio::sync::mpsc;
-use tokio::task::{JoinError, JoinHandle};
 use tracing::error;
 
-/// Queue-capacity input for encoded blocks awaiting shard writers; it is not a
-/// per-PUT or process-RSS memory limit.
 const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
 const ENV_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: &str = "RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS";
 const ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: &str = "RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST";
@@ -90,33 +87,6 @@ fn use_bytesmut_ingest() -> bool {
         rustfs_utils::get_env_bool(ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST, DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST)
     })
 }
-
-/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
-/// task when its `JoinHandle` is dropped, so the producer must be aborted when
-/// an upload is cancelled before the encode pipeline finishes.
-struct AbortOnDropTask<T>(JoinHandle<T>);
-
-impl<T> AbortOnDropTask<T> {
-    fn new(task: JoinHandle<T>) -> Self {
-        Self(task)
-    }
-
-    async fn abort_and_wait(&mut self) {
-        self.0.abort();
-        let _ = (&mut self.0).await;
-    }
-
-    async fn join(&mut self) -> Result<T, JoinError> {
-        (&mut self.0).await
-    }
-}
-
-impl<T> Drop for AbortOnDropTask<T> {
-    fn drop(&mut self) {
-        self.0.abort();
-    }
-}
-
 /// Read up to `limit` bytes into `buf`'s uninitialized spare capacity, appending after its
 /// current length, and distinguish a clean EOF from a short read.
 ///
@@ -163,65 +133,22 @@ fn queued_block_bytes(block: &[Bytes]) -> usize {
     block.iter().map(Bytes::len).sum()
 }
 
-/// Owns an encoded queue entry's gauge contribution until its consumer takes it.
-struct QueuedInflightBytes {
-    bytes: usize,
-}
-
-impl QueuedInflightBytes {
-    fn new(bytes: usize) -> Self {
-        rustfs_io_metrics::add_ec_encode_inflight_bytes(bytes);
-        Self { bytes }
+async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
+    while let Some(block) = rx.recv().await {
+        rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
     }
-
-    fn settle(&mut self) {
-        let bytes = std::mem::take(&mut self.bytes);
-        if bytes != 0 {
-            rustfs_io_metrics::remove_ec_encode_inflight_bytes(bytes);
-        }
-    }
-}
-
-impl Drop for QueuedInflightBytes {
-    fn drop(&mut self) {
-        self.settle();
-    }
-}
-
-/// Couples an encoded block with its queue gauge contribution. Keeping the
-/// guard in the queue entry also covers Tokio sends that complete through a
-/// permit after the receiver has closed.
-struct InflightEntry<T> {
-    entry: T,
-    accounting: QueuedInflightBytes,
-}
-
-impl<T> InflightEntry<T> {
-    fn new(entry: T, bytes: usize) -> Self {
-        Self {
-            entry,
-            accounting: QueuedInflightBytes::new(bytes),
-        }
-    }
-
-    fn into_inner(mut self) -> T {
-        self.accounting.settle();
-        self.entry
-    }
-}
-
-async fn send_queued<T>(
-    sender: &mpsc::Sender<InflightEntry<T>>,
-    entry: T,
-    bytes: usize,
-) -> Result<(), mpsc::error::SendError<InflightEntry<T>>> {
-    sender.send(InflightEntry::new(entry, bytes)).await
 }
 
 fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
     batch.iter().map(|block| queued_block_bytes(block)).sum()
 }
 
+async fn drain_queued_batched_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Vec<Bytes>>>) {
+    while let Some(batch) = rx.recv().await {
+        rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
+    }
+}
+
 fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
     summary.dominant_error_label
 }
@@ -577,7 +504,7 @@ impl Erasure {
         Ok((reader, total))
     }
 
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub async fn encode<R>(
         self: Arc<Self>,
         reader: R,
@@ -613,14 +540,13 @@ impl Erasure {
             ));
         }
 
-        // Bound queued encoded blocks by a queue budget; this does not bound
-        // reader, encoder, writer, allocator, or process-RSS memory.
+        // Bound queued encoded blocks by memory budget to avoid per-request spikes.
         let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
         let max_inflight_bytes = erasure_encode_max_inflight_bytes();
         let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
-        let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
+        let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
 
-        let mut task = AbortOnDropTask::new(tokio::spawn(async move {
+        let task = tokio::spawn(async move {
             let block_size = self.block_size;
             let mut total = 0;
             if use_bytesmut_ingest {
@@ -641,9 +567,10 @@ impl Erasure {
                             let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
                             buf = BytesMut::with_capacity(ingest_capacity);
                             let queued_bytes = queued_block_bytes(&res);
-                            let _producer_stage = rustfs_io_metrics::track_ec_encode_producer_bytes(queued_bytes);
+                            rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                             let send_wait_stage_start = stage_timer_if_enabled();
-                            if let Err(err) = send_queued(&tx, res, queued_bytes).await {
+                            if let Err(err) = tx.send(res).await {
+                                rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
                                 return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                             }
                             record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -671,9 +598,10 @@ impl Erasure {
                             let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
                             buf = returned_buf;
                             let queued_bytes = queued_block_bytes(&res);
-                            let _producer_stage = rustfs_io_metrics::track_ec_encode_producer_bytes(queued_bytes);
+                            rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
                             let send_wait_stage_start = stage_timer_if_enabled();
-                            if let Err(err) = send_queued(&tx, res, queued_bytes).await {
+                            if let Err(err) = tx.send(res).await {
+                                rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
                                 return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                             }
                             record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -698,7 +626,7 @@ impl Erasure {
             }
 
             Ok((reader, total))
-        }));
+        });
 
         let mut writers = MultiWriter::new(writers, quorum);
 
@@ -710,11 +638,11 @@ impl Erasure {
                 break;
             };
             record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
-            let block = block.into_inner();
             if block.is_empty() {
                 break;
             }
-            let _writer_stage = rustfs_io_metrics::track_ec_encode_writer_bytes(queued_block_bytes(&block));
+            let queued_bytes = queued_block_bytes(&block);
+            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
             let write_stage_start = stage_timer_if_enabled();
             if let Err(err) = writers.write(block).await {
                 write_err = Some(err);
@@ -724,8 +652,9 @@ impl Erasure {
         }
 
         if let Some(err) = write_err {
-            task.abort_and_wait().await;
-            drop(rx);
+            task.abort();
+            let _ = task.await;
+            drain_queued_inflight_bytes(&mut rx).await;
             let shutdown_stage_start = stage_timer_if_enabled();
             if let Err(shutdown_err) = writers.shutdown().await {
                 error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -734,14 +663,14 @@ impl Erasure {
             return Err(err);
         }
 
-        let (reader, total) = task.join().await??;
+        let (reader, total) = task.await??;
         let shutdown_stage_start = stage_timer_if_enabled();
         writers.shutdown().await?;
         record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
         Ok((reader, total))
     }
 
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub async fn encode_batched<R>(
         self: Arc<Self>,
         mut reader: R,
@@ -763,15 +692,14 @@ impl Erasure {
         let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
         let batch_blocks = encode_batch_block_count().min(inflight_blocks);
         let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
-        let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
+        let (tx, mut rx) = mpsc::channel::<Vec<Vec<Bytes>>>(channel_capacity);
 
-        let mut task = AbortOnDropTask::new(tokio::spawn(async move {
+        let task = tokio::spawn(async move {
             let block_size = self.block_size;
             let mut total = 0;
             let mut buf = vec![0u8; block_size];
             let mut pending_batch = Vec::with_capacity(batch_blocks);
             let mut pending_batch_bytes = 0usize;
-            let mut pending_batch_stage = None;
             loop {
                 match rustfs_utils::read_full_or_eof(&mut reader, &mut buf).await {
                     Ok(Some(n)) => {
@@ -783,16 +711,15 @@ impl Erasure {
                         let queued_bytes = queued_block_bytes(&res);
                         pending_batch_bytes = pending_batch_bytes.saturating_add(queued_bytes);
                         pending_batch.push(res);
-                        drop(pending_batch_stage.take());
-                        pending_batch_stage = Some(rustfs_io_metrics::track_ec_encode_producer_bytes(pending_batch_bytes));
 
                         if pending_batch.len() >= batch_blocks {
+                            rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
                             let send_wait_stage_start = stage_timer_if_enabled();
-                            if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
+                            if let Err(err) = tx.send(pending_batch).await {
+                                rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
                                 return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                             }
                             record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
-                            drop(pending_batch_stage.take());
                             pending_batch = Vec::with_capacity(batch_blocks);
                             pending_batch_bytes = 0;
                         }
@@ -815,16 +742,17 @@ impl Erasure {
             }
 
             if !pending_batch.is_empty() {
+                rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
                 let send_wait_stage_start = stage_timer_if_enabled();
-                if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
+                if let Err(err) = tx.send(pending_batch).await {
+                    rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
                     return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
                 }
                 record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
-                drop(pending_batch_stage);
             }
 
             Ok((reader, total))
-        }));
+        });
 
         let mut writers = MultiWriter::new(writers, quorum);
         let mut write_err = None;
@@ -835,8 +763,7 @@ impl Erasure {
                 break;
             };
             record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
-            let batch = batch.into_inner();
-            let _writer_stage = rustfs_io_metrics::track_ec_encode_writer_bytes(queued_batch_bytes(&batch));
+            rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
             let write_stage_start = stage_timer_if_enabled();
             for block in batch {
                 if let Err(err) = writers.write(block).await {
@@ -851,8 +778,9 @@ impl Erasure {
         }
 
         if let Some(err) = write_err {
-            task.abort_and_wait().await;
-            drop(rx);
+            task.abort();
+            let _ = task.await;
+            drain_queued_batched_inflight_bytes(&mut rx).await;
             let shutdown_stage_start = stage_timer_if_enabled();
             if let Err(shutdown_err) = writers.shutdown().await {
                 error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -861,7 +789,7 @@ impl Erasure {
             return Err(err);
         }
 
-        let (reader, total) = task.join().await??;
+        let (reader, total) = task.await??;
         let shutdown_stage_start = stage_timer_if_enabled();
         writers.shutdown().await?;
         record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
@@ -870,7 +798,7 @@ impl Erasure {
 
     /// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
     /// Reads all data, encodes directly, writes shards sequentially.
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub async fn encode_inline_small<R>(
         self: Arc<Self>,
         reader: R,
@@ -885,7 +813,7 @@ impl Erasure {
 
     /// Fast path for single-block non-inline objects: avoids the producer/consumer
     /// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub async fn encode_single_block_non_inline<R>(
         self: Arc<Self>,
         reader: R,
@@ -911,104 +839,7 @@ mod tests {
     use std::sync::{Arc, Mutex};
     use std::task::{Context, Poll};
     use std::time::Duration;
-    use tokio::io::{AsyncWrite, AsyncWriteExt, ReadBuf};
-    use tokio::sync::oneshot;
-
-    struct PendingReader {
-        entered: Option<oneshot::Sender<()>>,
-        dropped: Option<oneshot::Sender<()>>,
-    }
-
-    impl PendingReader {
-        fn new() -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>) {
-            let (entered_tx, entered_rx) = oneshot::channel();
-            let (dropped_tx, dropped_rx) = oneshot::channel();
-            (
-                Self {
-                    entered: Some(entered_tx),
-                    dropped: Some(dropped_tx),
-                },
-                entered_rx,
-                dropped_rx,
-            )
-        }
-    }
-
-    impl AsyncRead for PendingReader {
-        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
-            if let Some(entered) = self.entered.take() {
-                let _ = entered.send(());
-            }
-            Poll::Pending
-        }
-    }
-
-    impl Drop for PendingReader {
-        fn drop(&mut self) {
-            if let Some(dropped) = self.dropped.take() {
-                let _ = dropped.send(());
-            }
-        }
-    }
-
-    struct BlocksThenPendingReader {
-        blocks_remaining: usize,
-        block: Vec<u8>,
-        blocked: Option<oneshot::Sender<()>>,
-        dropped: Option<oneshot::Sender<()>>,
-        final_block: Option<oneshot::Sender<()>>,
-    }
-
-    impl BlocksThenPendingReader {
-        fn new(
-            blocks_remaining: usize,
-            block_size: usize,
-        ) -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>, oneshot::Receiver<()>) {
-            let (blocked_tx, blocked_rx) = oneshot::channel();
-            let (dropped_tx, dropped_rx) = oneshot::channel();
-            let (final_block_tx, final_block_rx) = oneshot::channel();
-            (
-                Self {
-                    blocks_remaining,
-                    block: vec![0x5a; block_size],
-                    blocked: Some(blocked_tx),
-                    dropped: Some(dropped_tx),
-                    final_block: Some(final_block_tx),
-                },
-                blocked_rx,
-                dropped_rx,
-                final_block_rx,
-            )
-        }
-    }
-
-    impl AsyncRead for BlocksThenPendingReader {
-        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
-            if self.blocks_remaining == 0 {
-                if let Some(blocked) = self.blocked.take() {
-                    let _ = blocked.send(());
-                }
-                return Poll::Pending;
-            }
-
-            if self.blocks_remaining == 1
-                && let Some(final_block) = self.final_block.take()
-            {
-                let _ = final_block.send(());
-            }
-            self.blocks_remaining -= 1;
-            buf.put_slice(&self.block);
-            Poll::Ready(Ok(()))
-        }
-    }
-
-    impl Drop for BlocksThenPendingReader {
-        fn drop(&mut self) {
-            if let Some(dropped) = self.dropped.take() {
-                let _ = dropped.send(());
-            }
-        }
-    }
+    use tokio::io::{AsyncWrite, AsyncWriteExt};
 
     fn erasure_with_zero_block_size() -> Erasure {
         let mut erasure = Erasure::default();
@@ -1066,54 +897,6 @@ mod tests {
         }
     }
 
-    struct FailAfterReaderBlocksWriter {
-        reader_blocked: oneshot::Receiver<()>,
-        writes: Arc<std::sync::atomic::AtomicUsize>,
-    }
-
-    impl AsyncWrite for FailAfterReaderBlocksWriter {
-        fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
-            match Pin::new(&mut self.reader_blocked).poll(cx) {
-                Poll::Pending => Poll::Pending,
-                Poll::Ready(_) => {
-                    self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                    Poll::Ready(Err(std::io::Error::other("injected write failure after producer blocks")))
-                }
-            }
-        }
-
-        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-
-        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-    }
-
-    struct StallOnWriteWithSignal {
-        entered: Option<oneshot::Sender<()>>,
-        writes: Arc<std::sync::atomic::AtomicUsize>,
-    }
-
-    impl AsyncWrite for StallOnWriteWithSignal {
-        fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
-            self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-            if let Some(entered) = self.entered.take() {
-                let _ = entered.send(());
-            }
-            Poll::Pending
-        }
-
-        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-
-        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-    }
-
     #[derive(Clone, Default)]
     struct ShortWriteWriter;
 
@@ -1263,262 +1046,6 @@ mod tests {
         BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::None)
     }
 
-    #[derive(Clone, Copy)]
-    enum EncodePipeline {
-        Vec,
-        BytesMut,
-        Batched,
-    }
-
-    async fn aborting_encode_drops_blocked_producer(pipeline: EncodePipeline) {
-        const BLOCK_SIZE: usize = 16;
-
-        let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let committed = Arc::new(Mutex::new(Vec::new()));
-        let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE))];
-        let (reader, entered, dropped) = PendingReader::new();
-        let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
-
-        let encode = match pipeline {
-            EncodePipeline::Vec => {
-                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
-            }
-            EncodePipeline::BytesMut => {
-                tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
-            }
-            EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
-        };
-
-        tokio::time::timeout(Duration::from_secs(1), entered)
-            .await
-            .expect("producer should enter the blocked reader before cancellation")
-            .expect("blocked reader should signal entry");
-        encode.abort();
-        assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
-        tokio::time::timeout(Duration::from_secs(1), dropped)
-            .await
-            .expect("cancelling encode should drop the producer reader")
-            .expect("blocked reader should signal producer drop");
-        assert!(
-            committed.lock().expect("committed buffer should be lockable").is_empty(),
-            "cancelling before the first encoded block must not make data visible"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            gauge_baseline,
-            "cancelling the encode pipeline must preserve the inflight queue gauge"
-        );
-    }
-
-    async fn writer_error_aborts_blocked_producer(pipeline: EncodePipeline) {
-        let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let producer_baseline = rustfs_io_metrics::current_ec_encode_producer_bytes();
-        let writer_baseline = rustfs_io_metrics::current_ec_encode_writer_bytes();
-        let producer_peak_before = rustfs_io_metrics::current_ec_encode_producer_bytes_peak();
-        let writer_peak_before = rustfs_io_metrics::current_ec_encode_writer_bytes_peak();
-        let block_size = match pipeline {
-            EncodePipeline::Batched => usize::try_from(producer_peak_before.max(writer_peak_before).saturating_add(1))
-                .expect("stage peak fits the test address space"),
-            EncodePipeline::Vec | EncodePipeline::BytesMut => 16,
-        };
-        rustfs_io_metrics::set_put_stage_metrics_enabled(true);
-        let erasure = Arc::new(Erasure::new(1, 0, block_size));
-        let batch_blocks = encode_batch_block_count().min(encode_channel_capacity(
-            erasure.shard_size().saturating_mul(erasure.total_shard_count()),
-            erasure_encode_max_inflight_bytes(),
-        ));
-        let blocks_before_pending = match pipeline {
-            EncodePipeline::Batched => batch_blocks,
-            EncodePipeline::Vec | EncodePipeline::BytesMut => 1,
-        };
-        let (reader, reader_blocked, reader_dropped, _final_block) =
-            BlocksThenPendingReader::new(blocks_before_pending, block_size);
-        let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let mut writers = vec![Some(bitrot_writer(
-            FailAfterReaderBlocksWriter {
-                reader_blocked,
-                writes: writes.clone(),
-            },
-            block_size,
-        ))];
-
-        let result = match pipeline {
-            EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
-            EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
-            EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
-        };
-
-        let err = match result {
-            Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
-            Err(err) => err,
-        };
-        assert!(err.to_string().contains("Failed to write data"));
-        tokio::time::timeout(Duration::from_secs(1), reader_dropped)
-            .await
-            .expect("writer failure should abort the blocked producer")
-            .expect("blocked producer should signal reader drop");
-        assert_eq!(
-            writes.load(std::sync::atomic::Ordering::SeqCst),
-            1,
-            "writer failure must stop the pipeline before any additional shard write"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            gauge_baseline,
-            "writer failure must settle all queued and pending encoded bytes"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_producer_bytes(),
-            producer_baseline,
-            "writer failure must settle producer stage bytes for every ingest pipeline"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_writer_bytes(),
-            writer_baseline,
-            "writer failure must settle writer stage bytes for every ingest pipeline"
-        );
-        if matches!(pipeline, EncodePipeline::Batched) {
-            let expected_batch_bytes = u64::try_from(block_size)
-                .expect("block size fits the stage gauge")
-                .checked_mul(u64::try_from(batch_blocks).expect("batch block count fits the stage gauge"))
-                .expect("test batch payload fits the stage gauge");
-            assert!(
-                rustfs_io_metrics::current_ec_encode_producer_bytes_peak() >= producer_peak_before.max(expected_batch_bytes),
-                "batched producer must expose its full pending batch before writer failure"
-            );
-            assert!(
-                rustfs_io_metrics::current_ec_encode_writer_bytes_peak() >= writer_peak_before.max(expected_batch_bytes),
-                "batched writer must expose its full batch before writer failure"
-            );
-        }
-        rustfs_io_metrics::set_put_stage_metrics_enabled(false);
-    }
-
-    async fn aborting_full_queue_settles_pending_send() {
-        const BLOCK_SIZE: usize = 16;
-
-        let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let producer_baseline = rustfs_io_metrics::current_ec_encode_producer_bytes();
-        let writer_baseline = rustfs_io_metrics::current_ec_encode_writer_bytes();
-        rustfs_io_metrics::set_put_stage_metrics_enabled(true);
-        let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
-        let inflight_blocks = encode_channel_capacity(
-            erasure.shard_size().saturating_mul(erasure.total_shard_count()),
-            erasure_encode_max_inflight_bytes(),
-        );
-        let (reader, _reader_blocked, reader_dropped, final_block) =
-            BlocksThenPendingReader::new(inflight_blocks + 2, BLOCK_SIZE);
-        let (writer_entered_tx, writer_entered) = oneshot::channel();
-        let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
-        let mut writers = vec![Some(bitrot_writer(
-            StallOnWriteWithSignal {
-                entered: Some(writer_entered_tx),
-                writes: writes.clone(),
-            },
-            BLOCK_SIZE,
-        ))];
-        let erasure_for_task = erasure.clone();
-        let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
-
-        tokio::time::timeout(Duration::from_secs(1), writer_entered)
-            .await
-            .expect("consumer should start the first writer call")
-            .expect("stalling writer should signal entry");
-        tokio::time::timeout(Duration::from_secs(1), final_block)
-            .await
-            .expect("producer should supply the block whose send fills the queue")
-            .expect("reader should signal final block");
-
-        let expected_queued_bytes =
-            u64::try_from((inflight_blocks + 1) * BLOCK_SIZE).expect("queued byte count should fit the gauge");
-        tokio::time::timeout(Duration::from_secs(1), async {
-            while rustfs_io_metrics::current_ec_encode_inflight_bytes() < gauge_baseline + expected_queued_bytes {
-                tokio::task::yield_now().await;
-            }
-        })
-        .await
-        .expect("producer should account for the pending send after the queue fills");
-        tokio::time::timeout(Duration::from_secs(1), async {
-            while rustfs_io_metrics::current_ec_encode_producer_bytes() == producer_baseline
-                || rustfs_io_metrics::current_ec_encode_writer_bytes() == writer_baseline
-            {
-                tokio::task::yield_now().await;
-            }
-        })
-        .await
-        .expect("full queue must retain producer and writer stage ownership before cancellation");
-
-        encode.abort();
-        assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
-        tokio::time::timeout(Duration::from_secs(1), reader_dropped)
-            .await
-            .expect("cancelling a full queue should abort its producer")
-            .expect("full-queue producer should signal reader drop");
-        assert_eq!(
-            writes.load(std::sync::atomic::Ordering::SeqCst),
-            1,
-            "cancellation must not resume the stalled writer"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            gauge_baseline,
-            "cancelling a full queue must settle queued and pending bytes"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_producer_bytes(),
-            producer_baseline,
-            "cancelling a full queue must settle the pending producer stage"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_writer_bytes(),
-            writer_baseline,
-            "cancelling a full queue must settle the stalled writer stage"
-        );
-        rustfs_io_metrics::set_put_stage_metrics_enabled(false);
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn cancelling_vec_encode_drops_blocked_producer() {
-        aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn cancelling_bytesmut_encode_drops_blocked_producer() {
-        aborting_encode_drops_blocked_producer(EncodePipeline::BytesMut).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn cancelling_batched_encode_drops_blocked_producer() {
-        aborting_encode_drops_blocked_producer(EncodePipeline::Batched).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn vec_writer_error_aborts_blocked_producer() {
-        writer_error_aborts_blocked_producer(EncodePipeline::Vec).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn bytesmut_writer_error_aborts_blocked_producer() {
-        writer_error_aborts_blocked_producer(EncodePipeline::BytesMut).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn batched_writer_error_aborts_blocked_producer() {
-        writer_error_aborts_blocked_producer(EncodePipeline::Batched).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn cancelling_full_queue_settles_pending_send() {
-        aborting_full_queue_settles_pending_send().await;
-    }
-
     #[tokio::test]
     async fn helper_writers_cover_flush_and_shutdown_paths() {
         let mut failing_write = FailingWriteWriter;
@@ -1802,99 +1329,25 @@ mod tests {
     }
 
     #[tokio::test]
-    #[serial_test::serial]
-    async fn queued_inflight_bytes_are_settled_on_all_queue_exit_paths() {
-        let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let (tx, rx) = mpsc::channel(1);
-        let queued = vec![Bytes::from_static(b"queued")];
-        let queued_bytes = queued_block_bytes(&queued);
+    async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
+        let (tx, mut rx) = mpsc::channel(2);
+        tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
+        drop(tx);
 
-        send_queued(&tx, queued, queued_bytes)
-            .await
-            .expect("first queue entry should fit");
+        drain_queued_inflight_bytes(&mut rx).await;
 
-        let blocked = vec![Bytes::from_static(b"blocked")];
-        let blocked_bytes = queued_block_bytes(&blocked);
-        {
-            let pending_send = send_queued(&tx, blocked, blocked_bytes);
-            tokio::pin!(pending_send);
-            assert!(
-                futures::poll!(pending_send.as_mut()).is_pending(),
-                "full queue must suspend producer send"
-            );
-        }
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            baseline + u64::try_from(queued_bytes).expect("queue bytes fit the gauge"),
-            "dropping a pending producer send must compensate its bytes"
-        );
-
-        drop(rx);
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            baseline,
-            "dropping the receiver must settle every buffered entry"
-        );
-
-        let rejected = vec![Bytes::from_static(b"rejected")];
-        let rejected_bytes = queued_block_bytes(&rejected);
-        assert!(
-            send_queued(&tx, rejected, rejected_bytes).await.is_err(),
-            "closed receiver must reject a new send"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            baseline,
-            "failed sends must compensate their bytes"
-        );
+        assert!(rx.recv().await.is_none());
     }
 
     #[tokio::test]
-    #[serial_test::serial]
-    async fn queued_batch_entry_settles_bytes_before_handoff() {
-        let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let (tx, rx) = mpsc::channel(2);
-        let mut rx = rx;
-        let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
-        let batch_bytes = queued_batch_bytes(&batch);
+    async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
+        let (tx, mut rx) = mpsc::channel(2);
+        tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
+        drop(tx);
 
-        send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
-        let batch = rx.recv().await.expect("queued batch should be received").into_inner();
-        assert_eq!(batch_bytes, queued_batch_bytes(&batch));
+        drain_queued_batched_inflight_bytes(&mut rx).await;
 
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            baseline,
-            "receiving a batch must settle all contained block bytes before shard writes"
-        );
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn queued_entry_settles_when_a_closed_receiver_accepts_an_outstanding_permit() {
-        let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let (tx, mut rx) = mpsc::channel(1);
-        let permit = tx
-            .clone()
-            .reserve_owned()
-            .await
-            .expect("open receiver should reserve queue capacity");
-        rx.close();
-        assert!(
-            matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
-            "an outstanding permit must leave the closed queue observably empty"
-        );
-
-        let block = vec![Bytes::from_static(b"late-permit")];
-        let block_bytes = queued_block_bytes(&block);
-        permit.send(InflightEntry::new(block, block_bytes));
-        drop(rx);
-
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            baseline,
-            "a queue entry sent through an outstanding permit must settle when Tokio drops it"
-        );
+        assert!(rx.recv().await.is_none());
     }
 
     #[tokio::test]
@@ -1941,61 +1394,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn bytesmut_streaming_encode_observes_all_payload_stage_peaks() {
-        let queue_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
-        let producer_peak_before = rustfs_io_metrics::current_ec_encode_producer_bytes_peak();
-        let queue_peak_before = rustfs_io_metrics::current_ec_encode_queue_bytes_peak();
-        let writer_peak_before = rustfs_io_metrics::current_ec_encode_writer_bytes_peak();
-        let prior_peak = producer_peak_before.max(queue_peak_before).max(writer_peak_before);
-        let block_size = usize::try_from(prior_peak.saturating_add(1)).expect("stage peak fits the test address space");
-        let erasure = Arc::new(Erasure::new(1, 0, block_size));
-        let encoded_block_bytes = erasure.shard_size() * erasure.total_shard_count();
-        let committed = Arc::new(Mutex::new(Vec::new()));
-        let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), block_size))];
-        let reader = tokio::io::BufReader::new(Cursor::new(vec![0x5a; block_size * 2]));
-
-        rustfs_io_metrics::set_put_stage_metrics_enabled(true);
-        let result = erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await;
-        rustfs_io_metrics::set_put_stage_metrics_enabled(false);
-
-        let (_reader, written) = result.expect("bytesmut streaming encode should complete");
-        assert_eq!(written, block_size * 2);
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_inflight_bytes(),
-            queue_baseline,
-            "completed streaming encode must not retain queue bytes"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_producer_bytes(),
-            0,
-            "completed streaming encode must not retain producer bytes"
-        );
-        assert_eq!(
-            rustfs_io_metrics::current_ec_encode_writer_bytes(),
-            0,
-            "completed streaming encode must not retain writer bytes"
-        );
-        let expected_peak = u64::try_from(encoded_block_bytes).expect("encoded block bytes fit the gauge");
-        assert!(
-            rustfs_io_metrics::current_ec_encode_producer_bytes_peak() >= producer_peak_before.max(expected_peak),
-            "producer peak must observe encoded bytes before queue hand-off"
-        );
-        assert!(
-            rustfs_io_metrics::current_ec_encode_queue_bytes_peak() >= queue_peak_before.max(expected_peak),
-            "queue peak must observe encoded bytes pending shard writers"
-        );
-        assert!(
-            rustfs_io_metrics::current_ec_encode_writer_bytes_peak() >= writer_peak_before.max(expected_peak),
-            "writer peak must observe encoded bytes after queue hand-off"
-        );
-        assert!(
-            !committed.lock().expect("committed buffer should be lockable").is_empty(),
-            "stage peak observation must not change writer commit behavior"
-        );
-    }
-
     #[tokio::test]
     async fn encode_streaming_write_quorum_failure_aborts_and_reports_error() {
         const DATA_SHARDS: usize = 2;

crates/ecstore/src/erasure/coding/erasure.rs

@@ -640,7 +640,7 @@ impl Erasure {
     /// # Returns
     /// A vector of encoded shards as `Bytes`.
     #[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
@@ -688,7 +688,7 @@ impl Erasure {
 
     /// Encode owned data, avoiding a copy when the caller already has a heap buffer.
     /// Falls back to copying into a new buffer if zero-copy conversion fails.
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
@@ -752,7 +752,7 @@ impl Erasure {
     /// block), the `resize(need_total_size)` below stays within capacity for every
     /// `data_len <= block_size` — both shard-size formulas are monotone in
     /// `data_len` — so this function never reallocates the buffer.
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
         let shard_size_fn = if self.uses_legacy {
             calc_shard_size_legacy
@@ -805,7 +805,7 @@ impl Erasure {
     ///
     /// # Returns
     /// Ok if reconstruction succeeds, error otherwise.
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {
@@ -825,7 +825,7 @@ impl Erasure {
     }
 
     /// Decode and reconstruct missing data shards, then regenerate parity shards.
-    #[hotpath::measure(impl_type = "Erasure")]
+    #[hotpath::measure]
     pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
         if self.parity_shards > 0 {
             if self.uses_legacy {

crates/ecstore/src/io_support/bitrot.rs

@@ -19,8 +19,6 @@ use crate::diagnostics::get::{
     GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
     GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
 };
-#[cfg(feature = "hotpath")]
-use crate::disk::FileWriter;
 use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
 use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
 use bytes::Bytes;
@@ -38,11 +36,6 @@ use std::time::Instant;
 use tokio::io::{AsyncRead, ReadBuf};
 use tracing::debug;
 
-#[cfg(all(test, feature = "hotpath"))]
-tokio::task_local! {
-    static FORCE_MMAP_COPY_FAILURE_FOR_TEST: ();
-}
-
 /// A shard source for the bitrot reader.
 ///
 /// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
@@ -367,21 +360,10 @@ async fn open_disk_reader(
             mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
             direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
         });
-        let mmap_result = {
-            #[cfg(all(test, feature = "hotpath"))]
-            if FORCE_MMAP_COPY_FAILURE_FOR_TEST.try_with(|_| ()).is_ok() {
-                Err(DiskError::other("forced mmap-copy failure for test"))
-            } else {
-                disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
-                    .await
-            }
-            #[cfg(not(all(test, feature = "hotpath")))]
-            {
-                disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
-                    .await
-            }
-        };
-        match mmap_result {
+        match disk
+            .read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
+            .await
+        {
             Ok(bytes) => {
                 let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
 
@@ -416,11 +398,7 @@ async fn open_disk_reader(
                 }
 
                 return match stream_result {
-                    Ok(reader) => {
-                        #[cfg(feature = "hotpath")]
-                        let reader = instrument_raw_shard_reader(reader, disk.is_local());
-                        Ok(wrap_first_read_metrics(reader, metrics_path))
-                    }
+                    Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
                     Err(_) => Err(err),
                 };
             }
@@ -429,8 +407,6 @@ async fn open_disk_reader(
 
     let stream_start = stage_metrics_enabled.then(Instant::now);
     let reader = disk.read_file_stream(bucket, path, offset, length).await?;
-    #[cfg(feature = "hotpath")]
-    let reader = instrument_raw_shard_reader(reader, disk.is_local());
     if let Some(metrics_path) = metrics_path {
         record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
     }
@@ -451,37 +427,6 @@ fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str
     ShardReader::Stream(reader)
 }
 
-// The labels are deliberately fixed: object keys, disk paths, and remote hosts
-// are all high-cardinality or sensitive and belong nowhere in a profiling report.
-#[cfg(feature = "hotpath")]
-const RAW_SHARD_READ_LOCAL_LABEL: &str = "EC raw shard read local";
-#[cfg(feature = "hotpath")]
-const RAW_SHARD_READ_REMOTE_LABEL: &str = "EC raw shard read remote";
-#[cfg(feature = "hotpath")]
-const RAW_SHARD_WRITE_LOCAL_LABEL: &str = "EC raw shard write local";
-#[cfg(feature = "hotpath")]
-const RAW_SHARD_WRITE_REMOTE_LABEL: &str = "EC raw shard write remote";
-
-#[cfg(feature = "hotpath")]
-fn instrument_raw_shard_reader(reader: FileReader, is_local: bool) -> FileReader {
-    // `io!` aggregates by call site, so the local and remote branches must remain distinct.
-    if is_local {
-        Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_LOCAL_LABEL))
-    } else {
-        Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_REMOTE_LABEL))
-    }
-}
-
-#[cfg(feature = "hotpath")]
-fn instrument_raw_shard_writer(writer: FileWriter, is_local: bool) -> FileWriter {
-    // `io!` aggregates by call site, so the local and remote branches must remain distinct.
-    if is_local {
-        Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_LOCAL_LABEL))
-    } else {
-        Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_REMOTE_LABEL))
-    }
-}
-
 fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
     (
         offset.div_ceil(shard_size) * checksum_algo.size() + offset,
@@ -741,8 +686,6 @@ pub async fn create_bitrot_writer(
         };
 
         let file = disk.create_file("", volume, path, length).await?;
-        #[cfg(feature = "hotpath")]
-        let file = instrument_raw_shard_writer(file, disk.is_local());
         CustomWriter::new_tokio_writer(file)
     } else {
         return Err(DiskError::DiskNotFound);
@@ -755,194 +698,6 @@ pub async fn create_bitrot_writer(
 mod tests {
     use super::*;
 
-    #[cfg(feature = "hotpath")]
-    use crate::cluster::rpc::RemoteDisk;
-    #[cfg(feature = "hotpath")]
-    use crate::cluster::rpc::internode_data_transport::{
-        InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
-    };
-    #[cfg(feature = "hotpath")]
-    use crate::disk::{Disk, DiskOption, error::Result};
-
-    #[cfg(feature = "hotpath")]
-    #[derive(Debug, Clone, Default)]
-    struct TestRemoteDataTransport {
-        bytes: Arc<Mutex<Vec<u8>>>,
-        write_error: Option<io::ErrorKind>,
-    }
-
-    #[cfg(feature = "hotpath")]
-    impl TestRemoteDataTransport {
-        fn with_write_error(kind: io::ErrorKind) -> Self {
-            Self {
-                bytes: Arc::default(),
-                write_error: Some(kind),
-            }
-        }
-
-        fn bytes(&self) -> Vec<u8> {
-            self.bytes
-                .lock()
-                .expect("test remote transport bytes lock should not be poisoned")
-                .clone()
-        }
-    }
-
-    #[cfg(feature = "hotpath")]
-    #[derive(Debug)]
-    struct TestRemoteWriter {
-        bytes: Arc<Mutex<Vec<u8>>>,
-        write_error: Option<io::ErrorKind>,
-    }
-
-    #[cfg(feature = "hotpath")]
-    impl tokio::io::AsyncWrite for TestRemoteWriter {
-        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
-            if let Some(kind) = self.write_error {
-                return Poll::Ready(Err(io::Error::from(kind)));
-            }
-            self.bytes
-                .lock()
-                .expect("test remote transport bytes lock should not be poisoned")
-                .extend_from_slice(buf);
-            Poll::Ready(Ok(buf.len()))
-        }
-
-        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-
-        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
-            Poll::Ready(Ok(()))
-        }
-    }
-
-    #[cfg(feature = "hotpath")]
-    #[async_trait::async_trait]
-    impl InternodeDataTransport for TestRemoteDataTransport {
-        async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
-            Ok(Box::new(Cursor::new(self.bytes())))
-        }
-
-        async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
-            Ok(Box::new(TestRemoteWriter {
-                bytes: Arc::clone(&self.bytes),
-                write_error: self.write_error,
-            }))
-        }
-
-        async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
-            panic!("open_walk_dir must not be used by the raw shard I/O test")
-        }
-
-        fn name(&self) -> &'static str {
-            "bitrot-test-remote"
-        }
-
-        fn capabilities(&self) -> InternodeDataTransportCapabilities {
-            InternodeDataTransportCapabilities::tcp_http()
-        }
-    }
-
-    async fn local_test_disk() -> (DiskStore, tempfile::TempDir) {
-        use crate::disk::endpoint::Endpoint;
-        use crate::disk::{DiskOption, new_disk};
-
-        let dir = tempfile::tempdir().expect("tempdir should be created");
-        let mut endpoint =
-            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(0);
-        let disk = new_disk(
-            &endpoint,
-            &DiskOption {
-                cleanup: false,
-                health_check: false,
-            },
-        )
-        .await
-        .expect("local disk should be created");
-
-        (disk, dir)
-    }
-
-    #[cfg(feature = "hotpath")]
-    async fn remote_test_disk(transport: TestRemoteDataTransport) -> (DiskStore, TestRemoteDataTransport) {
-        use crate::disk::endpoint::Endpoint;
-
-        let endpoint = Endpoint {
-            url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("test remote endpoint should parse"),
-            is_local: false,
-            pool_idx: 0,
-            set_idx: 0,
-            disk_idx: 0,
-        };
-        let remote = RemoteDisk::new(
-            &endpoint,
-            &DiskOption {
-                cleanup: false,
-                health_check: false,
-            },
-            Arc::new(transport.clone()),
-        )
-        .await
-        .expect("test remote disk should be created");
-
-        (Arc::new(Disk::Remote(Box::new(remote))), transport)
-    }
-
-    async fn round_trip_disk_bitrot(disk: &DiskStore, bucket: &str, path: &str, payload: &[u8], shard_size: usize) -> Vec<u8> {
-        disk.make_volume(bucket).await.expect("volume should be created");
-        let mut writer = create_bitrot_writer(
-            false,
-            Some(disk),
-            bucket,
-            path,
-            i64::try_from(payload.len()).expect("test payload length should fit i64"),
-            shard_size,
-            HashAlgorithm::None,
-        )
-        .await
-        .expect("disk bitrot writer should open the raw shard file");
-        for chunk in payload.chunks(shard_size) {
-            writer
-                .write(chunk)
-                .await
-                .expect("disk bitrot writer should preserve each shard block");
-        }
-        writer.shutdown().await.expect("disk bitrot writer should close cleanly");
-
-        let mut reader = create_bitrot_reader(
-            None,
-            Some(disk),
-            bucket,
-            path,
-            0,
-            payload.len(),
-            shard_size,
-            HashAlgorithm::None,
-            false,
-            false,
-        )
-        .await
-        .expect("disk bitrot reader should open the raw shard file")
-        .expect("disk bitrot reader should exist");
-        let mut actual = Vec::with_capacity(payload.len());
-        while actual.len() < payload.len() {
-            let remaining = payload.len() - actual.len();
-            let mut chunk = vec![0; remaining.min(shard_size)];
-            let read = reader
-                .read(&mut chunk)
-                .await
-                .expect("disk bitrot reader should return the complete shard body");
-            assert!(read > 0, "disk bitrot reader must not end before the expected shard body is complete");
-            actual.extend_from_slice(&chunk[..read]);
-        }
-
-        actual
-    }
-
     #[test]
     fn object_mmap_read_enabled_accepts_legacy_zero_copy_alias() {
         temp_env::with_vars(
@@ -969,214 +724,6 @@ mod tests {
         );
     }
 
-    #[cfg(feature = "hotpath")]
-    #[test]
-    fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes() {
-        const CHILD_ENV: &str = "RUSTFS_HOTPATH_RAW_SHARD_IO_TEST_CHILD";
-        if std::env::var_os(CHILD_ENV).is_none() {
-            let status = std::process::Command::new(std::env::current_exe().expect("test executable path should be available"))
-                .arg("--exact")
-                .arg("io_support::bitrot::tests::raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes")
-                .arg("--nocapture")
-                .env(CHILD_ENV, "1")
-                .status()
-                .expect("isolated HotPath I/O test process should start");
-            assert!(status.success(), "isolated HotPath I/O test process should pass");
-            return;
-        }
-
-        tokio::runtime::Builder::new_current_thread()
-            .enable_all()
-            .build()
-            .expect("test runtime should be created")
-            .block_on(raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process());
-    }
-
-    #[cfg(feature = "hotpath")]
-    async fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process() {
-        use hotpath::{Format, HotpathGuardBuilder, Section};
-        use tokio::io::AsyncReadExt;
-
-        let report_dir = tempfile::tempdir().expect("report tempdir should be created");
-        let report_path = report_dir.path().join("hotpath-io.json");
-        let guard = HotpathGuardBuilder::new("raw_shard_io_test")
-            .format(Format::Json)
-            .output_path(&report_path)
-            .sections(vec![Section::Io])
-            .build();
-
-        let (disk, _dir) = local_test_disk().await;
-        let bucket = "test-bucket";
-        let path = "obj/hotpath-part.1";
-        let payload = b"local shard bytes";
-        let shard_size = 4;
-        let local_read = round_trip_disk_bitrot(&disk, bucket, path, payload, shard_size).await;
-        assert_eq!(local_read, payload, "local raw shard I/O instrumentation must not alter stored bytes");
-
-        let fallback_path = "obj/hotpath-mmap-fallback-part.1";
-        let fallback_payload = b"mmap fallback shard bytes";
-        disk.write_all("test-bucket", fallback_path, Bytes::from_static(fallback_payload))
-            .await
-            .expect("fallback shard file should be written");
-        let mut fallback_reader = FORCE_MMAP_COPY_FAILURE_FOR_TEST
-            .scope((), open_disk_reader(&disk, bucket, fallback_path, 0, fallback_payload.len(), true, None))
-            .await
-            .expect("mmap-copy failure should fall back to a raw shard stream");
-        assert!(
-            matches!(fallback_reader, ShardReader::Stream(_)),
-            "forced mmap-copy failure must use the streaming fallback"
-        );
-        let mut fallback_read = Vec::new();
-        fallback_reader
-            .read_to_end(&mut fallback_read)
-            .await
-            .expect("mmap-copy fallback stream should preserve bytes");
-        assert_eq!(fallback_read, fallback_payload);
-
-        let (remote_disk, remote_transport) = remote_test_disk(TestRemoteDataTransport::default()).await;
-        let remote_payload = b"remote shard bytes";
-        let mut remote_writer = create_bitrot_writer(
-            false,
-            Some(&remote_disk),
-            bucket,
-            "obj/hotpath-remote-part.1",
-            i64::try_from(remote_payload.len()).expect("remote payload length should fit i64"),
-            shard_size,
-            HashAlgorithm::None,
-        )
-        .await
-        .expect("remote bitrot writer should use the production raw writer path");
-        for chunk in remote_payload.chunks(shard_size) {
-            remote_writer
-                .write(chunk)
-                .await
-                .expect("remote bitrot writer should preserve bytes");
-        }
-        remote_writer
-            .shutdown()
-            .await
-            .expect("remote bitrot writer should close cleanly");
-        assert_eq!(remote_transport.bytes(), remote_payload);
-
-        let mut reader = create_bitrot_reader(
-            None,
-            Some(&remote_disk),
-            bucket,
-            "obj/hotpath-remote-part.1",
-            0,
-            remote_payload.len(),
-            shard_size,
-            HashAlgorithm::None,
-            false,
-            true,
-        )
-        .await
-        .expect("remote bitrot reader should use the production raw reader path")
-        .expect("remote bitrot reader should exist");
-        let mut remote_read = Vec::with_capacity(remote_payload.len());
-        while remote_read.len() < remote_payload.len() {
-            let remaining = remote_payload.len() - remote_read.len();
-            let mut chunk = vec![0; remaining.min(shard_size)];
-            let read = reader
-                .read(&mut chunk)
-                .await
-                .expect("remote bitrot reader should preserve bytes");
-            assert!(read > 0, "remote bitrot reader must not end before the expected shard body is complete");
-            remote_read.extend_from_slice(&chunk[..read]);
-        }
-        assert_eq!(remote_read, remote_payload);
-
-        let (failing_remote_disk, _) = remote_test_disk(TestRemoteDataTransport::with_write_error(io::ErrorKind::Other)).await;
-        let mut failing_writer = create_bitrot_writer(
-            false,
-            Some(&failing_remote_disk),
-            bucket,
-            "obj/hotpath-remote-write-error-part.1",
-            4,
-            shard_size,
-            HashAlgorithm::None,
-        )
-        .await
-        .expect("failing remote bitrot writer should open before its first write");
-        let write_error = failing_writer
-            .write(b"fail")
-            .await
-            .expect_err("raw shard writer failures must remain visible through the bitrot writer");
-        assert_eq!(write_error.kind(), io::ErrorKind::Other);
-
-        let (would_block_remote_disk, _) =
-            remote_test_disk(TestRemoteDataTransport::with_write_error(io::ErrorKind::WouldBlock)).await;
-        let mut would_block_writer = create_bitrot_writer(
-            false,
-            Some(&would_block_remote_disk),
-            bucket,
-            "obj/hotpath-remote-write-would-block-part.1",
-            4,
-            shard_size,
-            HashAlgorithm::None,
-        )
-        .await
-        .expect("would-block remote bitrot writer should open before its first write");
-        let would_block = would_block_writer
-            .write(b"wait")
-            .await
-            .expect_err("would-block must remain visible to the caller");
-        assert_eq!(would_block.kind(), io::ErrorKind::WouldBlock);
-
-        drop(guard);
-        let report = std::fs::read_to_string(&report_path).expect("HotPath I/O report should be written");
-        let report: serde_json::Value = serde_json::from_str(&report).expect("HotPath I/O report should be valid JSON");
-        let entries = report["io"]["data"]
-            .as_array()
-            .expect("HotPath I/O report should include data rows");
-        let io_bytes = |label: &str, direction: &str| {
-            let byte_count: u64 = entries
-                .iter()
-                .filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
-                .filter_map(|entry| entry[direction]["bytes"].as_u64())
-                .sum();
-            assert!(byte_count > 0, "report must include fixed label {label}");
-            byte_count
-        };
-        let io_errors = |label: &str, direction: &str| {
-            entries
-                .iter()
-                .filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
-                .filter_map(|entry| entry[direction]["errors"].as_u64())
-                .sum::<u64>()
-        };
-        let payload_len = u64::try_from(payload.len()).expect("test payload length should fit u64");
-        assert_eq!(
-            io_bytes(RAW_SHARD_READ_LOCAL_LABEL, "read"),
-            payload_len + u64::try_from(fallback_payload.len()).expect("fallback payload length should fit u64")
-        );
-        assert_eq!(io_bytes(RAW_SHARD_WRITE_LOCAL_LABEL, "write"), payload_len);
-        assert_eq!(
-            io_bytes(RAW_SHARD_READ_REMOTE_LABEL, "read"),
-            u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
-        );
-        assert_eq!(
-            io_bytes(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
-            u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
-        );
-        assert_eq!(
-            io_errors(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
-            1,
-            "the wrapper must record the real remote writer failure without changing it, while excluding WouldBlock"
-        );
-        for label in [
-            RAW_SHARD_READ_LOCAL_LABEL,
-            RAW_SHARD_READ_REMOTE_LABEL,
-            RAW_SHARD_WRITE_LOCAL_LABEL,
-            RAW_SHARD_WRITE_REMOTE_LABEL,
-        ] {
-            assert!(
-                !label.contains(['/', ':', '?', '@']),
-                "raw shard I/O labels must not carry a path, host, query, or credential delimiter: {label}"
-            );
-        }
-    }
-
     #[test]
     fn object_mmap_read_max_length_defaults_and_env_override() {
         temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
@@ -1194,9 +741,25 @@ mod tests {
     // be materialized in memory by the mmap-copy path; over-cap reads stream.
     #[tokio::test]
     async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
+        use crate::disk::endpoint::Endpoint;
+        use crate::disk::{DiskOption, new_disk};
         use tokio::io::AsyncReadExt;
 
-        let (disk, _dir) = local_test_disk().await;
+        let dir = tempfile::tempdir().expect("tempdir should be created");
+        let mut endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(0);
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("local disk should be created");
 
         let payload = vec![7u8; 4096];
         disk.make_volume("test-bucket").await.expect("volume should be created");
@@ -1251,17 +814,6 @@ mod tests {
         .await;
     }
 
-    #[tokio::test]
-    async fn disk_bitrot_reader_and_writer_preserve_full_shard_body() {
-        let (disk, _dir) = local_test_disk().await;
-        let bucket = "test-bucket";
-        let path = "obj/wrapped-part.1";
-        let payload = b"wrapped shard body";
-        let actual = round_trip_disk_bitrot(&disk, bucket, path, payload, 4).await;
-
-        assert_eq!(actual, payload, "raw shard I/O instrumentation must not alter stored bytes");
-    }
-
     #[tokio::test]
     async fn test_create_bitrot_reader_with_inline_data() {
         let test_data = b"hello world test data";

crates/ecstore/src/layout/endpoints.rs

@@ -22,8 +22,6 @@ use rustfs_config::{
     ENV_STARTUP_TOPOLOGY_WAIT_MODE, ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, ENV_UNSAFE_BYPASS_DISK_CHECK,
 };
 use rustfs_utils::{XHost, check_local_server_addr, get_env_opt_str, get_host_ip, is_local_host};
-#[cfg(test)]
-use std::sync::{LazyLock, Mutex};
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
     future::Future,
@@ -600,58 +598,6 @@ const DNS_RETRY_JITTER_PERCENT: u64 = 20;
 /// wait does not flood the log with one line per backoff tick.
 const TOPOLOGY_WARN_THROTTLE: Duration = Duration::from_secs(30);
 
-#[cfg(test)]
-static FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS: LazyLock<Mutex<HashSet<String>>> = LazyLock::new(|| Mutex::new(HashSet::new()));
-
-#[cfg(test)]
-struct LocalHostResolutionTimeoutGuard {
-    hosts: Vec<String>,
-}
-
-#[cfg(test)]
-impl Drop for LocalHostResolutionTimeoutGuard {
-    fn drop(&mut self) {
-        let mut forced_hosts = FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
-            .lock()
-            .expect("local-host test resolver mutex poisoned");
-        for host in &self.hosts {
-            forced_hosts.remove(host);
-        }
-    }
-}
-
-#[cfg(test)]
-fn force_local_host_resolution_timeout_for_test(hosts: &[&str]) -> LocalHostResolutionTimeoutGuard {
-    let hosts = hosts.iter().map(|host| (*host).to_string()).collect::<Vec<_>>();
-    let mut forced_hosts = FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
-        .lock()
-        .expect("local-host test resolver mutex poisoned");
-    forced_hosts.extend(hosts.iter().cloned());
-    LocalHostResolutionTimeoutGuard { hosts }
-}
-
-#[cfg(test)]
-fn local_host_resolution_timeout_forced(host: &Host<&str>) -> bool {
-    let host = match host {
-        Host::Domain(domain) => (*domain).to_string(),
-        Host::Ipv4(ip) => ip.to_string(),
-        Host::Ipv6(ip) => ip.to_string(),
-    };
-    FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
-        .lock()
-        .expect("local-host test resolver mutex poisoned")
-        .contains(&host)
-}
-
-fn endpoint_is_local_host(host: Host<&str>, port: u16, local_port: u16) -> Result<bool> {
-    #[cfg(test)]
-    if local_host_resolution_timeout_forced(&host) {
-        return Err(Error::new(ErrorKind::TimedOut, "resolver timeout"));
-    }
-
-    is_local_host(host, port, local_port)
-}
-
 struct DnsRetryDeadline {
     started: Instant,
     timeout: Duration,
@@ -748,7 +694,7 @@ async fn resolve_local_host_with_retry(
     retry_dns_operation(
         || {
             let host = host.clone();
-            async move { endpoint_is_local_host(host, port, local_port) }
+            async move { is_local_host(host, port, local_port) }
         },
         async_sleep,
         dns_retry_deadline,
@@ -2285,9 +2231,6 @@ mod test {
     #[serial]
     #[tokio::test]
     async fn create_server_endpoints_bounds_kubernetes_alias_dns_fallback() {
-        let _resolution_timeout =
-            force_local_host_resolution_timeout_for_test(&["unrelated-0.example.invalid", "unrelated-1.example.invalid"]);
-
         async_with_vars(
             [
                 (ENV_LOCAL_ENDPOINT_HOST, None),

crates/ecstore/src/services/notification_sys.rs

@@ -337,15 +337,6 @@ pub struct NotificationPeerErr {
     pub err: Option<Error>,
 }
 
-/// One peer's answer to a KMS configuration fingerprint probe.
-pub struct PeerKmsConfigFingerprint {
-    pub host: String,
-    /// `None` when the peer has no KMS configuration of its own, or could not
-    /// be asked at all, in which case `err` carries the reason.
-    pub fingerprint: Option<String>,
-    pub err: Option<Error>,
-}
-
 fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
     NotificationPeerErr { host, err: result.err() }
 }
@@ -527,47 +518,6 @@ impl NotificationSys {
         self.signal_dynamic_config(sub_sys, false).await
     }
 
-    /// Ask every peer to re-read the cluster-persisted KMS configuration.
-    ///
-    /// Best-effort by contract: the caller has already switched locally, so a
-    /// peer that fails is reported rather than rolled back. Peers built before
-    /// the KMS subsystem existed reject the signal with an explicit error.
-    pub async fn reload_kms_config(&self) -> Vec<NotificationPeerErr> {
-        self.reload_dynamic_config(crate::cluster::rpc::KMS_SIGNAL_SUBSYSTEM).await
-    }
-
-    /// Collect the KMS configuration fingerprint each peer is running.
-    ///
-    /// A peer whose build predates the KMS subsystem rejects the probe, so it
-    /// is reported as an error rather than silently agreeing with this node.
-    pub async fn kms_config_fingerprints(&self) -> Vec<PeerKmsConfigFingerprint> {
-        let mut futures = Vec::with_capacity(self.peer_clients.len());
-        for client in self.peer_clients.iter() {
-            futures.push(async move {
-                let Some(client) = client else {
-                    return PeerKmsConfigFingerprint {
-                        host: String::new(),
-                        fingerprint: None,
-                        err: Some(Error::other("peer is not reachable")),
-                    };
-                };
-                match client.kms_config_fingerprint().await {
-                    Ok(fingerprint) => PeerKmsConfigFingerprint {
-                        host: client.host.to_string(),
-                        fingerprint,
-                        err: None,
-                    },
-                    Err(e) => PeerKmsConfigFingerprint {
-                        host: client.host.to_string(),
-                        fingerprint: None,
-                        err: Some(e),
-                    },
-                }
-            });
-        }
-        join_all(futures).await
-    }
-
     pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {

crates/ecstore/src/set_disk/read.rs

@@ -199,7 +199,7 @@ impl SetDisks {
         );
     }
 
-    #[hotpath::measure(impl_type = "SetDisks")]
+    #[hotpath::measure]
     pub async fn read_version_optimized(
         &self,
         bucket: &str,
@@ -238,7 +238,7 @@ impl SetDisks {
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
-    #[hotpath::measure(impl_type = "SetDisks")]
+    #[hotpath::measure]
     pub(super) async fn get_object_fileinfo(
         &self,
         bucket: &str,
@@ -410,7 +410,7 @@ impl SetDisks {
         Ok((fi, parts_metadata, op_online_disks))
     }
 
-    #[hotpath::measure(impl_type = "SetDisks")]
+    #[hotpath::measure]
     pub(super) async fn get_object_info_and_quorum(
         &self,
         bucket: &str,
@@ -605,7 +605,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[hotpath::measure(impl_type = "SetDisks")]
+    #[hotpath::measure]
     pub(super) async fn get_object_with_fileinfo<W>(
         // &self,
         bucket: &str,
@@ -1140,7 +1140,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[hotpath::measure(impl_type = "SetDisks")]
+    #[hotpath::measure]
     pub(super) async fn get_object_decode_reader_with_fileinfo(
         bucket: &str,
         object: &str,
@@ -1296,7 +1296,7 @@ impl SetDisks {
     }
 
     #[allow(clippy::too_many_arguments)]
-    #[hotpath::measure(impl_type = "SetDisks")]
+    #[hotpath::measure]
     async fn build_codec_streaming_part_reader(
         bucket: &str,
         object: &str,

crates/ecstore/src/store/list_objects.rs

@@ -4861,7 +4861,10 @@ async fn poll_merge_head(rx: &CancellationToken, in_channels: &mut [Receiver<Met
 
 async fn send_or_cancel(rx: &CancellationToken, out_channel: &Sender<MetaCacheEntry>, entry: MetaCacheEntry) -> Result<bool> {
     tokio::select! {
-        result = out_channel.send(entry) => Ok(result.is_ok()),
+        result = out_channel.send(entry) => {
+            result.map_err(Error::other)?;
+            Ok(true)
+        }
         _ = rx.cancelled() => Ok(false),
     }
 }
@@ -9855,28 +9858,6 @@ mod test {
         assert_eq!(results, vec!["obj-a", "obj-b"]);
     }
 
-    #[tokio::test]
-    async fn merge_entry_channels_treats_dropped_output_receiver_as_completion() {
-        let (tx_a, rx_a) = mpsc::channel(4);
-        let (tx_b, rx_b) = mpsc::channel(4);
-        let (out_tx, out_rx) = mpsc::channel(1);
-
-        tx_a.send(test_meta_entry("obj-a")).await.unwrap();
-        tx_b.send(test_meta_entry("obj-b")).await.unwrap();
-        drop(tx_a);
-        drop(tx_b);
-        drop(out_rx);
-
-        let result = timeout(
-            Duration::from_secs(1),
-            merge_entry_channels(CancellationToken::new(), vec![rx_a, rx_b], out_tx, 1),
-        )
-        .await
-        .expect("merge should stop promptly when its consumer disconnects");
-
-        assert!(result.is_ok(), "consumer disconnect must not surface as a merge worker error");
-    }
-
     #[test]
     fn walk_ascending_versions_contract_reverses_newest_first_metadata() {
         // Documents the invariant the walk `versions_sort` handling relies on:

crates/ecstore/src/store/multipart.rs

@@ -238,7 +238,7 @@ impl ECStore {
     }
 
     #[instrument(skip(self, data))]
-    #[hotpath::measure(impl_type = "ECStore")]
+    #[hotpath::measure]
     pub(super) async fn handle_put_object_part(
         &self,
         bucket: &str,

crates/ecstore/src/store/object.rs

@@ -815,7 +815,7 @@ impl ECStore {
     }
 
     #[instrument(level = "debug", skip(self))]
-    #[hotpath::measure(impl_type = "ECStore")]
+    #[hotpath::measure]
     pub(super) async fn handle_get_object_reader(
         &self,
         bucket: &str,
@@ -849,7 +849,7 @@ impl ECStore {
     }
 
     #[instrument(level = "debug", skip(self, data))]
-    #[hotpath::measure(impl_type = "ECStore")]
+    #[hotpath::measure]
     pub(super) async fn handle_put_object(
         &self,
         bucket: &str,

crates/ecstore/tests/hotpath_cpu_attribution_contract_test.rs

@@ -1,95 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-const STORE_MULTIPART: &str = include_str!("../src/store/multipart.rs");
-const STORE_OBJECT: &str = include_str!("../src/store/object.rs");
-const ERASURE: &str = include_str!("../src/erasure/coding/erasure.rs");
-const DECODE: &str = include_str!("../src/erasure/coding/decode.rs");
-const ENCODE: &str = include_str!("../src/erasure/coding/encode.rs");
-const LOCAL_DISK: &str = include_str!("../src/disk/local.rs");
-const BITROT: &str = include_str!("../src/erasure/coding/bitrot.rs");
-const SET_DISK_READ: &str = include_str!("../src/set_disk/read.rs");
-
-fn assert_measured_as(source: &str, impl_type: &str, function: &str) {
-    let attribute = format!("#[hotpath::measure(impl_type = \"{impl_type}\")]\n");
-    let function_start = format!("fn {function}");
-    let mut remaining = source;
-
-    while let Some(attribute_offset) = remaining.find(&attribute) {
-        let measured_source = &remaining[attribute_offset + attribute.len()..];
-        if measured_source.find(&function_start).is_some_and(|offset| offset < 256) {
-            return;
-        }
-        remaining = measured_source;
-    }
-
-    panic!("missing HotPath impl_type={impl_type} for {function}");
-}
-
-#[test]
-fn inherent_hotpath_measurements_have_cpu_attribution_types() {
-    assert_measured_as(STORE_MULTIPART, "ECStore", "handle_put_object_part");
-    for function in ["handle_get_object_reader", "handle_put_object"] {
-        assert_measured_as(STORE_OBJECT, "ECStore", function);
-    }
-    for function in [
-        "encode_data",
-        "encode_data_owned",
-        "encode_data_bytes_mut",
-        "decode_data",
-        "decode_data_and_parity",
-    ] {
-        assert_measured_as(ERASURE, "Erasure", function);
-    }
-    assert_measured_as(DECODE, "ParallelReader", "read");
-    assert_measured_as(DECODE, "Erasure", "decode");
-    for function in [
-        "encode",
-        "encode_batched",
-        "encode_inline_small",
-        "encode_single_block_non_inline",
-    ] {
-        assert_measured_as(ENCODE, "Erasure", function);
-    }
-    for function in ["read_metadata_with_dmtime", "read_all_data"] {
-        assert_measured_as(LOCAL_DISK, "LocalDisk", function);
-    }
-    assert_measured_as(BITROT, "BitrotReader", "read");
-    assert!(
-        BITROT.contains("#[hotpath::measure(label = \"BitrotWriter::write\", impl_type = \"BitrotWriter\")]"),
-        "BitrotWriter::write must retain its stable label and CPU impl_type",
-    );
-    for function in [
-        "read_version_optimized",
-        "get_object_fileinfo",
-        "get_object_info_and_quorum",
-        "get_object_with_fileinfo",
-        "get_object_decode_reader_with_fileinfo",
-        "build_codec_streaming_part_reader",
-    ] {
-        assert_measured_as(SET_DISK_READ, "SetDisks", function);
-    }
-}
-
-#[test]
-fn module_hotpath_measurements_remain_untyped() {
-    assert!(
-        BITROT.contains("#[hotpath::measure]\npub async fn bitrot_verify"),
-        "bitrot_verify is a module function and must not claim an impl type",
-    );
-    assert!(
-        SET_DISK_READ.contains("#[hotpath::measure]\nasync fn setup_multipart_part_readers"),
-        "setup_multipart_part_readers is a module function and must not claim an impl type",
-    );
-}

crates/iam/src/sys.rs

@@ -41,7 +41,7 @@ use rustfs_policy::policy::Args;
 use rustfs_policy::policy::opa;
 use rustfs_policy::policy::{Policy, PolicyDoc, iam_policy_claim_name_sa, policy_needs_existing_object_tag_for_args};
 use serde_json::Value;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::sync::OnceLock;
 use time::OffsetDateTime;
@@ -61,51 +61,10 @@ const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-polic
 const FEDERATED_POLICY_BOUNDARY_CLAIM: &str = "x-rustfs-internal-federated-boundary";
 pub(crate) const SITE_REPLICATOR_CLAIM: &str = "site-replicator";
 
-#[derive(Clone)]
-enum PolicyPluginState {
-    Disabled,
-    Initializing,
-    Ready(opa::AuthZPlugin),
-    Failed,
-}
+static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
 
-static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
-
-fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
-    POLICY_PLUGIN_STATE
-        .get_or_init(|| {
-            let configured = opa::is_configured();
-            let state = Arc::new(RwLock::new(if configured {
-                PolicyPluginState::Initializing
-            } else {
-                PolicyPluginState::Disabled
-            }));
-            if configured {
-                let state = Arc::clone(&state);
-                tokio::spawn(async move {
-                    let next_state = match opa::lookup_config().await {
-                        Ok(conf) if conf.enable() => {
-                            info!("OPA plugin enabled");
-                            PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
-                        }
-                        Ok(_) => PolicyPluginState::Failed,
-                        Err(e) => {
-                            error!(
-                                component = "iam",
-                                subsystem = "policy_plugin",
-                                result = "configuration_load_failed",
-                                error_kind = e.kind(),
-                                "OPA plugin configuration load failed"
-                            );
-                            PolicyPluginState::Failed
-                        }
-                    };
-                    *state.write().await = next_state;
-                });
-            }
-            state
-        })
-        .clone()
+fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
+    POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
 }
 
 pub struct IamSys<T> {
@@ -214,7 +173,19 @@ impl<T: Store> IamSys<T> {
     /// # Returns
     /// A new instance of IamSys
     pub fn new(store: Arc<IamCache<T>>) -> Self {
-        get_policy_plugin_state();
+        tokio::spawn(async move {
+            match opa::lookup_config().await {
+                Ok(conf) => {
+                    if conf.enable() {
+                        Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
+                        info!("OPA plugin enabled");
+                    }
+                }
+                Err(e) => {
+                    error!("Error loading OPA configuration err:{}", e);
+                }
+            };
+        });
 
         Self {
             store,
@@ -235,22 +206,15 @@ impl<T: Store> IamSys<T> {
     }
 
     pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
-        let policy_plugin_state = get_policy_plugin_state();
-        let mut guard = policy_plugin_state.write().await;
-        *guard = PolicyPluginState::Ready(client);
+        let policy_plugin_client = get_policy_plugin_client();
+        let mut guard = policy_plugin_client.write().await;
+        *guard = Some(client);
     }
 
     pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
-        let policy_plugin_state = get_policy_plugin_state();
-        let guard = policy_plugin_state.read().await;
-        match &*guard {
-            PolicyPluginState::Ready(client) => Some(client.clone()),
-            PolicyPluginState::Disabled | PolicyPluginState::Initializing | PolicyPluginState::Failed => None,
-        }
-    }
-
-    async fn policy_plugin_state() -> PolicyPluginState {
-        get_policy_plugin_state().read().await.clone()
+        let policy_plugin_client = get_policy_plugin_client();
+        let guard = policy_plugin_client.read().await;
+        guard.clone()
     }
 
     pub async fn load_group(&self, name: &str) -> Result<()> {
@@ -1096,20 +1060,11 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        match Self::policy_plugin_state().await {
-            PolicyPluginState::Ready(_) => {
-                return PreparedIamAuth {
-                    needs_existing_object_tag: false,
-                    mode: PreparedIamMode::Opa,
-                };
-            }
-            PolicyPluginState::Initializing | PolicyPluginState::Failed => {
-                return PreparedIamAuth {
-                    needs_existing_object_tag: false,
-                    mode: PreparedIamMode::Deny,
-                };
-            }
-            PolicyPluginState::Disabled => {}
+        if Self::get_policy_plugin_client().await.is_some() {
+            return PreparedIamAuth {
+                needs_existing_object_tag: false,
+                mode: PreparedIamMode::Opa,
+            };
         }
 
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
@@ -1208,22 +1163,7 @@ impl<T: Store> IamSys<T> {
             };
         }
 
-        let (resolved_policies, combined_policy) = self.store.merge_policies(&policies.join(",")).await;
-        if args.deny_only {
-            let resolved_policy_names: HashSet<&str> = resolved_policies
-                .split(',')
-                .filter(|policy_name| !policy_name.trim().is_empty())
-                .collect();
-            if policies
-                .iter()
-                .any(|policy_name| !resolved_policy_names.contains(policy_name.as_str()))
-            {
-                return PreparedIamAuth {
-                    needs_existing_object_tag: false,
-                    mode: PreparedIamMode::Deny,
-                };
-            }
-        }
+        let combined_policy = self.get_combined_policy(&policies).await;
         PreparedIamAuth {
             needs_existing_object_tag: policy_needs_existing_object_tag_for_args(&combined_policy, args).await,
             mode: PreparedIamMode::Regular { combined_policy },
@@ -1763,7 +1703,7 @@ mod tests {
     fn deprecated_list_polices_api_is_available() {
         let _ = IamSys::<StsTestMockStore>::list_polices;
     }
-    use rustfs_policy::policy::action::{Action, AdminAction, S3Action, StsAction};
+    use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
     use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
     use serde_json::Value;
     use std::{
@@ -3369,155 +3309,6 @@ mod tests {
         assert!(!iam_sys.eval_prepared(&prepared, &args).await);
     }
 
-    #[tokio::test]
-    async fn regular_deny_only_rejects_unresolved_mapped_policy() {
-        let iam_sys = test_iam_sys().await;
-        let user = "regular-unresolved-policy";
-        let identity = UserIdentity::from(Credentials {
-            access_key: user.to_string(),
-            secret_key: "longenoughsecret".to_string(),
-            status: ACCOUNT_ON.to_string(),
-            ..Default::default()
-        });
-        iam_sys.store.cache.with_write_lock(|cache| {
-            let now = OffsetDateTime::now_utc();
-            cache.add_or_update_user(user, &identity, now);
-            cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
-        });
-
-        let groups = None;
-        let claims = HashMap::new();
-        let conditions = HashMap::new();
-        let args = Args {
-            account: user,
-            groups: &groups,
-            action: Action::StsAction(StsAction::AssumeRoleAction),
-            bucket: "",
-            conditions: &conditions,
-            is_owner: false,
-            object: "",
-            claims: &claims,
-            deny_only: true,
-        };
-
-        let prepared = iam_sys.prepare_regular_auth(&args).await;
-        assert!(matches!(prepared.mode, PreparedIamMode::Deny));
-        assert!(
-            !iam_sys.eval_prepared(&prepared, &args).await,
-            "deny-only evaluation must fail closed when a mapped policy document is unresolved"
-        );
-    }
-
-    #[tokio::test]
-    async fn regular_full_evaluation_keeps_resolved_part_of_partial_mapping() {
-        let iam_sys = test_iam_sys().await;
-        let user = "regular-partial-policy";
-        let identity = UserIdentity::from(Credentials {
-            access_key: user.to_string(),
-            secret_key: "longenoughsecret".to_string(),
-            status: ACCOUNT_ON.to_string(),
-            ..Default::default()
-        });
-        iam_sys.store.cache.with_write_lock(|cache| {
-            let now = OffsetDateTime::now_utc();
-            cache.add_or_update_user(user, &identity, now);
-            cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
-        });
-
-        let groups = None;
-        let claims = HashMap::new();
-        let conditions = HashMap::new();
-        let args = Args {
-            account: user,
-            groups: &groups,
-            action: Action::S3Action(S3Action::GetObjectAction),
-            bucket: "bucket",
-            conditions: &conditions,
-            is_owner: false,
-            object: "object",
-            claims: &claims,
-            deny_only: false,
-        };
-
-        let prepared = iam_sys.prepare_regular_auth(&args).await;
-        assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
-        assert!(
-            iam_sys.eval_prepared(&prepared, &args).await,
-            "full evaluation must preserve the historical partial-resolution behavior"
-        );
-    }
-
-    #[tokio::test]
-    async fn regular_deny_only_honors_group_derived_explicit_deny() {
-        let iam_sys = test_iam_sys().await;
-        let deny_policy =
-            Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["sts:AssumeRole"]}]}"#)
-                .expect("group deny policy should parse");
-        let now = OffsetDateTime::now_utc();
-        iam_sys
-            .store
-            .cache
-            .add_or_update_policy_doc("deny-assume-role", &PolicyDoc::new(deny_policy), now);
-        iam_sys
-            .store
-            .cache
-            .add_or_update_group_policy("testgroup", &MappedPolicy::new("deny-assume-role"), now);
-
-        let groups = Some(vec!["testgroup".to_string()]);
-        let claims = HashMap::new();
-        let conditions = HashMap::new();
-        let args = Args {
-            account: "sts-fallback-test-parent",
-            groups: &groups,
-            action: Action::StsAction(StsAction::AssumeRoleAction),
-            bucket: "",
-            conditions: &conditions,
-            is_owner: false,
-            object: "",
-            claims: &claims,
-            deny_only: true,
-        };
-
-        let prepared = iam_sys.prepare_regular_auth(&args).await;
-        assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
-        assert!(
-            !iam_sys.eval_prepared(&prepared, &args).await,
-            "group-derived explicit Deny must remain authoritative"
-        );
-    }
-
-    #[tokio::test]
-    async fn regular_deny_only_rejects_unresolved_group_mapping() {
-        let iam_sys = test_iam_sys().await;
-        iam_sys.store.cache.add_or_update_group_policy(
-            "testgroup",
-            &MappedPolicy::new("readwrite,missing-policy"),
-            OffsetDateTime::now_utc(),
-        );
-
-        let groups = Some(vec!["testgroup".to_string()]);
-        let claims = HashMap::new();
-        let conditions = HashMap::new();
-        let args = Args {
-            account: "sts-fallback-test-parent",
-            groups: &groups,
-            action: Action::StsAction(StsAction::AssumeRoleAction),
-            bucket: "",
-            conditions: &conditions,
-            is_owner: false,
-            object: "",
-            claims: &claims,
-            deny_only: true,
-        };
-
-        let prepared = iam_sys.prepare_regular_auth(&args).await;
-        assert!(matches!(prepared.mode, PreparedIamMode::Deny));
-        assert!(
-            !iam_sys.eval_prepared(&prepared, &args).await,
-            "deny-only evaluation must fail closed for an unresolved group-derived mapping"
-        );
-    }
-
     #[tokio::test]
     async fn test_sts_claim_policy_ignores_unsafe_and_missing_policy_names() {
         let store = StsTestMockStore::new(true);

crates/io-metrics/src/lib.rs

@@ -49,10 +49,7 @@
 #[macro_use]
 extern crate metrics;
 
-use std::sync::{
-    Mutex,
-    atomic::{AtomicBool, AtomicU64, Ordering},
-};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 
 /// Global switch for detailed per-stage PUT metrics (path label, stage durations).
 /// When `false`, `record_put_object_path` and `record_put_object_stage_duration`
@@ -272,12 +269,6 @@ pub use collector::MetricsCollector;
 pub use performance::PerformanceMetrics;
 
 static EC_ENCODE_INFLIGHT_BYTES: AtomicU64 = AtomicU64::new(0);
-static EC_ENCODE_PRODUCER_BYTES_CURRENT: AtomicU64 = AtomicU64::new(0);
-static EC_ENCODE_PRODUCER_BYTES_PEAK: AtomicU64 = AtomicU64::new(0);
-static EC_ENCODE_QUEUE_BYTES_PEAK: AtomicU64 = AtomicU64::new(0);
-static EC_ENCODE_WRITER_BYTES_CURRENT: AtomicU64 = AtomicU64::new(0);
-static EC_ENCODE_WRITER_BYTES_PEAK: AtomicU64 = AtomicU64::new(0);
-static EC_ENCODE_PEAK_PUBLISH_LOCK: Mutex<()> = Mutex::new(());
 static GET_OBJECT_BUFFERED_BYTES: AtomicU64 = AtomicU64::new(0);
 const SHARD_READ_COST_LOCAL: &str = "local";
 const SHARD_READ_COST_REMOTE: &str = "remote";
@@ -322,49 +313,6 @@ fn saturating_sub_atomic(counter: &AtomicU64, bytes: u64) -> u64 {
     }
 }
 
-#[inline(always)]
-fn update_peak_atomic(counter: &AtomicU64, value: u64) -> Option<u64> {
-    let mut peak = counter.load(Ordering::Relaxed);
-    loop {
-        if value <= peak {
-            return None;
-        }
-        match counter.compare_exchange_weak(peak, value, Ordering::Relaxed, Ordering::Relaxed) {
-            Ok(_) => return Some(value),
-            Err(actual) => peak = actual,
-        }
-    }
-}
-
-#[derive(Clone, Copy)]
-enum EcEncodePeakMetric {
-    Producer,
-    Queue,
-    Writer,
-}
-
-fn publish_ec_encode_peak_with(counter: &AtomicU64, value: u64, before_publish_lock: impl FnOnce(), set_gauge: impl FnOnce(f64)) {
-    if update_peak_atomic(counter, value).is_none() {
-        return;
-    }
-    before_publish_lock();
-    let _guard = EC_ENCODE_PEAK_PUBLISH_LOCK.lock().unwrap_or_else(|error| error.into_inner());
-    set_gauge(counter.load(Ordering::Relaxed) as f64);
-}
-
-fn publish_ec_encode_peak(counter: &AtomicU64, metric: EcEncodePeakMetric, value: u64) {
-    publish_ec_encode_peak_with(
-        counter,
-        value,
-        || {},
-        |peak| match metric {
-            EcEncodePeakMetric::Producer => gauge_set_cached!("rustfs_ec_encode_producer_bytes_peak", peak),
-            EcEncodePeakMetric::Queue => gauge_set_cached!("rustfs_ec_encode_queue_bytes_peak", peak),
-            EcEncodePeakMetric::Writer => gauge_set_cached!("rustfs_ec_encode_writer_bytes_peak", peak),
-        },
-    );
-}
-
 #[inline(always)]
 fn usize_to_f64(value: usize) -> f64 {
     value as f64
@@ -2247,25 +2195,18 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
     }
 }
 
-/// Track encoded bytes in the queue hand-off between erasure encode and disk writers.
-///
-/// This is queue occupancy, not a per-request or process-RSS memory limit. The
-/// erasure encoder settles it on failed/cancelled sends, receiver drop, and
-/// consumer hand-off before shard writes begin.
+/// Track encoded bytes currently queued between erasure encode and disk writers.
 #[inline(always)]
 pub fn add_ec_encode_inflight_bytes(bytes: usize) {
     let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
-    gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
-    if put_stage_metrics_enabled() {
-        publish_ec_encode_peak(&EC_ENCODE_QUEUE_BYTES_PEAK, EcEncodePeakMetric::Queue, next);
-    }
+    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
 }
 
 /// Remove encoded bytes from the tracked erasure encode in-flight gauge.
 #[inline(always)]
 pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
     let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
-    gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
+    gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
 }
 
 /// Return the current tracked EC encode in-flight bytes.
@@ -2274,109 +2215,6 @@ pub fn current_ec_encode_inflight_bytes() -> u64 {
     EC_ENCODE_INFLIGHT_BYTES.load(Ordering::Relaxed)
 }
 
-/// Tracks encoded payload bytes held before queue hand-off or during shard writes.
-///
-/// Each guard contributes to a process-wide stage total until it is dropped. The
-/// reported peak therefore includes concurrent PUTs, but excludes reader,
-/// allocator, and transport buffers; it is not a per-PUT or process-RSS limit.
-pub struct EcEncodePayloadStageGuard {
-    counter: &'static AtomicU64,
-    bytes: u64,
-    enabled: bool,
-    current_metric: EcEncodePeakMetric,
-}
-
-impl Drop for EcEncodePayloadStageGuard {
-    fn drop(&mut self) {
-        if !self.enabled {
-            return;
-        }
-        let next = saturating_sub_atomic(self.counter, self.bytes);
-        match self.current_metric {
-            EcEncodePeakMetric::Producer => gauge_set_cached!("rustfs_ec_encode_producer_bytes_current", next as f64),
-            EcEncodePeakMetric::Queue => unreachable!("queue bytes use their own ownership guard"),
-            EcEncodePeakMetric::Writer => gauge_set_cached!("rustfs_ec_encode_writer_bytes_current", next as f64),
-        }
-    }
-}
-
-fn track_ec_encode_payload_stage(
-    bytes: usize,
-    counter: &'static AtomicU64,
-    peak: &'static AtomicU64,
-    metric: EcEncodePeakMetric,
-) -> EcEncodePayloadStageGuard {
-    let enabled = put_stage_metrics_enabled();
-    let bytes = bytes as u64;
-    if enabled {
-        let next = counter.fetch_add(bytes, Ordering::Relaxed) + bytes;
-        match metric {
-            EcEncodePeakMetric::Producer => gauge_set_cached!("rustfs_ec_encode_producer_bytes_current", next as f64),
-            EcEncodePeakMetric::Queue => unreachable!("queue bytes use their own ownership guard"),
-            EcEncodePeakMetric::Writer => gauge_set_cached!("rustfs_ec_encode_writer_bytes_current", next as f64),
-        }
-        publish_ec_encode_peak(peak, metric, next);
-    }
-    EcEncodePayloadStageGuard {
-        counter,
-        bytes,
-        enabled,
-        current_metric: metric,
-    }
-}
-
-/// Track encoded producer payload bytes until queue hand-off completes.
-#[inline(always)]
-pub fn track_ec_encode_producer_bytes(bytes: usize) -> EcEncodePayloadStageGuard {
-    track_ec_encode_payload_stage(
-        bytes,
-        &EC_ENCODE_PRODUCER_BYTES_CURRENT,
-        &EC_ENCODE_PRODUCER_BYTES_PEAK,
-        EcEncodePeakMetric::Producer,
-    )
-}
-
-/// Track encoded payload bytes while shard writers own the batch.
-#[inline(always)]
-pub fn track_ec_encode_writer_bytes(bytes: usize) -> EcEncodePayloadStageGuard {
-    track_ec_encode_payload_stage(
-        bytes,
-        &EC_ENCODE_WRITER_BYTES_CURRENT,
-        &EC_ENCODE_WRITER_BYTES_PEAK,
-        EcEncodePeakMetric::Writer,
-    )
-}
-
-/// Return the process-lifetime high-water mark of encoded producer payload bytes.
-#[inline(always)]
-pub fn current_ec_encode_producer_bytes_peak() -> u64 {
-    EC_ENCODE_PRODUCER_BYTES_PEAK.load(Ordering::Relaxed)
-}
-
-/// Return the current process-wide encoded producer payload bytes.
-#[inline(always)]
-pub fn current_ec_encode_producer_bytes() -> u64 {
-    EC_ENCODE_PRODUCER_BYTES_CURRENT.load(Ordering::Relaxed)
-}
-
-/// Return the process-lifetime high-water mark of encoded queue payload bytes.
-#[inline(always)]
-pub fn current_ec_encode_queue_bytes_peak() -> u64 {
-    EC_ENCODE_QUEUE_BYTES_PEAK.load(Ordering::Relaxed)
-}
-
-/// Return the process-lifetime high-water mark of encoded writer payload bytes.
-#[inline(always)]
-pub fn current_ec_encode_writer_bytes_peak() -> u64 {
-    EC_ENCODE_WRITER_BYTES_PEAK.load(Ordering::Relaxed)
-}
-
-/// Return the current process-wide encoded writer payload bytes.
-#[inline(always)]
-pub fn current_ec_encode_writer_bytes() -> u64 {
-    EC_ENCODE_WRITER_BYTES_CURRENT.load(Ordering::Relaxed)
-}
-
 /// Track whole-object buffering on the GET path.
 #[inline(always)]
 pub fn track_get_object_buffered_bytes(bytes: usize) -> Option<MemoryGaugeGuard> {
@@ -2543,9 +2381,7 @@ pub fn record_io_latency_p99(latency_ms: f64) {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use metrics_util::MetricKind;
-    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-    use std::sync::{Arc, Barrier, Mutex};
+    use std::sync::Mutex;
 
     // Serialize tests that mutate the process-global PUT_STAGE_METRICS_ENABLED flag.
     static METRICS_FLAG_LOCK: Mutex<()> = Mutex::new(());
@@ -2998,9 +2834,6 @@ mod tests {
 
     #[test]
     fn test_ec_encode_inflight_bytes_tracking() {
-        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
-        set_put_stage_metrics_enabled(true);
-        let queue_peak_before = current_ec_encode_queue_bytes_peak();
         EC_ENCODE_INFLIGHT_BYTES.store(0, Ordering::Relaxed);
         add_ec_encode_inflight_bytes(1024);
         add_ec_encode_inflight_bytes(2048);
@@ -3008,188 +2841,6 @@ mod tests {
         remove_ec_encode_inflight_bytes(2048);
         remove_ec_encode_inflight_bytes(4096);
         assert_eq!(current_ec_encode_inflight_bytes(), 0);
-        assert!(
-            current_ec_encode_queue_bytes_peak() >= queue_peak_before.max(3072),
-            "queue peak must retain the largest observed queue occupancy"
-        );
-        set_put_stage_metrics_enabled(false);
-    }
-
-    #[test]
-    fn test_ec_encode_producer_and_writer_stage_guards_aggregate_and_settle() {
-        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
-        set_put_stage_metrics_enabled(true);
-
-        let producer_bytes = 1024;
-        let producer_first = track_ec_encode_producer_bytes(producer_bytes);
-        let producer_second = track_ec_encode_producer_bytes(producer_bytes);
-        assert_eq!(current_ec_encode_producer_bytes(), 2048);
-        assert!(
-            current_ec_encode_producer_bytes_peak() >= 2048,
-            "producer peak must include simultaneous stage ownership"
-        );
-        drop((producer_first, producer_second));
-        assert_eq!(current_ec_encode_producer_bytes(), 0);
-
-        let writer_bytes = 2048;
-        let writer_first = track_ec_encode_writer_bytes(writer_bytes);
-        let writer_second = track_ec_encode_writer_bytes(writer_bytes);
-        assert_eq!(current_ec_encode_writer_bytes(), 4096);
-        assert!(
-            current_ec_encode_writer_bytes_peak() >= 4096,
-            "writer peak must include simultaneous stage ownership"
-        );
-        drop((writer_first, writer_second));
-        assert_eq!(current_ec_encode_writer_bytes(), 0);
-
-        set_put_stage_metrics_enabled(false);
-    }
-
-    #[test]
-    fn test_ec_encode_payload_stage_guards_are_noop_when_disabled() {
-        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
-        set_put_stage_metrics_enabled(false);
-
-        let producer_current_before = current_ec_encode_producer_bytes();
-        let producer_peak_before = current_ec_encode_producer_bytes_peak();
-        let writer_current_before = current_ec_encode_writer_bytes();
-        let writer_peak_before = current_ec_encode_writer_bytes_peak();
-
-        let producer = track_ec_encode_producer_bytes(1024);
-        let writer = track_ec_encode_writer_bytes(2048);
-        assert_eq!(current_ec_encode_producer_bytes(), producer_current_before);
-        assert_eq!(current_ec_encode_producer_bytes_peak(), producer_peak_before);
-        assert_eq!(current_ec_encode_writer_bytes(), writer_current_before);
-        assert_eq!(current_ec_encode_writer_bytes_peak(), writer_peak_before);
-
-        drop((producer, writer));
-        assert_eq!(current_ec_encode_producer_bytes(), producer_current_before);
-        assert_eq!(current_ec_encode_producer_bytes_peak(), producer_peak_before);
-        assert_eq!(current_ec_encode_writer_bytes(), writer_current_before);
-        assert_eq!(current_ec_encode_writer_bytes_peak(), writer_peak_before);
-    }
-
-    fn assert_concurrent_ec_encode_stage_guards_aggregate_and_settle(
-        track: fn(usize) -> EcEncodePayloadStageGuard,
-        current: fn() -> u64,
-        peak: fn() -> u64,
-    ) {
-        const WORKERS: usize = 4;
-        const STAGE_BYTES: usize = 1536;
-
-        let current_before = current();
-        let peak_before = peak();
-        let entered = Arc::new(Barrier::new(WORKERS + 1));
-        let release = Arc::new(Barrier::new(WORKERS + 1));
-        let mut workers = Vec::with_capacity(WORKERS);
-
-        for _ in 0..WORKERS {
-            let entered = Arc::clone(&entered);
-            let release = Arc::clone(&release);
-            workers.push(std::thread::spawn(move || {
-                let stage = track(STAGE_BYTES);
-                entered.wait();
-                release.wait();
-                drop(stage);
-            }));
-        }
-
-        entered.wait();
-        let expected_delta = u64::try_from(WORKERS).expect("worker count should fit in u64")
-            * u64::try_from(STAGE_BYTES).expect("stage bytes should fit in u64");
-        assert_eq!(current(), current_before + expected_delta);
-        assert!(
-            peak() >= peak_before.max(current_before + expected_delta),
-            "stage peak must expose process-wide concurrent ownership"
-        );
-
-        release.wait();
-        for worker in workers {
-            worker.join().expect("stage worker should not panic");
-        }
-        assert_eq!(current(), current_before);
-    }
-
-    #[test]
-    fn test_ec_encode_payload_stage_guards_track_concurrent_ownership() {
-        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
-        set_put_stage_metrics_enabled(true);
-
-        assert_concurrent_ec_encode_stage_guards_aggregate_and_settle(
-            track_ec_encode_producer_bytes,
-            current_ec_encode_producer_bytes,
-            current_ec_encode_producer_bytes_peak,
-        );
-        assert_concurrent_ec_encode_stage_guards_aggregate_and_settle(
-            track_ec_encode_writer_bytes,
-            current_ec_encode_writer_bytes,
-            current_ec_encode_writer_bytes_peak,
-        );
-
-        set_put_stage_metrics_enabled(false);
-    }
-
-    #[test]
-    fn test_ec_encode_producer_peak_exports_the_high_water_mark() {
-        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
-        assert_eq!(current_ec_encode_producer_bytes(), 0, "test must start without producer stage ownership");
-        let previous_peak = EC_ENCODE_PRODUCER_BYTES_PEAK.swap(0, Ordering::Relaxed);
-        let recorder = DebuggingRecorder::new();
-        let snapshotter = recorder.snapshotter();
-
-        metrics::with_local_recorder(&recorder, || {
-            set_put_stage_metrics_enabled(true);
-            let first = track_ec_encode_producer_bytes(1024);
-            let second = track_ec_encode_producer_bytes(2048);
-            drop((first, second));
-            set_put_stage_metrics_enabled(false);
-        });
-
-        let exported_peak = snapshotter
-            .snapshot()
-            .into_vec()
-            .into_iter()
-            .find_map(|(composite, _, _, value)| {
-                (composite.kind() == MetricKind::Gauge && composite.key().name() == "rustfs_ec_encode_producer_bytes_peak")
-                    .then_some(value)
-            });
-        assert!(
-            matches!(exported_peak, Some(DebugValue::Gauge(value)) if value.0 == 3072.0),
-            "exported producer peak must retain the aggregate high-water mark"
-        );
-        EC_ENCODE_PRODUCER_BYTES_PEAK.fetch_max(previous_peak, Ordering::Relaxed);
-    }
-
-    #[test]
-    fn test_ec_encode_peak_publish_does_not_regress_after_out_of_order_cas() {
-        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
-        let peak = Arc::new(AtomicU64::new(0));
-        let exported = Arc::new(AtomicU64::new(0));
-        let first_ready = Arc::new(Barrier::new(2));
-        let release_first = Arc::new(Barrier::new(2));
-        let first_peak = Arc::clone(&peak);
-        let first_exported = Arc::clone(&exported);
-        let first_ready_for_thread = Arc::clone(&first_ready);
-        let release_first_for_thread = Arc::clone(&release_first);
-
-        let first = std::thread::spawn(move || {
-            publish_ec_encode_peak_with(
-                &first_peak,
-                10,
-                || {
-                    first_ready_for_thread.wait();
-                    release_first_for_thread.wait();
-                },
-                |value| first_exported.store(value as u64, Ordering::Relaxed),
-            );
-        });
-        first_ready.wait();
-        publish_ec_encode_peak_with(&peak, 20, || {}, |value| exported.store(value as u64, Ordering::Relaxed));
-        release_first.wait();
-        first.join().expect("first publisher should not panic");
-
-        assert_eq!(peak.load(Ordering::Relaxed), 20);
-        assert_eq!(exported.load(Ordering::Relaxed), 20, "late publisher must reload the high-water mark");
     }
 
     #[test]

crates/kms/Cargo.toml

@@ -64,10 +64,6 @@ md-5 = { workspace = true }
 arc-swap = { workspace = true }
 rustfs-utils = { workspace = true }
 rustfs-security-governance = { workspace = true }
-# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
-# so the audit sink can live outside this crate without a second, drifting
-# copy of the event vocabulary.
-rustfs-s3-types = { workspace = true }
 
 # HTTP client for Vault
 reqwest = { workspace = true }
@@ -77,16 +73,6 @@ vaultrs = { workspace = true }
 rustify = { workspace = true }
 tokio-util = { workspace = true }
 
-# AWS KMS backend. Credentials come from the standard aws-config provider chain
-# (environment, shared profile, IMDS/container roles); this crate never handles
-# AWS credential material itself.
-aws-config = { workspace = true }
-aws-sdk-kms = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
-# SdkError variants and raw HTTP status are needed to classify AWS failures for
-# the operation policy's retry decisions.
-aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
-aws-smithy-types = { workspace = true }
-
 [dev-dependencies]
 anyhow = { workspace = true }
 # Debugging recorder for asserting emitted metrics in tests.
@@ -96,9 +82,6 @@ tempfile = { workspace = true }
 temp-env = { workspace = true }
 # "net" backs the scripted loopback Vault used by the policy wiring tests.
 tokio = { workspace = true, features = ["net", "test-util"] }
-# Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline.
-aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
-http = { workspace = true }
 
 [features]
 default = []

crates/kms/examples/kms_local_demo.rs

@@ -227,12 +227,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Step 12: Check cache statistics
     println!("12. Checking cache statistics...");
-    if let Some(stats) = encryption_service.cache_stats().await {
+    if let Some((hits, misses)) = encryption_service.cache_stats().await {
         println!("   ✓ Cache statistics:");
-        println!("     - Cached entries: {}", stats.entries);
-        println!("     - Cache hits: {}", stats.hits);
-        println!("     - Cache misses: {}", stats.misses);
-        println!("     - Entries evicted: {}\n", stats.evictions);
+        println!("     - Cache hits: {}", hits);
+        println!("     - Cache misses: {}\n", misses);
     } else {
         println!("   - Cache is disabled\n");
     }

crates/kms/examples/kms_vault_kv_demo.rs

@@ -263,12 +263,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Step 12: Check cache statistics
     println!("12. Checking cache statistics...");
-    if let Some(stats) = encryption_service.cache_stats().await {
+    if let Some((hits, misses)) = encryption_service.cache_stats().await {
         println!("   ✓ Cache statistics:");
-        println!("     - Cached entries: {}", stats.entries);
-        println!("     - Cache hits: {}", stats.hits);
-        println!("     - Cache misses: {}", stats.misses);
-        println!("     - Entries evicted: {}\n", stats.evictions);
+        println!("     - Cache hits: {}", hits);
+        println!("     - Cache misses: {}\n", misses);
     } else {
         println!("   - Cache is disabled\n");
     }

crates/kms/src/api_types.rs

@@ -15,9 +15,9 @@
 //! API types for KMS dynamic configuration
 
 use crate::config::{
-    BackendConfig, CacheConfig, DEFAULT_CACHE_TTL, DEFAULT_MAX_CACHED_KEYS, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
-    DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend, KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod,
-    VaultConfig, VaultTransitConfig, allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option,
+    BackendConfig, CacheConfig, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend,
+    KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, redacted_secret,
+    redacted_secret_option,
 };
 use crate::service_manager::KmsServiceStatus;
 use crate::types::{KeyMetadata, KeyUsage};
@@ -418,26 +418,14 @@ pub enum BackendSummary {
         /// Configured key identifier
         key_id: String,
     },
-    /// AWS KMS backend summary
-    Aws {
-        /// Configured region, when pinned instead of resolved by the AWS chain
-        region: Option<String>,
-        /// Endpoint override, when set for an emulator or private endpoint
-        endpoint_url: Option<String>,
-    },
 }
 
 impl From<&KmsConfig> for KmsConfigSummary {
     fn from(config: &KmsConfig) -> Self {
-        // Report the lifetime the cache was built with, not the raw configured
-        // value: an oversized `ttl` is clamped rather than rejected, and this
-        // response is what operators check the cache against.
-        let cache_ttl_seconds = config.cache_config.effective_ttl().as_secs();
-
         let cache_summary = if config.enable_cache {
             Some(CacheSummary {
                 max_keys: config.cache_config.max_keys,
-                ttl_seconds: cache_ttl_seconds,
+                ttl_seconds: config.cache_config.ttl.as_secs(),
                 enable_metrics: config.cache_config.enable_metrics,
             })
         } else {
@@ -479,10 +467,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
             BackendConfig::Static(static_config) => BackendSummary::Static {
                 key_id: static_config.key_id.clone(),
             },
-            BackendConfig::Aws(aws_config) => BackendSummary::Aws {
-                region: aws_config.region.clone(),
-                endpoint_url: aws_config.endpoint_url.clone(),
-            },
         };
 
         Self {
@@ -492,7 +476,7 @@ impl From<&KmsConfig> for KmsConfigSummary {
             retry_attempts: config.retry_attempts,
             enable_cache: config.enable_cache,
             max_cached_keys: config.cache_config.max_keys,
-            cache_ttl_seconds,
+            cache_ttl_seconds: config.cache_config.ttl.as_secs(),
             cache_summary,
             backend_summary,
         }
@@ -511,17 +495,13 @@ impl ConfigureLocalKmsRequest {
                 file_permissions: self.file_permissions,
             }),
             allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
-            // Read from server configuration, never from the request body: the
-            // gate must mean the same thing whether KMS was configured at
-            // startup or through this endpoint.
-            allow_immediate_deletion: allow_immediate_deletion_from_env(),
             timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
             retry_attempts: self.retry_attempts.unwrap_or(3),
             enable_cache: self.enable_cache.unwrap_or(true),
             cache_config: CacheConfig {
-                max_keys: self.max_cached_keys.unwrap_or(DEFAULT_MAX_CACHED_KEYS),
-                ttl: self.cache_ttl_seconds.map_or(DEFAULT_CACHE_TTL, Duration::from_secs),
-                ..CacheConfig::default()
+                max_keys: self.max_cached_keys.unwrap_or(1000),
+                ttl: Duration::from_secs(self.cache_ttl_seconds.unwrap_or(3600)),
+                enable_metrics: true,
             },
         }
     }
@@ -552,17 +532,13 @@ impl ConfigureVaultKmsRequest {
                 },
             })),
             allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
-            // Read from server configuration, never from the request body: the
-            // gate must mean the same thing whether KMS was configured at
-            // startup or through this endpoint.
-            allow_immediate_deletion: allow_immediate_deletion_from_env(),
             timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
             retry_attempts: self.retry_attempts.unwrap_or(3),
             enable_cache: self.enable_cache.unwrap_or(true),
             cache_config: CacheConfig {
-                max_keys: self.max_cached_keys.unwrap_or(DEFAULT_MAX_CACHED_KEYS),
-                ttl: self.cache_ttl_seconds.map_or(DEFAULT_CACHE_TTL, Duration::from_secs),
-                ..CacheConfig::default()
+                max_keys: self.max_cached_keys.unwrap_or(1000),
+                ttl: Duration::from_secs(self.cache_ttl_seconds.unwrap_or(3600)),
+                enable_metrics: true,
             },
         }
     }
@@ -593,17 +569,13 @@ impl ConfigureVaultTransitKmsRequest {
                 },
             })),
             allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
-            // Read from server configuration, never from the request body: the
-            // gate must mean the same thing whether KMS was configured at
-            // startup or through this endpoint.
-            allow_immediate_deletion: allow_immediate_deletion_from_env(),
             timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
             retry_attempts: self.retry_attempts.unwrap_or(3),
             enable_cache: self.enable_cache.unwrap_or(true),
             cache_config: CacheConfig {
-                max_keys: self.max_cached_keys.unwrap_or(DEFAULT_MAX_CACHED_KEYS),
-                ttl: self.cache_ttl_seconds.map_or(DEFAULT_CACHE_TTL, Duration::from_secs),
-                ..CacheConfig::default()
+                max_keys: self.max_cached_keys.unwrap_or(1000),
+                ttl: Duration::from_secs(self.cache_ttl_seconds.unwrap_or(3600)),
+                enable_metrics: true,
             },
         }
     }
@@ -620,17 +592,13 @@ impl ConfigureStaticKmsRequest {
                 secret_key: self.secret_key.clone(),
             }),
             allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
-            // Read from server configuration, never from the request body: the
-            // gate must mean the same thing whether KMS was configured at
-            // startup or through this endpoint.
-            allow_immediate_deletion: allow_immediate_deletion_from_env(),
             timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
             retry_attempts: self.retry_attempts.unwrap_or(3),
             enable_cache: self.enable_cache.unwrap_or(true),
             cache_config: CacheConfig {
-                max_keys: self.max_cached_keys.unwrap_or(DEFAULT_MAX_CACHED_KEYS),
-                ttl: self.cache_ttl_seconds.map_or(DEFAULT_CACHE_TTL, Duration::from_secs),
-                ..CacheConfig::default()
+                max_keys: self.max_cached_keys.unwrap_or(1000),
+                ttl: Duration::from_secs(self.cache_ttl_seconds.unwrap_or(3600)),
+                enable_metrics: true,
             },
         }
     }
@@ -879,7 +847,6 @@ mod tests {
                 tls: None,
             })),
             allow_insecure_dev_defaults: true,
-            allow_immediate_deletion: false,
             timeout: Duration::from_secs(30),
             retry_attempts: 3,
             enable_cache: true,
@@ -891,8 +858,8 @@ mod tests {
         assert_eq!(summary.backend_type, KmsBackend::VaultTransit);
         assert_eq!(summary.timeout_seconds, 30);
         assert_eq!(summary.retry_attempts, 3);
-        assert_eq!(summary.max_cached_keys, DEFAULT_MAX_CACHED_KEYS);
-        assert_eq!(summary.cache_ttl_seconds, DEFAULT_CACHE_TTL.as_secs());
+        assert_eq!(summary.max_cached_keys, 1000);
+        assert_eq!(summary.cache_ttl_seconds, 3600);
 
         match summary.backend_summary {
             BackendSummary::VaultTransit {

crates/kms/src/audit.rs

@@ -1,424 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Audit contract for KMS management operations.
-//!
-//! [`KmsManager`](crate::manager::KmsManager) builds a [`KmsAuditRecord`] for
-//! every management operation it serves and hands it to the installed
-//! [`KmsAuditSink`]. No sink is installed by default, so a deployment that
-//! does not consume KMS audit records pays nothing beyond the `Option` check.
-//!
-//! The record is deliberately a KMS-side type rather than an audit-crate
-//! entry: keeping the dependency edge out of this crate lets the server map
-//! records onto its existing audit pipeline (and its delivery semantics)
-//! without KMS having to know that pipeline exists.
-//!
-//! # Redaction
-//!
-//! A record has no field that can hold key material, and every value that
-//! originates from a caller passes through [`redact_encryption_context`]
-//! before it is stored. Adding a field here means re-checking that invariant.
-
-use crate::error::KmsError;
-use crate::types::OperationContext;
-use rustfs_s3_types::EventName;
-use sha2::{Digest, Sha256};
-use std::collections::{BTreeMap, HashMap};
-use std::time::Duration;
-use uuid::Uuid;
-
-/// Encryption-context keys that describe *where* an object lives rather than
-/// anything secret about it. Their values are recorded verbatim; every other
-/// key is reduced to a digest.
-const ENCRYPTION_CONTEXT_ALLOWLIST: [&str; 5] = ["bucket", "object", "object_key", "algorithm", "sse_type"];
-
-/// Prefix marking a value that was replaced by a digest of itself.
-const DIGEST_PREFIX: &str = "sha256:";
-
-/// Hex characters of the digest that are kept. Enough to correlate repeated
-/// values across records without being reversible in practice.
-const DIGEST_LEN: usize = 16;
-
-/// A KMS management operation that produces an audit record.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum KmsAuditOperation {
-    /// Creation of a new master key.
-    CreateKey,
-    /// Metadata lookup for a single key.
-    DescribeKey,
-    /// Enumeration of keys.
-    ListKeys,
-    /// Scheduling a key for deletion after the pending window.
-    ScheduleKeyDeletion,
-    /// Cancelling a previously scheduled deletion.
-    CancelKeyDeletion,
-    /// Returning a disabled key to service.
-    EnableKey,
-    /// Taking a key out of service without destroying it.
-    DisableKey,
-    /// Rotating a key to a new version.
-    RotateKey,
-    /// Irreversible removal of key material once the pending window expired.
-    DeleteKey,
-}
-
-impl KmsAuditOperation {
-    /// Stable operation name for audit consumers.
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            KmsAuditOperation::CreateKey => "CreateKey",
-            KmsAuditOperation::DescribeKey => "DescribeKey",
-            KmsAuditOperation::ListKeys => "ListKeys",
-            KmsAuditOperation::ScheduleKeyDeletion => "ScheduleKeyDeletion",
-            KmsAuditOperation::CancelKeyDeletion => "CancelKeyDeletion",
-            KmsAuditOperation::EnableKey => "EnableKey",
-            KmsAuditOperation::DisableKey => "DisableKey",
-            KmsAuditOperation::RotateKey => "RotateKey",
-            KmsAuditOperation::DeleteKey => "DeleteKey",
-        }
-    }
-
-    /// Notification/audit event name carried by records for this operation.
-    pub fn event_name(&self) -> EventName {
-        match self {
-            KmsAuditOperation::CreateKey => EventName::KmsKeyCreated,
-            KmsAuditOperation::DescribeKey | KmsAuditOperation::ListKeys => EventName::KmsKeyAccessed,
-            KmsAuditOperation::ScheduleKeyDeletion => EventName::KmsKeyDeletionScheduled,
-            KmsAuditOperation::CancelKeyDeletion => EventName::KmsKeyDeletionCancelled,
-            KmsAuditOperation::EnableKey => EventName::KmsKeyEnabled,
-            KmsAuditOperation::DisableKey => EventName::KmsKeyDisabled,
-            KmsAuditOperation::RotateKey => EventName::KmsKeyRotated,
-            KmsAuditOperation::DeleteKey => EventName::KmsKeyDeleted,
-        }
-    }
-}
-
-/// Whether the audited operation completed.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum KmsAuditOutcome {
-    /// The operation completed and its effect is durable.
-    Success,
-    /// The operation failed; `error_class` carries the reason category.
-    Failure,
-}
-
-impl KmsAuditOutcome {
-    /// Stable outcome name for audit consumers.
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            KmsAuditOutcome::Success => "success",
-            KmsAuditOutcome::Failure => "failure",
-        }
-    }
-}
-
-/// Coarse, stable classification of a failed KMS operation.
-///
-/// Audit consumers alert on these, so the strings are a wire contract: add
-/// new classes rather than renaming existing ones.
-pub fn error_class(error: &KmsError) -> &'static str {
-    match error {
-        KmsError::ConfigurationError { .. } => "configuration",
-        KmsError::KeyNotFound { .. } => "key_not_found",
-        KmsError::InvalidKey { .. } => "invalid_key",
-        KmsError::CryptographicError { .. } => "cryptographic",
-        KmsError::BackendError { .. } => "backend",
-        KmsError::AccessDenied { .. } => "access_denied",
-        KmsError::KeyAlreadyExists { .. } => "key_already_exists",
-        KmsError::InvalidOperation { .. } => "invalid_operation",
-        KmsError::InternalError { .. } => "internal",
-        KmsError::SerializationError { .. } => "serialization",
-        KmsError::IoError { .. } => "io",
-        KmsError::CacheError { .. } => "cache",
-        KmsError::ValidationError { .. } => "validation",
-        KmsError::UnsupportedAlgorithm { .. } => "unsupported_algorithm",
-        KmsError::InvalidKeySize { .. } => "invalid_key_size",
-        KmsError::ContextMismatch { .. } => "context_mismatch",
-        KmsError::OperationTimedOut { .. } => "timeout",
-        KmsError::OperationCancelled { .. } => "cancelled",
-        KmsError::MaterialMissing { .. } => "material_missing",
-        KmsError::MaterialCorrupt { .. } => "material_corrupt",
-        KmsError::MaterialAuthenticationFailed { .. } => "material_authentication_failed",
-        KmsError::UnsupportedFormatVersion { .. } => "unsupported_format_version",
-        KmsError::KeyVersionNotFound { .. } => "key_version_not_found",
-        KmsError::Backup(_) => "backup",
-        KmsError::UnsupportedCapability { .. } => "unsupported_capability",
-        KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
-        KmsError::BaselineVersionLost { .. } => "baseline_version_lost",
-    }
-}
-
-/// One audited KMS management operation.
-///
-/// Everything an audit consumer needs to answer "who did what to which key,
-/// against which backend, and did it work" — and nothing that could carry key
-/// material. See the module docs for the redaction rules.
-///
-/// Tenant attribution is intentionally absent: multi-tenancy is not modelled
-/// yet, and an invented tenant value is worse than a missing one. It will
-/// arrive as an entry in [`Self::context`] once tenancy lands.
-#[derive(Debug, Clone)]
-pub struct KmsAuditRecord {
-    /// Correlates every record emitted for one logical request.
-    pub operation_id: Uuid,
-    /// The audited operation.
-    pub operation: KmsAuditOperation,
-    /// Event name published to audit consumers.
-    pub event: EventName,
-    /// Authenticated identity that requested the operation, or
-    /// [`OperationContext::INTERNAL_PRINCIPAL`] for server-initiated work.
-    pub principal: String,
-    /// Client address, when the caller supplied one.
-    pub source_ip: Option<String>,
-    /// Client user agent, when the caller supplied one.
-    pub user_agent: Option<String>,
-    /// Key the operation acted on. `None` for operations that span keys, such
-    /// as listing.
-    pub key_id: Option<String>,
-    /// Key version the operation resolved to, when the result identifies one.
-    /// Operations on a key as a whole leave this unset.
-    pub key_version: Option<u32>,
-    /// Whether the operation succeeded.
-    pub outcome: KmsAuditOutcome,
-    /// Failure category; `None` on success. See [`error_class`].
-    pub error_class: Option<&'static str>,
-    /// Backend that served the operation, e.g. `local` or `vault-transit`.
-    pub backend: &'static str,
-    /// Wall-clock time spent in the audited operation.
-    pub latency: Duration,
-    /// Retries performed above the audit point. `None` means the number is
-    /// not observable here: backend-internal retries are accounted for by the
-    /// `rustfs_kms_backend_operation_attempts` metric instead of being
-    /// duplicated (and possibly contradicted) in the audit trail.
-    pub retry_count: Option<u32>,
-    /// Server-supplied correlation values carried by the operation context.
-    /// Also the reserved slot for tenant attribution.
-    pub context: BTreeMap<String, String>,
-    /// Caller-supplied encryption context, after [`redact_encryption_context`].
-    pub encryption_context: BTreeMap<String, String>,
-}
-
-impl KmsAuditRecord {
-    /// Start a record for `operation` performed under `context`.
-    ///
-    /// The outcome defaults to failure so that a record which somehow escapes
-    /// without [`Self::with_result`] under-reports success rather than
-    /// inventing it.
-    pub fn new(operation: KmsAuditOperation, context: &OperationContext, backend: &'static str) -> Self {
-        Self {
-            operation_id: context.operation_id,
-            operation,
-            event: operation.event_name(),
-            principal: context.principal.clone(),
-            source_ip: context.source_ip.clone(),
-            user_agent: context.user_agent.clone(),
-            key_id: None,
-            key_version: None,
-            outcome: KmsAuditOutcome::Failure,
-            error_class: None,
-            backend,
-            latency: Duration::ZERO,
-            retry_count: None,
-            context: context
-                .additional_context
-                .iter()
-                .map(|(k, v)| (k.clone(), v.clone()))
-                .collect(),
-            encryption_context: BTreeMap::new(),
-        }
-    }
-
-    /// Attach the key the operation acted on.
-    pub fn with_key_id(mut self, key_id: Option<impl Into<String>>) -> Self {
-        self.key_id = key_id.map(Into::into);
-        self
-    }
-
-    /// Attach the key version the operation resolved to.
-    pub fn with_key_version(mut self, key_version: Option<u32>) -> Self {
-        self.key_version = key_version;
-        self
-    }
-
-    /// Attach the measured duration of the operation.
-    pub fn with_latency(mut self, latency: Duration) -> Self {
-        self.latency = latency;
-        self
-    }
-
-    /// Derive outcome and error class from the operation result.
-    pub fn with_result<T>(mut self, result: &crate::error::Result<T>) -> Self {
-        match result {
-            Ok(_) => {
-                self.outcome = KmsAuditOutcome::Success;
-                self.error_class = None;
-            }
-            Err(error) => {
-                self.outcome = KmsAuditOutcome::Failure;
-                self.error_class = Some(error_class(error));
-            }
-        }
-        self
-    }
-
-    /// Attach the caller-supplied encryption context, redacted.
-    ///
-    /// Redaction happens here rather than at the call site so no caller can
-    /// place a raw context value into a record by accident.
-    pub fn with_encryption_context(mut self, encryption_context: &HashMap<String, String>) -> Self {
-        self.encryption_context = redact_encryption_context(encryption_context);
-        self
-    }
-}
-
-/// Reduce a caller-supplied encryption context to something safe to persist.
-///
-/// Keys naming the object's location are kept verbatim because they are the
-/// reason the context is audited at all. Every other value is replaced by a
-/// truncated digest, which still correlates repeated values across records
-/// but does not reproduce whatever the caller put there.
-pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -> BTreeMap<String, String> {
-    encryption_context
-        .iter()
-        .map(|(key, value)| {
-            let recorded = if ENCRYPTION_CONTEXT_ALLOWLIST.contains(&key.as_str()) {
-                value.clone()
-            } else {
-                digest_value(value)
-            };
-            (key.clone(), recorded)
-        })
-        .collect()
-}
-
-fn digest_value(value: &str) -> String {
-    let digest = hex::encode(Sha256::digest(value.as_bytes()));
-    format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
-}
-
-/// Receives KMS audit records.
-///
-/// Implementations must not block: they are called on the task that served
-/// the KMS operation. Delivery failures are the sink's problem — the KMS
-/// operation has already completed by the time a record is emitted, and its
-/// result is never changed by what the sink does.
-pub trait KmsAuditSink: Send + Sync {
-    /// Handle one audit record.
-    fn emit(&self, record: KmsAuditRecord);
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn every_operation_maps_to_a_kms_event() {
-        let operations = [
-            KmsAuditOperation::CreateKey,
-            KmsAuditOperation::DescribeKey,
-            KmsAuditOperation::ListKeys,
-            KmsAuditOperation::ScheduleKeyDeletion,
-            KmsAuditOperation::CancelKeyDeletion,
-            KmsAuditOperation::EnableKey,
-            KmsAuditOperation::DisableKey,
-            KmsAuditOperation::RotateKey,
-            KmsAuditOperation::DeleteKey,
-        ];
-
-        for operation in operations {
-            let event = operation.event_name();
-            assert!(event.is_kms(), "{} must map to a KMS event, got {event}", operation.as_str());
-        }
-    }
-
-    #[test]
-    fn record_defaults_to_failure_until_a_result_is_attached() {
-        let context = OperationContext::new("tester".to_string());
-        let record = KmsAuditRecord::new(KmsAuditOperation::CreateKey, &context, "local");
-
-        assert_eq!(record.outcome, KmsAuditOutcome::Failure);
-        assert!(record.error_class.is_none());
-
-        let ok: crate::error::Result<()> = Ok(());
-        assert_eq!(record.clone().with_result(&ok).outcome, KmsAuditOutcome::Success);
-
-        let denied: crate::error::Result<()> = Err(KmsError::access_denied("nope"));
-        let failed = record.with_result(&denied);
-        assert_eq!(failed.outcome, KmsAuditOutcome::Failure);
-        assert_eq!(failed.error_class, Some("access_denied"));
-    }
-
-    #[test]
-    fn record_carries_the_full_operation_context() {
-        let context = OperationContext::new("arn:aws:iam::user/alice".to_string())
-            .with_source_ip("10.0.0.7".to_string())
-            .with_user_agent("aws-cli/2".to_string())
-            .with_context("requestID".to_string(), "req-1".to_string());
-
-        let record = KmsAuditRecord::new(KmsAuditOperation::RotateKey, &context, "vault-transit")
-            .with_key_id(Some("key-1"))
-            .with_key_version(Some(3))
-            .with_latency(Duration::from_millis(12));
-
-        assert_eq!(record.operation_id, context.operation_id);
-        assert_eq!(record.principal, "arn:aws:iam::user/alice");
-        assert_eq!(record.source_ip.as_deref(), Some("10.0.0.7"));
-        assert_eq!(record.user_agent.as_deref(), Some("aws-cli/2"));
-        assert_eq!(record.key_id.as_deref(), Some("key-1"));
-        assert_eq!(record.key_version, Some(3));
-        assert_eq!(record.backend, "vault-transit");
-        assert_eq!(record.latency, Duration::from_millis(12));
-        assert_eq!(record.event, EventName::KmsKeyRotated);
-        assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-1"));
-    }
-
-    #[test]
-    fn encryption_context_keeps_only_allowlisted_values_verbatim() {
-        let secret = "eyJhbGciOiJIUzI1NiJ9.super-secret-grant-token";
-        let context = HashMap::from([
-            ("bucket".to_string(), "photos".to_string()),
-            ("object_key".to_string(), "2026/cat.png".to_string()),
-            ("algorithm".to_string(), "AES256".to_string()),
-            ("grant_token".to_string(), secret.to_string()),
-            ("customer_reference".to_string(), "acct-4711".to_string()),
-        ]);
-
-        let redacted = redact_encryption_context(&context);
-
-        assert_eq!(redacted.get("bucket").map(String::as_str), Some("photos"));
-        assert_eq!(redacted.get("object_key").map(String::as_str), Some("2026/cat.png"));
-        assert_eq!(redacted.get("algorithm").map(String::as_str), Some("AES256"));
-
-        for key in ["grant_token", "customer_reference"] {
-            let value = redacted.get(key).expect("non-allowlisted key should be kept as a digest");
-            assert!(value.starts_with(DIGEST_PREFIX), "{key} should be digested, got {value}");
-        }
-
-        let rendered = format!("{redacted:?}");
-        assert!(!rendered.contains(secret), "redacted context must not reproduce the raw value");
-        assert!(!rendered.contains("acct-4711"), "redacted context must not reproduce the raw value");
-    }
-
-    #[test]
-    fn identical_values_digest_identically_across_records() {
-        // Correlation is the reason a digest is preferable to a fixed
-        // placeholder; assert it actually holds.
-        let first = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
-        let second = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
-        let other = redact_encryption_context(&HashMap::from([("tenant".to_string(), "beta".to_string())]));
-
-        assert_eq!(first.get("tenant"), second.get("tenant"));
-        assert_ne!(first.get("tenant"), other.get("tenant"));
-    }
-}

crates/kms/src/backends/contract_tests.rs

@@ -108,7 +108,6 @@ fn schedule_request(key_id: &str) -> DeleteKeyRequest {
         key_id: key_id.to_string(),
         pending_window_in_days: Some(7),
         force_immediate: None,
-        confirm_key_id: None,
     }
 }
 

crates/kms/src/backends/local.rs

@@ -14,10 +14,7 @@
 
 //! Local file-based KMS backend implementation
 
-use crate::backends::{
-    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
-    ensure_tag_keys_are_mutable,
-};
+use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits};
 use crate::config::KmsConfig;
 use crate::config::LocalConfig;
 use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -1297,32 +1294,6 @@ impl LocalKmsClient {
         Ok(())
     }
 
-    /// Read-modify-write of a key's mutable metadata under the per-key write
-    /// lock, carrying the existing key material over untouched.
-    ///
-    /// `mutate` reports whether it changed anything; an unchanged record is
-    /// never rewritten, so a repeated no-op update neither rewrites the key
-    /// file nor risks a failed commit on an unrelated call.
-    async fn update_key_metadata<F>(&self, key_id: &str, mutate: F) -> Result<()>
-    where
-        F: FnOnce(&mut MasterKeyInfo) -> Result<bool>,
-    {
-        let _write_guard = self.lock_key_for_write(key_id).await;
-        let mut master_key = self.load_master_key(key_id).await?;
-        if !mutate(&mut master_key)? {
-            return Ok(());
-        }
-
-        // Preserve the existing key material (see enable_key): a metadata edit
-        // must never regenerate the master key, or every DEK wrapped by it
-        // becomes undecryptable.
-        let key_material = self.get_key_material(key_id).await?;
-        self.save_master_key(&master_key, &key_material).await?;
-
-        debug!(key_id, "Local KMS key metadata updated");
-        Ok(())
-    }
-
     /// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
     #[cfg(test)]
     pub(crate) async fn schedule_key_deletion(
@@ -1415,8 +1386,7 @@ impl LocalKmsBackend {
             crate::config::BackendConfig::Local(local_config) => local_config.clone(),
             crate::config::BackendConfig::VaultKv2(_)
             | crate::config::BackendConfig::VaultTransit(_)
-            | crate::config::BackendConfig::Static(_)
-            | crate::config::BackendConfig::Aws(_) => {
+            | crate::config::BackendConfig::Static(_) => {
                 return Err(KmsError::configuration_error("Expected Local backend configuration"));
             }
         };
@@ -1441,15 +1411,12 @@ impl KmsBackend for LocalKmsBackend {
             // Generate key material
             let key_material = generate_key_material(algorithm)?;
 
-            let mut master_key = MasterKeyInfo::new_with_description(
+            let master_key = MasterKeyInfo::new_with_description(
                 key_id.clone(),
                 algorithm.to_string(),
                 Some("local-kms".to_string()),
                 request.description.clone(),
             );
-            // Persist the caller's tags: the response below reports them, and
-            // describe_key reads them back out of this record.
-            master_key.metadata = request.tags.clone();
 
             // Save to disk
             self.client.save_new_master_key(&master_key, &key_material).await?;
@@ -1617,15 +1584,9 @@ impl KmsBackend for LocalKmsBackend {
             // Schedule for deletion (default 30 days)
             ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
 
-            // Defensive: KmsManager::delete_key is the enforcement point for the
-            // waiting window and rejects out-of-range requests before any
-            // backend runs. This repeats the bound for callers holding a backend
-            // handle directly (tests, maintenance tasks).
-            let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
-            if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
-                return Err(KmsError::invalid_parameter(format!(
-                    "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
-                )));
+            let days = request.pending_window_in_days.unwrap_or(30);
+            if !(7..=30).contains(&days) {
+                return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
             }
 
             let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -1723,52 +1684,10 @@ impl KmsBackend for LocalKmsBackend {
         self.client.disable_key(key_id, None).await
     }
 
-    async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
-        self.client
-            .update_key_metadata(key_id, |master_key| {
-                if master_key.description.as_deref() == description {
-                    return Ok(false);
-                }
-                master_key.description = description.map(str::to_string);
-                Ok(true)
-            })
-            .await
-    }
-
-    async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
-        ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
-        self.client
-            .update_key_metadata(key_id, |master_key| {
-                let mut changed = false;
-                for (tag_key, value) in tags {
-                    changed |= master_key.metadata.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
-                }
-                Ok(changed)
-            })
-            .await
-    }
-
-    async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
-        ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
-        self.client
-            .update_key_metadata(key_id, |master_key| {
-                let mut changed = false;
-                for tag_key in tag_keys {
-                    changed |= master_key.metadata.remove(tag_key).is_some();
-                }
-                Ok(changed)
-            })
-            .await
-    }
-
     async fn health_check(&self) -> Result<bool> {
         self.client.health_check().await.map(|_| true)
     }
 
-    fn local_backup_client(&self) -> Option<&LocalKmsClient> {
-        Some(&self.client)
-    }
-
     fn capabilities(&self) -> BackendCapabilities {
         // Rotation stays unadvertised until historical key versions can be
         // retained (see LocalKmsClient::rotate_key); without version history
@@ -1777,7 +1696,6 @@ impl KmsBackend for LocalKmsBackend {
             .with_enable_disable(true)
             .with_schedule_deletion(true)
             .with_physical_delete(true)
-            .with_update_key_metadata(true)
     }
 
     async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -2688,7 +2606,6 @@ mod tests {
                 key_id: "durable-key".to_string(),
                 pending_window_in_days: None,
                 force_immediate: Some(true),
-                confirm_key_id: None,
             })
             .await
             .expect("delete key");
@@ -2696,41 +2613,6 @@ mod tests {
         assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory");
     }
 
-    /// KmsManager::delete_key is the enforcement point for the waiting window;
-    /// this pins the backend's defensive copy of the same bound, which is all
-    /// that stands between a direct backend caller and a one-day window.
-    #[tokio::test]
-    async fn delete_key_refuses_a_window_outside_the_supported_range() {
-        let (client, _temp_dir) = create_test_client().await;
-        let key_id = "window-bounds-key";
-        client.create_key(key_id, "AES_256", None).await.expect("create key");
-        let backend = LocalKmsBackend { client };
-
-        for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
-            let result = backend
-                .delete_key(DeleteKeyRequest {
-                    key_id: key_id.to_string(),
-                    pending_window_in_days: Some(days),
-                    ..Default::default()
-                })
-                .await;
-            assert!(
-                matches!(result, Err(KmsError::InvalidOperation { .. })),
-                "a {days}-day window must be refused, got {result:?}"
-            );
-        }
-
-        let state = backend
-            .describe_key(DescribeKeyRequest {
-                key_id: key_id.to_string(),
-            })
-            .await
-            .expect("describe should succeed")
-            .key_metadata
-            .key_state;
-        assert_eq!(state, KeyState::Enabled, "a refused window must not schedule the key");
-    }
-
     #[tokio::test]
     async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() {
         use durable_file::{CommitStep, failpoint};

crates/kms/src/backends/mod.rs

@@ -19,9 +19,7 @@ use crate::types::*;
 use async_trait::async_trait;
 use jiff::Zoned;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
 
-pub mod aws;
 #[cfg(test)]
 mod contract_tests;
 pub mod local;
@@ -100,32 +98,6 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat
     ensure_key_state_permits(key_id, &state, operation)
 }
 
-/// Tag key that carries the key's identity rather than user metadata.
-///
-/// The key-creation path lifts `name` out of the caller's tag map and uses it
-/// as the key id, so a key's `name` tag and its id are the same string.
-/// Rewriting or dropping it after creation would leave the key addressable
-/// under an id its own metadata no longer states. Metadata updates therefore
-/// reject it; only creation may set it.
-pub const RESERVED_KEY_NAME_TAG: &str = "name";
-
-/// Reject a metadata update that would rewrite or remove
-/// [`RESERVED_KEY_NAME_TAG`].
-///
-/// Enforced by every backend that implements tag updates, so no call path —
-/// including a direct [`KmsBackend`] user that bypasses the manager — can
-/// detach a key from its identity.
-pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item = &'a str>) -> Result<()> {
-    for tag_key in tag_keys {
-        if tag_key == RESERVED_KEY_NAME_TAG {
-            return Err(KmsError::invalid_parameter(format!(
-                "Tag '{RESERVED_KEY_NAME_TAG}' identifies the key and cannot be updated or removed"
-            )));
-        }
-    }
-    Ok(())
-}
-
 /// Simplified KMS backend interface for manager
 #[async_trait]
 pub trait KmsBackend: Send + Sync {
@@ -181,38 +153,6 @@ pub trait KmsBackend: Send + Sync {
         Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
     }
 
-    /// Replace a key's free-form description; `None` clears it.
-    ///
-    /// Backends that advertise [`BackendCapabilities::update_key_metadata`]
-    /// must override this method; the default rejects the operation.
-    async fn update_key_description(&self, _key_id: &str, _description: Option<&str>) -> Result<()> {
-        Err(KmsError::unsupported_capability(
-            "backend without key metadata updates",
-            "update_key_description",
-        ))
-    }
-
-    /// Add or overwrite the given tags, leaving every other tag untouched.
-    ///
-    /// Implementations must run [`ensure_tag_keys_are_mutable`] before
-    /// persisting anything. Backends that advertise
-    /// [`BackendCapabilities::update_key_metadata`] must override this method;
-    /// the default rejects the operation.
-    async fn tag_key(&self, _key_id: &str, _tags: &HashMap<String, String>) -> Result<()> {
-        Err(KmsError::unsupported_capability("backend without key metadata updates", "tag_key"))
-    }
-
-    /// Remove the given tags.
-    ///
-    /// Tags that are not set are ignored, so repeating the call is a no-op
-    /// rather than an error. Implementations must run
-    /// [`ensure_tag_keys_are_mutable`] before persisting anything. Backends
-    /// that advertise [`BackendCapabilities::update_key_metadata`] must
-    /// override this method; the default rejects the operation.
-    async fn untag_key(&self, _key_id: &str, _tag_keys: &[String]) -> Result<()> {
-        Err(KmsError::unsupported_capability("backend without key metadata updates", "untag_key"))
-    }
-
     /// Health check
     async fn health_check(&self) -> Result<bool>;
 
@@ -241,19 +181,6 @@ pub trait KmsBackend: Send + Sync {
     async fn remove_expired_key(&self, _key_id: &str, _now: &Zoned) -> Result<ExpiredKeyRemoval> {
         Err(KmsError::unsupported_capability("backend without deletion support", "remove_expired_key"))
     }
-
-    /// The running client to export a full-material backup bundle from.
-    ///
-    /// Only the Local backend owns key material RustFS is allowed to export in
-    /// full (see [`crate::backup::BackupResponsibility`]); every other backend
-    /// keeps its cryptographic root outside RustFS and returns `None` here.
-    ///
-    /// The export must run against the *running* client so that its fence
-    /// actually blocks concurrent create/delete work — a second client opened
-    /// on the same key directory would fence nothing.
-    fn local_backup_client(&self) -> Option<&local::LocalKmsClient> {
-        None
-    }
 }
 
 /// Outcome of [`KmsBackend::remove_expired_key`].
@@ -295,8 +222,6 @@ pub struct BackendCapabilities {
     pub versioning: bool,
     /// Irreversible physical deletion of key material
     pub physical_delete: bool,
-    /// Updating a key's description and tags after creation
-    pub update_key_metadata: bool,
 }
 
 impl BackendCapabilities {
@@ -313,7 +238,6 @@ impl BackendCapabilities {
             schedule_deletion: false,
             versioning: false,
             physical_delete: false,
-            update_key_metadata: false,
         }
     }
 
@@ -364,12 +288,6 @@ impl BackendCapabilities {
         self.physical_delete = physical_delete;
         self
     }
-
-    /// Set whether description and tag updates are supported
-    pub const fn with_update_key_metadata(mut self, update_key_metadata: bool) -> Self {
-        self.update_key_metadata = update_key_metadata;
-        self
-    }
 }
 
 impl Default for BackendCapabilities {
@@ -448,7 +366,6 @@ mod tests {
         assert!(!capabilities.schedule_deletion);
         assert!(!capabilities.versioning);
         assert!(!capabilities.physical_delete);
-        assert!(!capabilities.update_key_metadata);
     }
 
     #[tokio::test]
@@ -457,12 +374,6 @@ mod tests {
             ("enable_key", MinimalBackend.enable_key("any-key").await),
             ("disable_key", MinimalBackend.disable_key("any-key").await),
             ("rotate_key", MinimalBackend.rotate_key("any-key").await),
-            (
-                "update_key_description",
-                MinimalBackend.update_key_description("any-key", Some("new")).await,
-            ),
-            ("tag_key", MinimalBackend.tag_key("any-key", &HashMap::new()).await),
-            ("untag_key", MinimalBackend.untag_key("any-key", &[]).await),
         ] {
             let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
             assert!(
@@ -484,20 +395,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn identity_tag_is_rejected_by_metadata_updates() {
-        let error = ensure_tag_keys_are_mutable([RESERVED_KEY_NAME_TAG])
-            .expect_err("the identity tag must not be writable through a metadata update");
-        assert!(
-            matches!(&error, KmsError::InvalidOperation { message } if message.contains(RESERVED_KEY_NAME_TAG)),
-            "expected a typed rejection naming the tag, got {error:?}"
-        );
-
-        // Ordinary tags — including ones that merely contain the reserved name
-        // — stay writable.
-        ensure_tag_keys_are_mutable(["team", "nickname", "Name"]).expect("ordinary tags must remain writable");
-    }
-
     #[tokio::test]
     async fn local_backend_capabilities_golden() {
         let temp_dir = tempfile::tempdir().expect("temp dir should be created");

crates/kms/src/backends/scripted_vault.rs

@@ -26,16 +26,16 @@ use std::sync::{Arc, Mutex};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};
 
-/// One scripted connection outcome.
-pub(crate) enum ScriptedResponse {
-    Http { status: u16, body: String },
-    Close,
+/// One canned HTTP response.
+pub(crate) struct ScriptedResponse {
+    status: u16,
+    body: String,
 }
 
 impl ScriptedResponse {
     /// A 200 response carrying `data` inside the standard Vault envelope.
     pub(crate) fn ok(data: serde_json::Value) -> Self {
-        Self::Http {
+        Self {
             status: 200,
             body: serde_json::json!({
                 "request_id": "scripted",
@@ -50,16 +50,11 @@ impl ScriptedResponse {
 
     /// An error response in Vault's `{"errors": [...]}` format.
     pub(crate) fn error(status: u16, message: &str) -> Self {
-        Self::Http {
+        Self {
             status,
             body: serde_json::json!({ "errors": [message] }).to_string(),
         }
     }
-
-    /// Close the connection after consuming a request without sending an HTTP response.
-    pub(crate) fn close() -> Self {
-        Self::Close
-    }
 }
 
 /// A scripted stand-in Vault listening on a loopback port.
@@ -93,14 +88,14 @@ impl ScriptedVault {
                 let response = responses
                     .next()
                     .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
-                if let ScriptedResponse::Http { status, body } = response {
-                    let payload = format!(
-                        "HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
-                        body.len(),
-                    );
-                    let _ = stream.write_all(payload.as_bytes()).await;
-                    let _ = stream.shutdown().await;
-                }
+                let payload = format!(
+                    "HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
+                    response.status,
+                    response.body.len(),
+                    response.body
+                );
+                let _ = stream.write_all(payload.as_bytes()).await;
+                let _ = stream.shutdown().await;
             }
         });
 

crates/kms/src/backends/snapshots/rustfs_kms__backends__aws__tests__aws_backend_capabilities.snap

@@ -1,15 +0,0 @@
----
-source: crates/kms/src/backends/aws.rs
-expression: capabilities_snapshot(backend.capabilities())
----
-{
-  "decrypt": true,
-  "enable_disable": true,
-  "encrypt": true,
-  "generate_data_key": true,
-  "physical_delete": false,
-  "rotate": true,
-  "schedule_deletion": true,
-  "update_key_metadata": false,
-  "versioning": true
-}

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__local_backend_capabilities.snap

@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": true,
   "rotate": false,
   "schedule_deletion": true,
-  "update_key_metadata": true,
   "versioning": false
 }

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__static_backend_capabilities.snap

@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": false,
   "rotate": false,
   "schedule_deletion": false,
-  "update_key_metadata": false,
   "versioning": false
 }

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_kv2_backend_capabilities.snap

@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": true,
   "rotate": true,
   "schedule_deletion": true,
-  "update_key_metadata": true,
   "versioning": true
 }

crates/kms/src/backends/snapshots/rustfs_kms__backends__tests__vault_transit_backend_capabilities.snap

@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
   "physical_delete": true,
   "rotate": true,
   "schedule_deletion": true,
-  "update_key_metadata": true,
   "versioning": true
 }

crates/kms/src/backends/static_kms.rs

@@ -431,32 +431,6 @@ mod tests {
         (backend, key_id, key)
     }
 
-    #[tokio::test]
-    async fn metadata_updates_report_the_capability_gap() {
-        let (backend, key_id, _key) = create_test_backend().await;
-        assert!(
-            !backend.capabilities().update_key_metadata,
-            "Static KMS owns no mutable key record, so it must not advertise metadata updates"
-        );
-
-        for (operation, result) in [
-            ("update_key_description", backend.update_key_description(&key_id, Some("new")).await),
-            (
-                "tag_key",
-                backend
-                    .tag_key(&key_id, &HashMap::from([("team".to_string(), "storage".to_string())]))
-                    .await,
-            ),
-            ("untag_key", backend.untag_key(&key_id, &["team".to_string()]).await),
-        ] {
-            let error = result.expect_err("Static KMS is read-only: metadata updates must be rejected");
-            assert!(
-                matches!(error, KmsError::UnsupportedCapability { .. }),
-                "expected UnsupportedCapability for {operation}, got {error:?}"
-            );
-        }
-    }
-
     #[tokio::test]
     async fn test_generate_and_decrypt_data_key() {
         let (backend, key_id, _key) = create_test_backend().await;
@@ -683,7 +657,6 @@ mod tests {
                 key_id: key_id.clone(),
                 pending_window_in_days: Some(7),
                 force_immediate: None,
-                confirm_key_id: None,
             },
         )
         .await;

crates/kms/src/backends/vault.rs

@@ -20,7 +20,6 @@ use crate::backends::vault_credentials::{
 };
 use crate::backends::{
     BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
-    ensure_tag_keys_are_mutable,
 };
 use crate::config::{KmsConfig, VaultConfig};
 use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -641,59 +640,26 @@ impl VaultKmsClient {
         .await
     }
 
-    /// The version numbers of a key's immutable version records.
+    /// Fail closed when the version history extends more than one step past
+    /// the current version.
     ///
-    /// Entries that are not version numbers are ignored: the rotation protocol
-    /// only ever creates numeric records under the versions directory, so
-    /// anything else is not part of the history this reasons about.
-    async fn recorded_version_numbers(&self, key_id: &str) -> Result<Vec<u32>> {
-        Ok(self
+    /// A record exactly one above current is the footprint of an interrupted
+    /// rotation (material persisted, pointer switch never committed) and is
+    /// recovered by the next rotation's adopt path. Anything further ahead
+    /// cannot come from the rotation protocol: it means the top-level record
+    /// regressed (for example a historical lost update rolled back committed
+    /// rotations), and extending the history from the rolled-back state would
+    /// re-mint version numbers that already have immutable records.
+    async fn ensure_current_version_not_behind(&self, key_id: &str, current_version: u32) -> Result<()> {
+        let max_recorded = self
             .list_key_version_records(key_id)
             .await?
             .unwrap_or_default()
             .iter()
             .filter_map(|entry| entry.trim_end_matches('/').parse::<u32>().ok())
-            .collect())
-    }
+            .max();
 
-    /// Fail closed when the version history contradicts the key record.
-    ///
-    /// Both contradictions below mean the record no longer describes the history
-    /// the rotation protocol produced, and rotating on top of either would make
-    /// the inconsistency permanent.
-    ///
-    /// * **Version records without a baseline.** The first rotation freezes the
-    ///   then-current material as a version record and pins `baseline_version` to
-    ///   it in the same commit, so records can only exist without a baseline if
-    ///   the baseline was dropped afterwards. A node older than versioned rotation
-    ///   does exactly that: it does not know the field, and every lifecycle write
-    ///   rewrites the whole record, so one enable/disable/tag from such a node
-    ///   erases it. Rotating here would freeze a *new* baseline at the current
-    ///   version and permanently resolve every pre-versioning envelope to material
-    ///   that never wrapped it — the point of no return this guard blocks.
-    /// * **A record more than one version above current.** A record exactly one
-    ///   above current is the footprint of an interrupted rotation (material
-    ///   persisted, pointer switch never committed) and is recovered by the next
-    ///   rotation's adopt path. Anything further ahead cannot come from the
-    ///   rotation protocol: it means the top-level record regressed (for example a
-    ///   historical lost update rolled back committed rotations), and extending the
-    ///   history from the rolled-back state would re-mint version numbers that
-    ///   already have immutable records.
-    async fn ensure_version_history_consistent(&self, key_id: &str, key_data: &VaultKeyData) -> Result<()> {
-        let recorded = self.recorded_version_numbers(key_id).await?;
-
-        if key_data.baseline_version.is_none()
-            && let Some(oldest_recorded) = recorded.iter().min().copied()
-        {
-            warn!(
-                key_id,
-                oldest_recorded, "Vault KMS key has version records but no baseline version; refusing to rotate"
-            );
-            return Err(KmsError::baseline_version_lost(key_id, oldest_recorded));
-        }
-
-        let current_version = key_data.version;
-        match recorded.iter().max().copied() {
+        match max_recorded {
             Some(max_recorded) if max_recorded > current_version.saturating_add(1) => Err(KmsError::internal_error(format!(
                 "current version {current_version} of key {key_id} is behind existing version record {max_recorded}; refusing to extend an inconsistent version history"
             ))),
@@ -863,74 +829,15 @@ impl VaultKmsClient {
         let key_material = self
             .get_key_material_for_version(&envelope.master_key_id, &key_data, version)
             .await?;
-        let plaintext = match self
+        let plaintext = self
             .dek_crypto
             .decrypt(&key_material, &envelope.encrypted_key, &envelope.nonce)
-            .await
-        {
-            Ok(plaintext) => plaintext,
-            Err(error) => {
-                return Err(self
-                    .explain_unwrap_failure(&envelope.master_key_id, &key_data, envelope.master_key_version, error)
-                    .await);
-            }
-        };
+            .await?;
 
         debug!("Vault KMS data decrypted");
         Ok(plaintext)
     }
 
-    /// Re-report a failure to unwrap a data key as the lost-baseline diagnosis
-    /// when that is what the key record shows.
-    ///
-    /// Only a pre-versioning envelope (no `master_key_version`) read against a key
-    /// with no `baseline_version` can qualify: every other envelope was unwrapped
-    /// with the version it or the baseline named. Such an envelope resolves to the
-    /// *current* version, which is correct for a key that was never rotated and
-    /// wrong for one whose baseline was erased — and the immutable version records
-    /// tell those two apart.
-    ///
-    /// Deliberately runs after the unwrap fails, never before it. A version-less
-    /// envelope written by an old node *after* a rotation is genuinely wrapped with
-    /// the current material and still decrypts, so refusing up front on the same
-    /// evidence would break reads that work today. Nothing is risked by trying
-    /// first: the wrapping is AES-256-GCM, so a wrong master key version cannot
-    /// yield plaintext, only this authentication failure. The extra listing is
-    /// therefore paid once per already-failing read instead of on every read of
-    /// pre-versioning data.
-    async fn explain_unwrap_failure(
-        &self,
-        key_id: &str,
-        key_data: &VaultKeyData,
-        envelope_version: Option<u32>,
-        error: KmsError,
-    ) -> KmsError {
-        if envelope_version.is_some() || key_data.baseline_version.is_some() {
-            return error;
-        }
-
-        match self.recorded_version_numbers(key_id).await {
-            Ok(recorded) => match recorded.iter().min().copied() {
-                Some(oldest_recorded) => {
-                    warn!(
-                        key_id,
-                        oldest_recorded,
-                        "Vault KMS pre-versioning data key failed to unwrap and the key has version records but no baseline version"
-                    );
-                    KmsError::baseline_version_lost(key_id, oldest_recorded)
-                }
-                // No version records: the key was never rotated by a versioning
-                // build, so the current version really is the right one and the
-                // failure has some other cause.
-                None => error,
-            },
-            Err(list_error) => {
-                warn!(key_id, %list_error, "Vault KMS could not list key version records while diagnosing a failed unwrap");
-                error
-            }
-        }
-    }
-
     pub(crate) async fn create_key(
         &self,
         key_id: &str,
@@ -1113,67 +1020,6 @@ impl VaultKmsClient {
         Ok(())
     }
 
-    /// Replace the key's description; `None` clears it.
-    ///
-    /// The write goes through the check-and-set read-modify-write loop, so a
-    /// rotation or state transition landing in between is carried over instead
-    /// of clobbered. A description that already matches is not rewritten.
-    pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
-        self.update_key_data_with_cas(key_id, |key_data| {
-            if key_data.description.as_deref() == description {
-                return Ok(CasMutation::Skip(()));
-            }
-            key_data.description = description.map(str::to_string);
-            Ok(CasMutation::Write(()))
-        })
-        .await?;
-
-        debug!(key_id, "Vault KMS key description updated");
-        Ok(())
-    }
-
-    /// Add or overwrite tags, leaving every other tag untouched.
-    pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
-        ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
-
-        self.update_key_data_with_cas(key_id, |key_data| {
-            let mut changed = false;
-            for (tag_key, value) in tags {
-                changed |= key_data.tags.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
-            }
-            Ok(if changed {
-                CasMutation::Write(())
-            } else {
-                CasMutation::Skip(())
-            })
-        })
-        .await?;
-
-        debug!(key_id, "Vault KMS key tags updated");
-        Ok(())
-    }
-
-    /// Remove tags; tags that are not set are ignored.
-    pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
-        ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
-
-        self.update_key_data_with_cas(key_id, |key_data| {
-            let mut changed = false;
-            for tag_key in tag_keys {
-                changed |= key_data.tags.remove(tag_key).is_some();
-            }
-            Ok(if changed {
-                CasMutation::Write(())
-            } else {
-                CasMutation::Skip(())
-            })
-        })
-        .await?;
-
-        debug!(key_id, "Vault KMS key tags removed");
-        Ok(())
-    }
-
     /// Rotate the master key while keeping every historical version decryptable.
     ///
     /// Commit protocol (all writes check-and-set, in this order):
@@ -1189,10 +1035,6 @@ impl VaultKmsClient {
     /// or interrupted rotation never exposes half-committed material. Concurrent
     /// rotations are serialized by the check-and-set writes: at most one caller
     /// commits each version and the losers fail without side effects on current.
-    ///
-    /// The whole protocol is refused up front, before any write, when the persisted
-    /// version history contradicts the key record — see
-    /// [`Self::ensure_version_history_consistent`].
     pub(crate) async fn rotate_key(&self, key_id: &str, _context: Option<&OperationContext>) -> Result<MasterKeyInfo> {
         debug!("Rotating master key: {}", key_id);
 
@@ -1205,11 +1047,10 @@ impl VaultKmsClient {
         decode_stored_key_material(key_id, &key_data.encrypted_key_material)
             .inspect_err(|error| warn!(key_id, %error, "Vault KMS key material failed validation"))?;
 
-        // The version history must still agree with the key record: a history
-        // that extends past what this rotation would commit means the current
-        // pointer regressed, and a history without a baseline means the baseline
-        // was erased after the fact. Both fail closed before any write.
-        self.ensure_version_history_consistent(key_id, &key_data).await?;
+        // A version history that already extends past what this rotation would
+        // commit means the current pointer regressed; fail closed instead of
+        // re-minting version numbers that have immutable records.
+        self.ensure_current_version_not_behind(key_id, key_data.version).await?;
 
         // Step 1: freeze the baseline on first rotation.
         if key_data.baseline_version.is_none() {
@@ -1322,8 +1163,7 @@ impl VaultKmsBackend {
             crate::config::BackendConfig::VaultKv2(vault_config) => (**vault_config).clone(),
             crate::config::BackendConfig::Local(_)
             | crate::config::BackendConfig::VaultTransit(_)
-            | crate::config::BackendConfig::Static(_)
-            | crate::config::BackendConfig::Aws(_) => {
+            | crate::config::BackendConfig::Static(_) => {
                 return Err(KmsError::configuration_error("Expected Vault KV2 backend configuration"));
             }
         };
@@ -1524,15 +1364,11 @@ impl KmsBackend for VaultKmsBackend {
             // Schedule for deletion (default 30 days)
             ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
 
-            // Defensive: KmsManager::delete_key is the enforcement point for the
-            // waiting window and rejects out-of-range requests before any
-            // backend runs. This repeats the bound for callers holding a backend
-            // handle directly (tests, maintenance tasks).
-            let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
-            if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
-                return Err(crate::error::KmsError::invalid_parameter(format!(
-                    "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
-                )));
+            let days = request.pending_window_in_days.unwrap_or(30);
+            if !(7..=30).contains(&days) {
+                return Err(crate::error::KmsError::invalid_parameter(
+                    "pending_window_in_days must be between 7 and 30".to_string(),
+                ));
             }
 
             let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -1610,18 +1446,6 @@ impl KmsBackend for VaultKmsBackend {
         self.client.rotate_key(key_id, None).await.map(|_| ())
     }
 
-    async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
-        self.client.update_key_description(key_id, description).await
-    }
-
-    async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
-        self.client.tag_key(key_id, tags).await
-    }
-
-    async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
-        self.client.untag_key(key_id, tag_keys).await
-    }
-
     async fn health_check(&self) -> Result<bool> {
         self.client.health_check().await.map(|_| true)
     }
@@ -1637,7 +1461,6 @@ impl KmsBackend for VaultKmsBackend {
             .with_schedule_deletion(true)
             .with_versioning(true)
             .with_physical_delete(true)
-            .with_update_key_metadata(true)
     }
 
     async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -1689,8 +1512,6 @@ mod tests {
     use crate::backends::scripted_vault::{ScriptedResponse, ScriptedVault};
     use crate::config::{VaultAuthMethod, VaultConfig};
 
-    const SCRIPTED_RETRY_ATTEMPTS: u32 = 3;
-
     /// Vault + KMS config pair pointing at a scripted loopback Vault.
     fn scripted_configs(address: &str) -> (VaultConfig, KmsConfig) {
         let vault_config = VaultConfig {
@@ -1706,7 +1527,7 @@ mod tests {
         };
         let kms_config = KmsConfig {
             timeout: Duration::from_secs(5),
-            retry_attempts: SCRIPTED_RETRY_ATTEMPTS,
+            retry_attempts: 3,
             ..KmsConfig::default()
         };
         (vault_config, kms_config)
@@ -1775,31 +1596,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn wired_read_retries_closed_connections_within_budget() {
-        let (vault, client) = scripted_client(vec![ScriptedResponse::close(), ScriptedResponse::close()]).await;
-
-        let error = client
-            .get_key_data("wired-key")
-            .await
-            .expect_err("closed connections must exhaust the retry budget");
-        assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
-
-        let requests = vault.requests();
-        let expected_requests = usize::try_from(SCRIPTED_RETRY_ATTEMPTS).expect("retry attempts must fit usize");
-        assert_eq!(
-            requests.len(),
-            expected_requests,
-            "all budgeted retry attempts must reach Vault: {requests:?}"
-        );
-        assert!(
-            requests
-                .iter()
-                .all(|line| line == "GET /v1/secret/data/rustfs/kms/keys/wired-key"),
-            "all attempts must hit the same read endpoint: {requests:?}"
-        );
-    }
-
     #[tokio::test]
     async fn wired_read_does_not_retry_permission_errors() {
         let (vault, client) = scripted_client(vec![ScriptedResponse::error(403, "permission denied")]).await;
@@ -2375,7 +2171,6 @@ mod tests {
                 key_id: key_id.clone(),
                 pending_window_in_days: Some(7),
                 force_immediate: Some(false),
-                confirm_key_id: None,
             })
             .await
             .expect("schedule delete");
@@ -2879,7 +2674,6 @@ mod tests {
                 key_id: "wired-key".to_string(),
                 pending_window_in_days: Some(7),
                 force_immediate: Some(false),
-                confirm_key_id: None,
             })
             .await
             .expect("the schedule must retry past the lost race and commit");
@@ -2896,45 +2690,6 @@ mod tests {
         );
     }
 
-    /// KmsManager::delete_key is the enforcement point for the waiting window;
-    /// this pins the backend's defensive copy of the same bound, which is all
-    /// that stands between a direct backend caller and a one-day window.
-    #[tokio::test]
-    async fn wired_schedule_deletion_refuses_a_window_outside_the_supported_range() {
-        for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
-            let vault = ScriptedVault::serve(vec![
-                // describe_key: key info plus stored metadata.
-                ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
-                ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
-            ])
-            .await;
-            let config = KmsConfig::vault(
-                url::Url::parse(&vault.address).expect("scripted vault address should parse"),
-                "scripted-token".to_string(),
-            )
-            .with_insecure_development_defaults();
-            let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
-
-            let result = backend
-                .delete_key(DeleteKeyRequest {
-                    key_id: "wired-key".to_string(),
-                    pending_window_in_days: Some(days),
-                    ..Default::default()
-                })
-                .await;
-            assert!(
-                matches!(result, Err(KmsError::InvalidOperation { .. })),
-                "a {days}-day window must be refused, got {result:?}"
-            );
-
-            let requests = vault.requests();
-            assert!(
-                !requests.iter().any(|line| line.starts_with("POST ")),
-                "a refused window must not write anything: {requests:?}"
-            );
-        }
-    }
-
     /// Conflict semantics are re-read *and* re-gate: when the re-read after a
     /// lost race shows the key was concurrently scheduled for deletion, the
     /// state gate rejects the retry instead of blindly re-applying it.
@@ -2967,7 +2722,6 @@ mod tests {
                 key_id: "wired-key".to_string(),
                 pending_window_in_days: Some(7),
                 force_immediate: Some(false),
-                confirm_key_id: None,
             })
             .await
             .expect_err("the retry must re-run the state gate against the fresh record");
@@ -3063,63 +2817,6 @@ mod tests {
         );
     }
 
-    /// Tag updates are check-and-set read-modify-writes over the live record,
-    /// never blind overwrites: they preserve the material and the tags they did
-    /// not address.
-    #[tokio::test]
-    async fn wired_tag_key_writeback_is_check_and_set() {
-        let mut key_data = healthy_key_data();
-        key_data.tags = HashMap::from([("name".to_string(), "wired-key".to_string())]);
-        let (vault, client) = scripted_client(vec![
-            ScriptedResponse::ok(kv2_metadata_read_data(1)),
-            ScriptedResponse::ok(kv2_read_data(&key_data)),
-            ScriptedResponse::ok(kv2_write_ack()),
-        ])
-        .await;
-
-        client
-            .tag_key("wired-key", &HashMap::from([("team".to_string(), "storage".to_string())]))
-            .await
-            .expect("tagging must succeed");
-
-        let bodies = vault.request_bodies();
-        let writeback = parse_write_body(&bodies[2]);
-        assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}");
-        assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}");
-        assert_eq!(
-            writeback["data"]["tags"]["name"],
-            serde_json::json!("wired-key"),
-            "a tag update must not drop tags it did not address: {writeback}"
-        );
-        assert_eq!(
-            writeback["data"]["encrypted_key_material"],
-            serde_json::json!(healthy_key_data().encrypted_key_material),
-            "the write-back must preserve the material of the freshly read record: {writeback}"
-        );
-    }
-
-    /// Rejecting the identity tag happens before any Vault call, so a rejected
-    /// request cannot leave a partial write behind.
-    #[tokio::test]
-    async fn wired_identity_tag_update_is_rejected_before_any_vault_call() {
-        let (vault, client) = scripted_client(Vec::new()).await;
-
-        for result in [
-            client
-                .tag_key("wired-key", &HashMap::from([("name".to_string(), "other".to_string())]))
-                .await,
-            client.untag_key("wired-key", &["name".to_string()]).await,
-        ] {
-            let error = result.expect_err("the identity tag must not be writable");
-            assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}");
-        }
-        assert!(
-            vault.request_bodies().is_empty(),
-            "a rejected metadata update must not reach Vault: {:?}",
-            vault.request_bodies()
-        );
-    }
-
     /// A version record above the current pointer means the top-level record
     /// regressed (a lost update rolled back a committed rotation). Resolving
     /// material through such a record must fail closed instead of quietly
@@ -3185,13 +2882,9 @@ mod tests {
     /// would collide with immutable records.
     #[tokio::test]
     async fn wired_rotate_fails_closed_when_version_history_regressed() {
-        // The baseline is intact, so this isolates the monotonicity guard from
-        // the lost-baseline guard that also inspects the version listing.
-        let mut key_data = healthy_key_data();
-        key_data.baseline_version = Some(1);
         let (vault, client) = scripted_client(vec![
             ScriptedResponse::ok(kv2_metadata_read_data(4)),
-            ScriptedResponse::ok(kv2_read_data(&key_data)),
+            ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
             // Version records reach 3 while the current pointer says 1.
             ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2", "3"] })),
         ])
@@ -3258,317 +2951,4 @@ mod tests {
             "{committed}"
         );
     }
-
-    /// The mixed-version corruption path: a node older than versioned rotation
-    /// performed a lifecycle write, which rewrites the whole key record and
-    /// silently drops the `baseline_version` it does not know. Version records
-    /// without a baseline can only mean that, so the next rotation must refuse
-    /// instead of freezing a fresh baseline at the current version — which would
-    /// resolve every pre-versioning envelope to material that never wrapped it.
-    #[tokio::test]
-    async fn wired_rotate_refuses_when_baseline_was_erased() {
-        let mut key_data = healthy_key_data();
-        key_data.version = 2;
-        key_data.baseline_version = None;
-
-        let (vault, client) = scripted_client(vec![
-            ScriptedResponse::ok(kv2_metadata_read_data(3)),
-            ScriptedResponse::ok(kv2_read_data(&key_data)),
-            // The key was rotated once, so records for versions 1 and 2 exist —
-            // the baseline the first rotation pinned is gone from the record.
-            ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })),
-        ])
-        .await;
-
-        let error = client
-            .rotate_key("wired-key", None)
-            .await
-            .expect_err("a key whose baseline was erased must not rotate");
-        assert!(
-            matches!(
-                &error,
-                KmsError::BaselineVersionLost { key_id, oldest_version: 1 } if key_id == "wired-key"
-            ),
-            "the refusal must name the baseline to restore: {error:?}"
-        );
-
-        let requests = vault.requests();
-        assert_eq!(
-            requests,
-            vec![
-                "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(),
-                "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=3".to_string(),
-                "LIST /v1/secret/metadata/rustfs/kms/keys/wired-key/versions".to_string(),
-            ],
-            "the refusal must be decided from reads alone"
-        );
-        assert!(
-            !requests.iter().any(|line| line.starts_with("POST ")),
-            "nothing may be written once the baseline is known lost: {requests:?}"
-        );
-    }
-
-    /// A key whose baseline is intact keeps rotating: the guard must key off the
-    /// contradiction, not off the presence of version records.
-    #[tokio::test]
-    async fn wired_rotate_with_intact_baseline_still_commits() {
-        let mut key_data = healthy_key_data();
-        key_data.version = 2;
-        key_data.baseline_version = Some(1);
-
-        let (vault, client) = scripted_client(vec![
-            ScriptedResponse::ok(kv2_metadata_read_data(3)),
-            ScriptedResponse::ok(kv2_read_data(&key_data)),
-            ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })),
-            // Version 3's material record, then the pointer switch.
-            ScriptedResponse::ok(kv2_write_ack()),
-            ScriptedResponse::ok(kv2_write_ack()),
-        ])
-        .await;
-
-        let rotated = client
-            .rotate_key("wired-key", None)
-            .await
-            .expect("a key with an intact baseline must still rotate");
-        assert_eq!(rotated.version, 3);
-
-        let requests = vault.requests();
-        assert_eq!(
-            requests,
-            vec![
-                "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(),
-                "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=3".to_string(),
-                "LIST /v1/secret/metadata/rustfs/kms/keys/wired-key/versions".to_string(),
-                "POST /v1/secret/data/rustfs/kms/keys/wired-key/versions/3".to_string(),
-                "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(),
-            ],
-            "an intact baseline skips the freeze step and commits the usual two writes"
-        );
-
-        let bodies = vault.request_bodies();
-        let record = parse_write_body(&bodies[3]);
-        assert_eq!(
-            record["options"]["cas"],
-            serde_json::json!(0),
-            "version records are create-only: {record}"
-        );
-        let committed = parse_write_body(&bodies[4]);
-        assert_eq!(committed["options"]["cas"], serde_json::json!(3), "{committed}");
-        assert_eq!(committed["data"]["version"], serde_json::json!(3), "{committed}");
-        assert_eq!(
-            committed["data"]["baseline_version"],
-            serde_json::json!(1),
-            "the existing baseline must be carried over untouched: {committed}"
-        );
-    }
-
-    /// A never-rotated key legitimately has no baseline and no version records,
-    /// so its first rotation must still freeze one. The guard must not read this
-    /// state as an erased baseline.
-    #[tokio::test]
-    async fn wired_first_rotate_of_never_rotated_key_still_freezes_baseline() {
-        let (vault, client) = scripted_client(vec![
-            ScriptedResponse::ok(kv2_metadata_read_data(7)),
-            ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
-            // The versions directory does not exist yet.
-            ScriptedResponse::error(404, "not found"),
-            // Freeze version 1, persist the baseline, create version 2, switch.
-            ScriptedResponse::ok(kv2_write_ack()),
-            ScriptedResponse::ok(kv2_write_ack()),
-            ScriptedResponse::ok(kv2_write_ack()),
-            ScriptedResponse::ok(kv2_write_ack()),
-        ])
-        .await;
-
-        let rotated = client
-            .rotate_key("wired-key", None)
-            .await
-            .expect("the first rotation of a never-rotated key must commit");
-        assert_eq!(rotated.version, 2);
-
-        let requests = vault.requests();
-        assert_eq!(
-            requests,
-            vec![
-                "GET /v1/secret/metadata/rustfs/kms/keys/wired-key".to_string(),
-                "GET /v1/secret/data/rustfs/kms/keys/wired-key?version=7".to_string(),
-                "LIST /v1/secret/metadata/rustfs/kms/keys/wired-key/versions".to_string(),
-                "POST /v1/secret/data/rustfs/kms/keys/wired-key/versions/1".to_string(),
-                "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(),
-                "POST /v1/secret/data/rustfs/kms/keys/wired-key/versions/2".to_string(),
-                "POST /v1/secret/data/rustfs/kms/keys/wired-key".to_string(),
-            ],
-            "{requests:?}"
-        );
-
-        let bodies = vault.request_bodies();
-        let frozen = parse_write_body(&bodies[3]);
-        assert_eq!(
-            frozen["data"]["encrypted_key_material"],
-            serde_json::json!(healthy_key_data().encrypted_key_material),
-            "the baseline record must freeze the pre-rotation material: {frozen}"
-        );
-        let baseline_commit = parse_write_body(&bodies[4]);
-        assert_eq!(
-            baseline_commit["data"]["baseline_version"],
-            serde_json::json!(1),
-            "the first rotation must pin the baseline: {baseline_commit}"
-        );
-    }
-
-    /// Reading a pre-versioning envelope against a key whose baseline was erased
-    /// resolves to the current version, whose material never wrapped it. The
-    /// unwrap therefore fails (AES-GCM cannot yield plaintext under the wrong
-    /// key); the failure must name the erased baseline instead of surfacing an
-    /// undiagnosable authentication error.
-    #[tokio::test]
-    async fn wired_decrypt_reports_erased_baseline_for_pre_versioning_envelope() {
-        let baseline_material = [0x41u8; 32];
-        let (encrypted_key, nonce) = AesDekCrypto::new()
-            .encrypt(&baseline_material, b"dek-plaintext")
-            .await
-            .expect("wrap test DEK under the baseline material");
-        // A pre-versioning envelope: no master_key_version field.
-        let envelope = DataKeyEnvelope {
-            key_id: "dek".to_string(),
-            master_key_id: "wired-key".to_string(),
-            key_spec: "AES_256".to_string(),
-            encrypted_key,
-            nonce,
-            encryption_context: HashMap::new(),
-            created_at: Zoned::now(),
-            master_key_version: None,
-        };
-        let ciphertext = serde_json::to_vec(&envelope).expect("serialize envelope");
-
-        // The key has been rotated (current material differs from the baseline's)
-        // and its baseline pointer was erased by an older node.
-        let mut key_data = healthy_key_data();
-        key_data.version = 2;
-        key_data.baseline_version = None;
-        key_data.encrypted_key_material = rotated_material();
-
-        let (vault, client) = scripted_client(vec![
-            ScriptedResponse::ok(kv2_read_data(&key_data)),
-            ScriptedResponse::ok(serde_json::json!({ "keys": ["1", "2"] })),
-        ])
-        .await;
-
-        let error = client
-            .decrypt(
-                &DecryptRequest {
-                    ciphertext,
-                    encryption_context: HashMap::new(),
-                    grant_tokens: Vec::new(),
-                },
-                None,
-            )
-            .await
-            .expect_err("the wrong master key version cannot unwrap the data key");
-        assert!(
-            matches!(
-                &error,
-                KmsError::BaselineVersionLost { key_id, oldest_version: 1 } if key_id == "wired-key"
-            ),
-            "the failure must point at the erased baseline: {error:?}"
-        );
-
-        let requests = vault.requests();
-        assert_eq!(requests.len(), 2, "the diagnosis costs one listing after the failure: {requests:?}");
-        assert!(requests[1].contains("/versions"), "{requests:?}");
-    }
-
-    /// Without version records the key was never rotated by a versioning build,
-    /// so the current version really is the right one for a pre-versioning
-    /// envelope and an unwrap failure has some other cause. Misreporting it as a
-    /// lost baseline would send operators after a baseline that never existed.
-    #[tokio::test]
-    async fn wired_decrypt_keeps_original_error_when_key_was_never_rotated() {
-        let (encrypted_key, nonce) = AesDekCrypto::new()
-            .encrypt(&[0x41u8; 32], b"dek-plaintext")
-            .await
-            .expect("wrap test DEK");
-        let envelope = DataKeyEnvelope {
-            key_id: "dek".to_string(),
-            master_key_id: "wired-key".to_string(),
-            key_spec: "AES_256".to_string(),
-            encrypted_key,
-            nonce,
-            encryption_context: HashMap::new(),
-            created_at: Zoned::now(),
-            master_key_version: None,
-        };
-        let ciphertext = serde_json::to_vec(&envelope).expect("serialize envelope");
-
-        let (vault, client) = scripted_client(vec![
-            // The key record holds different material than the envelope was
-            // wrapped with, but has no version history at all.
-            ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
-            ScriptedResponse::error(404, "not found"),
-        ])
-        .await;
-
-        let error = client
-            .decrypt(
-                &DecryptRequest {
-                    ciphertext,
-                    encryption_context: HashMap::new(),
-                    grant_tokens: Vec::new(),
-                },
-                None,
-            )
-            .await
-            .expect_err("the mismatched material must still fail the unwrap");
-        assert!(
-            matches!(error, KmsError::CryptographicError { .. }),
-            "a key with no version records must keep its original failure: {error:?}"
-        );
-        assert_eq!(vault.requests().len(), 2);
-    }
-
-    /// The common upgrade shape — pre-versioning envelopes against a key that was
-    /// never rotated — must keep decrypting with exactly one Vault read. The
-    /// diagnosis above may not add a listing to reads that succeed.
-    #[tokio::test]
-    async fn wired_decrypt_of_pre_versioning_envelope_adds_no_request() {
-        let key_data = healthy_key_data();
-        let key_material = general_purpose::STANDARD
-            .decode(&key_data.encrypted_key_material)
-            .expect("decode fixture material");
-        let (encrypted_key, nonce) = AesDekCrypto::new()
-            .encrypt(&key_material, b"dek-plaintext")
-            .await
-            .expect("wrap test DEK under the current material");
-        let envelope = DataKeyEnvelope {
-            key_id: "dek".to_string(),
-            master_key_id: "wired-key".to_string(),
-            key_spec: "AES_256".to_string(),
-            encrypted_key,
-            nonce,
-            encryption_context: HashMap::new(),
-            created_at: Zoned::now(),
-            master_key_version: None,
-        };
-        let ciphertext = serde_json::to_vec(&envelope).expect("serialize envelope");
-
-        let (vault, client) = scripted_client(vec![ScriptedResponse::ok(kv2_read_data(&key_data))]).await;
-
-        let plaintext = client
-            .decrypt(
-                &DecryptRequest {
-                    ciphertext,
-                    encryption_context: HashMap::new(),
-                    grant_tokens: Vec::new(),
-                },
-                None,
-            )
-            .await
-            .expect("a pre-versioning envelope on a never-rotated key must decrypt");
-        assert_eq!(plaintext, b"dek-plaintext".to_vec());
-        assert_eq!(
-            vault.requests(),
-            vec!["GET /v1/secret/data/rustfs/kms/keys/wired-key".to_string()],
-            "a successful read must not pay for the lost-baseline diagnosis"
-        );
-    }
 }

crates/kms/src/backends/vault_credentials.rs

@@ -57,42 +57,6 @@ const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5);
 /// Default seconds between token file re-reads for [`TokenFileSource`].
 const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30;
 
-// ---------------------------------------------------------------------------
-// Metrics
-//
-// Both gauges describe the one credential generation currently installed, so
-// they carry no labels: the Vault address, mount, auth path and token are all
-// off limits as label values, and there is exactly one generation to describe.
-// The renewal loop republishes them on a bounded cadence while it waits, so a
-// scrape landing between refresh cycles never reads a TTL frozen at the last
-// refresh, or a fail-closed state that flipped after it.
-// ---------------------------------------------------------------------------
-
-/// Gauge: seconds left before the active Vault token expires; `0` once it has.
-const METRIC_TOKEN_TTL_SECONDS: &str = "rustfs_kms_vault_token_ttl_seconds";
-/// Gauge: `1` while [`VaultCredentialProvider::current`] refuses to hand out
-/// the token because it is inside the fail-closed safety window, `0` otherwise.
-const METRIC_CREDENTIALS_FAIL_CLOSED: &str = "rustfs_kms_vault_credentials_fail_closed";
-
-/// How often the renewal loop republishes the credential gauges while waiting.
-/// Bounds how stale a scrape can be, without any additional Vault traffic.
-const CREDENTIAL_GAUGE_INTERVAL: Duration = Duration::from_secs(10);
-
-/// Register metric descriptions once per process.
-fn describe_credential_metrics() {
-    static DESCRIBE: std::sync::Once = std::sync::Once::new();
-    DESCRIBE.call_once(|| {
-        metrics::describe_gauge!(
-            METRIC_TOKEN_TTL_SECONDS,
-            "Seconds remaining before the Vault token backing the KMS backend expires"
-        );
-        metrics::describe_gauge!(
-            METRIC_CREDENTIALS_FAIL_CLOSED,
-            "1 while the Vault credential provider refuses to serve its token because the token is inside the fail-closed safety window"
-        );
-    });
-}
-
 /// A crate-owned secret value, zeroized on drop and redacted in Debug output.
 #[derive(Clone, Zeroize, ZeroizeOnDrop)]
 pub(crate) struct SecretString(String);
@@ -663,45 +627,6 @@ impl VaultCredentialProvider {
         Ok(handle)
     }
 
-    /// Publish the credential gauges for the generation currently installed.
-    ///
-    /// The fail-closed gauge re-evaluates the very gate
-    /// [`VaultCredentialProvider::current`] applies, so what operators see and
-    /// what the request path does cannot drift apart.
-    fn record_credential_gauges(&self) {
-        let handle = self.current.load();
-        let now = Instant::now();
-        let fail_closed = match handle.expires_at() {
-            Some(expires_at) => {
-                metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
-                now + self.policy.safety_window >= expires_at
-            }
-            // A generation without an expiry has no remaining TTL to report
-            // and can never lapse, so it can never fail closed either.
-            None => false,
-        };
-        metrics::gauge!(METRIC_CREDENTIALS_FAIL_CLOSED).set(if fail_closed { 1.0 } else { 0.0 });
-    }
-
-    /// Wait until `deadline`, republishing the credential gauges on the
-    /// observation cadence. Reports `false` when cancellation cut the wait
-    /// short.
-    async fn wait_publishing_gauges(&self, deadline: Instant, cancel: &CancellationToken) -> bool {
-        loop {
-            self.record_credential_gauges();
-            let now = Instant::now();
-            if now >= deadline {
-                return true;
-            }
-            let slice = (deadline - now).min(CREDENTIAL_GAUGE_INTERVAL);
-            tokio::select! {
-                biased;
-                _ = cancel.cancelled() => return false,
-                _ = tokio::time::sleep(slice) => {}
-            }
-        }
-    }
-
     /// Refresh the credentials if generation `observed` is still current.
     ///
     /// Single-flight: concurrent callers serialize on the refresh lock, and a
@@ -786,22 +711,17 @@ impl fmt::Debug for VaultCredentialProvider {
 /// again immediately (the renewal point is already in the past), so the
 /// provider keeps trying to recover even after the fail-closed window has
 /// been reached.
-///
-/// Both waits run through [`VaultCredentialProvider::wait_publishing_gauges`],
-/// which is the only place the credential gauges are published: the loop is
-/// already the component that tracks token expiry, and doing it here keeps the
-/// request path free of any metric work.
 async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: CancellationToken) {
-    describe_credential_metrics();
     loop {
         let handle = provider.snapshot();
         let Some(renew_at) = handle.renew_at() else {
             // The current generation never expires; nothing left to schedule.
-            provider.record_credential_gauges();
             return;
         };
-        if !provider.wait_publishing_gauges(renew_at, &cancel).await {
-            return;
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => return,
+            _ = tokio::time::sleep_until(renew_at) => {}
         }
 
         match provider.refresh(handle.generation, &cancel).await {
@@ -813,9 +733,10 @@ async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: Cancellati
                     error = %error,
                     "Vault credential refresh failed; retrying until the credentials recover"
                 );
-                let retry_at = Instant::now() + provider.policy.retry_interval;
-                if !provider.wait_publishing_gauges(retry_at, &cancel).await {
-                    return;
+                tokio::select! {
+                    biased;
+                    _ = cancel.cancelled() => return,
+                    _ = tokio::time::sleep(provider.policy.retry_interval) => {}
                 }
             }
         }
@@ -1433,125 +1354,4 @@ mod tests {
         assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}");
         assert!(rendered.contains("vault-token"), "the file path is not a secret");
     }
-
-    // -- Metric emission ----------------------------------------------------
-    //
-    // Each test installs a thread-local debugging recorder and drives a
-    // paused-clock current-thread runtime inside it, so the renewal loop's
-    // gauge publications land on exact virtual timestamps.
-
-    use metrics_util::MetricKind;
-    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-
-    type MetricEntry = (
-        metrics_util::CompositeKey,
-        Option<metrics::Unit>,
-        Option<metrics::SharedString>,
-        DebugValue,
-    );
-
-    /// Run `test` on a paused current-thread runtime under a debugging
-    /// recorder and return one snapshot of everything it emitted.
-    ///
-    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
-    /// the recorded state, so taking it per assertion would only show the
-    /// first assertion any data.
-    fn record_metrics<Out>(
-        test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
-    ) -> (Vec<MetricEntry>, Out) {
-        let recorder = DebuggingRecorder::new();
-        let snapshotter = recorder.snapshotter();
-        let out = metrics::with_local_recorder(&recorder, || {
-            let runtime = tokio::runtime::Builder::new_current_thread()
-                .enable_time()
-                .start_paused(true)
-                .build()
-                .expect("current-thread runtime must build");
-            runtime.block_on(test())
-        });
-        (snapshotter.snapshot().into_vec(), out)
-    }
-
-    /// Last value of a gauge, plus the labels it was published with.
-    fn gauge(snapshot: &[MetricEntry], name: &str) -> Option<(f64, Vec<String>)> {
-        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
-            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
-            match (matches, value) {
-                (true, DebugValue::Gauge(value)) => Some((
-                    value.into_inner(),
-                    composite.key().labels().map(|label| label.key().to_string()).collect(),
-                )),
-                _ => None,
-            }
-        })
-    }
-
-    #[test]
-    fn renewal_loop_republishes_token_ttl_while_it_waits() {
-        let (snapshot, ()) = record_metrics(|| {
-            Box::pin(async {
-                let (provider, _state) = scripted_provider(
-                    Duration::from_secs(60),
-                    true,
-                    test_policy(Duration::from_secs(10), Duration::from_secs(5)),
-                )
-                .await;
-                let task = provider.spawn_renewal_task().expect("lease-bound tokens need renewal");
-
-                // Well inside the first half of the lease: nothing has been
-                // renewed yet, so only the observation cadence can have moved
-                // the gauge off its initial 60s.
-                tokio::time::sleep(Duration::from_secs(25)).await;
-                task.shutdown().await;
-            })
-        });
-
-        let (ttl, labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
-        assert!(
-            (ttl - 40.0).abs() < 1.0,
-            "expected ~40s left of a 60s lease at the last observation, got {ttl}"
-        );
-        assert!(labels.is_empty(), "credential gauges must carry no labels, got {labels:?}");
-        assert_eq!(
-            gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
-            Some(0.0),
-            "a token outside its safety window must not report fail-closed"
-        );
-    }
-
-    #[test]
-    fn fail_closed_gauge_tracks_the_gate_the_request_path_applies() {
-        let (snapshot, refused) = record_metrics(|| {
-            Box::pin(async {
-                let (provider, state) = scripted_provider(
-                    Duration::from_secs(60),
-                    true,
-                    test_policy(Duration::from_secs(10), Duration::from_secs(5)),
-                )
-                .await;
-                state.fail_renew.store(true, Ordering::SeqCst);
-                state.fail_login.store(true, Ordering::SeqCst);
-                let task = provider.spawn_renewal_task().expect("renewal task");
-
-                // 60s lease minus the 10s safety window: by 51s the provider
-                // is refusing the token, and the gauge must already say so.
-                tokio::time::sleep(Duration::from_secs(51)).await;
-                let refused = provider.current().is_err();
-                task.shutdown().await;
-                refused
-            })
-        });
-
-        assert!(refused, "a token inside the safety window must be refused");
-        assert_eq!(
-            gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
-            Some(1.0),
-            "the gauge must report the same fail-closed state the request path enforces"
-        );
-        let (ttl, _labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
-        assert!(
-            (ttl - 10.0).abs() < 1.0,
-            "expected the TTL gauge to track the lease down into its safety window, got {ttl}"
-        );
-    }
 }

crates/kms/src/backends/vault_transit.rs

@@ -18,10 +18,7 @@ use crate::backends::vault_credentials::{
     CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider,
     token_source_for,
 };
-use crate::backends::{
-    BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
-    ensure_tag_keys_are_mutable,
-};
+use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits};
 use crate::config::{KmsConfig, VaultTransitConfig};
 use crate::encryption::{DataKeyEnvelope, generate_key_material};
 use crate::error::{KmsError, Result};
@@ -59,14 +56,7 @@ const METADATA_CAS_ATTEMPTS: usize = 3;
 /// TTL bound on cached metadata records. This caps how long one node can keep
 /// acting on lifecycle state another node has since changed (disable,
 /// schedule-deletion): the divergence window is one TTL instead of "until
-/// process restart".
-///
-/// Deliberately fixed rather than derived from `CacheConfig`: this cache gates
-/// cryptographic operations through `ensure_key_state_allows`, so its staleness
-/// window must not follow a knob an operator turns to tune the manager-level
-/// describe cache. It happens to equal `config::DEFAULT_CACHE_TTL` today, but
-/// that is a coincidence rather than a contract, and binding the two would let
-/// a later change to the operator-facing default silently widen this window.
+/// process restart". Matches the manager-level `KmsCache` TTL.
 const METADATA_CACHE_TTL: Duration = Duration::from_secs(300);
 
 /// Capacity bound on the metadata cache so an unbounded key namespace cannot
@@ -918,46 +908,6 @@ impl VaultTransitKmsClient {
         .map(|_| ())
     }
 
-    /// Replace the key's description; `None` clears it.
-    ///
-    /// Metadata edits carry no state gate: they neither use nor invalidate key
-    /// material, so they stay available for whatever lifecycle state the key
-    /// is in.
-    pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
-        self.mutate_key_metadata(key_id, |metadata| {
-            metadata.description = description.map(str::to_string);
-            Ok(())
-        })
-        .await
-        .map(|_| ())
-    }
-
-    /// Add or overwrite tags, leaving every other tag untouched.
-    pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
-        ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
-        self.mutate_key_metadata(key_id, |metadata| {
-            metadata
-                .tags
-                .extend(tags.iter().map(|(key, value)| (key.clone(), value.clone())));
-            Ok(())
-        })
-        .await
-        .map(|_| ())
-    }
-
-    /// Remove tags; tags that are not set are ignored.
-    pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
-        ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
-        self.mutate_key_metadata(key_id, |metadata| {
-            for tag_key in tag_keys {
-                metadata.tags.remove(tag_key);
-            }
-            Ok(())
-        })
-        .await
-        .map(|_| ())
-    }
-
     /// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
     #[cfg(test)]
     pub(crate) async fn schedule_key_deletion(
@@ -1062,9 +1012,7 @@ impl VaultTransitKmsBackend {
                 metadata_key_prefix: vault_config.key_path_prefix.clone(),
                 tls: vault_config.tls.clone(),
             },
-            crate::config::BackendConfig::Local(_)
-            | crate::config::BackendConfig::Static(_)
-            | crate::config::BackendConfig::Aws(_) => {
+            crate::config::BackendConfig::Local(_) | crate::config::BackendConfig::Static(_) => {
                 return Err(KmsError::configuration_error("Expected Vault Transit backend configuration"));
             }
         };
@@ -1226,15 +1174,9 @@ impl KmsBackend for VaultTransitKmsBackend {
         } else {
             ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
 
-            // Defensive: KmsManager::delete_key is the enforcement point for the
-            // waiting window and rejects out-of-range requests before any
-            // backend runs. This repeats the bound for callers holding a backend
-            // handle directly (tests, maintenance tasks).
-            let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
-            if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
-                return Err(KmsError::invalid_parameter(format!(
-                    "pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
-                )));
+            let days = request.pending_window_in_days.unwrap_or(30);
+            if !(7..=30).contains(&days) {
+                return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"));
             }
 
             let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
@@ -1293,18 +1235,6 @@ impl KmsBackend for VaultTransitKmsBackend {
         self.client.rotate_key(key_id, None).await.map(|_| ())
     }
 
-    async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
-        self.client.update_key_description(key_id, description).await
-    }
-
-    async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
-        self.client.tag_key(key_id, tags).await
-    }
-
-    async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
-        self.client.untag_key(key_id, tag_keys).await
-    }
-
     async fn health_check(&self) -> Result<bool> {
         self.client.health_check().await.map(|_| true)
     }
@@ -1319,7 +1249,6 @@ impl KmsBackend for VaultTransitKmsBackend {
             .with_schedule_deletion(true)
             .with_versioning(true)
             .with_physical_delete(true)
-            .with_update_key_metadata(true)
     }
 
     async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -1832,48 +1761,6 @@ mod tests {
         assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}");
     }
 
-    /// KmsManager::delete_key is the enforcement point for the waiting window;
-    /// this pins the backend's defensive copy of the same bound, which is all
-    /// that stands between a direct backend caller and a one-day window.
-    #[tokio::test]
-    async fn wired_backend_delete_refuses_a_window_outside_the_supported_range() {
-        for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
-            let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
-            let vault = ScriptedVault::serve(vec![
-                // The state gate reads the transit key, then its metadata record.
-                ScriptedResponse::ok(transit_key_read_data("wired-key")),
-                ScriptedResponse::ok(metadata_read_data(&metadata)),
-            ])
-            .await;
-            let config = KmsConfig::vault_transit(
-                url::Url::parse(&vault.address).expect("scripted vault address should parse"),
-                "scripted-token".to_string(),
-            )
-            .with_insecure_development_defaults();
-            let backend = VaultTransitKmsBackend::new(config)
-                .await
-                .expect("vault transit backend should build");
-
-            let result = backend
-                .delete_key(DeleteKeyRequest {
-                    key_id: "wired-key".to_string(),
-                    pending_window_in_days: Some(days),
-                    ..Default::default()
-                })
-                .await;
-            assert!(
-                matches!(result, Err(KmsError::InvalidOperation { .. })),
-                "a {days}-day window must be refused, got {result:?}"
-            );
-
-            let requests = vault.requests();
-            assert!(
-                !requests.iter().any(|line| line.starts_with("POST ")),
-                "a refused window must not write anything: {requests:?}"
-            );
-        }
-    }
-
     /// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the
     /// current secret version used as the check-and-set base.
     fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value {

crates/kms/src/backup/local_export.rs

@@ -74,8 +74,7 @@ pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json";
 const ARTIFACTS_DIR: &str = "artifacts";
 const KEYS_DIR: &str = "artifacts/keys";
 const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc";
-const CONFIG_ARTIFACT_PATH: &str = "artifacts/kms-config.json.enc";
-pub(crate) const AEAD_NONCE_LEN: usize = 12;
+const AEAD_NONCE_LEN: usize = 12;
 /// Domain-separation context for the artifact AAD binding.
 const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1";
 /// Domain-separation context for the master-key verifier.
@@ -119,7 +118,7 @@ impl BackupKek {
         }
     }
 
-    pub(crate) fn cipher(&self) -> Aes256Gcm {
+    fn cipher(&self) -> Aes256Gcm {
         Aes256Gcm::new(&Key::<Aes256Gcm>::from(*self.key))
     }
 }
@@ -142,15 +141,6 @@ pub struct LocalBackupExportRequest {
     pub snapshot_generation: u64,
     /// Bundle output directory; must not exist yet or must be empty.
     pub destination: PathBuf,
-    /// Serialized sanitized KMS configuration to seal into the bundle, if the
-    /// caller produced one.
-    ///
-    /// The persisted `KmsConfig` carries plaintext credentials (Vault token,
-    /// AppRole secret id, Local master key), so the sanitized projection is
-    /// owned by the admin layer and this module only seals the bytes it is
-    /// handed. The artifact is evidence for an operator decision: restore
-    /// verifies it opens but never applies a configuration.
-    pub sanitized_config: Option<Vec<u8>>,
 }
 
 impl LocalBackupExportRequest {
@@ -237,14 +227,11 @@ pub async fn export_local_backup(
     Ok(manifest)
 }
 
-/// Read and fully validate the manifest of a bundle directory, whatever
-/// backend produced it.
+/// Read and fully validate the manifest of a local bundle directory.
 ///
 /// A directory without a manifest is an interrupted export: the manifest is
-/// written last, so its absence means the bundle never sealed. The manifest
-/// file name and framing are bundle-wide, not Local-specific, so consumers of
-/// other backends' bundles read them through here too.
-pub async fn read_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
+/// written last, so its absence means the bundle never sealed.
+pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
     let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE);
     let bytes = match fs::read(&manifest_path).await {
         Ok(bytes) => bytes,
@@ -253,12 +240,7 @@ pub async fn read_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
         }
         Err(error) => return Err(error.into()),
     };
-    Ok(BackupManifest::decode(&bytes)?)
-}
-
-/// Read and fully validate the manifest of a *local* bundle directory.
-pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
-    let manifest = read_bundle_manifest(bundle_dir).await?;
+    let manifest = BackupManifest::decode(&bytes)?;
     if manifest.backend != BackupBackendKind::Local {
         return Err(
             BackupError::corrupted(format!("bundle manifest declares backend {:?}, expected Local", manifest.backend)).into(),
@@ -420,10 +402,6 @@ async fn build_and_write_bundle(
         let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::MasterKeySalt, SALT_ARTIFACT_PATH, salt).await?;
         artifacts.push(descriptor);
     }
-    if let Some(config) = &request.sanitized_config {
-        let descriptor = encrypt_and_write_artifact(kek, request, ArtifactKind::KmsConfig, CONFIG_ARTIFACT_PATH, config).await?;
-        artifacts.push(descriptor);
-    }
 
     // Make the artifact directory entries durable before sealing: the sealed
     // manifest must never survive a crash that its artifacts did not.
@@ -446,7 +424,6 @@ async fn build_and_write_bundle(
         backup_kek: kek.descriptor(),
         artifacts,
         local_kdf: Some(local_kdf_descriptor(snapshot, master_key_verifier)),
-        external_references: None,
         key_versions: None,
         capability_discovery: None,
         completeness: CompletenessState::InProgress,
@@ -525,7 +502,7 @@ async fn encrypt_and_write_artifact(
 
 /// AAD binding an artifact to its bundle identity and path. A JSON tuple
 /// gives unambiguous field boundaries without a hand-rolled framing format.
-pub(crate) fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
+fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
     serde_json::to_vec(&(BUNDLE_AAD_CONTEXT, backup_id, snapshot_generation, artifact_path))
         .expect("AAD tuple of strings and integers always serializes")
 }
@@ -687,7 +664,6 @@ mod tests {
             rustfs_version: "1.0.0-test".to_string(),
             snapshot_generation: 7,
             destination,
-            sanitized_config: None,
         }
     }
 

crates/kms/src/backup/local_restore.rs

@@ -522,7 +522,6 @@ async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result<DecodedBund
 
     let mut records = Vec::new();
     let mut salt: Option<Zeroizing<Vec<u8>>> = None;
-    let mut config_seen = false;
     for artifact in &manifest.artifacts {
         match artifact.kind {
             ArtifactKind::KeyMaterial => {
@@ -544,18 +543,6 @@ async fn decode_bundle(bundle_dir: &Path, kek: &BackupKek) -> Result<DecodedBund
                 }
                 salt = Some(plaintext);
             }
-            // Configuration is evidence, not restorable state: the artifact is
-            // verified so a tampered bundle still fails closed, but restore
-            // never writes a configuration into the target. Presenting the
-            // difference against the running configuration, and any decision to
-            // adopt it, belongs to the admin layer.
-            ArtifactKind::KmsConfig => {
-                if config_seen {
-                    return Err(BackupError::corrupted("bundle carries more than one KMS configuration artifact").into());
-                }
-                config_seen = true;
-                decrypt_bundle_artifact(bundle_dir, &manifest, artifact, kek).await?;
-            }
             // No producer emits these for a Local bundle yet; restoring a
             // bundle that carries state this implementation cannot apply
             // would silently drop it, so fail closed instead.
@@ -1085,14 +1072,6 @@ mod tests {
     }
 
     async fn export_bundle(client: &LocalKmsClient, dir: &Path) -> (PathBuf, BackupManifest) {
-        export_bundle_with_config(client, dir, None).await
-    }
-
-    async fn export_bundle_with_config(
-        client: &LocalKmsClient,
-        dir: &Path,
-        sanitized_config: Option<Vec<u8>>,
-    ) -> (PathBuf, BackupManifest) {
         let destination = dir.join("bundle");
         let manifest = export_local_backup(
             client,
@@ -1103,7 +1082,6 @@ mod tests {
                 rustfs_version: "1.0.0-test".to_string(),
                 snapshot_generation: SNAPSHOT_GENERATION,
                 destination: destination.clone(),
-                sanitized_config,
             },
         )
         .await
@@ -1989,71 +1967,4 @@ mod tests {
             .expect("a pre-marker abort only removes staging");
         assert_eq!(top_level_names(staging_only.path()).await, Vec::<String>::new());
     }
-
-    /// A bundle carrying the sanitized configuration artifact stays fully
-    /// restorable, and the configuration never becomes target state: restore
-    /// verifies it and moves on, leaving the decision to the admin layer.
-    #[tokio::test]
-    async fn a_config_artifact_is_verified_but_never_restored() {
-        let (client, _source_dir) = source_with_keys(Some(TEST_MASTER_KEY), &["alpha"]).await;
-        let bundle_parent = TempDir::new().expect("bundle parent");
-        let config_bytes = br#"{"backend":"local","note":"sanitized"}"#.to_vec();
-        let (bundle, manifest) = export_bundle_with_config(&client, bundle_parent.path(), Some(config_bytes.clone())).await;
-        drop(client);
-
-        assert!(
-            manifest
-                .artifacts
-                .iter()
-                .any(|artifact| artifact.kind == ArtifactKind::KmsConfig),
-            "the bundle must carry the configuration artifact"
-        );
-
-        let target = TempDir::new().expect("target");
-        let report = dry_run_local_restore(&test_kek(), &restore_request(&bundle, target.path()))
-            .await
-            .expect("dry-run should evaluate a bundle with a config artifact");
-        assert!(report.blockers.is_empty(), "unexpected blockers: {:?}", report.blockers);
-
-        let mut request = restore_request(&bundle, target.path());
-        request.conflict_policy = RestoreConflictPolicy::RestoreIntoEmptyTarget;
-        let report = restore_local_backup(&test_kek(), &request)
-            .await
-            .expect("restore should succeed");
-        assert_eq!(report.restored_key_ids, vec!["alpha".to_string()]);
-
-        // Only key material and the salt land in the target; the configuration
-        // is evidence that stays in the bundle.
-        let names = top_level_names(target.path()).await;
-        assert!(names.contains(&"alpha.key".to_string()), "{names:?}");
-        for name in &names {
-            assert!(
-                !name.contains("config"),
-                "restore must not write a configuration into the key directory: {names:?}"
-            );
-        }
-        let restored = std::fs::read(target.path().join("alpha.key")).expect("restored record");
-        assert_ne!(restored, config_bytes);
-
-        // A tampered config artifact must still fail the bundle closed.
-        let descriptor = manifest
-            .artifacts
-            .iter()
-            .find(|artifact| artifact.kind == ArtifactKind::KmsConfig)
-            .expect("config artifact");
-        let artifact_path = bundle.join(&descriptor.path);
-        let mut payload = std::fs::read(&artifact_path).expect("artifact bytes");
-        let last = payload.len() - 1;
-        payload[last] ^= 0xff;
-        std::fs::write(&artifact_path, &payload).expect("tamper artifact");
-
-        let fresh_target = TempDir::new().expect("target");
-        dry_run_local_restore(&test_kek(), &restore_request(&bundle, fresh_target.path()))
-            .await
-            .expect("dry-run reports rather than errors")
-            .blockers
-            .iter()
-            .find(|blocker| blocker.code == RestoreBlockerCode::BundleCorrupted)
-            .expect("a tampered config artifact must be reported as corruption");
-    }
 }

crates/kms/src/backup/manifest.rs

@@ -306,141 +306,6 @@ impl LocalKdfDescriptor {
     }
 }
 
-/// Transit engine reference recorded in a Vault-backed bundle.
-///
-/// Transit keys are non-exportable, so a bundle can only ever name the key and
-/// the version window its content depends on. Restore compares these values
-/// against the Transit key the operator's native Vault restore produced.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct VaultTransitReference {
-    /// Transit engine mount path.
-    pub mount_path: String,
-    /// Transit key name.
-    pub key_name: String,
-    /// Lowest Transit key version the bundle's content still needs. A target
-    /// whose `min_decryption_version` sits above this can no longer decrypt
-    /// the oldest state in the bundle.
-    pub required_min_version: u32,
-    /// Latest Transit key version at snapshot time. A target whose newest
-    /// version is below this was restored to a point before the bundle.
-    pub current_version: u32,
-    /// Operator-recorded immutable reference to the native Vault/HSM snapshot
-    /// protecting this key. RustFS neither produces nor consumes that
-    /// snapshot; the reference exists so restore evidence can name it.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub native_snapshot_reference: Option<String>,
-}
-
-/// KV generation of one key's record at snapshot time.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct VaultKvRecordReference {
-    /// Stable key id the record belongs to.
-    pub key_id: String,
-    /// KV2 secret version holding the record when the snapshot was taken.
-    /// Restore compares the target's current version against it: a lower
-    /// value means the target's Vault was restored to a point before the
-    /// bundle, a higher value means the target moved ahead of it.
-    pub kv_version: u64,
-}
-
-/// Immutable references to the external Vault state a bundle depends on.
-///
-/// A Vault-backed bundle never carries the cryptographic root: Transit keys
-/// cannot be exported and KV2 records live in Vault's own storage. What the
-/// bundle can own is a precise description of *which* external state it was
-/// captured against, so a restore refuses to proceed when the operator's
-/// native Vault restore landed somewhere else. Credentials are structurally
-/// absent — no token, AppRole secret id, or TLS material is representable in
-/// this type.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct VaultExternalReferences {
-    /// Opaque identity of the Vault cluster the snapshot was taken from.
-    pub cluster_id: String,
-    /// Vault Enterprise namespace, when the deployment uses one.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub namespace: Option<String>,
-    /// KV v2 mount holding the RustFS records.
-    pub kv_mount: String,
-    /// Path prefix under `kv_mount` holding the RustFS records.
-    pub kv_path_prefix: String,
-    /// Transit engine reference; present exactly when the bundle's material
-    /// is protected by a Transit key.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub transit: Option<VaultTransitReference>,
-    /// Per-key KV generation observed at snapshot time, one entry per key the
-    /// bundle carries a record for.
-    pub kv_records: Vec<VaultKvRecordReference>,
-}
-
-impl VaultExternalReferences {
-    fn validate(&self, backend: BackupBackendKind, protection: AtRestProtection) -> Result<(), BackupError> {
-        require_non_empty("external_references.cluster_id", &self.cluster_id)?;
-        require_non_empty("external_references.kv_mount", &self.kv_mount)?;
-        require_non_empty("external_references.kv_path_prefix", &self.kv_path_prefix)?;
-        if self.namespace.as_deref() == Some("") {
-            return Err(BackupError::corrupted("external Vault namespace must not be empty when present"));
-        }
-
-        // The Transit reference is required exactly where the cryptographic
-        // root lives in Transit; a storage-only KV2 bundle that claims one
-        // would misdescribe its own trust root.
-        let transit_required = matches!(
-            (backend, protection),
-            (BackupBackendKind::VaultTransit, _) | (BackupBackendKind::VaultKv2, AtRestProtection::TransitWrapped)
-        );
-        match (&self.transit, transit_required) {
-            (None, true) => {
-                return Err(BackupError::corrupted(format!(
-                    "({backend:?}, {protection:?}) bundles must reference the Transit key protecting them"
-                )));
-            }
-            (Some(_), false) => {
-                return Err(BackupError::corrupted(format!(
-                    "({backend:?}, {protection:?}) bundles have no Transit trust root to reference"
-                )));
-            }
-            (Some(transit), true) => {
-                require_non_empty("external_references.transit.mount_path", &transit.mount_path)?;
-                require_non_empty("external_references.transit.key_name", &transit.key_name)?;
-                if transit.required_min_version == 0 {
-                    return Err(BackupError::corrupted("Transit reference must require at least key version 1"));
-                }
-                if transit.current_version < transit.required_min_version {
-                    return Err(BackupError::corrupted(format!(
-                        "Transit reference declares current version {} below the required minimum {}",
-                        transit.current_version, transit.required_min_version
-                    )));
-                }
-                if transit.native_snapshot_reference.as_deref() == Some("") {
-                    return Err(BackupError::corrupted("Transit native snapshot reference must not be empty when present"));
-                }
-            }
-            (None, false) => {}
-        }
-
-        let mut seen = BTreeSet::new();
-        for record in &self.kv_records {
-            require_non_empty("external_references.kv_records[].key_id", &record.key_id)?;
-            if record.kv_version == 0 {
-                return Err(BackupError::corrupted(format!(
-                    "KV generation for key '{}' must be at least 1",
-                    record.key_id
-                )));
-            }
-            if !seen.insert(record.key_id.as_str()) {
-                return Err(BackupError::corrupted(format!(
-                    "KV reference for key '{}' appears more than once",
-                    record.key_id
-                )));
-            }
-        }
-        Ok(())
-    }
-}
-
 /// Completeness marker of a bundle.
 ///
 /// A producer writes `in-progress` state (or no manifest at all) until the
@@ -529,10 +394,6 @@ pub struct BackupManifest {
     /// Local backend section; required exactly when `backend` is `Local`.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub local_kdf: Option<LocalKdfDescriptor>,
-    /// External Vault references; required exactly when `backend` is a Vault
-    /// backend. Never carries credentials — see [`VaultExternalReferences`].
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub external_references: Option<VaultExternalReferences>,
     /// Reserved for the per-key version/envelope inventory defined by the
     /// backlog#1565 contract. May not carry data in format version 1.
     #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -711,7 +572,6 @@ impl BackupManifest {
         }
         self.validate_responsibility()?;
         self.validate_local_kdf()?;
-        self.validate_external_references()?;
         self.validate_artifacts()?;
         Ok(())
     }
@@ -741,19 +601,6 @@ impl BackupManifest {
         }
     }
 
-    fn validate_external_references(&self) -> Result<(), BackupError> {
-        let vault_backend = matches!(self.backend, BackupBackendKind::VaultKv2 | BackupBackendKind::VaultTransit);
-        match (&self.external_references, vault_backend) {
-            (None, true) => Err(BackupError::corrupted("Vault backend manifest must carry external Vault references")),
-            (Some(_), false) => Err(BackupError::corrupted(format!(
-                "external Vault references are not valid for backend {:?}",
-                self.backend
-            ))),
-            (Some(references), true) => references.validate(self.backend, self.at_rest_protection),
-            (None, false) => Ok(()),
-        }
-    }
-
     fn validate_artifacts(&self) -> Result<(), BackupError> {
         let mut paths = BTreeSet::new();
         for artifact in &self.artifacts {
@@ -1018,7 +865,6 @@ mod tests {
                 vec![AtRestProtection::EncryptedMasterKey],
                 Some("verifier-opaque-1".to_string()),
             )),
-            external_references: None,
             key_versions: None,
             capability_discovery: None,
             completeness: CompletenessState::InProgress,
@@ -1425,164 +1271,4 @@ mod tests {
         assert_eq!(decoded, sealed);
         assert_eq!(decoded.responsibility, BackupResponsibility::ReferenceOnly);
     }
-
-    fn transit_references() -> VaultExternalReferences {
-        VaultExternalReferences {
-            cluster_id: "vault-cluster-a".to_string(),
-            namespace: Some("tenant-a".to_string()),
-            kv_mount: "secret".to_string(),
-            kv_path_prefix: "rustfs/kms/transit-metadata".to_string(),
-            transit: Some(VaultTransitReference {
-                mount_path: "transit".to_string(),
-                key_name: "rustfs-master".to_string(),
-                required_min_version: 2,
-                current_version: 5,
-                native_snapshot_reference: Some("s3://dr/vault/2026-08-01.snap".to_string()),
-            }),
-            kv_records: vec![VaultKvRecordReference {
-                key_id: "object-key".to_string(),
-                kv_version: 4,
-            }],
-        }
-    }
-
-    fn transit_manifest_unsealed() -> BackupManifest {
-        BackupManifest {
-            backend: BackupBackendKind::VaultTransit,
-            at_rest_protection: AtRestProtection::ExternalNonExportable,
-            responsibility: BackupResponsibility::MetadataPlusExternalRoot,
-            artifacts: vec![artifact(ArtifactKind::KeyMetadata, "vault/records/object-key.json.enc")],
-            local_kdf: None,
-            external_references: Some(transit_references()),
-            ..local_manifest_unsealed()
-        }
-    }
-
-    #[test]
-    fn vault_bundle_round_trips_with_external_references() {
-        let sealed = seal(transit_manifest_unsealed());
-        let bytes = sealed.encode().expect("encoding should succeed");
-        let decoded = BackupManifest::decode(&bytes).expect("decoding should succeed");
-        assert_eq!(decoded, sealed);
-    }
-
-    #[test]
-    fn vault_backend_requires_external_references() {
-        let manifest = BackupManifest {
-            external_references: None,
-            ..transit_manifest_unsealed()
-        };
-        expect_corrupted(
-            BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
-            "must carry external Vault references",
-        );
-    }
-
-    #[test]
-    fn non_vault_backend_rejects_external_references() {
-        let manifest = BackupManifest {
-            external_references: Some(transit_references()),
-            ..local_manifest_unsealed()
-        };
-        expect_corrupted(
-            BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
-            "not valid for backend",
-        );
-    }
-
-    /// One way to contradict an otherwise valid set of Vault references.
-    type ReferenceContradiction = Box<dyn Fn(&mut VaultExternalReferences)>;
-
-    #[test]
-    fn external_reference_contradictions_fail_closed() {
-        let cases: [(&str, ReferenceContradiction); 7] = [
-            ("cluster_id", Box::new(|refs| refs.cluster_id.clear())),
-            ("kv_mount", Box::new(|refs| refs.kv_mount.clear())),
-            ("namespace must not be empty", Box::new(|refs| refs.namespace = Some(String::new()))),
-            (
-                "must reference the Transit key",
-                Box::new(|refs: &mut VaultExternalReferences| refs.transit = None),
-            ),
-            (
-                "at least key version 1",
-                Box::new(|refs: &mut VaultExternalReferences| {
-                    if let Some(transit) = refs.transit.as_mut() {
-                        transit.required_min_version = 0;
-                    }
-                }),
-            ),
-            (
-                "below the required minimum",
-                Box::new(|refs: &mut VaultExternalReferences| {
-                    if let Some(transit) = refs.transit.as_mut() {
-                        transit.current_version = 1;
-                    }
-                }),
-            ),
-            (
-                "appears more than once",
-                Box::new(|refs: &mut VaultExternalReferences| {
-                    let first = refs.kv_records[0].clone();
-                    refs.kv_records.push(first);
-                }),
-            ),
-        ];
-        for (needle, mutate) in cases {
-            let mut references = transit_references();
-            mutate(&mut references);
-            let manifest = BackupManifest {
-                external_references: Some(references),
-                ..transit_manifest_unsealed()
-            };
-            expect_corrupted(BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")), needle);
-        }
-    }
-
-    /// The reference schema is the only place a Vault coordinate reaches a
-    /// bundle, so its field set is pinned: a credential-carrying field could
-    /// only appear by editing this assertion.
-    #[test]
-    fn external_reference_field_set_is_frozen() {
-        let value = serde_json::to_value(transit_references()).expect("serialize");
-        let mut top: Vec<&str> = value.as_object().expect("object").keys().map(String::as_str).collect();
-        top.sort_unstable();
-        assert_eq!(
-            top,
-            [
-                "cluster_id",
-                "kv_mount",
-                "kv_path_prefix",
-                "kv_records",
-                "namespace",
-                "transit"
-            ]
-        );
-
-        let mut transit: Vec<&str> = value["transit"]
-            .as_object()
-            .expect("transit object")
-            .keys()
-            .map(String::as_str)
-            .collect();
-        transit.sort_unstable();
-        assert_eq!(
-            transit,
-            [
-                "current_version",
-                "key_name",
-                "mount_path",
-                "native_snapshot_reference",
-                "required_min_version"
-            ]
-        );
-
-        let mut record: Vec<&str> = value["kv_records"][0]
-            .as_object()
-            .expect("record object")
-            .keys()
-            .map(String::as_str)
-            .collect();
-        record.sort_unstable();
-        assert_eq!(record, ["key_id", "kv_version"]);
-    }
 }

crates/kms/src/backup/mod.rs

@@ -17,11 +17,9 @@
 //! The contract side defines the versioned backup manifest, the per-backend
 //! responsibility matrix, typed failure modes, and the restore dry-run
 //! report. [`local_export`] implements the producer side and
-//! [`local_restore`] the consumer side for the Local backend;
-//! [`vault_restore`] orchestrates the consumer side for the Vault backends,
-//! whose cryptographic root is restored by Vault's own disaster-recovery
-//! flow. All are crate-internal APIs; the admin API builds on these pieces in
-//! follow-up changes.
+//! [`local_restore`] the consumer side for the Local backend as
+//! crate-internal APIs; the admin API builds on these pieces in follow-up
+//! changes.
 //!
 //! # Bundle model
 //!
@@ -56,7 +54,6 @@ mod error;
 pub mod local_export;
 pub mod local_restore;
 mod manifest;
-pub mod vault_restore;
 
 pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
 pub use dry_run::{
@@ -65,7 +62,7 @@ pub use dry_run::{
 pub use error::BackupError;
 pub use local_export::{
     BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup,
-    read_bundle_manifest, read_local_bundle_manifest,
+    read_local_bundle_manifest,
 };
 pub use local_restore::{
     LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore,
@@ -73,11 +70,5 @@ pub use local_restore::{
 };
 pub use manifest::{
     AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest,
-    DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, VaultExternalReferences, VaultKvRecordReference,
-    VaultTransitReference,
-};
-pub use vault_restore::{
-    VaultRestoreAbortReport, VaultRestoreAbortSkip, VaultRestoreAbortSkipReason, VaultRestoreClient, VaultRestoreMismatch,
-    VaultRestoreReport, VaultRestoreRequest, VaultRestoreSequence, VaultRestoreStage, VaultRestoreTarget, abort_vault_restore,
-    dry_run_vault_restore, restore_vault_backup,
+    DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot,
 };

crates/kms/src/cache.rs

@@ -14,125 +14,30 @@
 
 //! Caching layer for KMS operations to improve performance
 
-use crate::config::CacheConfig;
 use crate::types::KeyMetadata;
 use moka::future::Cache;
-use moka::notification::RemovalCause;
-use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
-
-// ---------------------------------------------------------------------------
-// Metrics
-//
-// Lookup outcomes and removals are counted here because moka exposes neither
-// hit nor miss counts; the numbers below are the cache's own, not derived.
-// Label values are exclusively static strings (lookup result, removal cause) —
-// key identifiers and key metadata must never reach a metric label.
-//
-// Publication is gated by `CacheConfig::enable_metrics`; the atomics backing
-// `KmsCache::stats` are maintained regardless, so the admin status API keeps
-// reporting real numbers with the metrics switch off.
-// ---------------------------------------------------------------------------
-
-/// Counter: key metadata lookups, by `result` (`hit` or `miss`).
-const METRIC_CACHE_LOOKUPS_TOTAL: &str = "rustfs_kms_metadata_cache_lookups_total";
-/// Counter: entries dropped from the cache, by `cause` (`expired`, `size`,
-/// `explicit`, `replaced`); only `expired` and `size` are true evictions, the
-/// rest are invalidations driven by key lifecycle operations.
-const METRIC_CACHE_EVICTIONS_TOTAL: &str = "rustfs_kms_metadata_cache_evictions_total";
-/// Gauge: entries currently held by the cache. Republished whenever the entry
-/// set can have changed — every write path, plus the lookups that miss, since
-/// TTL expiry drops entries without any write taking place.
-const METRIC_CACHE_ENTRIES: &str = "rustfs_kms_metadata_cache_entries";
-
-/// Register metric descriptions once per process.
-fn describe_metrics() {
-    static DESCRIBE: std::sync::Once = std::sync::Once::new();
-    DESCRIBE.call_once(|| {
-        metrics::describe_counter!(METRIC_CACHE_LOOKUPS_TOTAL, "Total KMS key metadata cache lookups, by hit or miss");
-        metrics::describe_counter!(
-            METRIC_CACHE_EVICTIONS_TOTAL,
-            "Total entries dropped from the KMS key metadata cache, by removal cause"
-        );
-        metrics::describe_gauge!(METRIC_CACHE_ENTRIES, "Entries currently held by the KMS key metadata cache");
-    });
-}
-
-fn removal_cause_label(cause: RemovalCause) -> &'static str {
-    match cause {
-        RemovalCause::Expired => "expired",
-        RemovalCause::Explicit => "explicit",
-        RemovalCause::Replaced => "replaced",
-        RemovalCause::Size => "size",
-    }
-}
-
-/// Cumulative cache counters. Shared with moka's eviction listener, which runs
-/// outside any `&self` borrow, hence the `Arc` and the atomics.
-#[derive(Debug, Default)]
-struct CacheCounters {
-    hits: AtomicU64,
-    misses: AtomicU64,
-    evictions: AtomicU64,
-}
-
-/// Snapshot of the KMS key metadata cache counters.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct KmsCacheStats {
-    /// Entries currently held. Eventually consistent: moka applies pending
-    /// maintenance lazily.
-    pub entries: u64,
-    /// Lookups served from the cache since process start.
-    pub hits: u64,
-    /// Lookups that fell through to the backend since process start.
-    pub misses: u64,
-    /// Entries dropped since process start, whatever the cause (expiry,
-    /// capacity, invalidation, replacement).
-    pub evictions: u64,
-}
+use std::time::Duration;
 
 /// KMS cache for storing frequently accessed keys and metadata
 pub struct KmsCache {
     key_metadata_cache: Cache<String, KeyMetadata>,
-    counters: Arc<CacheCounters>,
-    metrics_enabled: bool,
 }
 
 impl KmsCache {
-    /// Create a new KMS cache from the operator-supplied cache configuration
-    ///
-    /// The entry lifetime is [`CacheConfig::effective_ttl`] rather than the raw
-    /// configured value: this value arrives straight from the admin configure
-    /// API, and moka panics when built with a time-to-live beyond 1000 years.
+    /// Create a new KMS cache with the specified capacity
     ///
     /// # Arguments
-    /// * `config` - Capacity, metadata lifetime and metrics switch to build with
+    /// * `capacity` - Maximum number of entries in the cache
     ///
     /// # Returns
     /// A new instance of `KmsCache`
     ///
-    pub fn new(config: &CacheConfig) -> Self {
-        let metrics_enabled = config.enable_metrics;
-        if metrics_enabled {
-            describe_metrics();
-        }
-
-        let counters = Arc::new(CacheCounters::default());
-        let eviction_counters = Arc::clone(&counters);
-
+    pub fn new(capacity: u64) -> Self {
         Self {
             key_metadata_cache: Cache::builder()
-                .max_capacity(config.max_keys as u64)
-                .time_to_live(config.effective_ttl())
-                .eviction_listener(move |_key: Arc<String>, _metadata: KeyMetadata, cause: RemovalCause| {
-                    eviction_counters.evictions.fetch_add(1, Ordering::Relaxed);
-                    if metrics_enabled {
-                        metrics::counter!(METRIC_CACHE_EVICTIONS_TOTAL, "cause" => removal_cause_label(cause)).increment(1);
-                    }
-                })
+                .max_capacity(capacity)
+                .time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
                 .build(),
-            counters,
-            metrics_enabled,
         }
     }
 
@@ -145,26 +50,7 @@ impl KmsCache {
     /// An `Option` containing the `KeyMetadata` if found, or `None` if not found
     ///
     pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
-        let cached = self.key_metadata_cache.get(key_id).await;
-        self.record_lookup(cached.is_some());
-
-        if cached.is_none() {
-            // TTL expiry is the one way the entry set shrinks without a write,
-            // and a miss is where it surfaces. moka expires lazily: the miss is
-            // reported at once, but the removal that decrements `entry_count`
-            // and reaches the eviction listener lands in the maintenance pass
-            // moka runs opportunistically on reads, on its own interval. So
-            // this converges the gauge within a housekeeping interval rather
-            // than on the first miss — enough to stop a cache that only ever
-            // expires, with no put, remove or clear to follow, from reporting a
-            // population that is gone. Forcing `run_pending_tasks` would
-            // tighten that at the cost of taking moka's maintenance lock on
-            // every miss, for a gauge `entry_count` only approximates anyway.
-            // Hits, the hot path, are left alone.
-            self.record_entry_count();
-        }
-
-        cached
+        self.key_metadata_cache.get(key_id).await
     }
 
     /// Put key metadata into cache
@@ -176,7 +62,6 @@ impl KmsCache {
     pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
         self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
         self.key_metadata_cache.run_pending_tasks().await;
-        self.record_entry_count();
     }
 
     /// Remove key metadata from cache
@@ -186,11 +71,6 @@ impl KmsCache {
     ///
     pub async fn remove_key_metadata(&mut self, key_id: &str) {
         self.key_metadata_cache.remove(key_id).await;
-
-        // Flush maintenance so the removal notification and the entry gauge
-        // describe the cache as it is once this call returns.
-        self.key_metadata_cache.run_pending_tasks().await;
-        self.record_entry_count();
     }
 
     /// Clear all cached entries
@@ -199,39 +79,18 @@ impl KmsCache {
 
         // Wait for invalidation to complete
         self.key_metadata_cache.run_pending_tasks().await;
-        self.record_entry_count();
     }
 
-    /// Get cache statistics
+    /// Get cache statistics (hit count, miss count)
     ///
     /// # Returns
-    /// A [`KmsCacheStats`] snapshot of entry count, hits, misses and evictions
+    /// A tuple containing total entries and total misses
     ///
-    pub fn stats(&self) -> KmsCacheStats {
-        KmsCacheStats {
-            entries: self.key_metadata_cache.entry_count(),
-            hits: self.counters.hits.load(Ordering::Relaxed),
-            misses: self.counters.misses.load(Ordering::Relaxed),
-            evictions: self.counters.evictions.load(Ordering::Relaxed),
-        }
-    }
-
-    fn record_lookup(&self, hit: bool) {
-        let (counter, result) = if hit {
-            (&self.counters.hits, "hit")
-        } else {
-            (&self.counters.misses, "miss")
-        };
-        counter.fetch_add(1, Ordering::Relaxed);
-        if self.metrics_enabled {
-            metrics::counter!(METRIC_CACHE_LOOKUPS_TOTAL, "result" => result).increment(1);
-        }
-    }
-
-    fn record_entry_count(&self) {
-        if self.metrics_enabled {
-            metrics::gauge!(METRIC_CACHE_ENTRIES).set(self.key_metadata_cache.entry_count() as f64);
-        }
+    pub fn stats(&self) -> (u64, u64) {
+        (
+            self.key_metadata_cache.entry_count(),
+            0u64, // moka doesn't provide miss count directly
+        )
     }
 }
 
@@ -240,101 +99,40 @@ mod tests {
     use super::*;
     use crate::types::{KeyState, KeyUsage};
     use jiff::Zoned;
-    use metrics_util::MetricKind;
-    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
     use std::time::Duration;
 
+    #[derive(Debug, Clone)]
+    struct CacheInfo {
+        key_metadata_count: u64,
+    }
+
+    impl CacheInfo {
+        fn total_entries(&self) -> u64 {
+            self.key_metadata_count
+        }
+    }
+
     impl KmsCache {
+        fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
+            Self {
+                key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
+            }
+        }
+
+        fn info_for_tests(&self) -> CacheInfo {
+            CacheInfo {
+                key_metadata_count: self.key_metadata_cache.entry_count(),
+            }
+        }
+
         fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
             self.key_metadata_cache.contains_key(key_id)
         }
     }
 
-    /// A cache holding `max_keys` entries for the default lifetime.
-    fn sized(max_keys: usize) -> KmsCache {
-        KmsCache::new(&CacheConfig {
-            max_keys,
-            ..Default::default()
-        })
-    }
-
-    fn test_metadata(key_id: &str) -> KeyMetadata {
-        KeyMetadata {
-            key_id: key_id.to_string(),
-            key_state: KeyState::Enabled,
-            key_usage: KeyUsage::EncryptDecrypt,
-            description: None,
-            creation_date: Zoned::now(),
-            deletion_date: None,
-            origin: "KMS".to_string(),
-            key_manager: "CUSTOMER".to_string(),
-            tags: std::collections::HashMap::new(),
-        }
-    }
-
-    type MetricEntry = (
-        metrics_util::CompositeKey,
-        Option<metrics::Unit>,
-        Option<metrics::SharedString>,
-        DebugValue,
-    );
-
-    /// Drive `test` on a current-thread runtime under a thread-local debugging
-    /// recorder and return its output plus one snapshot of everything emitted.
-    ///
-    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
-    /// the recorded state, so taking it per assertion would leave every
-    /// assertion after the first with nothing to look at.
-    fn record_metrics<F, Fut, T>(test: F) -> (T, Vec<MetricEntry>)
-    where
-        F: FnOnce() -> Fut,
-        Fut: std::future::Future<Output = T>,
-    {
-        let recorder = DebuggingRecorder::new();
-        let snapshotter = recorder.snapshotter();
-        let output = metrics::with_local_recorder(&recorder, || {
-            let runtime = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("current-thread runtime must build");
-            runtime.block_on(test())
-        });
-        (output, snapshotter.snapshot().into_vec())
-    }
-
-    /// Sum of counters with `name` whose labels include every `(key, value)` pair.
-    fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
-        snapshot
-            .iter()
-            .filter_map(|(composite, _unit, _description, value)| {
-                let key = composite.key();
-                let matches = composite.kind() == MetricKind::Counter
-                    && key.name() == name
-                    && labels
-                        .iter()
-                        .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected));
-                match (matches, value) {
-                    (true, DebugValue::Counter(count)) => Some(*count),
-                    _ => None,
-                }
-            })
-            .sum()
-    }
-
-    /// Last value recorded for the unlabelled gauge `name`.
-    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
-        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
-            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
-            match (matches, value) {
-                (true, DebugValue::Gauge(gauge)) => Some(**gauge),
-                _ => None,
-            }
-        })
-    }
-
     #[tokio::test]
     async fn test_cache_operations() {
-        let mut cache = sized(100);
+        let mut cache = KmsCache::new(100);
 
         // Test key metadata caching
         let metadata = KeyMetadata {
@@ -356,20 +154,22 @@ mod tests {
         assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
 
         // Test cache info
-        assert_eq!(cache.stats().entries, 1);
+        let info = cache.info_for_tests();
+        assert_eq!(info.key_metadata_count, 1);
+        assert_eq!(info.total_entries(), 1);
 
         // Test cache clearing
         cache.clear().await;
-        assert_eq!(cache.stats().entries, 0);
+        let info_after_clear = cache.info_for_tests();
+        assert_eq!(info_after_clear.total_entries(), 0);
     }
 
     #[tokio::test]
     async fn test_cache_with_custom_ttl() {
-        let mut cache = KmsCache::new(&CacheConfig {
-            max_keys: 100,
-            ttl: Duration::from_millis(100), // Short TTL for testing
-            ..Default::default()
-        });
+        let mut cache = KmsCache::with_ttl_for_tests(
+            100,
+            Duration::from_millis(100), // Short TTL for testing
+        );
 
         let metadata = KeyMetadata {
             key_id: "ttl-test-key".to_string(),
@@ -397,7 +197,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_cache_contains_methods() {
-        let mut cache = sized(100);
+        let mut cache = KmsCache::new(100);
 
         assert!(!cache.contains_key_metadata_for_tests("nonexistent"));
 
@@ -417,125 +217,4 @@ mod tests {
 
         assert!(cache.contains_key_metadata_for_tests("contains-test"));
     }
-
-    #[test]
-    fn lookups_report_real_hit_and_miss_counts() {
-        let (stats, snapshot) = record_metrics(|| async {
-            let mut cache = sized(100);
-
-            assert!(cache.get_key_metadata("absent").await.is_none());
-            cache.put_key_metadata("present", &test_metadata("present")).await;
-            assert!(cache.get_key_metadata("present").await.is_some());
-            assert!(cache.get_key_metadata("absent").await.is_none());
-
-            cache.stats()
-        });
-
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 2);
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "hit")]), 1);
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "miss")]), 2);
-    }
-
-    #[test]
-    fn disabling_metrics_stops_publication_without_blinding_the_status_api() {
-        let (stats, snapshot) = record_metrics(|| async {
-            let mut cache = KmsCache::new(&CacheConfig {
-                max_keys: 1,
-                enable_metrics: false,
-                ..Default::default()
-            });
-
-            assert!(cache.get_key_metadata("absent").await.is_none());
-            cache.put_key_metadata("first", &test_metadata("first")).await;
-            assert!(cache.get_key_metadata("first").await.is_some());
-            // Capacity is one, so this insert evicts "first".
-            cache.put_key_metadata("second", &test_metadata("second")).await;
-
-            cache.stats()
-        });
-
-        // The counters behind the admin status API keep moving...
-        assert_eq!(stats.hits, 1);
-        assert_eq!(stats.misses, 1);
-        assert_eq!(stats.evictions, 1);
-
-        // ...while nothing reaches the metrics recorder.
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[]), 0);
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[]), 0);
-        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), None);
-    }
-
-    #[test]
-    fn removals_report_their_cause_and_the_resulting_entry_count() {
-        let (stats, snapshot) = record_metrics(|| async {
-            let mut cache = sized(100);
-
-            cache.put_key_metadata("key", &test_metadata("key")).await;
-            cache.put_key_metadata("key", &test_metadata("key")).await;
-            cache.remove_key_metadata("key").await;
-
-            cache.stats()
-        });
-
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "replaced")]), 1);
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "explicit")]), 1);
-        assert_eq!(stats.evictions, 2);
-        assert_eq!(stats.entries, 0);
-        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
-    }
-
-    #[test]
-    fn capacity_pressure_reports_size_evictions() {
-        let (stats, snapshot) = record_metrics(|| async {
-            let mut cache = sized(1);
-
-            cache.put_key_metadata("first", &test_metadata("first")).await;
-            cache.put_key_metadata("second", &test_metadata("second")).await;
-
-            cache.stats()
-        });
-
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "size")]), 1);
-        assert_eq!(stats.evictions, 1);
-        assert_eq!(stats.entries, 1);
-        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(1.0));
-    }
-
-    /// Entries that expire are dropped outside every write path, so the gauge
-    /// has to be republished from the lookup that observes the expiry — nothing
-    /// else runs afterwards to correct it.
-    ///
-    /// moka's clock is internal and cannot be driven by tokio's paused time,
-    /// hence the one short real sleep. The explicit `run_pending_tasks` stands
-    /// in for the maintenance moka schedules by itself on reads, so the test
-    /// does not depend on moka's internal 300ms housekeeping interval.
-    #[test]
-    fn expiry_without_further_writes_converges_the_entry_gauge() {
-        let ttl = Duration::from_millis(100);
-
-        let (stats, snapshot) = record_metrics(move || async move {
-            let mut cache = KmsCache::new(&CacheConfig {
-                max_keys: 100,
-                ttl,
-                ..Default::default()
-            });
-
-            cache.put_key_metadata("expiring", &test_metadata("expiring")).await;
-            assert_eq!(cache.stats().entries, 1);
-
-            tokio::time::sleep(Duration::from_millis(150)).await;
-            cache.key_metadata_cache.run_pending_tasks().await;
-
-            // The lookup that observes the expiry is the last thing to touch
-            // the cache: no put, remove or clear follows it.
-            assert!(cache.get_key_metadata("expiring").await.is_none());
-
-            cache.stats()
-        });
-
-        assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "expired")]), 1);
-        assert_eq!(stats.entries, 0);
-        assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
-    }
 }

crates/kms/src/config.rs

@@ -24,7 +24,6 @@ use std::time::Duration;
 use url::Url;
 
 pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
-pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
 pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
 pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
 pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
@@ -35,8 +34,6 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
 pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
 pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
 pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
-pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
-pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
 pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
 pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
 pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
@@ -50,19 +47,6 @@ pub(crate) const MAX_OPERATION_TIMEOUT: Duration = Duration::from_secs(300);
 /// Upper bound applied to `KmsConfig::retry_attempts` when deriving backend behavior.
 pub(crate) const MAX_RETRY_ATTEMPTS: u32 = 10;
 
-/// Default number of key metadata entries the cache holds.
-pub const DEFAULT_MAX_CACHED_KEYS: usize = 1000;
-
-/// Default lifetime of a cached key metadata entry.
-pub const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(300);
-
-/// Upper bound applied to `CacheConfig::ttl` when building the metadata cache.
-///
-/// Out-of-range values are clamped at use rather than rejected, matching
-/// `MAX_OPERATION_TIMEOUT`, so existing deployments with an oversized setting
-/// keep starting after an upgrade.
-pub(crate) const MAX_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
-
 fn default_vault_transit_metadata_kv_mount() -> String {
     DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT.to_string()
 }
@@ -144,26 +128,6 @@ pub enum KmsBackend {
     /// Static single-key backend that derives DEKs from a pre-configured key
     #[serde(rename = "Static")]
     Static,
-    /// AWS KMS backend: AWS is the cryptographic source of truth and owns key
-    /// state, versioning, and the deletion window.
-    #[serde(rename = "AWS", alias = "AwsKms")]
-    Aws,
-}
-
-impl KmsBackend {
-    /// Stable identifier for logs, metrics, and audit records.
-    ///
-    /// External consumers key off these values, so treat them as a wire
-    /// contract rather than a rendering detail.
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            KmsBackend::VaultKv2 => "vault-kv2",
-            KmsBackend::VaultTransit => "vault-transit",
-            KmsBackend::Local => "local",
-            KmsBackend::Static => "static",
-            KmsBackend::Aws => "aws",
-        }
-    }
 }
 
 /// Main KMS configuration
@@ -178,24 +142,6 @@ pub struct KmsConfig {
     /// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault.
     #[serde(default)]
     pub allow_insecure_dev_defaults: bool,
-    /// Allow `DeleteKey` requests to skip the pending-deletion waiting window and
-    /// destroy key material right away.
-    ///
-    /// Off by default: an immediate deletion is unrecoverable and takes every
-    /// object encrypted under the key with it, so the waiting window (plus
-    /// `CancelKeyDeletion`) is the only recovery path. Operators who genuinely
-    /// need immediate deletion — throwaway test clusters, key material that was
-    /// never used — must turn it on through server configuration
-    /// ([`ENV_KMS_ALLOW_IMMEDIATE_DELETION`]); the request must still echo the
-    /// key id back for confirmation.
-    ///
-    /// Not part of the serialized configuration, and not settable through the
-    /// admin configure API. It is per-server operator state that has to be
-    /// re-stated to survive a restart: persisting it would carry one operator's
-    /// one-time enablement into the cluster-wide config that every node reloads,
-    /// long after the deletion it was turned on for.
-    #[serde(skip)]
-    pub allow_immediate_deletion: bool,
     /// Timeout for a single backend attempt.
     ///
     /// This bounds one outbound request, not the whole operation: the operation
@@ -219,7 +165,6 @@ impl Default for KmsConfig {
             default_key_id: None,
             backend_config: BackendConfig::default(),
             allow_insecure_dev_defaults: false,
-            allow_immediate_deletion: false,
             timeout: Duration::from_secs(30),
             retry_attempts: 3,
             enable_cache: true,
@@ -240,9 +185,6 @@ pub enum BackendConfig {
     VaultTransit(Box<VaultTransitConfig>),
     /// Static single-key backend configuration
     Static(StaticConfig),
-    /// AWS KMS backend configuration
-    #[serde(rename = "AWS", alias = "AwsKms")]
-    Aws(Box<AwsKmsConfig>),
 }
 
 impl Default for BackendConfig {
@@ -258,7 +200,6 @@ impl fmt::Debug for BackendConfig {
             Self::VaultKv2(config) => f.debug_tuple("VaultKv2").field(config).finish(),
             Self::VaultTransit(config) => f.debug_tuple("VaultTransit").field(config).finish(),
             Self::Static(config) => f.debug_tuple("Static").field(config).finish(),
-            Self::Aws(config) => f.debug_tuple("Aws").field(config).finish(),
         }
     }
 }
@@ -568,59 +509,22 @@ pub struct TlsConfig {
 pub struct CacheConfig {
     /// Maximum number of keys to cache
     pub max_keys: usize,
-    /// Lifetime of a cached key metadata entry.
-    ///
-    /// This bounds how long a describe can answer from metadata that another
-    /// node has since changed (disable, schedule-deletion); encrypt, decrypt
-    /// and data key generation never read the cache. Values above 24 hours are
-    /// clamped at use (see [`CacheConfig::effective_ttl`]).
+    /// TTL for cached keys
     pub ttl: Duration,
-    /// Publish the `rustfs_kms_metadata_cache_*` metrics.
-    ///
-    /// Only metrics-recorder output is gated: the counters behind the admin
-    /// status API are maintained either way.
+    /// Enable cache metrics
     pub enable_metrics: bool,
 }
 
 impl Default for CacheConfig {
     fn default() -> Self {
         Self {
-            max_keys: DEFAULT_MAX_CACHED_KEYS,
-            ttl: DEFAULT_CACHE_TTL,
+            max_keys: 1000,
+            ttl: Duration::from_secs(3600), // 1 hour
             enable_metrics: true,
         }
     }
 }
 
-impl CacheConfig {
-    /// Metadata lifetime with the configured value clamped to the supported maximum.
-    ///
-    /// This is the value the cache is built with and the value reported back to
-    /// operators, so what the admin API advertises is what the cache does.
-    pub fn effective_ttl(&self) -> Duration {
-        self.ttl.min(MAX_CACHE_TTL)
-    }
-}
-
-/// AWS KMS backend configuration.
-///
-/// Deliberately holds no credential material: the backend resolves credentials
-/// through the standard `aws-config` provider chain (environment, shared
-/// profile, container/IMDS role), so RustFS never stores, persists, or redacts
-/// AWS secrets of its own.
-#[derive(Clone, Debug, Default, Serialize, Deserialize)]
-pub struct AwsKmsConfig {
-    /// AWS region hosting the KMS keys. When unset, the region is resolved by
-    /// the standard chain (`AWS_REGION`, profile, IMDS).
-    #[serde(default)]
-    pub region: Option<String>,
-    /// Override for the KMS endpoint, for local emulators and private
-    /// endpoints. Unset in production, where the SDK derives the regional
-    /// endpoint.
-    #[serde(default)]
-    pub endpoint_url: Option<String>,
-}
-
 impl KmsConfig {
     /// Create a new KMS configuration for local backend (for development and testing only)
     pub fn local(key_dir: PathBuf) -> Self {
@@ -730,29 +634,6 @@ impl KmsConfig {
         }
     }
 
-    /// Create a new KMS configuration for the AWS KMS backend.
-    ///
-    /// Credentials are resolved by the standard `aws-config` provider chain;
-    /// only the region is configured here.
-    pub fn aws(region: Option<String>) -> Self {
-        Self {
-            backend: KmsBackend::Aws,
-            backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
-                region,
-                endpoint_url: None,
-            })),
-            ..Default::default()
-        }
-    }
-
-    /// Get the AWS configuration if backend is AWS KMS
-    pub fn aws_kms_config(&self) -> Option<&AwsKmsConfig> {
-        match &self.backend_config {
-            BackendConfig::Aws(config) => Some(config),
-            _ => None,
-        }
-    }
-
     /// Set default key ID
     pub fn with_default_key(mut self, key_id: String) -> Self {
         self.default_key_id = Some(key_id);
@@ -765,12 +646,6 @@ impl KmsConfig {
         self
     }
 
-    /// Explicitly allow deletions that bypass the pending-deletion waiting window.
-    pub fn with_immediate_deletion_allowed(mut self) -> Self {
-        self.allow_immediate_deletion = true;
-        self
-    }
-
     /// Set operation timeout
     pub fn with_timeout(mut self, timeout: Duration) -> Self {
         self.timeout = timeout;
@@ -915,39 +790,11 @@ impl KmsConfig {
                 // Validate that the key can be decoded (right length, valid base64)
                 config.decode_key()?;
             }
-            BackendConfig::Aws(config) => {
-                if let Some(region) = &config.region
-                    && region.is_empty()
-                {
-                    return Err(KmsError::configuration_error("AWS KMS region cannot be empty when set"));
-                }
-
-                if let Some(endpoint) = &config.endpoint_url {
-                    if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
-                        return Err(KmsError::configuration_error("AWS KMS endpoint URL must use http or https scheme"));
-                    }
-                    // A plaintext endpoint override exposes every KMS request,
-                    // including plaintext data keys, so it stays gated on the
-                    // explicit development opt-in.
-                    if endpoint.starts_with("http://") && !self.allow_insecure_dev_defaults {
-                        return Err(development_default_error("AWS KMS endpoint URL must use https"));
-                    }
-                }
-            }
         }
 
         // Validate cache configuration
-        if self.enable_cache {
-            if self.cache_config.max_keys == 0 {
-                return Err(KmsError::configuration_error("Cache max_keys must be greater than 0"));
-            }
-
-            // A zero TTL expires every entry on insert, which is a cache that
-            // cannot serve anything; reject it instead of silently disabling
-            // the cache the operator asked for.
-            if self.cache_config.ttl.is_zero() {
-                return Err(KmsError::configuration_error("Cache ttl must be greater than 0"));
-            }
+        if self.enable_cache && self.cache_config.max_keys == 0 {
+            return Err(KmsError::configuration_error("Cache max_keys must be greater than 0"));
         }
 
         Ok(())
@@ -964,7 +811,6 @@ impl KmsConfig {
                 "vault" | "vault-kv2" | "vault_kv2" => KmsBackend::VaultKv2,
                 "vault-transit" | "vault_transit" => KmsBackend::VaultTransit,
                 "static" => KmsBackend::Static,
-                "aws" | "aws-kms" | "aws_kms" => KmsBackend::Aws,
                 _ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {backend_type}"))),
             };
         }
@@ -993,7 +839,6 @@ impl KmsConfig {
         config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache);
         config.allow_insecure_dev_defaults =
             get_env_bool(ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS, config.allow_insecure_dev_defaults);
-        config.allow_immediate_deletion = get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, config.allow_immediate_deletion);
 
         // Backend-specific configuration
         match config.backend {
@@ -1094,14 +939,6 @@ impl KmsConfig {
                 });
                 config.default_key_id = Some(key_id);
             }
-            KmsBackend::Aws => {
-                // Only non-credential settings are read here; access keys,
-                // profiles, and role assumption stay with the aws-config chain.
-                config.backend_config = BackendConfig::Aws(Box::new(AwsKmsConfig {
-                    region: get_env_opt_str(ENV_KMS_AWS_REGION),
-                    endpoint_url: get_env_opt_str(ENV_KMS_AWS_ENDPOINT_URL),
-                }));
-            }
         }
 
         config.validate()?;
@@ -1109,15 +946,6 @@ impl KmsConfig {
     }
 }
 
-/// Read the immediate-deletion gate from the environment.
-///
-/// Callers that assemble a [`KmsConfig`] field by field instead of going
-/// through [`KmsConfig::from_env`] use this, so the gate keeps one name, one
-/// default, and one place to look it up.
-pub fn allow_immediate_deletion_from_env() -> bool {
-    get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, false)
-}
-
 fn vault_tls_config(skip_tls_verify: bool) -> Option<TlsConfig> {
     skip_tls_verify.then_some(TlsConfig {
         ca_cert_path: None,
@@ -1278,60 +1106,6 @@ mod tests {
         assert_eq!(local_config.key_dir, temp_dir.path());
     }
 
-    #[test]
-    fn oversized_cache_ttl_is_clamped_not_rejected() {
-        let temp_dir = TempDir::new().expect("Failed to create temp dir");
-        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
-
-        assert_eq!(config.cache_config.ttl, DEFAULT_CACHE_TTL);
-        assert_eq!(config.cache_config.effective_ttl(), DEFAULT_CACHE_TTL);
-
-        // An oversized lifetime must not keep the service from starting, and
-        // must not reach moka, whose builder panics beyond 1000 years.
-        let config = KmsConfig {
-            cache_config: CacheConfig {
-                ttl: Duration::from_secs(u64::MAX),
-                ..Default::default()
-            },
-            ..config
-        };
-        assert!(config.validate().is_ok());
-        assert_eq!(config.cache_config.effective_ttl(), MAX_CACHE_TTL);
-
-        // In-range values pass through unchanged.
-        let config = KmsConfig {
-            cache_config: CacheConfig {
-                ttl: Duration::from_secs(600),
-                ..Default::default()
-            },
-            ..config
-        };
-        assert!(config.validate().is_ok());
-        assert_eq!(config.cache_config.effective_ttl(), Duration::from_secs(600));
-    }
-
-    #[test]
-    fn zero_cache_ttl_is_rejected_only_while_caching_is_enabled() {
-        let temp_dir = TempDir::new().expect("Failed to create temp dir");
-        let config = KmsConfig {
-            cache_config: CacheConfig {
-                ttl: Duration::ZERO,
-                ..Default::default()
-            },
-            ..KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults()
-        };
-
-        // A cache that expires every entry on insert is a misconfiguration, not
-        // a way to turn caching off.
-        assert!(config.validate().is_err());
-
-        let config = KmsConfig {
-            enable_cache: false,
-            ..config
-        };
-        assert!(config.validate().is_ok());
-    }
-
     #[test]
     fn test_oversized_timeout_and_retries_clamped_not_rejected() {
         let temp_dir = TempDir::new().expect("Failed to create temp dir");
@@ -1766,43 +1540,6 @@ mod tests {
         );
     }
 
-    /// The AWS backend reads only non-credential settings from the
-    /// environment; access keys, profiles, and role assumption stay with the
-    /// aws-config provider chain.
-    #[test]
-    fn test_from_env_selects_aws_backend_without_credentials() {
-        with_vars(
-            vec![
-                ("RUSTFS_KMS_BACKEND", Some("aws")),
-                (ENV_KMS_AWS_REGION, Some("eu-central-1")),
-                (ENV_KMS_AWS_ENDPOINT_URL, None::<&str>),
-            ],
-            || {
-                let config = KmsConfig::from_env().expect("kms config should load from env");
-                assert_eq!(config.backend, KmsBackend::Aws);
-                let aws = config.aws_kms_config().expect("aws backend config");
-                assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
-                assert_eq!(aws.endpoint_url, None);
-            },
-        );
-    }
-
-    /// A plaintext endpoint override exposes every KMS request, including the
-    /// plaintext data keys, so it stays behind the explicit development opt-in.
-    #[test]
-    fn test_from_env_rejects_plaintext_aws_endpoint() {
-        with_vars(
-            vec![
-                ("RUSTFS_KMS_BACKEND", Some("aws")),
-                (ENV_KMS_AWS_REGION, Some("us-east-1")),
-                (ENV_KMS_AWS_ENDPOINT_URL, Some("http://localhost:4566")),
-            ],
-            || {
-                KmsConfig::from_env().expect_err("a plaintext AWS endpoint must be rejected by default");
-            },
-        );
-    }
-
     #[test]
     fn test_from_env_approle_requires_secret_id_or_file() {
         with_vars(
@@ -1932,38 +1669,6 @@ mod tests {
         assert_eq!(refresh_safety_window_secs, None);
     }
 
-    /// The gate lives in server configuration only: it never rides along in a
-    /// serialized config, and a stored config that claims it must not be
-    /// believed. Otherwise one operator's one-time enablement would reach every
-    /// node that later reloads that config.
-    #[test]
-    fn immediate_deletion_gate_is_server_local_and_never_persisted() {
-        let persisted =
-            serde_json::to_value(KmsConfig::default().with_immediate_deletion_allowed()).expect("kms config should serialize");
-        assert!(
-            persisted.get("allow_immediate_deletion").is_none(),
-            "the gate must not be written into a persisted config: {persisted}"
-        );
-
-        let mut forged = persisted;
-        forged
-            .as_object_mut()
-            .expect("a persisted config must be a JSON object")
-            .insert("allow_immediate_deletion".to_string(), serde_json::json!(true));
-        let restored: KmsConfig = serde_json::from_value(forged).expect("an unknown gate field must not break loading");
-        assert!(!restored.allow_immediate_deletion, "a stored gate must fail closed");
-
-        with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, Some("true"))], || {
-            assert!(
-                allow_immediate_deletion_from_env(),
-                "the gate must be reachable from server configuration"
-            );
-        });
-        with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, None::<&str>)], || {
-            assert!(!allow_immediate_deletion_from_env());
-        });
-    }
-
     #[test]
     fn test_validate_rejects_incomplete_approle() {
         let mut config = KmsConfig::vault_approle(

crates/kms/src/deletion_worker.rs

@@ -22,123 +22,18 @@
 //! every node of a deployment concurrently — a key is only ever removed while
 //! its (re-read) record is an expired pending deletion or a tombstone.
 
-use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
 use crate::backends::{ExpiredKeyRemoval, KmsBackend};
-use crate::error::Result;
-use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
+use crate::types::{KeyStatus, ListKeysRequest};
 use async_trait::async_trait;
 use jiff::Zoned;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Duration;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};
 
 /// How often the worker looks for expired pending deletions.
 pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
 
-// ---------------------------------------------------------------------------
-// Metrics
-//
-// The sweep already pages through the whole key set, so everything below is
-// derived from the pages it has in hand: observing the key lifecycle costs no
-// extra backend call. Each metric is an aggregate — the gauges carry no labels
-// at all and the counter only a static outcome — because a per-key label would
-// carry key identifiers into the metric stream and grow the series count with
-// the key set. "This key is overdue for rotation" is a threshold on the
-// aggregate age and belongs in an alerting rule, not in a label.
-// ---------------------------------------------------------------------------
-
-/// Gauge: keys scheduled for deletion whose deadline has not passed, as of the
-/// end of the last sweep that saw the whole key set.
-const METRIC_PENDING_DELETION_KEYS: &str = "rustfs_kms_pending_deletion_keys";
-/// Gauge: keys left tombstoned by an interrupted removal and still awaiting
-/// the sweep, as of the end of the last sweep that saw the whole key set.
-const METRIC_TOMBSTONE_KEYS: &str = "rustfs_kms_deletion_tombstone_keys";
-/// Gauge: seconds since the least recently rotated usable key was rotated
-/// (its creation time when it was never rotated); `0` when there are none.
-const METRIC_OLDEST_ROTATION_AGE_SECONDS: &str = "rustfs_kms_oldest_key_rotation_age_seconds";
-/// Counter: keys the sweep acted on, by `outcome` (`removed`, `blocked`,
-/// `skipped`, `failed`).
-const METRIC_SWEEP_KEYS_TOTAL: &str = "rustfs_kms_deletion_sweep_keys_total";
-
-/// Register metric descriptions once per process.
-fn describe_metrics() {
-    static DESCRIBE: std::sync::Once = std::sync::Once::new();
-    DESCRIBE.call_once(|| {
-        metrics::describe_gauge!(
-            METRIC_PENDING_DELETION_KEYS,
-            "KMS keys scheduled for deletion whose deadline has not passed yet"
-        );
-        metrics::describe_gauge!(
-            METRIC_TOMBSTONE_KEYS,
-            "KMS keys left tombstoned by an interrupted removal, still awaiting the deletion sweep"
-        );
-        metrics::describe_gauge!(
-            METRIC_OLDEST_ROTATION_AGE_SECONDS,
-            "Seconds since the least recently rotated usable KMS key was last rotated, counting from creation for keys that were never rotated"
-        );
-        metrics::describe_counter!(METRIC_SWEEP_KEYS_TOTAL, "Total keys acted on by the KMS deletion sweep, by outcome");
-    });
-}
-
-/// Aggregate view of the key set, accumulated while the sweep pages through it.
-///
-/// Counts and one maximum age only: these feed label-less gauges, so no key
-/// identifier can reach the metric stream through them.
-#[derive(Debug, Default, Clone, Copy, PartialEq)]
-struct KeyCensus {
-    pending_deletion: usize,
-    tombstones: usize,
-    /// Longest time since last rotation across keys that are still usable,
-    /// counting from creation for keys that were never rotated. Keys on their
-    /// way out are excluded: they will never be rotated again, and would
-    /// otherwise pin the gauge high until the sweep finishes removing them.
-    oldest_rotation_age_seconds: f64,
-}
-
-impl KeyCensus {
-    fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
-        match key.status {
-            KeyStatus::PendingDeletion => self.pending_deletion += 1,
-            KeyStatus::Deleted => self.tombstones += 1,
-            KeyStatus::Active | KeyStatus::Disabled => {
-                let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
-                self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
-            }
-        }
-    }
-}
-
-/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
-/// timestamp persisted by a node running ahead) cannot produce a negative age.
-fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
-    (now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
-}
-
-/// Publish one sweep's counters, plus the lifecycle gauges when `census` covers
-/// the whole key set.
-fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
-    describe_metrics();
-    for (outcome, count) in [
-        ("removed", report.removed.len()),
-        ("blocked", report.blocked.len()),
-        ("skipped", report.skipped),
-        ("failed", report.failed),
-    ] {
-        // Emitted even at zero so every outcome series exists from the first
-        // sweep on and a rate over it is defined.
-        metrics::counter!(METRIC_SWEEP_KEYS_TOTAL, "outcome" => outcome).increment(count as u64);
-    }
-
-    // A sweep that could not finish listing saw only part of the key set;
-    // publishing its counts would understate every gauge, so the previous
-    // (complete) values are left standing instead.
-    let Some(census) = census else { return };
-    metrics::gauge!(METRIC_PENDING_DELETION_KEYS).set(census.pending_deletion as f64);
-    metrics::gauge!(METRIC_TOMBSTONE_KEYS).set(census.tombstones as f64);
-    metrics::gauge!(METRIC_OLDEST_ROTATION_AGE_SECONDS).set(census.oldest_rotation_age_seconds);
-}
-
 /// Reports configuration that still references a KMS key.
 ///
 /// Consulted before any material is destroyed; a non-empty result blocks the
@@ -173,8 +68,6 @@ pub(crate) struct DeletionWorker {
     default_key_id: Option<String>,
     reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
     interval: Duration,
-    backend_kind: &'static str,
-    audit_sink: Option<Arc<dyn KmsAuditSink>>,
 }
 
 impl DeletionWorker {
@@ -182,24 +75,15 @@ impl DeletionWorker {
         backend: Arc<dyn KmsBackend>,
         default_key_id: Option<String>,
         reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
-        backend_kind: &'static str,
     ) -> Self {
         Self {
             backend,
             default_key_id,
             reference_checker,
-            backend_kind,
-            audit_sink: None,
             interval: DEFAULT_SWEEP_INTERVAL,
         }
     }
 
-    /// Audit each irreversible key removal to `sink`.
-    pub(crate) fn with_audit_sink(mut self, sink: Option<Arc<dyn KmsAuditSink>>) -> Self {
-        self.audit_sink = sink;
-        self
-    }
-
     pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
         tokio::spawn(async move { self.run(cancel).await })
     }
@@ -230,15 +114,10 @@ impl DeletionWorker {
 
     /// Run one sweep at the given time. Exposed separately so tests can drive
     /// the expiry logic deterministically.
-    ///
-    /// Also feeds the lifecycle gauges: every key it lists is counted into a
-    /// [`KeyCensus`] on the way past, so the observability comes out of the
-    /// pages the sweep already had to fetch.
     pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
         let mut report = SweepReport::default();
-        let mut census = KeyCensus::default();
         let mut marker: Option<String> = None;
-        let listed_everything = loop {
+        loop {
             let request = ListKeysRequest {
                 limit: Some(100),
                 marker: marker.clone(),
@@ -250,99 +129,54 @@ impl DeletionWorker {
                 Err(error) => {
                     warn!(%error, "KMS deletion sweep could not list keys");
                     report.failed += 1;
-                    break false;
+                    return report;
                 }
             };
             for key in &response.keys {
-                // Keys this sweep destroys are left out of the census: the
-                // gauges describe the key set as it stands once the sweep is
-                // done, not as it was when the page was listed.
-                let removed = if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
-                    self.process_key(&key.key_id, now, &mut report).await
-                } else {
-                    false
-                };
-                if !removed {
-                    census.observe(key, now);
+                if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
+                    self.process_key(&key.key_id, now, &mut report).await;
                 }
             }
             if !response.truncated {
-                break true;
+                break;
             }
             match response.next_marker {
                 Some(next_marker) => marker = Some(next_marker),
-                // Truncated without a marker: the rest of the key set is out
-                // of reach, so the census is incomplete.
-                None => break false,
+                None => break,
             }
-        };
-        record_sweep(&report, listed_everything.then_some(census));
+        }
         report
     }
 
-    /// Handle one key that is on its way out; reports whether it was removed.
-    async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
+    async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) {
         // Never remove a key that live configuration still points at. The
         // default key check is built in; broader references (bucket
         // encryption settings, ...) come from the injected checker.
         if self.default_key_id.as_deref() == Some(key_id) {
             warn!(key_id, "expired KMS key is still the default key; refusing removal");
             report.blocked.push(key_id.to_string());
-            return false;
+            return;
         }
         if let Some(checker) = &self.reference_checker {
             let references = checker.references(key_id).await;
             if !references.is_empty() {
                 warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal");
                 report.blocked.push(key_id.to_string());
-                return false;
+                return;
             }
         }
 
         // The backend re-checks state and deadline under its own write
         // synchronization, so a cancellation racing this sweep wins there.
-        let started = Instant::now();
-        let outcome = self.backend.remove_expired_key(key_id, now).await;
-        match &outcome {
-            Ok(ExpiredKeyRemoval::Removed) => {
-                report.removed.push(key_id.to_string());
-                self.audit_removal(key_id, started, &outcome);
-                true
-            }
-            Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => {
-                report.skipped += 1;
-                false
-            }
+        match self.backend.remove_expired_key(key_id, now).await {
+            Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()),
+            Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1,
             Err(error) => {
                 warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep");
                 report.failed += 1;
-                self.audit_removal(key_id, started, &outcome);
-                false
             }
         }
     }
-
-    /// Record an attempted destruction of key material.
-    ///
-    /// Only attempts that reached the backend are audited: a key the sweep
-    /// declined to touch (not yet due, still referenced, state changed under
-    /// us) was never at risk, and recording it as a deletion event would
-    /// drown the real ones.
-    fn audit_removal(&self, key_id: &str, started: Instant, outcome: &Result<ExpiredKeyRemoval>) {
-        let Some(sink) = self.audit_sink.as_ref() else {
-            return;
-        };
-
-        // Removal runs on a background sweep, so there is no request
-        // principal to attribute it to.
-        let context = OperationContext::internal();
-        sink.emit(
-            KmsAuditRecord::new(KmsAuditOperation::DeleteKey, &context, self.backend_kind)
-                .with_key_id(Some(key_id))
-                .with_latency(started.elapsed())
-                .with_result(outcome),
-        );
-    }
 }
 
 #[cfg(test)]
@@ -376,14 +210,13 @@ mod tests {
                 key_id: key_id.to_string(),
                 pending_window_in_days: Some(7),
                 force_immediate: None,
-                confirm_key_id: None,
             })
             .await
             .expect("deletion should be scheduled");
     }
 
     fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker {
-        DeletionWorker::new(backend, None, None, "local")
+        DeletionWorker::new(backend, None, None)
     }
 
     fn after_window() -> Zoned {
@@ -474,7 +307,7 @@ mod tests {
         schedule(&backend, &key_id).await;
 
         // Blocked while it is the configured default key.
-        let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None, "local");
+        let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None);
         let report = as_default.sweep(&after_window()).await;
         assert_eq!(report.blocked, vec![key_id.clone()]);
         assert!(report.removed.is_empty());
@@ -484,7 +317,6 @@ mod tests {
             backend.clone(),
             None,
             Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))),
-            "local",
         );
         let report = with_references.sweep(&after_window()).await;
         assert_eq!(report.blocked, vec![key_id.clone()]);
@@ -495,53 +327,11 @@ mod tests {
             .expect("blocked key must still exist");
 
         // Removed once nothing references it anymore.
-        let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))), "local");
+        let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))));
         let report = unreferenced.sweep(&after_window()).await;
         assert_eq!(report.removed, vec![key_id.clone()]);
     }
 
-    #[tokio::test]
-    async fn only_attempted_removals_are_audited() {
-        #[derive(Default)]
-        struct CapturingSink {
-            records: std::sync::Mutex<Vec<KmsAuditRecord>>,
-        }
-
-        impl KmsAuditSink for CapturingSink {
-            fn emit(&self, record: KmsAuditRecord) {
-                self.records.lock().expect("sink lock should not be poisoned").push(record);
-            }
-        }
-
-        let temp_dir = tempfile::tempdir().expect("temp dir");
-        let backend = local_backend(&temp_dir).await;
-        let key_id = create_key(&backend, "audited-removal").await;
-        schedule(&backend, &key_id).await;
-
-        let sink = Arc::new(CapturingSink::default());
-        let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
-
-        // A key that is not yet due was never at risk, so it produces no record.
-        worker.sweep(&Zoned::now()).await;
-        assert!(
-            sink.records.lock().expect("sink lock").is_empty(),
-            "a key the sweep declined to touch must not be audited as a deletion"
-        );
-
-        worker.sweep(&after_window()).await;
-
-        let records = sink.records.lock().expect("sink lock");
-        assert_eq!(records.len(), 1, "the removal should be audited exactly once");
-        let record = &records[0];
-        assert_eq!(record.operation, crate::audit::KmsAuditOperation::DeleteKey);
-        assert_eq!(record.event, rustfs_s3_types::EventName::KmsKeyDeleted);
-        assert_eq!(record.outcome, crate::audit::KmsAuditOutcome::Success);
-        assert_eq!(record.key_id.as_deref(), Some(key_id.as_str()));
-        assert_eq!(record.backend, "local");
-        // Background work has no request principal to attribute.
-        assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
-    }
-
     #[tokio::test]
     async fn deadline_survives_backend_restart_and_sweep_completes_it() {
         let temp_dir = tempfile::tempdir().expect("temp dir");
@@ -605,156 +395,4 @@ mod tests {
         cancel.cancel();
         task.await.expect("worker task must stop after cancellation");
     }
-
-    // -- Metric emission ----------------------------------------------------
-
-    use metrics_util::MetricKind;
-    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-
-    type MetricEntry = (
-        metrics_util::CompositeKey,
-        Option<metrics::Unit>,
-        Option<metrics::SharedString>,
-        DebugValue,
-    );
-
-    /// Run `test` on a current-thread runtime under a debugging recorder and
-    /// return one snapshot of everything it emitted.
-    ///
-    /// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
-    /// the recorded state, so taking it per assertion would only show the
-    /// first assertion any data.
-    fn record_metrics<Out>(
-        test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
-    ) -> (Vec<MetricEntry>, Out) {
-        let recorder = DebuggingRecorder::new();
-        let snapshotter = recorder.snapshotter();
-        let out = metrics::with_local_recorder(&recorder, || {
-            let runtime = tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("current-thread runtime must build");
-            runtime.block_on(test())
-        });
-        (snapshotter.snapshot().into_vec(), out)
-    }
-
-    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
-        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
-            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
-            match (matches, value) {
-                (true, DebugValue::Gauge(value)) => Some(value.into_inner()),
-                _ => None,
-            }
-        })
-    }
-
-    fn counter_value(snapshot: &[MetricEntry], name: &str, outcome: &str) -> u64 {
-        snapshot
-            .iter()
-            .filter_map(|(composite, _unit, _description, value)| {
-                let matches = composite.kind() == MetricKind::Counter
-                    && composite.key().name() == name
-                    && composite
-                        .key()
-                        .labels()
-                        .any(|label| label.key() == "outcome" && label.value() == outcome);
-                match (matches, value) {
-                    (true, DebugValue::Counter(count)) => Some(*count),
-                    _ => None,
-                }
-            })
-            .sum()
-    }
-
-    fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
-        KeyInfo {
-            key_id: key_id.to_string(),
-            description: None,
-            algorithm: "AES-256".to_string(),
-            usage: KeyUsage::EncryptDecrypt,
-            status,
-            version: 1,
-            metadata: std::collections::HashMap::new(),
-            tags: std::collections::HashMap::new(),
-            created_at,
-            rotated_at,
-            created_by: None,
-        }
-    }
-
-    #[test]
-    fn census_ages_from_rotation_and_ignores_departing_keys() {
-        let now = Zoned::now();
-        let day = Duration::from_secs(86400);
-        let mut census = KeyCensus::default();
-
-        // Rotation beats creation as the age baseline...
-        census.observe(
-            &key_info("rotated", KeyStatus::Active, now.clone() - 30 * day, Some(now.clone() - 2 * day)),
-            &now,
-        );
-        // ...and a key that was never rotated ages from its creation.
-        census.observe(&key_info("never-rotated", KeyStatus::Disabled, now.clone() - 5 * day, None), &now);
-        // Keys on their way out only ever move the counts: they will not be
-        // rotated again, so their age must not drive the rotation gauge.
-        census.observe(&key_info("pending", KeyStatus::PendingDeletion, now.clone() - 400 * day, None), &now);
-        census.observe(&key_info("tombstone", KeyStatus::Deleted, now.clone() - 400 * day, None), &now);
-
-        assert_eq!(census.pending_deletion, 1);
-        assert_eq!(census.tombstones, 1);
-        let expected = 5.0 * 86400.0;
-        assert!(
-            (census.oldest_rotation_age_seconds - expected).abs() < 1.0,
-            "expected the never-rotated key to set the age, got {}",
-            census.oldest_rotation_age_seconds
-        );
-    }
-
-    #[test]
-    fn sweep_publishes_lifecycle_gauges_without_key_labels() {
-        let (snapshot, key_ids) = record_metrics(|| {
-            Box::pin(async {
-                let temp_dir = tempfile::tempdir().expect("temp dir");
-                let backend = local_backend(&temp_dir).await;
-                let live = create_key(&backend, "live-key").await;
-                let doomed = create_key(&backend, "doomed-key").await;
-                schedule(&backend, &doomed).await;
-
-                // Three days in, still inside the seven-day window: the sweep
-                // observes the scheduled key instead of removing it.
-                let report = worker(backend.clone())
-                    .sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
-                    .await;
-                assert_eq!(report.skipped, 1);
-                assert!(report.removed.is_empty());
-                vec![live, doomed]
-            })
-        });
-
-        assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(1.0));
-        assert_eq!(gauge_value(&snapshot, METRIC_TOMBSTONE_KEYS), Some(0.0));
-        let age = gauge_value(&snapshot, METRIC_OLDEST_ROTATION_AGE_SECONDS).expect("rotation age gauge must be published");
-        // Both keys were just created, and only the usable one counts, so the
-        // reported age is the sweep's own offset into the future.
-        assert!(
-            (age - 3.0 * 86400.0).abs() < 60.0,
-            "expected the sweep offset as the rotation age, got {age}"
-        );
-        assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "skipped"), 1);
-        assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "removed"), 0);
-
-        for (composite, ..) in &snapshot {
-            for label in composite.key().labels() {
-                for key_id in &key_ids {
-                    assert!(
-                        !label.value().contains(key_id.as_str()),
-                        "metric {} leaked a key identifier through label {}",
-                        composite.key().name(),
-                        label.key()
-                    );
-                }
-            }
-        }
-    }
 }

crates/kms/src/error.rs

@@ -137,13 +137,6 @@ pub enum KmsError {
     /// fail closed instead of being sent with credentials that may lapse mid-flight
     #[error("KMS credentials unavailable: {message}")]
     CredentialsUnavailable { message: String },
-
-    /// Key has master key version records but no baseline version, so envelopes
-    /// written before versioned rotation can no longer be resolved
-    #[error(
-        "Baseline version lost for key {key_id}: master key version records exist (oldest {oldest_version}) but the key record carries no baseline version, so data keys written before versioned rotation can no longer be resolved to the master key version that wrapped them. A node older than versioned rotation rewrote the key record and dropped the field. Finish upgrading every node, restore baseline_version to {oldest_version} on the key record, then retry"
-    )]
-    BaselineVersionLost { key_id: String, oldest_version: u32 },
 }
 
 impl KmsError {
@@ -298,18 +291,6 @@ impl KmsError {
     pub fn credentials_unavailable<S: Into<String>>(message: S) -> Self {
         Self::CredentialsUnavailable { message: message.into() }
     }
-
-    /// Create a baseline version lost error
-    ///
-    /// `oldest_version` is the lowest master key version that still has a
-    /// material record; it is exactly the baseline that was dropped, because
-    /// version records start at the baseline the first rotation froze.
-    pub fn baseline_version_lost<S: Into<String>>(key_id: S, oldest_version: u32) -> Self {
-        Self::BaselineVersionLost {
-            key_id: key_id.into(),
-            oldest_version,
-        }
-    }
 }
 
 /// Convert from standard library errors

crates/kms/src/lib.rs

@@ -65,7 +65,6 @@
 
 // Core modules
 pub mod api_types;
-pub mod audit;
 pub mod backends;
 pub mod backup;
 mod cache;
@@ -75,7 +74,6 @@ mod encryption;
 mod error;
 pub mod manager;
 mod policy;
-pub mod probe;
 pub mod service;
 pub mod service_manager;
 mod time_serde;
@@ -88,14 +86,11 @@ pub use api_types::{
     StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
     UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
-pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
-pub use cache::KmsCacheStats;
 pub use config::*;
 pub use deletion_worker::DeletionReferenceChecker;
 pub use encryption::is_data_key_envelope;
 pub use error::{KmsError, KmsUnavailableError, Result};
 pub use manager::KmsManager;
-pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
     KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,

crates/kms/src/policy.rs

@@ -104,9 +104,7 @@ pub(crate) fn classify_vaultrs(error: &vaultrs::error::ClientError) -> ErrorClas
     }
 }
 
-/// Retry classification of an HTTP status, shared by every backend that talks
-/// to an external KMS over HTTP.
-pub(crate) fn classify_status(code: u16) -> ErrorClass {
+fn classify_status(code: u16) -> ErrorClass {
     match code {
         429 | 500 | 502 | 503 | 504 => ErrorClass::RetryableStatus,
         _ => ErrorClass::Fatal,

crates/kms/src/probe.rs

@@ -1,894 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Background synthetic probe for the configured KMS backend.
-//!
-//! A health check that only reports "the backend answered" cannot distinguish a
-//! usable KMS from one that is reachable but no longer returns material that
-//! round-trips. Each probe round therefore generates a data key under a
-//! dedicated key, decrypts the returned ciphertext, and compares the plaintext,
-//! so a backend that answers without being usable becomes visible before object
-//! traffic depends on it.
-//!
-//! The probe key uses a fixed reserved id ([`PROBE_KEY_ID`]) and is created only
-//! when missing, so every node of a deployment converges on the same key and a
-//! create that loses the race against another node is a success, not an error.
-//!
-//! Backends that cannot host such a key — single-key read-only backends, or
-//! backends without data key generation — are reported as
-//! [`ProbeResult::Unsupported`] exactly once and then left alone: an
-//! unsupported backend is a configuration fact, not an outage, and must never
-//! raise failure metrics or alerts.
-//!
-//! Results are published two ways: as a lock-free [`ProbeStatus`] snapshot that
-//! readiness reporting can read without touching the backend, and as
-//! `rustfs_kms_probe_*` metrics. As everywhere else in this crate, metric labels
-//! carry only static outcome classes — never key identifiers, key material, or
-//! ciphertext.
-
-use crate::backends::KmsBackend;
-use crate::error::KmsError;
-use crate::types::{CreateKeyRequest, DecryptRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyUsage};
-use arc_swap::ArcSwap;
-use std::collections::HashMap;
-use std::sync::Arc;
-use std::time::Duration;
-use subtle::ConstantTimeEq;
-use tokio::time::Instant;
-use tokio_util::sync::CancellationToken;
-use tracing::{debug, info, warn};
-use zeroize::Zeroize;
-
-/// Reserved id of the key every probe round runs through.
-///
-/// Fixed rather than per-node or per-process: all nodes must converge on one
-/// key instead of littering the backend with probe keys, and the reserved
-/// `rustfs-internal-` prefix marks it as RustFS-owned for operators reviewing
-/// the key list.
-pub const PROBE_KEY_ID: &str = "rustfs-internal-kms-probe";
-
-/// Description stored on the probe key when this node creates it.
-const PROBE_KEY_DESCRIPTION: &str = "RustFS internal KMS health probe key; created and used only by the synthetic probe";
-
-/// Default interval between probe rounds.
-pub const DEFAULT_PROBE_INTERVAL: Duration = Duration::from_secs(60);
-
-/// Floor for the configured interval. Probing faster buys no additional signal
-/// and multiplies backend load by the size of the deployment.
-pub const MIN_PROBE_INTERVAL: Duration = Duration::from_secs(5);
-
-/// Probe interval in whole seconds; `0` disables the probe entirely.
-pub const ENV_KMS_PROBE_INTERVAL_SECS: &str = "RUSTFS_KMS_PROBE_INTERVAL_SECS";
-
-/// Result of the most recent probe round.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ProbeResult {
-    /// No round has completed yet for this service version.
-    Pending,
-    /// The full generate/decrypt/compare cycle succeeded.
-    Success,
-    /// The backend cannot host the probe key; no further rounds run and no
-    /// failures are recorded.
-    Unsupported,
-    /// The round failed with the given classification.
-    Failure(ProbeFailureKind),
-}
-
-impl ProbeResult {
-    /// Static metric label for this result.
-    fn as_label(self) -> &'static str {
-        match self {
-            ProbeResult::Pending => "pending",
-            ProbeResult::Success => "success",
-            ProbeResult::Unsupported => "unsupported",
-            ProbeResult::Failure(_) => "failure",
-        }
-    }
-}
-
-/// Stage of the round trip that failed, used as a static metric label.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum ProbeFailureKind {
-    /// The dedicated probe key could not be described or created.
-    KeyProvisioning,
-    /// Data key generation failed.
-    Generate,
-    /// The freshly generated ciphertext could not be decrypted.
-    Decrypt,
-    /// Decryption succeeded but returned material that differs from what was
-    /// generated — the backend answers without round-tripping.
-    Mismatch,
-}
-
-impl ProbeFailureKind {
-    /// Static metric label for this failure class.
-    fn as_label(self) -> &'static str {
-        match self {
-            ProbeFailureKind::KeyProvisioning => "key_provisioning",
-            ProbeFailureKind::Generate => "generate",
-            ProbeFailureKind::Decrypt => "decrypt",
-            ProbeFailureKind::Mismatch => "mismatch",
-        }
-    }
-}
-
-/// Snapshot of what the probe has observed so far.
-///
-/// Consumed by readiness reporting, which owns the staleness and consecutive
-/// failure thresholds; the probe itself only records what happened.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ProbeStatus {
-    /// Result of the most recent completed round.
-    pub result: ProbeResult,
-    /// When the most recent round completed.
-    pub last_round_at: Option<Instant>,
-    /// When the most recent successful round completed.
-    pub last_success_at: Option<Instant>,
-    /// Wall-clock timestamp of the most recent success, in Unix seconds, for
-    /// display and for the exported gauge. Use [`ProbeStatus::last_success_age`]
-    /// for staleness decisions: it is monotonic and immune to clock steps.
-    pub last_success_unix_secs: Option<i64>,
-    /// Failed rounds since the last success; reset by any non-failure result.
-    pub consecutive_failures: u32,
-}
-
-impl ProbeStatus {
-    /// Initial snapshot published before the first round completes.
-    fn pending() -> Self {
-        Self {
-            result: ProbeResult::Pending,
-            last_round_at: None,
-            last_success_at: None,
-            last_success_unix_secs: None,
-            consecutive_failures: 0,
-        }
-    }
-
-    /// How long ago the most recent round completed.
-    pub fn last_round_age(&self) -> Option<Duration> {
-        self.last_round_at.map(|at| at.elapsed())
-    }
-
-    /// How long ago the most recent successful round completed.
-    pub fn last_success_age(&self) -> Option<Duration> {
-        self.last_success_at.map(|at| at.elapsed())
-    }
-}
-
-/// Lock-free publication slot for [`ProbeStatus`].
-///
-/// Written by the single worker task and read by anyone; readers never block
-/// the probe and the probe never blocks a readiness handler.
-struct ProbeState {
-    status: ArcSwap<ProbeStatus>,
-}
-
-impl ProbeState {
-    fn new() -> Self {
-        Self {
-            status: ArcSwap::from_pointee(ProbeStatus::pending()),
-        }
-    }
-
-    fn status(&self) -> Arc<ProbeStatus> {
-        self.status.load_full()
-    }
-
-    /// Fold one round result into the published snapshot.
-    ///
-    /// Safe as a plain load-then-store because the owning worker task is the
-    /// only writer.
-    fn publish(&self, result: ProbeResult) {
-        let previous = self.status.load();
-        let now = Instant::now();
-        let (last_success_at, last_success_unix_secs) = match result {
-            ProbeResult::Success => (Some(now), Some(jiff::Timestamp::now().as_second())),
-            _ => (previous.last_success_at, previous.last_success_unix_secs),
-        };
-        let consecutive_failures = match result {
-            ProbeResult::Failure(_) => previous.consecutive_failures.saturating_add(1),
-            _ => 0,
-        };
-        self.status.store(Arc::new(ProbeStatus {
-            result,
-            last_round_at: Some(now),
-            last_success_at,
-            last_success_unix_secs,
-            consecutive_failures,
-        }));
-        record_state(result, last_success_unix_secs, consecutive_failures);
-    }
-}
-
-/// Owner of one service version's probe worker.
-///
-/// Mirrors the deletion worker's ownership model: cancelling stops the loop,
-/// and dropping the handle — a service version replaced without an explicit
-/// stop — cancels as a safety net.
-pub struct ProbeHandle {
-    state: Arc<ProbeState>,
-    cancel: CancellationToken,
-    task: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
-}
-
-impl ProbeHandle {
-    /// Latest published snapshot.
-    pub fn status(&self) -> Arc<ProbeStatus> {
-        self.state.status()
-    }
-
-    /// Stop the worker. The task observes the cancelled token on its next poll.
-    pub(crate) fn shutdown(&self) {
-        self.cancel.cancel();
-        if let Ok(mut task) = self.task.lock() {
-            drop(task.take());
-        }
-    }
-
-    #[cfg(test)]
-    pub(crate) fn is_cancelled(&self) -> bool {
-        self.cancel.is_cancelled()
-    }
-}
-
-impl Drop for ProbeHandle {
-    fn drop(&mut self) {
-        self.cancel.cancel();
-    }
-}
-
-/// Start the probe for a service version.
-///
-/// Returns `None` when the backend advertises no data key round trip, or when
-/// the operator disabled the probe; both cases leave no background task behind.
-pub(crate) fn spawn_probe_worker(backend: Arc<dyn KmsBackend>) -> Option<ProbeHandle> {
-    let capabilities = backend.capabilities();
-    if !capabilities.generate_data_key || !capabilities.decrypt {
-        return None;
-    }
-    let interval = configured_interval()?;
-    let state = Arc::new(ProbeState::new());
-    let cancel = CancellationToken::new();
-    let task = ProbeWorker::new(backend, state.clone(), interval).spawn(cancel.clone());
-    Some(ProbeHandle {
-        state,
-        cancel,
-        task: std::sync::Mutex::new(Some(task)),
-    })
-}
-
-/// Probe interval from the environment, or `None` when the probe is disabled.
-fn configured_interval() -> Option<Duration> {
-    parse_interval(std::env::var(ENV_KMS_PROBE_INTERVAL_SECS).ok().as_deref())
-}
-
-/// Interpret the configured interval: unset or unparsable falls back to the
-/// default, `0` disables the probe, and anything below [`MIN_PROBE_INTERVAL`]
-/// is raised to it so a misconfigured value cannot turn the probe into a load
-/// generator against the backend.
-fn parse_interval(value: Option<&str>) -> Option<Duration> {
-    let Some(value) = value else {
-        return Some(DEFAULT_PROBE_INTERVAL);
-    };
-    let Ok(seconds) = value.trim().parse::<u64>() else {
-        warn!(
-            variable = ENV_KMS_PROBE_INTERVAL_SECS,
-            "ignoring unparsable KMS probe interval; falling back to the default"
-        );
-        return Some(DEFAULT_PROBE_INTERVAL);
-    };
-    if seconds == 0 {
-        return None;
-    }
-    Some(Duration::from_secs(seconds).max(MIN_PROBE_INTERVAL))
-}
-
-/// Encryption context bound to every probe data key.
-///
-/// Static and free of per-node values, so material generated by one node stays
-/// decryptable by every other node running the same probe.
-fn probe_encryption_context() -> HashMap<String, String> {
-    HashMap::from([("rustfs-purpose".to_string(), "kms-probe".to_string())])
-}
-
-/// Outcome of making sure the dedicated probe key exists.
-enum KeyProvisioning {
-    Ready,
-    Unsupported,
-    Failed,
-}
-
-pub(crate) struct ProbeWorker {
-    backend: Arc<dyn KmsBackend>,
-    state: Arc<ProbeState>,
-    interval: Duration,
-}
-
-impl ProbeWorker {
-    fn new(backend: Arc<dyn KmsBackend>, state: Arc<ProbeState>, interval: Duration) -> Self {
-        Self {
-            backend,
-            state,
-            interval,
-        }
-    }
-
-    fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
-        tokio::spawn(async move { self.run(cancel).await })
-    }
-
-    async fn run(self, cancel: CancellationToken) {
-        describe_metrics();
-        let mut ticker = tokio::time::interval(self.interval);
-        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-        loop {
-            tokio::select! {
-                _ = cancel.cancelled() => {
-                    debug!("KMS probe worker stopped");
-                    return;
-                }
-                _ = ticker.tick() => {}
-            }
-            let started = Instant::now();
-            let result = self.round().await;
-            record_round(result, started.elapsed());
-            self.state.publish(result);
-            if result == ProbeResult::Unsupported {
-                // A capability gap does not change while a service version
-                // lives, so retrying it forever would only produce noise.
-                info!("KMS backend cannot host the synthetic probe key; probe disabled for this service version");
-                return;
-            }
-        }
-    }
-
-    /// Run one round. Exposed separately so tests can drive the round trip
-    /// deterministically without the interval loop.
-    pub(crate) async fn round(&self) -> ProbeResult {
-        let capabilities = self.backend.capabilities();
-        if !capabilities.generate_data_key || !capabilities.decrypt {
-            return ProbeResult::Unsupported;
-        }
-        match self.ensure_probe_key().await {
-            KeyProvisioning::Ready => {}
-            KeyProvisioning::Unsupported => return ProbeResult::Unsupported,
-            KeyProvisioning::Failed => return ProbeResult::Failure(ProbeFailureKind::KeyProvisioning),
-        }
-
-        let mut generated = match self
-            .backend
-            .generate_data_key(GenerateDataKeyRequest {
-                key_id: PROBE_KEY_ID.to_string(),
-                key_spec: KeySpec::Aes256,
-                encryption_context: probe_encryption_context(),
-            })
-            .await
-        {
-            Ok(generated) => generated,
-            Err(error) => {
-                warn!(%error, "KMS probe could not generate a data key");
-                return ProbeResult::Failure(ProbeFailureKind::Generate);
-            }
-        };
-
-        let decrypted = self
-            .backend
-            .decrypt(DecryptRequest {
-                ciphertext: generated.ciphertext_blob.clone(),
-                encryption_context: probe_encryption_context(),
-                grant_tokens: Vec::new(),
-            })
-            .await;
-
-        let result = match decrypted {
-            Ok(mut response) => {
-                let round_trips = bool::from(response.plaintext.ct_eq(&generated.plaintext_key));
-                response.plaintext.zeroize();
-                if round_trips {
-                    debug!("KMS probe round trip succeeded");
-                    ProbeResult::Success
-                } else {
-                    // The backend answered both calls but the material does not
-                    // survive the round trip: report it as loudly as an outage.
-                    warn!("KMS probe decrypted material does not match the generated data key");
-                    ProbeResult::Failure(ProbeFailureKind::Mismatch)
-                }
-            }
-            Err(error) => {
-                warn!(%error, "KMS probe could not decrypt its generated data key");
-                ProbeResult::Failure(ProbeFailureKind::Decrypt)
-            }
-        };
-        generated.plaintext_key.zeroize();
-        result
-    }
-
-    /// Create the probe key if it is missing.
-    ///
-    /// Idempotent by construction: an existing key is used as is, and a create
-    /// that loses the race against another node reports the key as ready
-    /// instead of failing the round.
-    async fn ensure_probe_key(&self) -> KeyProvisioning {
-        match self
-            .backend
-            .describe_key(DescribeKeyRequest {
-                key_id: PROBE_KEY_ID.to_string(),
-            })
-            .await
-        {
-            Ok(_) => return KeyProvisioning::Ready,
-            Err(KmsError::KeyNotFound { .. }) => {}
-            Err(error) if is_capability_gap(&error) => return KeyProvisioning::Unsupported,
-            Err(error) => {
-                warn!(%error, "KMS probe could not describe its probe key");
-                return KeyProvisioning::Failed;
-            }
-        }
-
-        match self
-            .backend
-            .create_key(CreateKeyRequest {
-                key_name: Some(PROBE_KEY_ID.to_string()),
-                key_usage: KeyUsage::EncryptDecrypt,
-                description: Some(PROBE_KEY_DESCRIPTION.to_string()),
-                ..Default::default()
-            })
-            .await
-        {
-            Ok(_) => {
-                info!(key_id = PROBE_KEY_ID, "created the KMS synthetic probe key");
-                KeyProvisioning::Ready
-            }
-            // Another node created the key between the describe and the create.
-            Err(KmsError::KeyAlreadyExists { .. }) => KeyProvisioning::Ready,
-            Err(error) if is_capability_gap(&error) => KeyProvisioning::Unsupported,
-            Err(error) => {
-                warn!(%error, "KMS probe could not create its probe key");
-                KeyProvisioning::Failed
-            }
-        }
-    }
-}
-
-/// Whether an error means "this backend will never host a probe key".
-///
-/// Read-only single-key backends reject key creation outright, and backends
-/// without the operation report an unsupported capability. Both are properties
-/// of the configuration rather than symptoms of an outage, so they stop the
-/// probe instead of counting as failures.
-fn is_capability_gap(error: &KmsError) -> bool {
-    matches!(error, KmsError::UnsupportedCapability { .. } | KmsError::InvalidOperation { .. })
-}
-
-// ---------------------------------------------------------------------------
-// Metrics
-//
-// Label values are exclusively static outcome classes; the probe key id is a
-// compile-time constant and still never becomes a label, so the metric shape
-// stays independent of how many keys or nodes exist.
-// ---------------------------------------------------------------------------
-
-/// Counter: probe rounds completed, by `result`.
-const METRIC_PROBE_ROUNDS_TOTAL: &str = "rustfs_kms_probe_rounds_total";
-/// Counter: failed probe rounds, by `failure_kind`.
-const METRIC_PROBE_FAILURES_TOTAL: &str = "rustfs_kms_probe_failures_total";
-/// Histogram: wall-clock duration of one probe round, in seconds, by `result`.
-const METRIC_PROBE_DURATION_SECONDS: &str = "rustfs_kms_probe_duration_seconds";
-/// Gauge: Unix timestamp of the most recent successful round.
-const METRIC_PROBE_LAST_SUCCESS_TIMESTAMP: &str = "rustfs_kms_probe_last_success_timestamp_seconds";
-/// Gauge: failed rounds since the last success.
-const METRIC_PROBE_CONSECUTIVE_FAILURES: &str = "rustfs_kms_probe_consecutive_failures";
-
-/// Register metric descriptions once per process.
-fn describe_metrics() {
-    static DESCRIBE: std::sync::Once = std::sync::Once::new();
-    DESCRIBE.call_once(|| {
-        metrics::describe_counter!(METRIC_PROBE_ROUNDS_TOTAL, "Total synthetic KMS probe rounds completed, by result");
-        metrics::describe_counter!(
-            METRIC_PROBE_FAILURES_TOTAL,
-            "Total failed synthetic KMS probe rounds, by the round-trip stage that failed"
-        );
-        metrics::describe_histogram!(
-            METRIC_PROBE_DURATION_SECONDS,
-            "Wall-clock duration of one synthetic KMS probe round, in seconds"
-        );
-        metrics::describe_gauge!(
-            METRIC_PROBE_LAST_SUCCESS_TIMESTAMP,
-            "Unix timestamp of the most recent successful synthetic KMS probe round"
-        );
-        metrics::describe_gauge!(
-            METRIC_PROBE_CONSECUTIVE_FAILURES,
-            "Synthetic KMS probe rounds that have failed since the last success"
-        );
-    });
-}
-
-/// Record one completed round.
-///
-/// An unsupported backend is deliberately counted as its own result and never
-/// as a failure, so alerts on the failure counter stay silent for deployments
-/// whose backend cannot host the probe.
-fn record_round(result: ProbeResult, elapsed: Duration) {
-    metrics::counter!(METRIC_PROBE_ROUNDS_TOTAL, "result" => result.as_label()).increment(1);
-    if let ProbeResult::Failure(kind) = result {
-        metrics::counter!(METRIC_PROBE_FAILURES_TOTAL, "failure_kind" => kind.as_label()).increment(1);
-    }
-    metrics::histogram!(METRIC_PROBE_DURATION_SECONDS, "result" => result.as_label()).record(elapsed.as_secs_f64());
-}
-
-/// Mirror the published snapshot into the gauges.
-fn record_state(result: ProbeResult, last_success_unix_secs: Option<i64>, consecutive_failures: u32) {
-    if result == ProbeResult::Success
-        && let Some(timestamp) = last_success_unix_secs
-    {
-        // Only ever moved forward by a success, so a failing probe leaves the
-        // gauge at the last known good time and its age keeps growing.
-        metrics::gauge!(METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).set(timestamp as f64);
-    }
-    metrics::gauge!(METRIC_PROBE_CONSECUTIVE_FAILURES).set(f64::from(consecutive_failures));
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::backends::local::LocalKmsBackend;
-    use crate::backends::static_kms::StaticKmsBackend;
-    use crate::config::KmsConfig;
-    use crate::types::{
-        CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyResponse, DecryptResponse, DeleteKeyRequest,
-        DeleteKeyResponse, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyResponse, KeyMetadata, KeyState,
-        ListKeysRequest, ListKeysResponse,
-    };
-    use async_trait::async_trait;
-    use base64::Engine as _;
-    use metrics_util::MetricKind;
-    use metrics_util::debugging::{DebugValue, DebuggingRecorder};
-    use std::future::Future;
-
-    async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<dyn KmsBackend> {
-        let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
-        Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build"))
-    }
-
-    async fn static_backend() -> Arc<dyn KmsBackend> {
-        let config =
-            KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
-        Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
-    }
-
-    fn worker(backend: Arc<dyn KmsBackend>) -> ProbeWorker {
-        ProbeWorker::new(backend, Arc::new(ProbeState::new()), DEFAULT_PROBE_INTERVAL)
-    }
-
-    /// Backend that answers every probe call but corrupts the round trip in a
-    /// configurable way, so each failure classification can be provoked without
-    /// a real backend outage.
-    struct BrokenBackend {
-        fault: Fault,
-    }
-
-    #[derive(Clone, Copy)]
-    enum Fault {
-        Generate,
-        Decrypt,
-        /// Decrypt succeeds but returns material other than the generated key.
-        Mismatch,
-    }
-
-    impl BrokenBackend {
-        fn arc(fault: Fault) -> Arc<dyn KmsBackend> {
-            Arc::new(Self { fault })
-        }
-
-        fn probe_key_metadata() -> KeyMetadata {
-            KeyMetadata {
-                key_id: PROBE_KEY_ID.to_string(),
-                key_state: KeyState::Enabled,
-                key_usage: KeyUsage::EncryptDecrypt,
-                description: None,
-                creation_date: jiff::Zoned::now(),
-                deletion_date: None,
-                origin: "test".to_string(),
-                key_manager: "test".to_string(),
-                tags: HashMap::new(),
-            }
-        }
-    }
-
-    #[async_trait]
-    impl KmsBackend for BrokenBackend {
-        async fn create_key(&self, _request: CreateKeyRequest) -> crate::error::Result<CreateKeyResponse> {
-            unimplemented!("the probe key always exists on this backend")
-        }
-
-        async fn encrypt(&self, _request: EncryptRequest) -> crate::error::Result<EncryptResponse> {
-            unimplemented!("not exercised by probe tests")
-        }
-
-        async fn decrypt(&self, _request: DecryptRequest) -> crate::error::Result<DecryptResponse> {
-            match self.fault {
-                Fault::Decrypt => Err(KmsError::backend_error("decrypt unavailable")),
-                Fault::Mismatch => Ok(DecryptResponse {
-                    plaintext: vec![0xffu8; 32],
-                    key_id: PROBE_KEY_ID.to_string(),
-                    encryption_algorithm: None,
-                }),
-                Fault::Generate => unreachable!("generation fails before decrypt is reached"),
-            }
-        }
-
-        async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> crate::error::Result<GenerateDataKeyResponse> {
-            match self.fault {
-                Fault::Generate => Err(KmsError::backend_error("data key generation unavailable")),
-                _ => Ok(GenerateDataKeyResponse {
-                    key_id: PROBE_KEY_ID.to_string(),
-                    plaintext_key: vec![0x11u8; 32],
-                    ciphertext_blob: vec![0x22u8; 48],
-                }),
-            }
-        }
-
-        async fn describe_key(&self, _request: DescribeKeyRequest) -> crate::error::Result<DescribeKeyResponse> {
-            Ok(DescribeKeyResponse {
-                key_metadata: Self::probe_key_metadata(),
-            })
-        }
-
-        async fn list_keys(&self, _request: ListKeysRequest) -> crate::error::Result<ListKeysResponse> {
-            unimplemented!("not exercised by probe tests")
-        }
-
-        async fn delete_key(&self, _request: DeleteKeyRequest) -> crate::error::Result<DeleteKeyResponse> {
-            unimplemented!("not exercised by probe tests")
-        }
-
-        async fn cancel_key_deletion(
-            &self,
-            _request: CancelKeyDeletionRequest,
-        ) -> crate::error::Result<CancelKeyDeletionResponse> {
-            unimplemented!("not exercised by probe tests")
-        }
-
-        async fn health_check(&self) -> crate::error::Result<bool> {
-            Ok(true)
-        }
-    }
-
-    #[tokio::test]
-    async fn round_creates_the_probe_key_once_and_round_trips_it() {
-        let temp_dir = tempfile::tempdir().expect("temp dir");
-        let backend = local_backend(&temp_dir).await;
-        let worker = worker(backend.clone());
-
-        assert_eq!(worker.round().await, ProbeResult::Success);
-        let created = backend
-            .describe_key(DescribeKeyRequest {
-                key_id: PROBE_KEY_ID.to_string(),
-            })
-            .await
-            .expect("the first round must create the probe key");
-        assert_eq!(created.key_metadata.key_id, PROBE_KEY_ID);
-
-        // A second round reuses the key instead of failing on the existing one,
-        // which is also what a second node in the deployment would do.
-        assert_eq!(worker.round().await, ProbeResult::Success);
-        assert_eq!(
-            backend
-                .list_keys(ListKeysRequest {
-                    limit: Some(100),
-                    marker: None,
-                    usage_filter: None,
-                    status_filter: None,
-                })
-                .await
-                .expect("list keys")
-                .keys
-                .iter()
-                .filter(|key| key.key_id == PROBE_KEY_ID)
-                .count(),
-            1,
-            "probe key creation must be idempotent"
-        );
-    }
-
-    #[test]
-    fn read_only_backend_is_reported_unsupported_without_failures() {
-        let (snapshot, result) = record_metrics(|| {
-            Box::pin(async {
-                let worker = worker(static_backend().await);
-                let result = worker.round().await;
-                record_round(result, Duration::from_millis(1));
-                result
-            })
-        });
-
-        assert_eq!(result, ProbeResult::Unsupported);
-        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "unsupported")]), 1);
-        assert_eq!(
-            counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[]),
-            0,
-            "an unsupported backend must never raise probe failures"
-        );
-    }
-
-    #[tokio::test]
-    async fn each_round_trip_stage_gets_its_own_failure_classification() {
-        for (fault, expected) in [
-            (Fault::Generate, ProbeFailureKind::Generate),
-            (Fault::Decrypt, ProbeFailureKind::Decrypt),
-            (Fault::Mismatch, ProbeFailureKind::Mismatch),
-        ] {
-            assert_eq!(worker(BrokenBackend::arc(fault)).round().await, ProbeResult::Failure(expected));
-        }
-    }
-
-    #[test]
-    fn metrics_and_status_record_success_then_failure() {
-        let (snapshot, status) = record_metrics(|| {
-            Box::pin(async {
-                let temp_dir = tempfile::tempdir().expect("temp dir");
-                let state = Arc::new(ProbeState::new());
-                let healthy = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL);
-                let round = healthy.round().await;
-                record_round(round, Duration::from_millis(10));
-                state.publish(round);
-
-                let broken = ProbeWorker::new(BrokenBackend::arc(Fault::Mismatch), state.clone(), DEFAULT_PROBE_INTERVAL);
-                for _ in 0..2 {
-                    let round = broken.round().await;
-                    record_round(round, Duration::from_millis(10));
-                    state.publish(round);
-                }
-                state.status()
-            })
-        });
-
-        assert_eq!(status.result, ProbeResult::Failure(ProbeFailureKind::Mismatch));
-        assert_eq!(status.consecutive_failures, 2);
-        assert!(
-            status.last_success_at.is_some() && status.last_success_unix_secs.is_some(),
-            "a failing probe must keep reporting when it last succeeded"
-        );
-
-        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "success")]), 1);
-        assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "failure")]), 2);
-        assert_eq!(counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[("failure_kind", "mismatch")]), 2);
-        assert_eq!(histogram_values(&snapshot, METRIC_PROBE_DURATION_SECONDS, &[]).len(), 3);
-        assert!(
-            gauge_value(&snapshot, METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).is_some_and(|timestamp| timestamp > 0.0),
-            "the last success gauge must carry a wall-clock timestamp"
-        );
-        assert_eq!(gauge_value(&snapshot, METRIC_PROBE_CONSECUTIVE_FAILURES), Some(2.0));
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn worker_publishes_status_and_stops_on_cancel() {
-        let temp_dir = tempfile::tempdir().expect("temp dir");
-        let state = Arc::new(ProbeState::new());
-        assert_eq!(state.status().result, ProbeResult::Pending);
-
-        let cancel = CancellationToken::new();
-        let task = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
-
-        // The paused clock auto-advances through the worker's interval ticks.
-        let mut observed = None;
-        for _ in 0..100 {
-            tokio::time::sleep(Duration::from_secs(1)).await;
-            if state.status().result != ProbeResult::Pending {
-                observed = Some(state.status());
-                break;
-            }
-        }
-        let observed = observed.expect("the worker loop must publish a round result");
-        assert_eq!(observed.result, ProbeResult::Success);
-        assert_eq!(observed.consecutive_failures, 0);
-        assert!(observed.last_success_at.is_some());
-
-        cancel.cancel();
-        task.await.expect("worker task must stop after cancellation");
-    }
-
-    #[tokio::test(start_paused = true)]
-    async fn unsupported_backend_stops_the_worker_without_cancellation() {
-        let state = Arc::new(ProbeState::new());
-        let cancel = CancellationToken::new();
-        let task = ProbeWorker::new(static_backend().await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
-
-        task.await.expect("worker must exit on its own for an unsupported backend");
-        assert_eq!(state.status().result, ProbeResult::Unsupported);
-        assert!(!cancel.is_cancelled());
-    }
-
-    #[test]
-    fn interval_configuration_defaults_clamps_and_disables() {
-        assert_eq!(parse_interval(None), Some(DEFAULT_PROBE_INTERVAL));
-        assert_eq!(parse_interval(Some("not-a-number")), Some(DEFAULT_PROBE_INTERVAL));
-        assert_eq!(parse_interval(Some(" 120 ")), Some(Duration::from_secs(120)));
-        assert_eq!(parse_interval(Some("1")), Some(MIN_PROBE_INTERVAL));
-        assert_eq!(parse_interval(Some("0")), None, "zero must disable the probe");
-    }
-
-    // -- Metric assertion helpers, mirroring the policy engine's tests --------
-
-    type MetricEntry = (
-        metrics_util::CompositeKey,
-        Option<metrics::Unit>,
-        Option<metrics::SharedString>,
-        DebugValue,
-    );
-
-    /// Run `test` on a paused current-thread runtime under a debugging recorder
-    /// and return one snapshot of everything it emitted.
-    fn record_metrics<Out>(test: impl FnOnce() -> std::pin::Pin<Box<dyn Future<Output = Out>>>) -> (Vec<MetricEntry>, Out) {
-        let recorder = DebuggingRecorder::new();
-        let snapshotter = recorder.snapshotter();
-        let out = metrics::with_local_recorder(&recorder, || {
-            let runtime = tokio::runtime::Builder::new_current_thread()
-                .enable_time()
-                .start_paused(true)
-                .build()
-                .expect("current-thread runtime must build");
-            runtime.block_on(test())
-        });
-        (snapshotter.snapshot().into_vec(), out)
-    }
-
-    fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
-        labels
-            .iter()
-            .all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected))
-    }
-
-    fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
-        snapshot
-            .iter()
-            .filter_map(|(composite, _unit, _description, value)| {
-                let matches = composite.kind() == MetricKind::Counter
-                    && composite.key().name() == name
-                    && labels_match(composite.key(), labels);
-                match (matches, value) {
-                    (true, DebugValue::Counter(count)) => Some(*count),
-                    _ => None,
-                }
-            })
-            .sum()
-    }
-
-    fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
-        snapshot
-            .iter()
-            .filter_map(|(composite, _unit, _description, value)| {
-                let matches = composite.kind() == MetricKind::Histogram
-                    && composite.key().name() == name
-                    && labels_match(composite.key(), labels);
-                match (matches, value) {
-                    (true, DebugValue::Histogram(values)) => Some(values),
-                    _ => None,
-                }
-            })
-            .flatten()
-            .map(|value| value.into_inner())
-            .collect()
-    }
-
-    fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
-        snapshot.iter().find_map(|(composite, _unit, _description, value)| {
-            let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
-            match (matches, value) {
-                (true, DebugValue::Gauge(gauge)) => Some(gauge.into_inner()),
-                _ => None,
-            }
-        })
-    }
-}

crates/kms/src/service.rs

@@ -14,10 +14,6 @@
 
 //! Object encryption service for S3-compatible encryption
 
-use crate::api_types::{
-    TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
-};
-use crate::cache::KmsCacheStats;
 use crate::encryption::ciphers::{create_cipher, generate_iv};
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
@@ -116,23 +112,6 @@ impl ObjectEncryptionService {
         self.kms_manager.create_key(request).await
     }
 
-    /// Create a new master key on behalf of `context`'s principal
-    ///
-    /// # Arguments
-    /// * `request` - CreateKeyRequest with key parameters
-    /// * `context` - Identity and correlation data recorded in the audit trail
-    ///
-    /// # Returns
-    /// CreateKeyResponse with created key details
-    ///
-    pub async fn create_key_with_context(
-        &self,
-        request: CreateKeyRequest,
-        context: &OperationContext,
-    ) -> Result<CreateKeyResponse> {
-        self.kms_manager.create_key_with_context(request, context).await
-    }
-
     /// Describe a master key (delegates to KMS manager)
     ///
     /// # Arguments
@@ -145,23 +124,6 @@ impl ObjectEncryptionService {
         self.kms_manager.describe_key(request).await
     }
 
-    /// Describe a master key on behalf of `context`'s principal
-    ///
-    /// # Arguments
-    /// * `request` - DescribeKeyRequest with key ID
-    /// * `context` - Identity and correlation data recorded in the audit trail
-    ///
-    /// # Returns
-    /// DescribeKeyResponse with key metadata
-    ///
-    pub async fn describe_key_with_context(
-        &self,
-        request: DescribeKeyRequest,
-        context: &OperationContext,
-    ) -> Result<DescribeKeyResponse> {
-        self.kms_manager.describe_key_with_context(request, context).await
-    }
-
     /// List master keys (delegates to KMS manager)
     ///
     /// # Arguments
@@ -174,77 +136,6 @@ impl ObjectEncryptionService {
         self.kms_manager.list_keys(request).await
     }
 
-    /// List master keys on behalf of `context`'s principal
-    ///
-    /// # Arguments
-    /// * `request` - ListKeysRequest with listing parameters
-    /// * `context` - Identity and correlation data recorded in the audit trail
-    ///
-    /// # Returns
-    /// ListKeysResponse with list of keys
-    ///
-    pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
-        self.kms_manager.list_keys_with_context(request, context).await
-    }
-
-    /// Replace a master key's description (delegates to KMS manager)
-    ///
-    /// # Arguments
-    /// * `request` - UpdateKeyDescriptionRequest with key ID and the new
-    ///   description; an empty description clears the stored value
-    ///
-    /// # Returns
-    /// UpdateKeyDescriptionResponse acknowledging the update
-    ///
-    pub async fn update_key_description(&self, request: UpdateKeyDescriptionRequest) -> Result<UpdateKeyDescriptionResponse> {
-        let description = (!request.description.is_empty()).then_some(request.description.as_str());
-        self.kms_manager.update_key_description(&request.key_id, description).await?;
-
-        Ok(UpdateKeyDescriptionResponse {
-            success: true,
-            message: "key description updated".to_string(),
-            key_id: request.key_id,
-        })
-    }
-
-    /// Add or overwrite master key tags (delegates to KMS manager)
-    ///
-    /// # Arguments
-    /// * `request` - TagKeyRequest with key ID and the tags to set; tags
-    ///   outside the request are left untouched
-    ///
-    /// # Returns
-    /// TagKeyResponse acknowledging the update
-    ///
-    pub async fn tag_key(&self, request: TagKeyRequest) -> Result<TagKeyResponse> {
-        self.kms_manager.tag_key(&request.key_id, &request.tags).await?;
-
-        Ok(TagKeyResponse {
-            success: true,
-            message: "key tags updated".to_string(),
-            key_id: request.key_id,
-        })
-    }
-
-    /// Remove master key tags (delegates to KMS manager)
-    ///
-    /// # Arguments
-    /// * `request` - UntagKeyRequest with key ID and the tag keys to remove;
-    ///   tag keys that are not set are ignored, so the call is idempotent
-    ///
-    /// # Returns
-    /// UntagKeyResponse acknowledging the removal
-    ///
-    pub async fn untag_key(&self, request: UntagKeyRequest) -> Result<UntagKeyResponse> {
-        self.kms_manager.untag_key(&request.key_id, &request.tag_keys).await?;
-
-        Ok(UntagKeyResponse {
-            success: true,
-            message: "key tags removed".to_string(),
-            key_id: request.key_id,
-        })
-    }
-
     /// Generate a data encryption key (delegates to KMS manager)
     ///
     /// # Arguments
@@ -269,9 +160,9 @@ impl ObjectEncryptionService {
     /// Get cache statistics
     ///
     /// # Returns
-    /// A [`KmsCacheStats`] snapshot if caching is enabled, `None` otherwise
+    /// Option with (hits, misses) if caching is enabled
     ///
-    pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
+    pub async fn cache_stats(&self) -> Option<(u64, u64)> {
         self.kms_manager.cache_stats().await
     }
 
@@ -1027,128 +918,6 @@ mod tests {
         );
     }
 
-    async fn describe(service: &ObjectEncryptionService, key_id: &str) -> KeyMetadata {
-        service
-            .describe_key(DescribeKeyRequest {
-                key_id: key_id.to_string(),
-            })
-            .await
-            .expect("describe should succeed")
-            .key_metadata
-    }
-
-    async fn create_metadata_test_key(service: &ObjectEncryptionService, key_id: &str) {
-        service
-            .create_key(CreateKeyRequest {
-                key_name: Some(key_id.to_string()),
-                key_usage: KeyUsage::EncryptDecrypt,
-                description: Some("original".to_string()),
-                policy: None,
-                tags: HashMap::from([
-                    ("name".to_string(), key_id.to_string()),
-                    ("team".to_string(), "storage".to_string()),
-                ]),
-                origin: None,
-            })
-            .await
-            .expect("test key should be created");
-    }
-
-    #[tokio::test]
-    async fn key_metadata_updates_are_visible_to_describe() {
-        let (service, _temp_dir) = create_test_service().await;
-        create_metadata_test_key(&service, "metadata-key").await;
-
-        service
-            .update_key_description(UpdateKeyDescriptionRequest {
-                key_id: "metadata-key".to_string(),
-                description: "updated".to_string(),
-            })
-            .await
-            .expect("description update should succeed");
-        service
-            .tag_key(TagKeyRequest {
-                key_id: "metadata-key".to_string(),
-                tags: HashMap::from([
-                    ("team".to_string(), "platform".to_string()),
-                    ("env".to_string(), "prod".to_string()),
-                ]),
-            })
-            .await
-            .expect("tagging should succeed");
-
-        // Reading through the manager's metadata cache must observe the
-        // updates, not the snapshot cached when the key was created.
-        let metadata = describe(&service, "metadata-key").await;
-        assert_eq!(metadata.description.as_deref(), Some("updated"));
-        assert_eq!(metadata.tags.get("team").map(String::as_str), Some("platform"));
-        assert_eq!(metadata.tags.get("env").map(String::as_str), Some("prod"));
-        assert_eq!(
-            metadata.tags.get("name").map(String::as_str),
-            Some("metadata-key"),
-            "tags set at creation must survive a later tag update"
-        );
-
-        // Untagging is idempotent: removing an absent tag is a no-op, not an
-        // error, so a retried request stays safe.
-        for attempt in 1..=2 {
-            service
-                .untag_key(UntagKeyRequest {
-                    key_id: "metadata-key".to_string(),
-                    tag_keys: vec!["env".to_string()],
-                })
-                .await
-                .unwrap_or_else(|error| panic!("untag attempt {attempt} should succeed: {error:?}"));
-        }
-        let metadata = describe(&service, "metadata-key").await;
-        assert!(!metadata.tags.contains_key("env"), "untagged tag must be gone: {:?}", metadata.tags);
-        assert_eq!(
-            metadata.tags.get("team").map(String::as_str),
-            Some("platform"),
-            "untagging must not touch other tags"
-        );
-
-        // An empty description clears the stored value.
-        service
-            .update_key_description(UpdateKeyDescriptionRequest {
-                key_id: "metadata-key".to_string(),
-                description: String::new(),
-            })
-            .await
-            .expect("clearing the description should succeed");
-        assert_eq!(describe(&service, "metadata-key").await.description, None);
-    }
-
-    #[tokio::test]
-    async fn identity_tag_is_not_writable_through_metadata_updates() {
-        let (service, _temp_dir) = create_test_service().await;
-        create_metadata_test_key(&service, "identity-key").await;
-
-        let rewrite = service
-            .tag_key(TagKeyRequest {
-                key_id: "identity-key".to_string(),
-                tags: HashMap::from([("name".to_string(), "other-key".to_string())]),
-            })
-            .await
-            .expect_err("rewriting the identity tag must be rejected");
-        assert!(matches!(rewrite, KmsError::InvalidOperation { .. }), "got {rewrite:?}");
-
-        let removal = service
-            .untag_key(UntagKeyRequest {
-                key_id: "identity-key".to_string(),
-                tag_keys: vec!["team".to_string(), "name".to_string()],
-            })
-            .await
-            .expect_err("removing the identity tag must be rejected");
-        assert!(matches!(removal, KmsError::InvalidOperation { .. }), "got {removal:?}");
-
-        // Both rejections are pre-write: the record is untouched, including
-        // the ordinary tag that shared the rejected untag request.
-        let metadata = describe(&service, "identity-key").await;
-        assert_eq!(metadata.tags.get("name").map(String::as_str), Some("identity-key"));
-        assert_eq!(metadata.tags.get("team").map(String::as_str), Some("storage"));
-    }
-
     #[tokio::test]
     async fn test_decrypt_data_key_uses_object_encryption_context() {
         let (service, _temp_dir) = create_test_service().await;

crates/kms/src/service_manager.rs

@@ -14,14 +14,12 @@
 
 //! KMS service manager for dynamic configuration and runtime management
 
-use crate::audit::KmsAuditSink;
 use crate::backends::vault_credentials::CredentialTaskHandle;
 use crate::backends::{KmsBackend, local::LocalKmsBackend};
 use crate::config::{BackendConfig, KmsConfig};
 use crate::deletion_worker::{DeletionReferenceChecker, DeletionWorker};
 use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
-use crate::probe::{ProbeHandle, ProbeStatus, spawn_probe_worker};
 use crate::service::ObjectEncryptionService;
 use arc_swap::ArcSwap;
 use sha2::{Digest, Sha256};
@@ -122,23 +120,13 @@ struct ServiceVersion {
     /// Background deletion worker owned by this service version, if the
     /// backend supports deletion scheduling
     deletion_worker: Option<Arc<DeletionWorkerHandle>>,
-    /// Background synthetic probe owned by this service version, if the
-    /// backend can round-trip a data key and the probe is enabled
-    probe_worker: Option<Arc<ProbeHandle>>,
 }
 
 impl ServiceVersion {
-    /// Stop the background workers this version owns.
-    ///
-    /// The credential renewal task is recycled separately because its shutdown
-    /// is asynchronous.
-    fn shutdown_background_workers(&self) {
+    fn shutdown_deletion_worker(&self) {
         if let Some(worker) = &self.deletion_worker {
             worker.shutdown();
         }
-        if let Some(probe) = &self.probe_worker {
-            probe.shutdown();
-        }
     }
 }
 
@@ -183,8 +171,6 @@ pub struct KmsServiceManager {
     lifecycle_mutex: Arc<Mutex<()>>,
     /// External reference checker consulted before expired keys are removed
     deletion_reference_checker: std::sync::RwLock<Option<Arc<dyn DeletionReferenceChecker>>>,
-    /// External sink receiving audit records for management operations
-    audit_sink: std::sync::RwLock<Option<Arc<dyn KmsAuditSink>>>,
 }
 
 impl KmsServiceManager {
@@ -199,26 +185,9 @@ impl KmsServiceManager {
             version_counter: Arc::new(AtomicU64::new(0)),
             lifecycle_mutex: Arc::new(Mutex::new(())),
             deletion_reference_checker: std::sync::RwLock::new(None),
-            audit_sink: std::sync::RwLock::new(None),
         }
     }
 
-    /// Install the sink that receives an audit record for every KMS
-    /// management operation. Takes effect for services built by the next
-    /// start or reconfigure.
-    ///
-    /// Without a sink no records are built, so KMS operations are unaffected
-    /// when auditing is not configured.
-    pub fn set_audit_sink(&self, sink: Arc<dyn KmsAuditSink>) {
-        if let Ok(mut slot) = self.audit_sink.write() {
-            *slot = Some(sink);
-        }
-    }
-
-    fn audit_sink(&self) -> Option<Arc<dyn KmsAuditSink>> {
-        self.audit_sink.read().ok().and_then(|slot| slot.clone())
-    }
-
     /// Install the reference checker consulted before the deletion worker
     /// removes an expired key. Takes effect for workers spawned by the next
     /// start or reconfigure.
@@ -410,7 +379,7 @@ impl KmsServiceManager {
         // Note: Existing Arc references will keep the service alive until operations complete
         let state = self.state.load_full();
         if let Some(current) = state.current_service.as_ref() {
-            current.shutdown_background_workers();
+            current.shutdown_deletion_worker();
         }
         self.state.store(Arc::new(RuntimeState {
             config: state.config.clone(),
@@ -545,19 +514,6 @@ impl KmsServiceManager {
         self.state.load().current_service.as_ref().map(|sv| sv.version)
     }
 
-    /// Latest synthetic probe snapshot of the running service version.
-    ///
-    /// `None` when KMS is not running, when the backend cannot round-trip a
-    /// data key, or when the probe is disabled. The read is lock-free and
-    /// performs no backend I/O, so a health endpoint can consult it without
-    /// turning an external KMS hiccup into probe pressure of its own; the
-    /// staleness and consecutive-failure thresholds belong to the caller.
-    pub fn probe_status(&self) -> Option<Arc<ProbeStatus>> {
-        let state = self.state.load();
-        let service_version = state.current_service.as_ref()?;
-        Some(service_version.probe_worker.as_ref()?.status())
-    }
-
     /// Health check for the KMS service
     pub async fn health_check(&self) -> Result<bool> {
         let checked_state = self.state.load_full();
@@ -626,19 +582,10 @@ impl KmsServiceManager {
                 let backend = crate::backends::static_kms::StaticKmsBackend::new(config.clone()).await?;
                 Arc::new(backend) as Arc<dyn KmsBackend>
             }
-            BackendConfig::Aws(_) => {
-                info!("Creating AWS KMS backend for version {}", version);
-                let backend = crate::backends::aws::AwsKmsBackend::new(config.clone()).await?;
-                Arc::new(backend) as Arc<dyn KmsBackend>
-            }
         };
 
         // Create KMS manager
-        let mut kms_manager = KmsManager::new(backend, config.clone());
-        if let Some(sink) = self.audit_sink() {
-            kms_manager = kms_manager.with_audit_sink(sink);
-        }
-        let kms_manager = Arc::new(kms_manager);
+        let kms_manager = Arc::new(KmsManager::new(backend, config.clone()));
 
         // Create encryption service
         let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
@@ -649,7 +596,6 @@ impl KmsServiceManager {
             manager: kms_manager,
             credential_task,
             deletion_worker: None,
-            probe_worker: None,
         })
     }
 
@@ -663,13 +609,9 @@ impl KmsServiceManager {
 
     fn publish_running(&self, config: KmsConfig, mut service_version: ServiceVersion) {
         if let Some(previous) = self.state.load().current_service.as_ref() {
-            previous.shutdown_background_workers();
+            previous.shutdown_deletion_worker();
         }
         service_version.deletion_worker = self.spawn_deletion_worker(&config, &service_version);
-        // Started at publish time for the same reason as the deletion worker:
-        // a start or reconfigure candidate that never becomes current must not
-        // leak a running task or publish probe results nobody asked for.
-        service_version.probe_worker = spawn_probe_worker(service_version.manager.backend()).map(Arc::new);
         self.state.store(Arc::new(RuntimeState {
             config: Some(config),
             status: KmsServiceStatus::Running,
@@ -687,13 +629,7 @@ impl KmsServiceManager {
             return None;
         }
         let cancel = CancellationToken::new();
-        let worker = DeletionWorker::new(
-            backend,
-            config.default_key_id.clone(),
-            self.deletion_reference_checker(),
-            config.backend.as_str(),
-        )
-        .with_audit_sink(self.audit_sink());
+        let worker = DeletionWorker::new(backend, config.default_key_id.clone(), self.deletion_reference_checker());
         let task = worker.spawn(cancel.clone());
         Some(Arc::new(DeletionWorkerHandle {
             cancel,
@@ -1044,49 +980,6 @@ mod tests {
         );
     }
 
-    #[tokio::test]
-    async fn probe_worker_follows_the_service_lifecycle() {
-        use tempfile::TempDir;
-
-        let key_dir = TempDir::new().expect("create local KMS directory");
-        let mut config = KmsConfig::local(key_dir.path().to_path_buf());
-        config.allow_insecure_dev_defaults = true;
-        let manager = KmsServiceManager::new();
-        manager.configure(config).await.expect("configure local KMS");
-        manager.start().await.expect("start local KMS");
-
-        let first_probe = manager
-            .state
-            .load()
-            .current_service
-            .as_ref()
-            .expect("running service")
-            .probe_worker
-            .clone()
-            .expect("a backend with a data key round trip must run a probe");
-        assert!(!first_probe.is_cancelled());
-        assert!(manager.probe_status().is_some(), "a running service must publish probe status");
-
-        // Replacing the service version replaces (and cancels) its probe.
-        manager.restart().await.expect("restart");
-        assert!(first_probe.is_cancelled(), "replaced version's probe must be cancelled");
-        let second_probe = manager
-            .state
-            .load()
-            .current_service
-            .as_ref()
-            .expect("running service")
-            .probe_worker
-            .clone()
-            .expect("restarted service must run a fresh probe");
-        assert!(!second_probe.is_cancelled());
-
-        // Stopping the service stops its probe and withdraws the snapshot.
-        manager.stop().await.expect("stop");
-        assert!(second_probe.is_cancelled(), "stop must cancel the probe worker");
-        assert!(manager.probe_status().is_none(), "a stopped service must publish no status");
-    }
-
     #[tokio::test]
     async fn reconfigure_allows_safe_local_runtime_settings_only() {
         use tempfile::TempDir;