feat(kms): orchestrate Vault restore ordering, verification, and mismatch detection

Restoring a Vault-backed deployment is an operator action for the trust root and a RustFS action for everything else. This adds the orchestration around that split. Ordering is structural, not documented: VaultRestoreSequence refuses any stage that is not the next one, so material and metadata can never be published before the Transit/HSM trust root is verified back. Every way the recovered Vault can disagree with the bundle is a distinct typed mismatch: a missing Transit key, a min_decryption_version raised above what the bundle still needs, a Transit version history restored below the snapshot point, a rolled-back KV generation, and cluster, namespace, or mount drift. A dry-run reports them all and issues reads only; an actual restore refuses on the first one, before any KV path is probed. Writes are create-only check-and-set throughout, so version regression, generation rollback, and reviving deleted state are impossible rather than merely rejected, and a lost race stops the restore instead of overwriting. A reserved KV record is the single commit point: published before the first write and removed after the last, it names exactly the records this restore owns, so re-running the same bundle rolls forward idempotently and abort takes back precisely what was written — never a record that pre-existed. Every interruption converges to the complete old or the complete new state. Tests drive the whole surface offline through the scripted Vault responder, including the zero-write contract asserted at the wire; the live-Vault drill is ignored but kept compiling. Refs rustfs/backlog#1572 (part of rustfs/backlog#1562)
feat(kms): record external Vault references in the backup manifest
2026-08-01 11:02:14 +00:00 · 2026-08-01 12:03:11 +08:00 · 2026-08-01 12:02:58 +08:00 · 2026-08-01 11:35:34 +08:00 · 2026-08-01 11:34:04 +08:00 · 2026-08-01 11:32:00 +08:00
49 changed files with 4181 additions and 274 deletions
@@ -25,9 +25,13 @@ inputs:
    required: false
    default: "rustfs-deps"
  cache-save-if:
-    description: "Condition for saving cache"
+    description: >-
+      Whether to save the cache. The fail-safe default is 'false': a caller that
+      wants to populate a cache must opt in explicitly, so a forgotten input
+      costs a cold cache (minutes) rather than silently consuming the
+      repository-wide 10GB Actions cache quota and evicting other lanes.
    required: false
-    default: "true"
+    default: "false"
  install-cross-tools:
    description: "Install cross-compilation tools"
    required: false
@@ -36,28 +40,43 @@ inputs:
    description: "Target architecture to add"
    required: false
    default: ""
-  github-token:
-    description: "GitHub token for API access"
+  install-build-packaging-tools:
+    description: >-
+      Install musl-tools/zip/unzip, needed for musl linking and release
+      packaging. Off for CI test lanes, which use none of them.
    required: false
-    default: ""
+    default: "true"
+  install-test-tools:
+    description: >-
+      Install cargo-nextest and the rustfmt/clippy components. Off for release
+      and audit lanes, which run no tests and no lints.
+    required: false
+    default: "true"

 runs:
  using: "composite"
  steps:
+    # protobuf-compiler is deliberately absent: the setup-protoc step below
+    # installs 34.1 into the tool cache and prepends it to PATH, so the apt
+    # build (older, and never version-matched) was shadowed on every run and
+    # simply never used.
    - name: Install system dependencies (Ubuntu)
      if: runner.os == 'Linux'
      shell: bash
      run: |
        sudo apt-get update
        sudo apt-get install -y \
-          musl-tools \
          build-essential \
          pkg-config \
          libssl-dev \
-          ripgrep \
-          unzip \
-          zip \
-          protobuf-compiler
+          ripgrep
+
+    # musl-gcc is needed by the native musl release leg, and zip/unzip by the
+    # release packaging steps. No CI test lane touches any of them.
+    - name: Install packaging and cross-linking dependencies (Ubuntu)
+      if: runner.os == 'Linux' && inputs.install-build-packaging-tools == 'true'
+      shell: bash
+      run: sudo apt-get install -y musl-tools zip unzip

    - name: Install protoc
      uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
@@ -75,7 +94,7 @@ runs:
      with:
        toolchain: ${{ inputs.rust-version }}
        targets: ${{ inputs.target }}
-        components: rustfmt, clippy
+        components: ${{ inputs.install-test-tools == 'true' && 'rustfmt, clippy' || '' }}

    - name: Install Zig
      if: inputs.install-cross-tools == 'true'
@@ -86,6 +105,7 @@ runs:
      uses: taiki-e/install-action@a21ae4029b089b9ddc45704028756f51ab8abe48 # cargo-zigbuild

    - name: Install cargo-nextest
+      if: inputs.install-test-tools == 'true'
      uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest

    - name: Setup Rust cache
@@ -37,6 +37,7 @@ jobs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request' && github.event.action == 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -45,6 +46,7 @@ jobs:
    name: Architecture Migration Rules
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

@@ -39,7 +39,12 @@ on:
      - 'scripts/security/check_preview_release_workflow.sh'
      - 'scripts/security/check_workflow_pins.sh'
  schedule:
-    - cron: '0 3 * * 0' # Weekly on Sunday 03:00 UTC (staggered after the midnight ci/build crons)
+    # Daily, not weekly. This schedule exists to catch RustSec advisories
+    # published against an unchanged dependency tree; at weekly cadence a new
+    # advisory could sit unnoticed for seven days. The check list is unchanged —
+    # splitting it into a light daily advisories-only run and a weekly full run
+    # would create runs where sources/bans/licenses go unverified.
+    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
  workflow_dispatch:

 permissions:
@@ -59,6 +64,7 @@ jobs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request' && github.event.action == 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -75,10 +81,27 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
+      # cargo-deny compiles nothing, so the full setup composite (apt packages,
+      # protoc, flatc, nextest, rustfmt/clippy) was pure overhead here. It does
+      # still need a real cargo: `cargo deny check` runs `cargo metadata`, and
+      # Cargo.toml pins datafusion and s3s as git dependencies, which must be
+      # materialised into ~/.cargo/git — a cold clone is hundreds of MB, so the
+      # cache stays.
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+      # Was relying on the composite's default, which used to be "true": every
+      # PR touching Cargo.toml/Cargo.lock saved a second, PR-scoped copy of this
+      # cache and pushed the main-scoped lanes out of the 10GB quota. The
+      # default is now "false", but state it explicitly — see
+      # scripts/security/check_cache_save_if.sh.
+      - name: Setup Rust cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
        with:
-          cache-shared-key: rustfs-cargo-deny
+          cache-all-crates: true
+          cache-on-failure: true
+          shared-key: rustfs-cargo-deny
+          save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Install cargo-deny
        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
@@ -100,12 +123,19 @@ jobs:
      - name: Report unpinned GitHub Actions
        run: ./scripts/security/check_workflow_pins.sh --enforce

+      - name: Check setup cache-save-if is explicit
+        run: ./scripts/security/check_cache_save_if.sh
+
+      - name: Check every job declares a timeout
+        run: ./scripts/security/check_job_timeouts.sh
+
      - name: Check preview release workflow policy
        run: ./scripts/security/check_preview_release_workflow.sh

  dependency-review:
    name: Dependency Review
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    if: github.event_name == 'pull_request' && github.event.action != 'closed'
    permissions:
      contents: read
@@ -125,3 +155,26 @@ jobs:
          # conscious re-review of the license/provenance claim (backlog#1181).
          allow-dependencies-licenses: pkg:cargo/rustfs-uring@0.1.0
          comment-summary-in-pr: always
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    # dependency-review is deliberately excluded: it only runs on pull_request,
+    # so it can never contribute a failure to a scheduled run.
+    needs: [cargo-deny, workflow-pin-report]
+    # A scheduled cargo-deny failure usually means the dependency tree just
+    # matched a newly published advisory — the single most important signal this
+    # workflow produces, and until now it was only visible to whoever happened to
+    # open the Actions tab. Same ci-8 mechanism coverage.yml and
+    # e2e-replication-nightly.yml already use.
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -50,12 +50,18 @@ on:
      - "**/*.svg"
      - ".gitignore"
      - ".dockerignore"
+      - "flake.lock"
  schedule:
    - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
  workflow_dispatch:
    inputs:
      build_docker:
-        description: "Build and push Docker images after binary build"
+        # Advisory only. docker.yml triggers on workflow_run and its job-level
+        # condition requires the triggering event to be a tag push, so a manual
+        # dispatch of this workflow never produces images regardless of this
+        # value. Kept because the summary step reports it; wiring it up would
+        # mean teaching docker.yml's version parser a second event shape.
+        description: "Build and push Docker images after binary build (ignored: dispatch runs never reach docker.yml)"
        required: false
        default: true
        type: boolean
@@ -83,6 +89,7 @@ jobs:
  build-check:
    name: Build Strategy Check
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    outputs:
      should_build: ${{ steps.check.outputs.should_build }}
      build_type: ${{ steps.check.outputs.build_type }}
@@ -164,6 +171,7 @@ jobs:
    name: Prepare Platform Matrix
    needs: build-check
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    outputs:
      matrix: ${{ steps.select.outputs.matrix }}
      selected: ${{ steps.select.outputs.selected }}
@@ -171,10 +179,14 @@ jobs:
      - name: Select target platforms
        id: select
        shell: bash
+        env:
+          # via env, not interpolation: a dispatch input is free-form text and
+          # would otherwise be pasted into the script for bash to evaluate.
+          RAW_PLATFORMS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}
        run: |
          set -euo pipefail

-          selected="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}"
+          selected="$RAW_PLATFORMS"
          selected="$(echo "${selected}" | tr -d '[:space:]')"
          if [[ -z "${selected}" ]]; then
            selected="all"
@@ -253,9 +265,17 @@ jobs:
          rust-version: stable
          target: ${{ matrix.target }}
          cache-shared-key: build-${{ matrix.target }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }}
+          # main only. A cache saved on refs/tags/X is scoped to that tag: no
+          # other tag, no main run and no PR can restore it, so every release
+          # cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
+          # six legs each) that nobody could read, evicting the hot lanes from
+          # the repo-wide 10GB quota. Tag builds still restore the main-scoped
+          # cache, since default-branch caches are readable from every ref.
+          # The one real cost: re-running a failed leg of the same tag no longer
+          # finds that tag's own warm cache and falls back to main's.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          install-cross-tools: ${{ matrix.cross }}
+          install-test-tools: 'false'

      - name: Download static console assets
        shell: bash
@@ -702,9 +722,14 @@ jobs:
    needs: [ build-check, build-rustfs ]
    if: always() && needs.build-check.outputs.should_build == 'true'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Build completion summary
        shell: bash
+        env:
+          # dispatch input via env: free-form text must not be pasted into the
+          # script for bash to evaluate.
+          INPUT_BUILD_DOCKER: ${{ github.event.inputs.build_docker }}
        run: |
          BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
          VERSION="${{ needs.build-check.outputs.version }}"
@@ -746,7 +771,7 @@ jobs:
          echo "🐳 Docker Images:"
          if [[ "$BUILD_TYPE" == "preview" ]]; then
            echo "⏭️  Preview tags do not publish Docker images"
-          elif [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then
+          elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then
            echo "⏭️  Docker image build was skipped (binary only build)"
          elif [[ "$BUILD_STATUS" == "success" ]]; then
            echo "🔄 Docker images will be built and pushed automatically via workflow_run event"
@@ -760,6 +785,7 @@ jobs:
    needs: [ build-check, build-rustfs ]
    if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    permissions:
      contents: write
    outputs:
@@ -819,6 +845,7 @@ jobs:
    needs: [ build-check, build-rustfs, create-release ]
    if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    permissions:
      contents: write
      actions: read
@@ -910,6 +937,7 @@ jobs:
    needs: [ build-check, publish-release ]
    if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    steps:
      - name: Update latest.json
        env:
@@ -969,6 +997,7 @@ jobs:
    needs: [ build-check, create-release, upload-release-assets ]
    if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    permissions:
      contents: write
    steps:
@@ -0,0 +1,196 @@
+# Copyright 2026 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sole writer of the Rust dependency caches that ci.yml restores.
+#
+# Why this is a separate workflow rather than steps inside ci.yml: ci.yml's
+# concurrency group cancels in-progress runs on main pushes, and merges land far
+# faster than its 70-minute pipeline. Measured over 15 consecutive main pushes:
+# 12 cancelled, 2 failed, 0 succeeded. A cancelled run never reaches
+# Swatinem/rust-cache's post step (cache-on-failure does not cover cancellation),
+# so the writer lanes were saving nothing and every PR paid a cold restore —
+# 11.8-20.9 minutes of "Setup Rust environment" against 0.7-3.4 warm.
+#
+# Splitting cache writing out of the test pipeline lets ci.yml keep cancelling
+# superseded runs (which is correct — nobody needs test results for a commit
+# that is already three merges behind) while the caches still get written.
+#
+# The group below deliberately does NOT cancel in progress. GitHub keeps at most
+# one running plus one pending run per group, so a burst of merges collapses
+# into "current run finishes, newest queued run follows" rather than a pile-up.
+# That also bounds this workflow to one self-hosted runner at a time.
+#
+# Each job below owns exactly one shared-key and is the only place that sets
+# cache-save-if to anything but 'false' for it; every lane in ci.yml reads.
+# scripts/security/check_cache_save_if.sh keeps the declarations explicit.
+#
+# The builds are supersets of what the reading lanes compile, because a reader
+# restores only what the writer saved. Feature resolution matters here: a lane
+# built with e2e-test-hooks resolves dependency features differently, which
+# changes -Cmetadata, so the plain build does not cover it. See
+# rustfs/backlog#1600.
+
+name: Cache Warm
+
+on:
+  push:
+    branches: [ main ]
+    # Mirrors ci.yml's push paths-ignore: if a commit cannot change what ci.yml
+    # compiles, it cannot change what ci.yml needs restored either.
+    paths-ignore:
+      - "**.md"
+      - "docs/**"
+      - "deploy/**"
+      - "scripts/dev_*.sh"
+      - "scripts/probe.sh"
+      - "LICENSE*"
+      - ".gitignore"
+      - ".dockerignore"
+      - "README*"
+      - "**/*.png"
+      - "**/*.jpg"
+      - "**/*.svg"
+      - ".github/workflows/build.yml"
+      - ".github/workflows/docker.yml"
+      - ".github/workflows/audit.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: cache-warm
+  cancel-in-progress: false
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
+  # e2e-tests, e2e-full.
+  warm-ci-dev:
+    name: Warm ci-dev
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-dev
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      # --all-targets covers the test binaries nextest builds, including
+      # e2e_test, which test-and-lint's own run excludes. The second build adds
+      # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
+      # and that no lint lane enables.
+      - name: Build ci-dev superset
+        env:
+          # Same limit ci.yml puts on its nextest step: this builds the same
+          # ~100 workspace test binaries, and three concurrent links saturate the
+          # self-hosted runner's overlay I/O and can wedge Cargo (#5394).
+          CARGO_BUILD_JOBS: "2"
+        run: |
+          cargo build --workspace --all-targets
+          cargo build -p rustfs --bins --features e2e-test-hooks
+
+  # Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
+  warm-ci-feat-rio:
+    name: Warm ci-feat-rio
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-feat-rio
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      - name: Build ci-feat-rio superset
+        run: |
+          cargo build -p rustfs -p rustfs-ecstore --all-targets --features rio-v2
+          cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
+
+  # Readers: the swift and sftp legs of test-and-lint-protocols. Built in
+  # sequence rather than as `--features swift,sftp`, which is a combination no
+  # lane actually compiles; running both leaves the union in target/.
+  warm-ci-feat-proto:
+    name: Warm ci-feat-proto
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-feat-proto
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      - name: Build ci-feat-proto superset
+        run: |
+          cargo build -p rustfs -p rustfs-protocols --all-targets --features swift
+          cargo build -p rustfs -p rustfs-protocols --all-targets --features sftp
+
+  # Reader: uring-integration. Runs on ubuntu-latest to match it: rust-cache's
+  # key covers runner.os and arch but not the runner label or image, so a cache
+  # written on sm-standard-4 would be restored by the hosted runner as if it
+  # belonged to it.
+  warm-ci-uring:
+    name: Warm ci-uring
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-uring
+          cache-save-if: 'true'
+          install-build-packaging-tools: 'false'
+
+      - name: Install build dependencies
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+
+      - name: Build ci-uring superset
+        run: cargo build -p rustfs-ecstore --all-targets
@@ -12,18 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-# Companion to ci.yml for the required "Test and Lint" status check.
+# Companion to ci.yml for required status checks.
 #
-# ci.yml skips docs-only pull requests via paths-ignore, but the branch
-# ruleset requires a check named "Test and Lint" — without this workflow a
-# docs-only PR would wait on that check forever. This workflow triggers on
-# exactly the paths ci.yml ignores and reports an instant success under the
-# same job name. Mixed PRs trigger both workflows and the real check still
-# gates: a required check with any failing run blocks the merge.
+# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
+# requires a check named "Test and Lint" — without this workflow a docs-only PR
+# would wait on it forever. This workflow triggers on exactly the paths ci.yml
+# ignores and reports success under the same job name. Mixed PRs trigger both
+# workflows and the real check still gates: a required check with any failing
+# run blocks the merge.
 # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
 #
+# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
+# required too (rustfs/backlog#1599). Until that change lands this job is
+# inert; mirroring it first is what lets the ruleset change happen without
+# stranding docs-only PRs on a check nobody reports.
+#
 # Keep the paths list below in sync with the pull_request paths-ignore list
-# in ci.yml.
+# in ci.yml, and keep the quick-checks steps below byte-identical to the
+# quick-checks job in ci.yml.

 name: Continuous Integration (docs only)

@@ -47,14 +53,75 @@ on:
      - ".github/workflows/build.yml"
      - ".github/workflows/docker.yml"
      - ".github/workflows/audit.yml"
+      - "flake.lock"

 permissions:
  contents: read

 jobs:
+  # Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
+  # check, ci.yml gates every expensive job behind it, so a mixed PR reports
+  # two check runs with this name: the real one (45-51s) and this companion.
+  # GitHub has no written contract for how it picks between same-named
+  # required check runs ("latest wins" vs "any failure blocks"), so instead of
+  # relying on ordering we make both runs execute the same commands against
+  # the same merge ref — their conclusions are then necessarily identical and
+  # the choice does not matter. Keep these steps byte-identical to the
+  # quick-checks job in ci.yml (a guard script that asserts this, and the paths
+  # sync below, is tracked in rustfs/backlog#1603).
+  #
+  # For a genuinely docs-only PR this adds no strictness (no code changed, so
+  # fmt and the guards always pass) and costs ~50s of ubuntu-latest.
+  quick-checks:
+    name: Quick Checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Install ripgrep
+        run: sudo apt-get update && sudo apt-get install -y ripgrep
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          components: rustfmt
+
+      - name: Check code formatting
+        run: cargo fmt --all --check
+
+      - name: Check unsafe code allowances
+        run: ./scripts/check_unsafe_code_allowances.sh
+
+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
+      - name: Check architecture migration rules
+        run: ./scripts/check_architecture_migration_rules.sh
+
+      - name: Check tokio io-uring feature guard
+        run: ./scripts/check_no_tokio_io_uring.sh
+
+      - name: Check extension schema boundaries
+        run: ./scripts/check_extension_schema_boundaries.sh
+
+      - name: Check body-cache whitelist guard
+        run: ./scripts/check_body_cache_whitelist.sh
+
+      - name: Check no planning docs committed
+        run: ./scripts/check_no_planning_docs.sh
+
+      - name: Check CI paths stay in sync
+        run: ./scripts/check_ci_paths_sync.sh
+
+      - name: Check io_uring lane --lib precondition
+        run: ./scripts/check_uring_lane_lib_only.sh
+
  test-and-lint:
    name: Test and Lint
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -33,6 +33,7 @@ on:
      - ".github/workflows/build.yml"
      - ".github/workflows/docker.yml"
      - ".github/workflows/audit.yml"
+      - "flake.lock"
  pull_request:
    types: [ opened, synchronize, reopened, closed ]
    branches: [ main ]
@@ -54,6 +55,7 @@ on:
      - ".github/workflows/build.yml"
      - ".github/workflows/docker.yml"
      - ".github/workflows/audit.yml"
+      - "flake.lock"
  merge_group:
    types: [ checks_requested ]
  schedule:
@@ -81,6 +83,7 @@ jobs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request' && github.event.action == 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -89,6 +92,7 @@ jobs:
    name: Typos
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
      - name: Typos check with custom config file
@@ -96,6 +100,10 @@ jobs:

  # Fast, compile-free checks that fail early so contributors get feedback in
  # ~1 minute instead of waiting for the full test job.
+  #
+  # These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
+  # PR, which reports two check runs named "Quick Checks", cannot get one red
+  # and one green. Edit both jobs together.
  quick-checks:
    name: Quick Checks
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -137,24 +145,48 @@ jobs:
      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh

+      - name: Check CI paths stay in sync
+        run: ./scripts/check_ci_paths_sync.sh
+
+      - name: Check io_uring lane --lib precondition
+        run: ./scripts/check_uring_lane_lib_only.sh
+
  test-and-lint:
    name: Test and Lint
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 90
+    # Both lines are required. Job-level `permissions` replaces the workflow
+    # block rather than merging with it, so declaring only `actions: write`
+    # would drop `contents: read` and break this job's checkout and the
+    # repo-token the setup action hands to setup-protoc.
+    permissions:
+      contents: read
+      actions: write
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          # This job's token can cancel runs and delete Actions caches. Checkout
+          # otherwise writes it into .git/config, where a PR's own build.rs or
+          # proc-macro could read it back out.
+          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-test
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          # Every lane in this workflow reads its cache and none writes it.
+          # cache-warm.yml is the sole writer for all four keys: this workflow
+          # cancels superseded runs on main, and a cancelled run never reaches
+          # rust-cache's post step, so writing from here saved nothing (12 of 15
+          # consecutive main-push runs were cancelled). See rustfs/backlog#1600.
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      - name: Prepare test evidence
        run: |
@@ -169,8 +201,14 @@ jobs:
      # Clippy runs before the test pass: lint failures are the most common
      # CI-only breakage and should surface in minutes, not after 20+ minutes
      # of tests.
+      # Sampled too: clippy is the natural control arm for any CARGO_BUILD_JOBS
+      # experiment, since --all-targets is check-only for workspace members and
+      # never links the ~100 test binaries the limit exists to throttle.
      - name: Run clippy lints
-        run: cargo clippy --all-targets -- -D warnings
+        run: |
+          ./scripts/ci/resource_sampler.sh start clippy
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          cargo clippy --all-targets -- -D warnings

      - name: Run nextest tests
        env:
@@ -179,33 +217,8 @@ jobs:
          CARGO_BUILD_JOBS: "2"
        run: |
          mkdir -p artifacts/test-and-lint
-          # Evidence sampler for issue #5394: the post-mortem pgrep below runs
-          # only after `timeout` has already TERM'd the whole cargo process
-          # group, so it cannot name a wedged process. Sample system and
-          # process state every 60s instead; the last samples before the
-          # timeout show what was stuck (rustc, linker, build script, memory
-          # pressure, ...). The log rides along in the existing artifact.
-          (
-            while true; do
-              {
-                echo "=== $(date --utc --iso-8601=seconds)"
-                echo "--- load";  cat /proc/loadavg
-                echo "--- psi";   grep -H . /proc/pressure/* 2>/dev/null || true
-                echo "--- mem";   free -m
-                echo "--- disk";  df -h / /home/runner 2>/dev/null || df -h /
-                echo "--- top-rss"
-                ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
-                echo "--- build/test processes"
-                ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
-                echo "--- d-state (uninterruptible IO)"
-                ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
-                echo
-              } >> artifacts/test-and-lint/sampler.log 2>&1 || true
-              sleep 60
-            done
-          ) &
-          sampler_pid=$!
-          trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
+          ./scripts/ci/resource_sampler.sh start nextest
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
          set +e
          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
            cargo nextest run --profile ci --all --exclude e2e_test \
@@ -277,6 +290,48 @@ jobs:
      - name: Run rebalance/decommission migration proofs
        run: ./scripts/check_migration_gate_count.sh

+      # Early stop. Once this job has failed the PR cannot merge, so the sibling
+      # lanes are burning runners on a result nobody can act on: on run
+      # 30674613104 three lanes had already failed while Test and Lint and the
+      # rio-v2 variant kept going past 70 minutes.
+      #
+      # Only this job may cancel. The lanes that are NOT required checks
+      # (protocols, ILM, e2e, s3-tests) must never hold that power: a flake in
+      # one of them would turn the required "Test and Lint" into `cancelled`,
+      # which blocks the merge. Today a maintainer can merge with sftp red, and
+      # that has to stay true.
+      #
+      # These steps run last so the `if: always()` artifact upload above still
+      # captures logs and diagnostics before the run goes away.
+      - name: Annotate early-stop reason
+        if: failure() && github.event_name == 'pull_request'
+        run: |
+          echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
+          echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
+          echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
+
+      # curl rather than `gh`: every existing `gh` call in this repo runs on
+      # ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
+      # ship no C toolchain, see the e2e job below), so `gh` is not known to
+      # exist here.
+      #
+      # Fork PRs are excluded explicitly instead of relying on the error path:
+      # their GITHUB_TOKEN is forced read-only and job-level permissions cannot
+      # raise it, so the call would always 403. Skipping keeps their logs clean.
+      - name: Cancel run on failure (same-repo PR only)
+        if: >-
+          failure() && github.event_name == 'pull_request'
+          && github.event.pull_request.head.repo.full_name == github.repository
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          curl -fsS -X POST \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" || true
+
  # Dedicated serial lane for the ILM / lifecycle integration tests. These tests
  # drive the object layer through process-global singletons (the GLOBAL_ENV
  # ECStore, the global tier-config manager, background-expiry workers) and bind
@@ -290,6 +345,7 @@ jobs:
  test-ilm-integration-serial:
    name: ILM Integration (serial)
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 45
    env:
@@ -302,9 +358,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-ilm-serial
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
      # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
@@ -327,6 +383,7 @@ jobs:
  test-and-lint-rio-v2:
    name: Test and Lint (rio-v2)
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 60
    env:
@@ -339,9 +396,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-test-rio-v2
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-feat-rio
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      - name: Run rio-v2 clippy lints
        run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
@@ -354,10 +411,17 @@ jobs:
  test-and-lint-protocols:
    name: "Test and Lint (${{ matrix.features.name }})"
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 60
    strategy:
-      fail-fast: false
+      # On a PR, one failing protocol leg is enough to know the PR is not ready,
+      # so stop the sibling leg instead of paying another ~40 minutes for it.
+      # Everywhere else (main pushes, the merge queue, the weekly schedule) keep
+      # the full signal: there we want to know whether swift AND sftp are broken,
+      # not just whichever failed first. This is the only part of the early-stop
+      # work that also covers fork PRs, since it needs no token.
+      fail-fast: ${{ github.event_name == 'pull_request' }}
      matrix:
        features:
          - name: swift
@@ -374,9 +438,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-test-${{ matrix.features.name }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-feat-proto
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      - name: Run clippy with ${{ matrix.features.name }}
        run: |
@@ -389,6 +453,7 @@ jobs:
  build-rustfs-debug-binary:
    name: Build RustFS Debug Binary
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 30
    env:
@@ -401,9 +466,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-rustfs-debug-binary
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      - name: Build debug binary
        run: cargo build -p rustfs --bins --features e2e-test-hooks
@@ -419,6 +484,7 @@ jobs:
  build-rustfs-debug-binary-rio-v2:
    name: Build RustFS Debug Binary (rio-v2)
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 30
    env:
@@ -431,9 +497,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-rustfs-debug-binary-rio-v2
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache-shared-key: ci-feat-rio
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      - name: Build debug binary with rio-v2
        run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
@@ -448,6 +514,14 @@ jobs:

  uring-integration:
    name: io_uring Integration (real)
+    # The pull_request trigger includes `closed` purely so the concurrency
+    # group cancels in-flight runs of a closed PR; every other job opts out of
+    # that run with this guard (or is skipped through its `needs` chain). This
+    # job had neither, so each closed/merged PR really ran the whole io_uring
+    # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
+    # 30662728539) and kept the cancellation run in progress for minutes.
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
+    needs: [ quick-checks ]
    # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
    # a container, applies no seccomp filter that would block io_uring_setup — so
    # the probe succeeds and the tests exercise the real UringBackend/FdCache/
@@ -463,12 +537,17 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
+          # Keeps its own key rather than joining ci-dev. rust-cache's key is
+          # built from runner.os/arch plus rustc and lockfile fingerprints — it
+          # does NOT include the runner label or image. ubuntu-latest and
+          # sm-standard-4 are therefore indistinguishable to it, so sharing a key
+          # would let two different system images overwrite each other's
+          # artifacts, and would make a 2-core hosted runner unpack ci-dev's ~3GB
+          # instead of this lane's ~1.3GB. cache-warm.yml warms this key on
+          # ubuntu-latest for the same reason.
          cache-shared-key: ci-uring
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Install build dependencies
-        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      # ext4 supports O_DIRECT; the runner's default TMPDIR may sit on tmpfs or
      # overlayfs, where open(O_DIRECT) returns EINVAL/EOPNOTSUPP and the native
@@ -495,7 +574,17 @@ jobs:
          RUSTFS_IO_URING_READ_ENABLE: "true"
          RUSTFS_URING_TESTS_MUST_RUN: "1"
          TMPDIR: /mnt/rustfs-odirect
-        run: cargo test -p rustfs-ecstore uring_ -- --test-threads=1 --nocapture
+        # --lib narrows what gets compiled, not what gets run: every selected
+        # test lives in the lib target. The 7 integration binaries under
+        # crates/ecstore/tests/ each reported "running 0 tests" here, so they
+        # were compiled and linked for nothing.
+        #
+        # The `uring_` filter must stay exactly as it is. libtest matches on
+        # substring, so it also selects names containing `during_` — 6 of the 18
+        # selected tests are such incidental matches. Narrowing the filter to
+        # `io_uring` would silently drop them, which is a coverage change.
+        # scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
+        run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture

  e2e-tests:
    name: End-to-End Tests
@@ -513,9 +602,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-e2e
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      # Download after the cache restore so the freshly built binary from the
      # build job always wins over anything restored into target/debug.
@@ -594,9 +683,9 @@ jobs:
        uses: ./.github/actions/setup
        with:
          rust-version: stable
-          cache-shared-key: ci-e2e
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: ci-dev
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'

      # Download after the cache restore so the freshly built binary from the
      # build job always wins over anything restored into target/debug.
@@ -746,7 +835,13 @@ jobs:
  # evaluates ILM within ~2s of the due time, well inside the poll window.
  s3-lifecycle-behavior-tests:
    name: S3 Lifecycle Behavior Tests
-    needs: [ build-rustfs-debug-binary ]
+    # Also gated on e2e-tests, matching s3-implemented-tests: when the e2e smoke
+    # suite is already red this lane cannot tell us anything new, and it holds a
+    # sm-standard-4 for up to 30 minutes doing so. Both lanes only download the
+    # prebuilt debug binary (no cargo build), and s3-implemented-tests — which
+    # already waits on e2e-tests — finishes later anyway, so a green PR's total
+    # wall clock is unchanged.
+    needs: [ build-rustfs-debug-binary, e2e-tests ]
    runs-on: sm-standard-4
    timeout-minutes: 30
    steps:
@@ -37,6 +37,7 @@ jobs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -44,6 +45,7 @@ jobs:
  cla:
    if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }}
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    steps:
      - name: Report CLA result for merge queue
        if: github.event_name == 'merge_group'
@@ -68,8 +68,8 @@ jobs:
        with:
          rust-version: stable
          cache-shared-key: ci-coverage
-          github-token: ${{ secrets.GITHUB_TOKEN }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'

      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
@@ -85,6 +85,7 @@ jobs:
       github.event.workflow_run.head_branch != 'main' &&
       !contains(github.event.workflow_run.head_branch, '-preview'))
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    outputs:
      should_build: ${{ steps.check.outputs.should_build }}
      should_push: ${{ steps.check.outputs.should_push }}
@@ -102,6 +103,12 @@ jobs:

      - name: Check build conditions
        id: check
+        env:
+          # dispatch inputs via env, not `${{ }}` interpolation: they are
+          # free-form strings and would otherwise be evaluated by bash.
+          INPUT_VERSION: ${{ github.event.inputs.version }}
+          INPUT_PUSH_IMAGES: ${{ github.event.inputs.push_images }}
+          INPUT_FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
        run: |
          should_build=false
          should_push=false
@@ -202,9 +209,9 @@ jobs:

          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            # Manual trigger
-            input_version="${{ github.event.inputs.version }}"
+            input_version="$INPUT_VERSION"
            version="${input_version}"
-            should_push="${{ github.event.inputs.push_images }}"
+            should_push="$INPUT_PUSH_IMAGES"
            should_build=true

            # Get short SHA
@@ -212,7 +219,7 @@ jobs:

            echo "🎯 Manual Docker build triggered:"
            echo "   📋 Requested version: $input_version"
-            echo "   🔧 Force rebuild: ${{ github.event.inputs.force_rebuild }}"
+            echo "   🔧 Force rebuild: $INPUT_FORCE_REBUILD"
            echo "   🚀 Push images: $should_push"

            case "$input_version" in
@@ -333,32 +340,28 @@ jobs:
          CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
          VARIANT_SUFFIX="${{ matrix.suffix }}"

-          # Convert version format for Dockerfile compatibility
+          # Convert version format for Dockerfile compatibility. The former
+          # DOCKER_CHANNEL was "release" down every branch and was passed as a
+          # build-arg no Dockerfile declares, so it is gone.
          case "$VERSION" in
            "latest")
-              # For stable latest, use RELEASE=latest + release CHANNEL
              DOCKER_RELEASE="latest"
-              DOCKER_CHANNEL="release"
              ;;
            v*)
              # For versioned releases (v1.0.0), remove 'v' prefix for Dockerfile
              DOCKER_RELEASE="${VERSION#v}"
-              DOCKER_CHANNEL="release"
              ;;
            *)
              # For other versions, pass as-is
              DOCKER_RELEASE="${VERSION}"
-              DOCKER_CHANNEL="release"
              ;;
          esac

          echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT"
-          echo "docker_channel=$DOCKER_CHANNEL" >> "$GITHUB_OUTPUT"

          echo "🐳 Docker build parameters:"
          echo "  - Original version: $VERSION"
          echo "  - Docker RELEASE: $DOCKER_RELEASE"
-          echo "  - Docker CHANNEL: $DOCKER_CHANNEL"

          # Generate tags based on build type
          # Only support release and prerelease builds (no development builds)
@@ -412,18 +415,24 @@ jobs:
          push: ${{ needs.build-check.outputs.should_push == 'true' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: |
-            type=gha,scope=docker-${{ matrix.variant }}
-          cache-to: |
-            type=gha,mode=max,scope=docker-${{ matrix.variant }}
+          # No layer cache. This build compiles nothing — it downloads a
+          # release zip and runs apk/apt — so the cache could only save the
+          # minute or two those take, while creating a correctness problem: with
+          # RELEASE=latest the binary URL is resolved by curl *inside* a RUN
+          # layer, and the layer key does not include what that resolved to. A
+          # rebuild at the same RELEASE value (dispatch with version=latest, or
+          # a re-run of the same version) would hit the old layer and ship the
+          # previous release's binary. mode=max also consumed the same 10GB
+          # Actions cache quota the Rust lanes are fighting over.
+          #
+          # Only RELEASE is passed: it is the sole build-arg the Dockerfiles
+          # declare besides TARGETARCH. BUILDTIME, VERSION, BUILD_TYPE, REVISION
+          # and CHANNEL were never read by any stage (and BUILDTIME's $(date ...)
+          # was a literal here, not a shell substitution). BUILD_DATE and VCS_REF
+          # are declared by the Dockerfiles but deliberately left unset —
+          # supplying them would change the published image labels.
          build-args: |
-            BUILDTIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-            VERSION=${{ needs.build-check.outputs.version }}
-            BUILD_TYPE=${{ needs.build-check.outputs.build_type }}
-            REVISION=${{ github.sha }}
            RELEASE=${{ steps.meta.outputs.docker_release }}
-            CHANNEL=${{ steps.meta.outputs.docker_channel }}
-            BUILDKIT_INLINE_CACHE=1
          provenance: true
          sbom: true
          # Add retry mechanism by splitting the build process
@@ -439,6 +448,7 @@ jobs:
    needs: [ build-check, build-docker ]
    if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    permissions:
      contents: read
      security-events: write
@@ -493,6 +503,7 @@ jobs:
    needs: [ build-check, build-docker ]
    if: always() && needs.build-check.outputs.should_build == 'true'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Docker build completion summary
        run: |
@@ -70,8 +70,8 @@ jobs:
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-repl
-          github-token: ${{ secrets.GITHUB_TOKEN }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'

      # awscurl lets the STS dual-node test actually exercise its path. Without
      # it the test skips gracefully with a visible log line
@@ -45,6 +45,13 @@
 # The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
 # via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: e2e-s3tests

 on:
@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Fuzz

 on:
@@ -59,6 +66,7 @@ jobs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request' && github.event.action == 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -85,7 +93,6 @@ jobs:
        with:
          rust-version: nightly
          cache-shared-key: fuzz-${{ hashFiles('fuzz/Cargo.lock') }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}

      - name: Install cargo-fuzz
@@ -32,6 +32,7 @@ permissions:
 jobs:
  build-helm-package:
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    if: |
      (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
      (
@@ -50,15 +51,23 @@ jobs:
      - name: Checkout helm chart repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

+      # Both inputs reach the shell through env rather than `${{ }}`
+      # interpolation. A git ref name may contain `$(...)` — anything without a
+      # space is a legal tag — and interpolation pastes it into the script
+      # verbatim, where bash would run it. Reading "$RAW_INPUT" instead makes it
+      # data.
      - name: Normalize release version
        id: version
+        env:
+          RAW_INPUT: ${{ github.event.inputs.version }}
+          RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
        run: |
          set -eux

          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            RAW="${{ github.event.inputs.version }}"
+            RAW="$RAW_INPUT"
          else
-            RAW="${{ github.event.workflow_run.head_branch }}"
+            RAW="$RAW_BRANCH"
          fi

          case "$RAW" in
@@ -73,10 +82,13 @@ jobs:
          ./scripts/helm_chart_version.sh "$RAW_TAG"

      - name: Replace chart version and app version
+        env:
+          CHART_VERSION: ${{ steps.version.outputs.chart_version }}
+          APP_VERSION: ${{ steps.version.outputs.app_version }}
        run: |
          set -eux
-          sed -i -E 's/^version:.*/version: "${{ steps.version.outputs.chart_version }}"/' helm/rustfs/Chart.yaml
-          sed -i -E 's/^appVersion:.*/appVersion: "${{ steps.version.outputs.app_version }}"/' helm/rustfs/Chart.yaml
+          sed -i -E "s/^version:.*/version: \"${CHART_VERSION}\"/" helm/rustfs/Chart.yaml
+          sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" helm/rustfs/Chart.yaml

      - name: Set up Helm
        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
@@ -101,6 +113,7 @@ jobs:

  publish-helm-package:
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    needs: [ build-helm-package ]
    if: needs.build-helm-package.result == 'success'

@@ -123,11 +136,19 @@ jobs:
      - name: Generate index
        run: helm repo index . --url https://charts.rustfs.com

+      # app_version is derived from the triggering tag name, and this job holds
+      # the cross-repository push token with rustfs/helm already checked out —
+      # the worst place in the repo to paste an attacker-influenced string into
+      # a shell line. Passed through env so bash treats it as data.
      - name: Push helm package and index file
+        env:
+          GIT_USERNAME: ${{ secrets.USERNAME }}
+          GIT_EMAIL: ${{ secrets.EMAIL_ADDRESS }}
+          APP_VERSION: ${{ needs.build-helm-package.outputs.app_version }}
        run: |
          set -eux
-          git config --global user.name "${{ secrets.USERNAME }}"
-          git config --global user.email "${{ secrets.EMAIL_ADDRESS }}"
+          git config --global user.name "${GIT_USERNAME}"
+          git config --global user.email "${GIT_EMAIL}"
          git add .
-          git commit -m "Update rustfs helm package with ${{ needs.build-helm-package.outputs.app_version }}." || echo "No changes to commit"
+          git commit -m "Update rustfs helm package with ${APP_VERSION}." || echo "No changes to commit"
          git push origin main
@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: "issue-translator"
 on:
  issue_comment:
@@ -26,6 +33,7 @@ permissions:
 jobs:
  build:
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - uses: usthe/issues-translate-action@b41f55ddc81d7d54bd542a4f289fe28ec081898e # v2.7
        with:
@@ -23,6 +23,13 @@
 # Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
 # unlike the self-hosted fleet, whose pods drift in Docker/pip availability
 # (see the infra note in e2e-s3tests.yml). Nightly + manual only.
+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: minio-interop

 on:
@@ -57,7 +64,6 @@ jobs:
        with:
          rust-version: stable
          cache-shared-key: ci-minio-interop
-          github-token: ${{ secrets.GITHUB_TOKEN }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}

      - name: Generate real MinIO fixtures via Docker
@@ -45,6 +45,13 @@
 # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
 # has fewer cores and reintroduces fleet-state risk for no reliability gain.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: mint

 on:
@@ -19,9 +19,12 @@ on:
  schedule:
    - cron: '0 5 * * 0' # Weekly on Sunday 05:00 UTC (staggered after the midnight ci/build crons)

+# GITHUB_TOKEN only needs to read the repository here: the branch push and the
+# pull request are both created by update-flake-lock using the
+# FLAKE_UPDATE_TOKEN PAT below, not by this token. Leaving write on it hands a
+# repo-write credential to an unattended weekly job that does not use it.
 permissions:
-  contents: write
-  pull-requests: write
+  contents: read

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Nix CI

 on:
@@ -46,6 +53,7 @@ jobs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request' && github.event.action == 'closed'
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
@@ -22,6 +22,13 @@
 # a deliberate correctness cost (e.g. the #4221 fsync durability fix) is
 # recorded but does not block (rustfs/backlog#935 correction 1).

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Performance A/B

 on:
@@ -99,7 +106,6 @@ jobs:
          rust-version: stable
          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Build release rustfs
        run: cargo build --release --bin rustfs
@@ -150,7 +156,6 @@ jobs:
          rust-version: stable
          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install warp
        run: |
@@ -162,13 +167,15 @@ jobs:

      - name: Decide exemption
        id: exempt
+        env:
+          INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
        run: |
          allow="false"
          if [[ "${{ github.event_name }}" == "pull_request" ]] \
            && ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then
            allow="true"
          fi
-          if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then
+          if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
            allow="true"
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
@@ -224,6 +231,8 @@ jobs:

      - name: Run warp A/B and gate
        id: ab
+        env:
+          INPUT_DURATION: ${{ github.event.inputs.duration }}
        run: |
          set -euo pipefail
          # Budget note: with perf-3's cached baseline the nightly does no source
@@ -234,7 +243,7 @@ jobs:
          # budget, which the rig's previous 60s health poll undershot (the first
          # two nightly failures). perf-6 recalibrates these once the noise study
          # lands.
-          duration="${{ github.event.inputs.duration || '12s' }}"
+          duration="${INPUT_DURATION:-12s}"
          baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
          candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
          baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
@@ -24,6 +24,13 @@
 # The run itself is expected to end red (the forced failure); only the
 # alert-on-failure job result matters.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: Schedule Failure Alert Drill

 on:
@@ -12,6 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# DISABLED. This workflow is switched off in the repository's Actions settings
+# (state: disabled_manually) and does not run on any trigger, including its cron
+# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
+# reading this file, which has already misled at least one audit — hence this
+# banner. Re-enabling is a UI action; anyone doing so should first check that the
+# workflow still matches the current CI layout. See rustfs/backlog#1603.
+#
 name: "Mark stale issues"
 on:
  schedule:
@@ -20,6 +27,7 @@ on:
 jobs:
  stale:
    runs-on: ubuntu-latest
+    timeout-minutes: 30
    steps:
      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9
        with:
@@ -15,6 +15,7 @@ concurrency:
 jobs:
  update:
    runs-on: ubuntu-latest
+    timeout-minutes: 10
    steps:
      - uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0
        with:
@@ -10027,6 +10027,7 @@ dependencies = [
 "rustfs-data-usage",
 "rustfs-ecstore",
 "rustfs-filemeta",
+ "rustfs-lock",
 "rustfs-storage-api",
 "rustfs-utils",
 "s3s",
@@ -6641,6 +6641,49 @@ impl LocalDisk {
        let xl_path = object_dir.join(STORAGE_FORMAT_FILE);
        restore_delete_rollback_after_error(object_dir, &xl_path, Some(rollback_dir), volume, object, stage, err).await
    }
+
+    /// Execute every deferred data-dir deletion pending on `volume` right now,
+    /// even while snapshot leases are still held. Bucket deletion requires it:
+    /// a streaming reader defers the physical cleanup of an already-deleted
+    /// version, and a non-force `delete_volume` would otherwise fail closed
+    /// with `VolumeNotEmpty` on those remnants even though the bucket is
+    /// logically empty. The still-active readers keep their open descriptors;
+    /// only path-based reopens observe the removal.
+    async fn settle_pending_snapshot_deletes(&self, volume: &str) {
+        let pending: Vec<(SnapshotLeaseKey, DeleteOptions)> = {
+            let mut registry = self.snapshot_leases.lock().await;
+            registry
+                .entries
+                .iter_mut()
+                .filter(|(key, entry)| key.volume == volume && !entry.deleting && entry.pending_delete.is_some())
+                .map(|(key, entry)| {
+                    entry.deleting = true;
+                    (key.clone(), entry.pending_delete.clone().expect("filtered on Some"))
+                })
+                .collect()
+        };
+
+        for (key, opts) in pending {
+            let result = self.delete_unleased(&key.volume, &key.path, &opts).await;
+            let mut registry = self.snapshot_leases.lock().await;
+            match result {
+                Ok(()) => {
+                    registry.entries.remove(&key);
+                }
+                Err(err) => {
+                    if let Some(entry) = registry.entries.get_mut(&key) {
+                        entry.deleting = false;
+                    }
+                    warn!(
+                        volume = %key.volume,
+                        path = %key.path,
+                        error = %err,
+                        "failed to settle deferred data-dir deletion before volume removal"
+                    );
+                }
+            }
+        }
+    }
 }

 #[async_trait::async_trait]
@@ -9122,6 +9165,14 @@ impl DiskAPI for LocalDisk {
        let p = self.get_bucket_path(volume)?;
        let _volume_mutation_guard = os::disk_volume_mutation_lock(&self.root, volume).write_owned().await;

+        // A streaming reader's snapshot lease defers the physical cleanup of
+        // data dirs whose version delete already committed. Those remnants are
+        // logically deleted, so run the parked cleanups now instead of letting
+        // the non-force removal below fail closed on them (the s3-tests SSE-C
+        // teardown races exactly this way: DeleteObjects, then DeleteBucket
+        // while an abandoned GET body still pins the lease).
+        self.settle_pending_snapshot_deletes(volume).await;
+
        // Non-force removes empty directory remnants children-first with
        // non-recursive rmdir calls. A file that exists during the scan, or
        // appears before its parent is removed, fails closed with
@@ -15981,6 +16032,74 @@ mod test {
        assert!(matches!(disk.read_all(volume, &first_part).await, Err(DiskError::FileNotFound)));
    }

+    #[tokio::test]
+    async fn delete_volume_settles_lease_deferred_cleanup() {
+        use tempfile::tempdir;
+
+        let root_dir = tempdir().expect("temp dir should be created");
+        let endpoint = Endpoint::try_from(root_dir.path().to_string_lossy().as_ref()).expect("endpoint should parse");
+        let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
+        let volume = "snapshot-lease-bucket-delete";
+        let object = "multipart_enc";
+        let version_id = Uuid::new_v4();
+        let data_dir = Uuid::new_v4();
+        let rollback_dir = Uuid::new_v4();
+        let data_path = path_join_buf(&[object, &data_dir.to_string()]);
+        let part = path_join_buf(&[&data_path, "part.1"]);
+        ensure_test_volume(&disk, volume).await;
+        disk.write_all(volume, &part, Bytes::from_static(b"payload"))
+            .await
+            .expect("shard should be written");
+        let fi = test_file_info(object, version_id, Some(data_dir), None);
+        disk.write_all(volume, &path_join_buf(&[object, STORAGE_FORMAT_FILE]), test_meta(fi.clone()).into())
+            .await
+            .expect("metadata should be written");
+
+        // An abandoned streaming GET pins the data dir with a snapshot lease.
+        let snapshot = disk
+            .acquire_snapshot_lease(volume, &data_path)
+            .await
+            .expect("snapshot lease should be acquired");
+        disk.delete_version(
+            volume,
+            object,
+            fi.clone(),
+            false,
+            DeleteOptions {
+                old_data_dir: Some(rollback_dir),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("version delete should commit metadata");
+        disk.delete(
+            volume,
+            &format!("{object}/{rollback_dir}"),
+            DeleteOptions {
+                recursive: true,
+                immediate: true,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("version delete should schedule physical cleanup");
+
+        // The bucket is logically empty; a non-force volume delete must settle
+        // the deferred data-dir cleanup instead of failing with VolumeNotEmpty.
+        disk.delete_volume(volume, false)
+            .await
+            .expect("bucket delete must not observe lease-deferred remnants");
+        assert!(matches!(
+            disk.read_all(volume, &part).await,
+            Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
+        ));
+
+        // The late lease release finds nothing pending and stays idempotent.
+        disk.release_snapshot_lease(volume, &data_path, snapshot)
+            .await
+            .expect("releasing the lease after bucket deletion should be a no-op");
+    }
+
    #[tokio::test]
    async fn version_delete_cleanup_intent_survives_local_disk_restart() {
        use tempfile::tempdir;
@@ -1645,15 +1645,28 @@ impl Erasure {
            }
            Err(e) => {
                record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_EMIT, emit_stage_start);
-                error!(
-                    block_offset,
-                    block_length,
-                    bytes_written = *written,
-                    stage = GET_STAGE_EMIT,
-                    reason = classify_io_error(&e).as_str(),
-                    error = ?e,
-                    "Erasure decode failed to emit reconstructed data"
-                );
+                let reason = classify_io_error(&e);
+                if reason == GetObjectFailureReason::DownstreamClosed {
+                    debug!(
+                        block_offset,
+                        block_length,
+                        bytes_written = *written,
+                        stage = GET_STAGE_EMIT,
+                        reason = reason.as_str(),
+                        error = ?e,
+                        "Erasure decode stopped after downstream closed"
+                    );
+                } else {
+                    error!(
+                        block_offset,
+                        block_length,
+                        bytes_written = *written,
+                        stage = GET_STAGE_EMIT,
+                        reason = reason.as_str(),
+                        error = ?e,
+                        "Erasure decode failed to emit reconstructed data"
+                    );
+                }
                *ret_err = Some(e);
                return StripeFlow::Stop;
            }
@@ -1945,7 +1958,7 @@ mod tests {
    use std::io::Cursor;
    use std::pin::Pin;
    use std::sync::{
-        Arc,
+        Arc, Mutex,
        atomic::{AtomicUsize, Ordering},
    };
    use std::task::{Context, Poll};
@@ -2120,6 +2133,59 @@ mod tests {
        }
    }

+    struct DownstreamClosedWriter;
+
+    impl AsyncWrite for DownstreamClosedWriter {
+        fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
+            Poll::Ready(Err(crate::diagnostics::get::mark_get_object_downstream_closed(io::Error::new(
+                ErrorKind::BrokenPipe,
+                "injected downstream close",
+            ))))
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[derive(Clone, Default)]
+    struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
+
+    struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
+
+    impl CapturedLogs {
+        fn contents(&self) -> String {
+            String::from_utf8(self.0.lock().expect("captured logs mutex should not be poisoned").clone())
+                .expect("captured logs should be valid UTF-8")
+        }
+    }
+
+    impl std::io::Write for CapturedLogWriter {
+        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+            self.0
+                .lock()
+                .expect("captured logs mutex should not be poisoned")
+                .extend_from_slice(buf);
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> std::io::Result<()> {
+            Ok(())
+        }
+    }
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
+        type Writer = CapturedLogWriter;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            CapturedLogWriter(Arc::clone(&self.0))
+        }
+    }
+
    #[test]
    fn parallel_reader_constructor_variants_preserve_read_cost_and_verification_flags() {
        let erasure = Erasure::new(2, 1, 64);
@@ -2215,6 +2281,47 @@ mod tests {
        assert_eq!(err.to_string(), "injected emit failure");
    }

+    #[tokio::test(flavor = "current_thread")]
+    async fn erasure_decode_logs_reconstructed_downstream_close_at_debug() {
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_max_level(tracing::Level::DEBUG)
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let _guard = tracing::subscriber::set_default(subscriber);
+
+        let erasure = Erasure::new(2, 1, 64);
+        let data: Vec<u8> = (0..64).collect();
+        let shard_size = erasure.shard_size();
+        let encoded = erasure.encode_data(&data).expect("test data should encode");
+        let readers = vec![
+            None,
+            Some(BitrotReader::new(
+                Cursor::new(encoded[1].to_vec()),
+                shard_size,
+                HashAlgorithm::None,
+                false,
+            )),
+            Some(BitrotReader::new(
+                Cursor::new(encoded[2].to_vec()),
+                shard_size,
+                HashAlgorithm::None,
+                false,
+            )),
+        ];
+
+        let mut writer = DownstreamClosedWriter;
+        let (written, err) = erasure.decode(&mut writer, readers, 0, data.len(), data.len()).await;
+
+        assert_eq!(written, 0);
+        assert_eq!(err.expect("downstream close must still terminate the GET").kind(), ErrorKind::BrokenPipe);
+        let captured = logs.contents();
+        assert!(captured.contains("Erasure decode stopped after downstream closed"));
+        assert!(!captured.contains("Erasure decode failed to emit reconstructed data"));
+    }
+
    #[tokio::test]
    async fn test_erasure_decode_rejects_reader_count_and_range_overflow() {
        let erasure = Erasure::new(2, 1, 64);
@@ -22,6 +22,8 @@ use rustfs_config::{
    ENV_STARTUP_TOPOLOGY_WAIT_MODE, ENV_STARTUP_TOPOLOGY_WAIT_TIMEOUT, ENV_UNSAFE_BYPASS_DISK_CHECK,
 };
 use rustfs_utils::{XHost, check_local_server_addr, get_env_opt_str, get_host_ip, is_local_host};
+#[cfg(test)]
+use std::sync::{LazyLock, Mutex};
 use std::{
    collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
    future::Future,
@@ -598,6 +600,58 @@ const DNS_RETRY_JITTER_PERCENT: u64 = 20;
 /// wait does not flood the log with one line per backoff tick.
 const TOPOLOGY_WARN_THROTTLE: Duration = Duration::from_secs(30);

+#[cfg(test)]
+static FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS: LazyLock<Mutex<HashSet<String>>> = LazyLock::new(|| Mutex::new(HashSet::new()));
+
+#[cfg(test)]
+struct LocalHostResolutionTimeoutGuard {
+    hosts: Vec<String>,
+}
+
+#[cfg(test)]
+impl Drop for LocalHostResolutionTimeoutGuard {
+    fn drop(&mut self) {
+        let mut forced_hosts = FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
+            .lock()
+            .expect("local-host test resolver mutex poisoned");
+        for host in &self.hosts {
+            forced_hosts.remove(host);
+        }
+    }
+}
+
+#[cfg(test)]
+fn force_local_host_resolution_timeout_for_test(hosts: &[&str]) -> LocalHostResolutionTimeoutGuard {
+    let hosts = hosts.iter().map(|host| (*host).to_string()).collect::<Vec<_>>();
+    let mut forced_hosts = FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
+        .lock()
+        .expect("local-host test resolver mutex poisoned");
+    forced_hosts.extend(hosts.iter().cloned());
+    LocalHostResolutionTimeoutGuard { hosts }
+}
+
+#[cfg(test)]
+fn local_host_resolution_timeout_forced(host: &Host<&str>) -> bool {
+    let host = match host {
+        Host::Domain(domain) => (*domain).to_string(),
+        Host::Ipv4(ip) => ip.to_string(),
+        Host::Ipv6(ip) => ip.to_string(),
+    };
+    FORCED_LOCAL_HOST_RESOLUTION_TIMEOUTS
+        .lock()
+        .expect("local-host test resolver mutex poisoned")
+        .contains(&host)
+}
+
+fn endpoint_is_local_host(host: Host<&str>, port: u16, local_port: u16) -> Result<bool> {
+    #[cfg(test)]
+    if local_host_resolution_timeout_forced(&host) {
+        return Err(Error::new(ErrorKind::TimedOut, "resolver timeout"));
+    }
+
+    is_local_host(host, port, local_port)
+}
+
 struct DnsRetryDeadline {
    started: Instant,
    timeout: Duration,
@@ -694,7 +748,7 @@ async fn resolve_local_host_with_retry(
    retry_dns_operation(
        || {
            let host = host.clone();
-            async move { is_local_host(host, port, local_port) }
+            async move { endpoint_is_local_host(host, port, local_port) }
        },
        async_sleep,
        dns_retry_deadline,
@@ -2231,6 +2285,9 @@ mod test {
    #[serial]
    #[tokio::test]
    async fn create_server_endpoints_bounds_kubernetes_alias_dns_fallback() {
+        let _resolution_timeout =
+            force_local_host_resolution_timeout_for_test(&["unrelated-0.example.invalid", "unrelated-1.example.invalid"]);
+
        async_with_vars(
            [
                (ENV_LOCAL_ENDPOINT_HOST, None),
@@ -4861,10 +4861,7 @@ async fn poll_merge_head(rx: &CancellationToken, in_channels: &mut [Receiver<Met

 async fn send_or_cancel(rx: &CancellationToken, out_channel: &Sender<MetaCacheEntry>, entry: MetaCacheEntry) -> Result<bool> {
    tokio::select! {
-        result = out_channel.send(entry) => {
-            result.map_err(Error::other)?;
-            Ok(true)
-        }
+        result = out_channel.send(entry) => Ok(result.is_ok()),
        _ = rx.cancelled() => Ok(false),
    }
 }
@@ -9858,6 +9855,28 @@ mod test {
        assert_eq!(results, vec!["obj-a", "obj-b"]);
    }

+    #[tokio::test]
+    async fn merge_entry_channels_treats_dropped_output_receiver_as_completion() {
+        let (tx_a, rx_a) = mpsc::channel(4);
+        let (tx_b, rx_b) = mpsc::channel(4);
+        let (out_tx, out_rx) = mpsc::channel(1);
+
+        tx_a.send(test_meta_entry("obj-a")).await.unwrap();
+        tx_b.send(test_meta_entry("obj-b")).await.unwrap();
+        drop(tx_a);
+        drop(tx_b);
+        drop(out_rx);
+
+        let result = timeout(
+            Duration::from_secs(1),
+            merge_entry_channels(CancellationToken::new(), vec![rx_a, rx_b], out_tx, 1),
+        )
+        .await
+        .expect("merge should stop promptly when its consumer disconnects");
+
+        assert!(result.is_ok(), "consumer disconnect must not surface as a merge worker error");
+    }
+
    #[test]
    fn walk_ascending_versions_contract_reverses_newest_first_metadata() {
        // Documents the invariant the walk `versions_sort` handling relies on:
@@ -26,16 +26,16 @@ use std::sync::{Arc, Mutex};
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};

-/// One canned HTTP response.
-pub(crate) struct ScriptedResponse {
-    status: u16,
-    body: String,
+/// One scripted connection outcome.
+pub(crate) enum ScriptedResponse {
+    Http { status: u16, body: String },
+    Close,
 }

 impl ScriptedResponse {
    /// A 200 response carrying `data` inside the standard Vault envelope.
    pub(crate) fn ok(data: serde_json::Value) -> Self {
-        Self {
+        Self::Http {
            status: 200,
            body: serde_json::json!({
                "request_id": "scripted",
@@ -50,11 +50,16 @@ impl ScriptedResponse {

    /// An error response in Vault's `{"errors": [...]}` format.
    pub(crate) fn error(status: u16, message: &str) -> Self {
-        Self {
+        Self::Http {
            status,
            body: serde_json::json!({ "errors": [message] }).to_string(),
        }
    }
+
+    /// Close the connection after consuming a request without sending an HTTP response.
+    pub(crate) fn close() -> Self {
+        Self::Close
+    }
 }

 /// A scripted stand-in Vault listening on a loopback port.
@@ -88,14 +93,14 @@ impl ScriptedVault {
                let response = responses
                    .next()
                    .unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
-                let payload = format!(
-                    "HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
-                    response.status,
-                    response.body.len(),
-                    response.body
-                );
-                let _ = stream.write_all(payload.as_bytes()).await;
-                let _ = stream.shutdown().await;
+                if let ScriptedResponse::Http { status, body } = response {
+                    let payload = format!(
+                        "HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
+                        body.len(),
+                    );
+                    let _ = stream.write_all(payload.as_bytes()).await;
+                    let _ = stream.shutdown().await;
+                }
            }
        });

@@ -1512,6 +1512,8 @@ mod tests {
    use crate::backends::scripted_vault::{ScriptedResponse, ScriptedVault};
    use crate::config::{VaultAuthMethod, VaultConfig};

+    const SCRIPTED_RETRY_ATTEMPTS: u32 = 3;
+
    /// Vault + KMS config pair pointing at a scripted loopback Vault.
    fn scripted_configs(address: &str) -> (VaultConfig, KmsConfig) {
        let vault_config = VaultConfig {
@@ -1527,7 +1529,7 @@ mod tests {
        };
        let kms_config = KmsConfig {
            timeout: Duration::from_secs(5),
-            retry_attempts: 3,
+            retry_attempts: SCRIPTED_RETRY_ATTEMPTS,
            ..KmsConfig::default()
        };
        (vault_config, kms_config)
@@ -1596,6 +1598,31 @@ mod tests {
        );
    }

+    #[tokio::test]
+    async fn wired_read_retries_closed_connections_within_budget() {
+        let (vault, client) = scripted_client(vec![ScriptedResponse::close(), ScriptedResponse::close()]).await;
+
+        let error = client
+            .get_key_data("wired-key")
+            .await
+            .expect_err("closed connections must exhaust the retry budget");
+        assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
+
+        let requests = vault.requests();
+        let expected_requests = usize::try_from(SCRIPTED_RETRY_ATTEMPTS).expect("retry attempts must fit usize");
+        assert_eq!(
+            requests.len(),
+            expected_requests,
+            "all budgeted retry attempts must reach Vault: {requests:?}"
+        );
+        assert!(
+            requests
+                .iter()
+                .all(|line| line == "GET /v1/secret/data/rustfs/kms/keys/wired-key"),
+            "all attempts must hit the same read endpoint: {requests:?}"
+        );
+    }
+
    #[tokio::test]
    async fn wired_read_does_not_retry_permission_errors() {
        let (vault, client) = scripted_client(vec![ScriptedResponse::error(403, "permission denied")]).await;
@@ -74,7 +74,7 @@ pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json";
 const ARTIFACTS_DIR: &str = "artifacts";
 const KEYS_DIR: &str = "artifacts/keys";
 const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc";
-const AEAD_NONCE_LEN: usize = 12;
+pub(crate) const AEAD_NONCE_LEN: usize = 12;
 /// Domain-separation context for the artifact AAD binding.
 const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1";
 /// Domain-separation context for the master-key verifier.
@@ -118,7 +118,7 @@ impl BackupKek {
        }
    }

-    fn cipher(&self) -> Aes256Gcm {
+    pub(crate) fn cipher(&self) -> Aes256Gcm {
        Aes256Gcm::new(&Key::<Aes256Gcm>::from(*self.key))
    }
 }
@@ -227,11 +227,14 @@ pub async fn export_local_backup(
    Ok(manifest)
 }

-/// Read and fully validate the manifest of a local bundle directory.
+/// Read and fully validate the manifest of a bundle directory, whatever
+/// backend produced it.
 ///
 /// A directory without a manifest is an interrupted export: the manifest is
-/// written last, so its absence means the bundle never sealed.
-pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
+/// written last, so its absence means the bundle never sealed. The manifest
+/// file name and framing are bundle-wide, not Local-specific, so consumers of
+/// other backends' bundles read them through here too.
+pub async fn read_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
    let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE);
    let bytes = match fs::read(&manifest_path).await {
        Ok(bytes) => bytes,
@@ -240,7 +243,12 @@ pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManif
        }
        Err(error) => return Err(error.into()),
    };
-    let manifest = BackupManifest::decode(&bytes)?;
+    Ok(BackupManifest::decode(&bytes)?)
+}
+
+/// Read and fully validate the manifest of a *local* bundle directory.
+pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
+    let manifest = read_bundle_manifest(bundle_dir).await?;
    if manifest.backend != BackupBackendKind::Local {
        return Err(
            BackupError::corrupted(format!("bundle manifest declares backend {:?}, expected Local", manifest.backend)).into(),
@@ -424,6 +432,7 @@ async fn build_and_write_bundle(
        backup_kek: kek.descriptor(),
        artifacts,
        local_kdf: Some(local_kdf_descriptor(snapshot, master_key_verifier)),
+        external_references: None,
        key_versions: None,
        capability_discovery: None,
        completeness: CompletenessState::InProgress,
@@ -502,7 +511,7 @@ async fn encrypt_and_write_artifact(

 /// AAD binding an artifact to its bundle identity and path. A JSON tuple
 /// gives unambiguous field boundaries without a hand-rolled framing format.
-fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
+pub(crate) fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
    serde_json::to_vec(&(BUNDLE_AAD_CONTEXT, backup_id, snapshot_generation, artifact_path))
        .expect("AAD tuple of strings and integers always serializes")
 }
@@ -306,6 +306,141 @@ impl LocalKdfDescriptor {
    }
 }

+/// Transit engine reference recorded in a Vault-backed bundle.
+///
+/// Transit keys are non-exportable, so a bundle can only ever name the key and
+/// the version window its content depends on. Restore compares these values
+/// against the Transit key the operator's native Vault restore produced.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct VaultTransitReference {
+    /// Transit engine mount path.
+    pub mount_path: String,
+    /// Transit key name.
+    pub key_name: String,
+    /// Lowest Transit key version the bundle's content still needs. A target
+    /// whose `min_decryption_version` sits above this can no longer decrypt
+    /// the oldest state in the bundle.
+    pub required_min_version: u32,
+    /// Latest Transit key version at snapshot time. A target whose newest
+    /// version is below this was restored to a point before the bundle.
+    pub current_version: u32,
+    /// Operator-recorded immutable reference to the native Vault/HSM snapshot
+    /// protecting this key. RustFS neither produces nor consumes that
+    /// snapshot; the reference exists so restore evidence can name it.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub native_snapshot_reference: Option<String>,
+}
+
+/// KV generation of one key's record at snapshot time.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct VaultKvRecordReference {
+    /// Stable key id the record belongs to.
+    pub key_id: String,
+    /// KV2 secret version holding the record when the snapshot was taken.
+    /// Restore compares the target's current version against it: a lower
+    /// value means the target's Vault was restored to a point before the
+    /// bundle, a higher value means the target moved ahead of it.
+    pub kv_version: u64,
+}
+
+/// Immutable references to the external Vault state a bundle depends on.
+///
+/// A Vault-backed bundle never carries the cryptographic root: Transit keys
+/// cannot be exported and KV2 records live in Vault's own storage. What the
+/// bundle can own is a precise description of *which* external state it was
+/// captured against, so a restore refuses to proceed when the operator's
+/// native Vault restore landed somewhere else. Credentials are structurally
+/// absent — no token, AppRole secret id, or TLS material is representable in
+/// this type.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct VaultExternalReferences {
+    /// Opaque identity of the Vault cluster the snapshot was taken from.
+    pub cluster_id: String,
+    /// Vault Enterprise namespace, when the deployment uses one.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub namespace: Option<String>,
+    /// KV v2 mount holding the RustFS records.
+    pub kv_mount: String,
+    /// Path prefix under `kv_mount` holding the RustFS records.
+    pub kv_path_prefix: String,
+    /// Transit engine reference; present exactly when the bundle's material
+    /// is protected by a Transit key.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub transit: Option<VaultTransitReference>,
+    /// Per-key KV generation observed at snapshot time, one entry per key the
+    /// bundle carries a record for.
+    pub kv_records: Vec<VaultKvRecordReference>,
+}
+
+impl VaultExternalReferences {
+    fn validate(&self, backend: BackupBackendKind, protection: AtRestProtection) -> Result<(), BackupError> {
+        require_non_empty("external_references.cluster_id", &self.cluster_id)?;
+        require_non_empty("external_references.kv_mount", &self.kv_mount)?;
+        require_non_empty("external_references.kv_path_prefix", &self.kv_path_prefix)?;
+        if self.namespace.as_deref() == Some("") {
+            return Err(BackupError::corrupted("external Vault namespace must not be empty when present"));
+        }
+
+        // The Transit reference is required exactly where the cryptographic
+        // root lives in Transit; a storage-only KV2 bundle that claims one
+        // would misdescribe its own trust root.
+        let transit_required = matches!(
+            (backend, protection),
+            (BackupBackendKind::VaultTransit, _) | (BackupBackendKind::VaultKv2, AtRestProtection::TransitWrapped)
+        );
+        match (&self.transit, transit_required) {
+            (None, true) => {
+                return Err(BackupError::corrupted(format!(
+                    "({backend:?}, {protection:?}) bundles must reference the Transit key protecting them"
+                )));
+            }
+            (Some(_), false) => {
+                return Err(BackupError::corrupted(format!(
+                    "({backend:?}, {protection:?}) bundles have no Transit trust root to reference"
+                )));
+            }
+            (Some(transit), true) => {
+                require_non_empty("external_references.transit.mount_path", &transit.mount_path)?;
+                require_non_empty("external_references.transit.key_name", &transit.key_name)?;
+                if transit.required_min_version == 0 {
+                    return Err(BackupError::corrupted("Transit reference must require at least key version 1"));
+                }
+                if transit.current_version < transit.required_min_version {
+                    return Err(BackupError::corrupted(format!(
+                        "Transit reference declares current version {} below the required minimum {}",
+                        transit.current_version, transit.required_min_version
+                    )));
+                }
+                if transit.native_snapshot_reference.as_deref() == Some("") {
+                    return Err(BackupError::corrupted("Transit native snapshot reference must not be empty when present"));
+                }
+            }
+            (None, false) => {}
+        }
+
+        let mut seen = BTreeSet::new();
+        for record in &self.kv_records {
+            require_non_empty("external_references.kv_records[].key_id", &record.key_id)?;
+            if record.kv_version == 0 {
+                return Err(BackupError::corrupted(format!(
+                    "KV generation for key '{}' must be at least 1",
+                    record.key_id
+                )));
+            }
+            if !seen.insert(record.key_id.as_str()) {
+                return Err(BackupError::corrupted(format!(
+                    "KV reference for key '{}' appears more than once",
+                    record.key_id
+                )));
+            }
+        }
+        Ok(())
+    }
+}
+
 /// Completeness marker of a bundle.
 ///
 /// A producer writes `in-progress` state (or no manifest at all) until the
@@ -394,6 +529,10 @@ pub struct BackupManifest {
    /// Local backend section; required exactly when `backend` is `Local`.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub local_kdf: Option<LocalKdfDescriptor>,
+    /// External Vault references; required exactly when `backend` is a Vault
+    /// backend. Never carries credentials — see [`VaultExternalReferences`].
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub external_references: Option<VaultExternalReferences>,
    /// Reserved for the per-key version/envelope inventory defined by the
    /// backlog#1565 contract. May not carry data in format version 1.
    #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -572,6 +711,7 @@ impl BackupManifest {
        }
        self.validate_responsibility()?;
        self.validate_local_kdf()?;
+        self.validate_external_references()?;
        self.validate_artifacts()?;
        Ok(())
    }
@@ -601,6 +741,19 @@ impl BackupManifest {
        }
    }

+    fn validate_external_references(&self) -> Result<(), BackupError> {
+        let vault_backend = matches!(self.backend, BackupBackendKind::VaultKv2 | BackupBackendKind::VaultTransit);
+        match (&self.external_references, vault_backend) {
+            (None, true) => Err(BackupError::corrupted("Vault backend manifest must carry external Vault references")),
+            (Some(_), false) => Err(BackupError::corrupted(format!(
+                "external Vault references are not valid for backend {:?}",
+                self.backend
+            ))),
+            (Some(references), true) => references.validate(self.backend, self.at_rest_protection),
+            (None, false) => Ok(()),
+        }
+    }
+
    fn validate_artifacts(&self) -> Result<(), BackupError> {
        let mut paths = BTreeSet::new();
        for artifact in &self.artifacts {
@@ -865,6 +1018,7 @@ mod tests {
                vec![AtRestProtection::EncryptedMasterKey],
                Some("verifier-opaque-1".to_string()),
            )),
+            external_references: None,
            key_versions: None,
            capability_discovery: None,
            completeness: CompletenessState::InProgress,
@@ -1271,4 +1425,161 @@ mod tests {
        assert_eq!(decoded, sealed);
        assert_eq!(decoded.responsibility, BackupResponsibility::ReferenceOnly);
    }
+
+    fn transit_references() -> VaultExternalReferences {
+        VaultExternalReferences {
+            cluster_id: "vault-cluster-a".to_string(),
+            namespace: Some("tenant-a".to_string()),
+            kv_mount: "secret".to_string(),
+            kv_path_prefix: "rustfs/kms/transit-metadata".to_string(),
+            transit: Some(VaultTransitReference {
+                mount_path: "transit".to_string(),
+                key_name: "rustfs-master".to_string(),
+                required_min_version: 2,
+                current_version: 5,
+                native_snapshot_reference: Some("s3://dr/vault/2026-08-01.snap".to_string()),
+            }),
+            kv_records: vec![VaultKvRecordReference {
+                key_id: "object-key".to_string(),
+                kv_version: 4,
+            }],
+        }
+    }
+
+    fn transit_manifest_unsealed() -> BackupManifest {
+        BackupManifest {
+            backend: BackupBackendKind::VaultTransit,
+            at_rest_protection: AtRestProtection::ExternalNonExportable,
+            responsibility: BackupResponsibility::MetadataPlusExternalRoot,
+            artifacts: vec![artifact(ArtifactKind::KeyMetadata, "vault/records/object-key.json.enc")],
+            local_kdf: None,
+            external_references: Some(transit_references()),
+            ..local_manifest_unsealed()
+        }
+    }
+
+    #[test]
+    fn vault_bundle_round_trips_with_external_references() {
+        let sealed = seal(transit_manifest_unsealed());
+        let bytes = sealed.encode().expect("encoding should succeed");
+        let decoded = BackupManifest::decode(&bytes).expect("decoding should succeed");
+        assert_eq!(decoded, sealed);
+    }
+
+    #[test]
+    fn vault_backend_requires_external_references() {
+        let manifest = BackupManifest {
+            external_references: None,
+            ..transit_manifest_unsealed()
+        };
+        expect_corrupted(
+            BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
+            "must carry external Vault references",
+        );
+    }
+
+    #[test]
+    fn non_vault_backend_rejects_external_references() {
+        let manifest = BackupManifest {
+            external_references: Some(transit_references()),
+            ..local_manifest_unsealed()
+        };
+        expect_corrupted(
+            BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
+            "not valid for backend",
+        );
+    }
+
+    #[test]
+    fn external_reference_contradictions_fail_closed() {
+        let cases: [(&str, Box<dyn Fn(&mut VaultExternalReferences)>); 7] = [
+            ("cluster_id", Box::new(|refs| refs.cluster_id.clear())),
+            ("kv_mount", Box::new(|refs| refs.kv_mount.clear())),
+            ("namespace must not be empty", Box::new(|refs| refs.namespace = Some(String::new()))),
+            (
+                "must reference the Transit key",
+                Box::new(|refs: &mut VaultExternalReferences| refs.transit = None),
+            ),
+            (
+                "at least key version 1",
+                Box::new(|refs: &mut VaultExternalReferences| {
+                    if let Some(transit) = refs.transit.as_mut() {
+                        transit.required_min_version = 0;
+                    }
+                }),
+            ),
+            (
+                "below the required minimum",
+                Box::new(|refs: &mut VaultExternalReferences| {
+                    if let Some(transit) = refs.transit.as_mut() {
+                        transit.current_version = 1;
+                    }
+                }),
+            ),
+            (
+                "appears more than once",
+                Box::new(|refs: &mut VaultExternalReferences| {
+                    let first = refs.kv_records[0].clone();
+                    refs.kv_records.push(first);
+                }),
+            ),
+        ];
+        for (needle, mutate) in cases {
+            let mut references = transit_references();
+            mutate(&mut references);
+            let manifest = BackupManifest {
+                external_references: Some(references),
+                ..transit_manifest_unsealed()
+            };
+            expect_corrupted(BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")), needle);
+        }
+    }
+
+    /// The reference schema is the only place a Vault coordinate reaches a
+    /// bundle, so its field set is pinned: a credential-carrying field could
+    /// only appear by editing this assertion.
+    #[test]
+    fn external_reference_field_set_is_frozen() {
+        let value = serde_json::to_value(transit_references()).expect("serialize");
+        let mut top: Vec<&str> = value.as_object().expect("object").keys().map(String::as_str).collect();
+        top.sort_unstable();
+        assert_eq!(
+            top,
+            [
+                "cluster_id",
+                "kv_mount",
+                "kv_path_prefix",
+                "kv_records",
+                "namespace",
+                "transit"
+            ]
+        );
+
+        let mut transit: Vec<&str> = value["transit"]
+            .as_object()
+            .expect("transit object")
+            .keys()
+            .map(String::as_str)
+            .collect();
+        transit.sort_unstable();
+        assert_eq!(
+            transit,
+            [
+                "current_version",
+                "key_name",
+                "mount_path",
+                "native_snapshot_reference",
+                "required_min_version"
+            ]
+        );
+
+        let mut record: Vec<&str> = value["kv_records"][0]
+            .as_object()
+            .expect("record object")
+            .keys()
+            .map(String::as_str)
+            .collect();
+        record.sort_unstable();
+        assert_eq!(record, ["key_id", "kv_version"]);
+    }
 }
@@ -17,9 +17,11 @@
 //! The contract side defines the versioned backup manifest, the per-backend
 //! responsibility matrix, typed failure modes, and the restore dry-run
 //! report. [`local_export`] implements the producer side and
-//! [`local_restore`] the consumer side for the Local backend as
-//! crate-internal APIs; the admin API builds on these pieces in follow-up
-//! changes.
+//! [`local_restore`] the consumer side for the Local backend;
+//! [`vault_restore`] orchestrates the consumer side for the Vault backends,
+//! whose cryptographic root is restored by Vault's own disaster-recovery
+//! flow. All are crate-internal APIs; the admin API builds on these pieces in
+//! follow-up changes.
 //!
 //! # Bundle model
 //!
@@ -54,6 +56,7 @@ mod error;
 pub mod local_export;
 pub mod local_restore;
 mod manifest;
+pub mod vault_restore;

 pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
 pub use dry_run::{
@@ -62,7 +65,7 @@ pub use dry_run::{
 pub use error::BackupError;
 pub use local_export::{
    BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup,
-    read_local_bundle_manifest,
+    read_bundle_manifest, read_local_bundle_manifest,
 };
 pub use local_restore::{
    LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore,
@@ -70,5 +73,10 @@ pub use local_restore::{
 };
 pub use manifest::{
    AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest,
-    DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot,
+    DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, VaultExternalReferences, VaultKvRecordReference,
+    VaultTransitReference,
+};
+pub use vault_restore::{
+    VaultRestoreClient, VaultRestoreMismatch, VaultRestoreReport, VaultRestoreRequest, VaultRestoreSequence, VaultRestoreStage,
+    VaultRestoreTarget, abort_vault_restore, dry_run_vault_restore, restore_vault_backup,
 };
@@ -14,8 +14,8 @@

 //! Fault-injection matrix for the Vault backend operation policy.
 //!
-//! Offline cases run against locally injected transport faults (a closed
-//! port, a listener that never responds) — deterministic, no external
+//! Offline cases run against locally injected transport faults (a listener
+//! that never responds) — deterministic, no external
 //! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault
 //! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`).
 //!
@@ -118,44 +118,6 @@ fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)])
        .sum()
 }

-/// Connection refused: connection-class failures are retried up to the
-/// configured budget, then surface as a backend error.
-#[test]
-fn connection_refused_is_retried_within_budget() {
-    // Reserve a loopback port and release it so nothing is listening there.
-    let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve a loopback port");
-    let address = format!("http://{}", listener.local_addr().expect("reserved port addr"));
-    drop(listener);
-
-    let snapshot = record_metrics(|| {
-        Box::pin(async move {
-            let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_secs(2), 2))
-                .await
-                .expect("client construction performs no network calls");
-            let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-refused"))
-                .await
-                .expect_err("a refused connection must fail the operation");
-            assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
-        })
-    });
-
-    assert_eq!(
-        counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]),
-        2,
-        "both budgeted attempts must observe the refused connection"
-    );
-    assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1);
-    // The static-token login records its own success; the Vault read must not.
-    assert_eq!(
-        counter_value(
-            &snapshot,
-            OPERATIONS_TOTAL,
-            &[("operation", "vault_kv2_read_key"), ("outcome", "success")]
-        ),
-        0
-    );
-}
-
 /// Stalled connection: a server that accepts but never responds is cut off by
 /// the per-attempt timeout (either the policy timer or the equally sized HTTP
 /// client timeout, whichever fires first) instead of hanging forever.
@@ -88,6 +88,7 @@ rmp-serde = { workspace = true }
 hmac = { workspace = true }
 sha2 = { workspace = true }
 rustfs-filemeta = { workspace = true }
+rustfs-lock.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
 rustfs-ecstore = { workspace = true }
 rustfs-storage-api = { workspace = true }
@@ -12,17 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#[cfg(test)]
+use crate::RUSTFS_META_BUCKET;
 use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
 use crate::scanner_io::{
-    DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, current_cache_root_or_prepare,
-    scanner_cache_lock_resource, scanner_cache_lock_timeout, scanner_set_disk_inventory,
+    DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, cache_root_entry_info,
+    current_cache_root_or_prepare, scanner_set_disk_inventory,
 };
 use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
-use crate::storage_api::scan::NamespaceLocking as _;
 use crate::{
    DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource,
-    DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, EcstoreError, RUSTFS_META_BUCKET, ScannerDiskExt as _, ScannerError,
-    ScannerObjectIO, StorageError, is_reserved_or_invalid_bucket, read_config, resolve_scanner_object_store_handle,
+    DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, EcstoreError, ScannerDiskExt as _, ScannerError, ScannerObjectIO,
+    StorageError, is_reserved_or_invalid_bucket, read_config, resolve_scanner_object_store_handle,
 };
 use hmac::{Hmac, KeyInit, Mac};
 use rustfs_common::heal_channel::HealScanMode;
@@ -63,6 +64,7 @@ const NS_SCANNER_DISK_HEALTH_TIMEOUT: Duration = Duration::from_secs(5);
 const NS_SCANNER_VALIDATED_CYCLE_TTL: Duration = Duration::from_secs(1);
 const NS_SCANNER_MAX_REPLAY_SESSIONS: usize = 65_536;
 const NS_SCANNER_MAX_ERROR_CHARS: usize = 4096;
+const NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX: &str = "retry_bucket:";
 const NS_SCANNER_FRAME_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-frame-v3";

 pub const NS_SCANNER_MAX_REQUEST_BODY_SIZE: usize = 16 * 1024;
@@ -317,6 +319,13 @@ impl RemoteScannerServerError {
        }
    }

+    fn retry_bucket(message: impl Into<String>) -> Self {
+        Self {
+            scope: RemoteScannerErrorScope::Bucket,
+            error: ScannerError::Other(format!("{}{}", NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX, message.into())),
+        }
+    }
+
    fn into_frame(self) -> RemoteScannerErrorFrame {
        RemoteScannerErrorFrame {
            scope: self.scope,
@@ -336,6 +345,7 @@ struct RemoteScannerStreamError {
    error: StorageError,
    progress_fully_reported: bool,
    retire_worker: bool,
+    retry_bucket: bool,
 }

 impl RemoteScannerStreamError {
@@ -344,6 +354,7 @@ impl RemoteScannerStreamError {
            error,
            progress_fully_reported: false,
            retire_worker: true,
+            retry_bucket: false,
        }
    }

@@ -352,6 +363,7 @@ impl RemoteScannerStreamError {
            error,
            progress_fully_reported: true,
            retire_worker: true,
+            retry_bucket: false,
        }
    }

@@ -360,6 +372,16 @@ impl RemoteScannerStreamError {
            error,
            progress_fully_reported: true,
            retire_worker: false,
+            retry_bucket: false,
+        }
+    }
+
+    fn retry_bucket(error: StorageError) -> Self {
+        Self {
+            error,
+            progress_fully_reported: true,
+            retire_worker: false,
+            retry_bucket: true,
        }
    }

@@ -951,16 +973,14 @@ async fn scan_and_persist_local_bucket(
            ))
        })?;
    let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
-    let lock_resource = scanner_cache_lock_resource(&cache_name);
-    let ns_lock = set
-        .new_ns_lock(RUSTFS_META_BUCKET, &lock_resource)
-        .await
-        .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache lock creation failed: {err}")))?;
-    let guard = ns_lock
-        .get_write_lock_quiet(scanner_cache_lock_timeout())
+    let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
        .await
        .map_err(|err| {
-            RemoteScannerServerError::worker(format!("remote namespace scanner cache lock acquisition failed: {err}"))
+            if err.is_contention() {
+                RemoteScannerServerError::retry_bucket(format!("remote namespace scanner cache lock contention: {err}"))
+            } else {
+                RemoteScannerServerError::worker(format!("remote namespace scanner cache lock acquisition failed: {err}"))
+            }
        })?;
    let mut cache = DataUsageCache::default();
    let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
@@ -1216,7 +1236,9 @@ fn finish_remote_scanner_stream(
            if !error.progress_fully_reported {
                budget.cancel_after_unreported_remote_progress();
            }
-            let failure = if error.retire_worker {
+            let failure = if error.retry_bucket {
+                RemoteScannerFailure::retry_bucket(error.error)
+            } else if error.retire_worker {
                RemoteScannerFailure::transport(error.error)
            } else {
                RemoteScannerFailure::bucket(error.error)
@@ -1374,9 +1396,16 @@ where
                return Ok(RemoteScannerOutcome::CycleAhead(required_cycle));
            }
            RemoteScannerFrameResult::Error(error_frame) => {
+                let retry_bucket = error_frame.message.starts_with(NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX);
+                let message = error_frame
+                    .message
+                    .strip_prefix(NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX)
+                    .map(str::trim_start)
+                    .unwrap_or(error_frame.message.as_str());
                let error =
-                    StorageError::other(format!("remote namespace scanner failed: {}", limit_error_message(error_frame.message)));
+                    StorageError::other(format!("remote namespace scanner failed: {}", limit_error_message(message.to_string())));
                return Err(match error_frame.scope {
+                    RemoteScannerErrorScope::Bucket if retry_bucket => RemoteScannerStreamError::retry_bucket(error),
                    RemoteScannerErrorScope::Bucket => RemoteScannerStreamError::bucket(error),
                    RemoteScannerErrorScope::Worker => RemoteScannerStreamError::reconciled(error),
                });
@@ -2491,6 +2520,42 @@ mod tests {
        assert!(!budget.budget_elapsed());
    }

+    #[tokio::test]
+    async fn retry_bucket_error_terminal_frame_requeues_bucket_work() {
+        let request_id = Uuid::new_v4();
+        let writer_auth = FrameAuthenticator::for_test(request_id);
+        let reader_auth = FrameAuthenticator::for_test(request_id);
+        let (mut writer, reader) = tokio::io::duplex(4096);
+        tokio::spawn(async move {
+            let mut sequence = 0;
+            write_frame(
+                &mut writer,
+                &writer_auth,
+                &mut sequence,
+                &RemoteScannerFrame::terminal(
+                    RemoteScannerProgress::default(),
+                    RemoteScannerFrameResult::Error(RemoteScannerErrorFrame {
+                        scope: RemoteScannerErrorScope::Bucket,
+                        message: format!("{NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX} cache lock contention"),
+                    }),
+                ),
+            )
+            .await
+            .expect("retry-bucket error terminal frame should write");
+        });
+
+        let parent = CancellationToken::new();
+        let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default());
+        let stream_result =
+            consume_remote_scanner_stream(reader, parent, budget.clone(), "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth)
+                .await;
+        let error = finish_remote_scanner_stream(stream_result, budget.as_ref()).expect_err("retry-bucket frame must fail");
+
+        assert!(error.retry_bucket_work());
+        assert!(!error.retire_worker());
+        assert!(error.to_string().contains("cache lock contention"));
+    }
+
    #[tokio::test]
    async fn worker_error_terminal_frame_retires_worker_without_cancelling_reported_budget() {
        let request_id = Uuid::new_v4();
@@ -949,7 +949,7 @@ pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool
                }
                SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => {
                    return Err(format!(
-                        "scanner activity peer {host} cannot acknowledge distributed dirty usage with protocol {}",
+                        "scanner activity peer {host} cannot safely share scanner cache locks with protocol {}",
                        SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION
                    ));
                }
@@ -7114,9 +7114,17 @@ mod tests {
                ..scanner_node_activity("epoch-a", 7, 3)
            },
        )]);
+        let previous = BTreeMap::from([(
+            "node-2".to_string(),
+            ScannerNodeActivity {
+                protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
+                ..scanner_node_activity("epoch-a", 7, 3)
+            },
+        )]);
        let current = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);

        assert_ne!(scanner_activity_snapshot_digest(&legacy), scanner_activity_snapshot_digest(&current));
+        assert_ne!(scanner_activity_snapshot_digest(&previous), scanner_activity_snapshot_digest(&current));
    }

    #[test]
@@ -30,6 +30,7 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e
 use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
 use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
 use rustfs_filemeta::FileMeta;
+use rustfs_lock::{LockError, NamespaceLockGuard};
 use rustfs_utils::path::path_join_buf;
 use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
 use sha2::{Digest as _, Sha256};
@@ -944,14 +945,85 @@ fn checked_bucket_usage_info(entry: &DataUsageEntry) -> Option<BucketUsageInfo>
    Some(usage)
 }

-pub(crate) fn scanner_cache_lock_resource(cache_name: &str) -> String {
-    path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, SCANNER_CACHE_LOCK_SUFFIX])
+pub(crate) fn scanner_cache_lock_resource(cache_name: &str, source: DataUsageCacheSource) -> String {
+    let lock_name = format!("{SCANNER_CACHE_LOCK_SUFFIX}.pool-{}.set-{}", source.pool_index, source.set_index);
+    path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name, &lock_name])
 }

 pub(crate) fn scanner_cache_lock_timeout() -> Duration {
    Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
 }

+#[derive(Debug)]
+pub(crate) struct ScannerCacheLockGuards {
+    scoped: NamespaceLockGuard,
+}
+
+impl ScannerCacheLockGuards {
+    pub(crate) fn is_lock_lost(&self) -> bool {
+        self.scoped.is_lock_lost()
+    }
+}
+
+#[derive(Debug)]
+pub(crate) enum ScannerCacheLockError {
+    Create { resource: String, source: Error },
+    Acquire { resource: String, source: LockError },
+}
+
+impl ScannerCacheLockError {
+    pub(crate) fn state(&self) -> &'static str {
+        match self {
+            Self::Create { .. } => "lock_create_failed",
+            Self::Acquire { .. } => "lock_acquire_failed",
+        }
+    }
+
+    pub(crate) fn is_contention(&self) -> bool {
+        matches!(
+            self,
+            Self::Acquire {
+                source: LockError::Timeout { .. } | LockError::AlreadyLocked { .. },
+                ..
+            }
+        )
+    }
+}
+
+impl std::fmt::Display for ScannerCacheLockError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Create { resource, source } => write!(formatter, "create scanner cache lock {resource}: {source}"),
+            Self::Acquire { resource, source } => write!(formatter, "acquire scanner cache lock {resource}: {source}"),
+        }
+    }
+}
+
+pub(crate) async fn acquire_scanner_cache_locks(
+    store: &SetDisks,
+    cache_name: &str,
+    source: DataUsageCacheSource,
+) -> std::result::Result<ScannerCacheLockGuards, ScannerCacheLockError> {
+    let timeout = scanner_cache_lock_timeout();
+    let scoped_resource = scanner_cache_lock_resource(cache_name, source);
+    let scoped_lock = store
+        .new_ns_lock(RUSTFS_META_BUCKET, &scoped_resource)
+        .await
+        .map_err(|source| ScannerCacheLockError::Create {
+            resource: scoped_resource.clone(),
+            source,
+        })?;
+    let scoped = scoped_lock
+        .get_write_lock_quiet(timeout)
+        .await
+        .map_err(|source| ScannerCacheLockError::Acquire {
+            resource: scoped_resource,
+            source,
+        })?;
+
+    Ok(ScannerCacheLockGuards { scoped })
+}
+
 async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)
 where
    F: Future,
@@ -1697,6 +1769,19 @@ mod publish_gate_tests {
        assert!(!cache_snapshot_is_current(&cache, "photos", source, 11, 0, second));
    }

+    #[test]
+    fn scanner_cache_lock_resource_is_scoped_to_cache_source() {
+        let cache_name = "photos/.usage-cache.bin";
+        let first_source = DataUsageCacheSource::new(0, 1);
+        let same_source = DataUsageCacheSource::new(0, 1);
+        let other_source = DataUsageCacheSource::new(1, 0);
+
+        let first = scanner_cache_lock_resource(cache_name, first_source);
+        assert_eq!(first, scanner_cache_lock_resource(cache_name, same_source));
+        assert_ne!(first, scanner_cache_lock_resource(cache_name, other_source));
+        assert!(first.ends_with(".scanner-cycle.lock.pool-0.set-1"));
+    }
+
    #[test]
    fn count_budget_serializes_set_and_disk_work() {
        assert_eq!(scanner_budgeted_concurrency_limit(8, true), 1);
@@ -1797,24 +1882,7 @@ async fn persist_and_publish_cache_snapshot(
    cache_cycle_floor: &AtomicU64,
 ) -> Option<SystemTime> {
    let source = cache_snapshot.info.source?;
-    let lock_resource = scanner_cache_lock_resource(DATA_USAGE_CACHE_NAME);
-    let ns_lock = match store.new_ns_lock(RUSTFS_META_BUCKET, &lock_resource).await {
-        Ok(lock) => lock,
-        Err(err) => {
-            error!(
-                target: "rustfs::scanner::io",
-                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                component = LOG_COMPONENT_SCANNER,
-                subsystem = LOG_SUBSYSTEM_IO,
-                cache_name = DATA_USAGE_CACHE_NAME,
-                state = "lock_create_failed",
-                error = %err,
-                "Scanner cache snapshot lock creation failed"
-            );
-            return None;
-        }
-    };
-    let guard = match ns_lock.get_write_lock_quiet(scanner_cache_lock_timeout()).await {
+    let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
        Ok(guard) => guard,
        Err(err) => {
            error!(
@@ -1823,7 +1891,7 @@ async fn persist_and_publish_cache_snapshot(
                component = LOG_COMPONENT_SCANNER,
                subsystem = LOG_SUBSYSTEM_IO,
                cache_name = DATA_USAGE_CACHE_NAME,
-                state = "lock_acquire_failed",
+                state = err.state(),
                error = %err,
                "Scanner cache snapshot lock acquisition failed"
            );
@@ -3080,32 +3148,35 @@ impl ScannerIOCache for SetDisks {
                        None
                    };

-                    // Lock order: scanner leader fence -> per-bucket cache lock ->
-                    // cache object read/write. The cache lock stays outermost for
-                    // local and rolling-upgrade workers so leader failover cannot
-                    // execute the same bucket concurrently.
-                    let lock_resource = scanner_cache_lock_resource(&cache_name);
-                    let ns_lock = match store_clone_clone.new_ns_lock(RUSTFS_META_BUCKET, &lock_resource).await {
-                        Ok(lock) => lock,
-                        Err(e) => {
-                            record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
-                            error!(
-                                target: "rustfs::scanner::io",
-                                event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                                component = LOG_COMPONENT_SCANNER,
-                                subsystem = LOG_SUBSYSTEM_IO,
-                                bucket = %bucket.name,
-                                cache_name = %cache_name,
-                                state = "lock_create_failed",
-                                error = %e,
-                                "Scanner bucket cache lock creation failed"
-                            );
-                            continue;
-                        }
-                    };
-                    let cache_guard = match ns_lock.get_write_lock_quiet(scanner_cache_lock_timeout()).await {
+                    // Lock order: scanner leader fence -> set-scoped per-bucket cache lock ->
+                    // cache object read/write.
+                    let cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
                        Ok(guard) => guard,
                        Err(e) => {
+                            if e.is_contention() {
+                                if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
+                                    increment_disk_bucket_scans_queued(
+                                        &queued_disk_bucket_scans_clone,
+                                        &pool_label_clone,
+                                        &set_label_clone,
+                                    );
+                                } else {
+                                    record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
+                                }
+                                debug!(
+                                    target: "rustfs::scanner::io",
+                                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                                    component = LOG_COMPONENT_SCANNER,
+                                    subsystem = LOG_SUBSYSTEM_IO,
+                                    bucket = %bucket.name,
+                                    cache_name = %cache_name,
+                                    state = "lock_contention_requeued",
+                                    error = %e,
+                                    "Scanner bucket cache lock contention requeued bucket work"
+                                );
+                                break;
+                            }
+
                            record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
                            error!(
                                target: "rustfs::scanner::io",
@@ -3114,7 +3185,7 @@ impl ScannerIOCache for SetDisks {
                                subsystem = LOG_SUBSYSTEM_IO,
                                bucket = %bucket.name,
                                cache_name = %cache_name,
-                                state = "lock_acquire_failed",
+                                state = e.state(),
                                error = %e,
                                "Scanner bucket cache lock acquisition failed"
                            );
@@ -3894,6 +3965,52 @@ mod tests {
        (temp_dir, store)
    }

+    #[tokio::test]
+    #[serial]
+    async fn scanner_cache_locks_block_same_source_workers() {
+        let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+        let set = &store.pools[0].disk_set[0];
+        let source = DataUsageCacheSource::new(0, 0);
+        let cache_name = "photos/.usage-cache.bin";
+
+        let guards = acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
+            .await
+            .expect("scanner cache locks should be acquired");
+        let scoped_lock = set
+            .new_ns_lock(RUSTFS_META_BUCKET, &scanner_cache_lock_resource(cache_name, source))
+            .await
+            .expect("scoped scanner cache lock should be created");
+        let scoped_err = scoped_lock
+            .get_write_lock_quiet(Duration::from_millis(100))
+            .await
+            .expect_err("same-source workers must be blocked while scanner cache lock is held");
+        assert!(matches!(scoped_err, LockError::Timeout { .. } | LockError::AlreadyLocked { .. }));
+
+        drop(guards);
+        acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
+            .await
+            .expect("scanner cache locks should be released when guards drop");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn scanner_cache_locks_allow_cross_source_workers() {
+        let (_temp_dir, store) = setup_two_pool_scanner_store().await;
+        let first_set = &store.pools[0].disk_set[0];
+        let second_set = &store.pools[1].disk_set[0];
+        let cache_name = "photos/.usage-cache.bin";
+
+        let first = acquire_scanner_cache_locks(first_set.as_ref(), cache_name, DataUsageCacheSource::new(0, 0))
+            .await
+            .expect("first source scanner cache locks should be acquired");
+        let second = acquire_scanner_cache_locks(second_set.as_ref(), cache_name, DataUsageCacheSource::new(1, 0))
+            .await
+            .expect("different source scanner cache locks should not contend");
+
+        assert!(!first.is_lock_lost());
+        assert!(!second.is_lock_lost());
+    }
+
    #[tokio::test]
    async fn data_usage_publish_fails_when_receiver_is_closed() {
        let (updates, receiver) = mpsc::channel(1);
@@ -28,8 +28,8 @@ pub const NS_SCANNER_SESSION_SEQUENCE_QUERY: &str = "ns_scanner_session_sequence
 pub const NS_SCANNER_PROTOCOL_VERSION_QUERY: &str = "ns_scanner_protocol";
 pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3;
 pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0;
-pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 4;
-pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 5;
+pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
+pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 6;

 #[derive(Debug, serde::Deserialize, serde::Serialize)]
 #[serde(deny_unknown_fields)]
@@ -17,7 +17,7 @@ for later deletion.
 - `rustfs-5416-kubernetes-alias-dns` Kubernetes endpoint identity fallback: deployments created before explicit local endpoint identity may use resolvable aliases that do not match the Pod hostname. An implicit auto-mode zero match retains legacy DNS locality with a bounded deadline, while ambiguous matches and invalid explicit anchors still fail closed. Remove the fallback after every supported direct-upgrade chart and deployment manifest provides a canonical RUSTFS_LOCAL_ENDPOINT_HOST for domain-based distributed topologies.
 - `rustfs-5416-zero-retry-delay` startup retry-delay validation: releases before bounded topology convergence accept RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY values of 0 or 0ms. New servers replace those values with the safe nonzero default so a direct upgrade neither fails startup nor enters a busy loop. Reject zero after the minimum supported direct-upgrade release validates or rewrites this setting before rollout.
 - `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`.
- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Current protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while the distributed scanner publishes usage only after every peer returns authenticated protocol v5 state. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, and remove the protocol-v4 activity codec after every supported peer implements protocol v5; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
+- `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state, but predates set-scoped scanner cache locks. Current protocol v6 additionally fences scanner cache lock-domain changes, so distributed scanner cycles publish usage only after every peer reports protocol v6 state. Servers retain protocol-0 and protocol-v4 codecs for rolling upgrades, while protocol-v5 peers are treated as previous-version peers that cannot safely participate in the new cache lock domain. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, remove the protocol-v4 activity codec after every supported peer implements protocol v5, and remove protocol-v5 previous-version rejection after every supported peer implements protocol v6; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version.
 - `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
 - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
 - `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC.
@@ -2,7 +2,7 @@

 RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone.

-For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md).
+For how the Vault backends authenticate (static token, AppRole, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md).

 ## Backend comparison

@@ -69,6 +69,77 @@ Decryption loads exactly the version recorded in the envelope and fails closed w

 Do not rotate any key until **every** RustFS node in the cluster runs a build that understands the `master_key_version` envelope field. Older binaries ignore the field and always decrypt with the current material: harmless while nothing has been rotated, but after a rotation they will fail to decrypt every object wrapped by an earlier key version. Complete the rolling upgrade of the entire cluster first, then rotate.

+This is the sharpest instance of a broader class of constraints; the rest are collected in [Mixed-version clusters during a rolling upgrade](#mixed-version-clusters-during-a-rolling-upgrade).
+
+## Mixed-version clusters during a rolling upgrade
+
+During a rolling upgrade the cluster runs two RustFS builds at once. That window matters more for KMS than for most subsystems, because KMS state is shared three ways: **Vault** holds the key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and — for configuration — can disagree for as long as the operator leaves them running.
+
+This section states only what is true of the current implementation. It is written for the KV2 and Transit backends; the Local backend is unsupported for multi-node deployments regardless of version (see the [deployment support matrix](#deployment-support-matrix)).
+
+### Persisted formats are backward compatible in both directions
+
+Nothing in this list requires a coordinated format cutover. The compatibility is deliberate and is covered by decode tests.
+
+- **DEK envelopes.** `DataKeyEnvelope::master_key_version` is optional and omitted when absent, so envelopes written by non-rotating backends stay byte-identical to the historical seven-field JSON shape. An upgraded node reading a pre-versioning envelope resolves `None` to the key's recorded baseline version, or — for a key that was never rotated, and so has no baseline — to the current version, which is exactly the pre-versioning behavior.
+- **KV2 key records.** `baseline_version` is read with a serde default, so records written by older builds deserialize unchanged, and `None` correctly means "never rotated".
+- **Transit metadata records.** Metadata persisted in KV v2 by either build decodes on the other.
+
+The one-way hazard is the rotation constraint above: an older binary reading a *new* envelope silently ignores the version field and decrypts with the current material.
+
+### Guarantees that hold only once every node is upgraded
+
+These are properties of the upgraded code, so a single node left behind removes them for the whole cluster.
+
+- **Check-and-set lifecycle writes.** Upgraded builds write every KV2 lifecycle mutation — create, enable, disable, tag metadata, schedule deletion, cancel deletion — as a versioned read followed by a check-and-set write, retrying on conflict by re-reading and re-validating the state gate (rustfs/rustfs#5518). Transit metadata writes got the same treatment (rustfs/rustfs#5520). Builds older than those write blind. A blind write from an old node can overwrite a check-and-set commit from an upgraded node without any conflict being reported, which is precisely the lost update the change was made to eliminate.
+- **`baseline_version` survives a write-back.** The KV2 key record does not deny unknown fields, so an old build reads a new record without error — and drops `baseline_version` when it writes that record back for any reason. A key that loses its baseline resolves pre-versioning envelopes to the current version again, which after a rotation means the wrong master key material. Any lifecycle operation issued to an old node is enough to trigger this.
+- **Version-record awareness.** Rotation stores each historical version under `{prefix}/{key_id}/versions/{N}` as a create-only record (check-and-set of 0), so two nodes racing the same version number produce exactly one creator; the loser adopts the persisted, never-current material or fails without touching the current pointer. Old builds have no concept of that sub-path: they never read or write it, and their key listing reports the KV2 directory entry (`my-key/`) as though it were a key, because the directory filter only exists in upgraded builds.
+
+### Windows in which nodes can legitimately disagree
+
+Even with every node on the same build, some state is process-local. These windows are bounded by design, except the last one.
+
+| What can diverge | Bound | Mechanism |
+| --- | --- | --- |
+| Transit key lifecycle state used by the `encrypt` and `generate_data_key` gates | ≤ 300 s (`METADATA_CACHE_TTL`) | Each node caches Transit metadata in process, TTL- and capacity-bounded, with targeted invalidation when a data-path call reports the key is gone server-side. A disable or schedule-deletion performed on one node is enforced on the others within one TTL at the latest, sooner if they hit that signal. |
+| `describe_key` output | ≤ 300 s | The manager-level key metadata cache. This is a reporting cache; the KV2 state gates do not read it. |
+| KV2 key lifecycle state | None | The KV2 backend re-reads the key record from Vault for every lifecycle and data-key operation, so a committed disable is effective on every upgraded node immediately. |
+| Active KMS configuration | Until the remaining nodes are restarted | See below. |
+
+Builds older than rustfs/rustfs#5520 held the Transit metadata cache with no TTL and no capacity bound. On such a node the divergence window is not 300 seconds but "until the process restarts": it can keep encrypting under a key that another node disabled, indefinitely.
+
+### Configuration is persisted cluster-wide but applied per node
+
+`POST /rustfs/admin/v3/kms/reconfigure` currently does two things: it switches the KMS service **on the node that handled the request**, and it persists the new configuration to cluster storage at `config/kms_config.json`. It does not notify peers. Every other node keeps running the configuration it started with until it is restarted, at which point it loads the persisted configuration during startup.
+
+The practical consequences today:
+
+- The configuration-split window has no upper bound other than the operator restarting the remaining nodes. Convergence is a restart, not a timeout.
+- During the window both configurations are live. If the reconfiguration changed backends, or changed the Vault mount or key prefix, different nodes write new key material to different places, and a key created through one node is invisible to the others.
+- `kms status` reflects the node that answered the request, so a single successful status response is not evidence that the cluster is consistent. Query every node.
+
+Treat `reconfigure` as the first step of a cluster-wide operation, not as the operation itself.
+
+### Recommended rolling upgrade order
+
+Follow the node-at-a-time procedure in the [multi-node restart runbook](rolling-restart.md); this adds the KMS-specific sequencing around it.
+
+1. **Freeze KMS administrative traffic** for the duration: no key creation, enable, disable, tagging, schedule-deletion, cancel-deletion, rotation, or reconfiguration. Object read and write traffic continues normally.
+2. **Upgrade one node at a time**, waiting for each to report ready before starting the next.
+3. **Verify no node is left behind** before unfreezing. A single old node is enough to reintroduce blind writes and to strip `baseline_version` on its next lifecycle write.
+4. **Resume administrative traffic.**
+5. **Only then perform the first rotation of any key.** Once the whole cluster understands `master_key_version`, rotation is safe; before that it is not.
+6. **If the KMS configuration was changed at any point**, restart the nodes that did not handle the request so they reload the persisted configuration, and confirm each one reports the intended backend.
+
+### Do not do these during a mixed-version window
+
+- **Rotate any key.** This is the hard constraint stated above; a rotation is unrecoverable for objects an old node must read.
+- **Issue any KV2 lifecycle write to an old node.** Its blind write can clobber a concurrent check-and-set commit and will drop `baseline_version` from the record.
+- **Create the same key ID from two nodes.** The create path is create-only on upgraded builds, but an old node's blind write does not honor that: the later writer's material wins and every DEK already wrapped with the earlier material becomes permanently unwrappable.
+- **Assume a disable or schedule-deletion took effect cluster-wide.** Old Transit nodes cache lifecycle state without expiry; confirm per node, or restart the old nodes, before treating a key as no longer in use.
+- **Reconfigure the KMS backend and consider it done.** The change applies to one node and is persisted; the rest need a restart.
+- **Delete or prune version records** under `{prefix}/{key_id}/versions/*` for any reason. This is never safe, mixed-version or not; see [Retention and destruction preconditions](#retention-and-destruction-preconditions).
+
 ## Choosing between Vault KV2 and Vault Transit

 Use **Vault Transit** (`VaultTransit`) when key material must be cryptographically isolated from anyone holding storage-level read access: Transit keeps key-encryption keys inside Vault and only ever returns ciphertext, and supports server-side key versioning/rotation.
@@ -0,0 +1,148 @@
+# Cryptographic compliance positioning
+
+This document records where RustFS stands on cryptographic module validation, what may and may not be said about it in external material, and what each possible route to a stronger position would actually cost. It exists so that the question is answered once, from the code, instead of being re-litigated from assumptions about crate names and feature flags.
+
+For where master key material lives per backend and how rotation retention works, see [KMS backend security properties](kms-backend-security.md).
+
+## Status: not FIPS 140-3 validated
+
+**RustFS is not FIPS 140-3 (or 140-2) validated, and no component it links is running as a validated cryptographic module.** There is no CMVP certificate covering RustFS or the libraries it uses in the shipped configuration.
+
+This is a deliberate position, not an oversight. It is also not a statement about algorithm strength: the algorithms in use are standard, well-reviewed AEADs. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS has or currently pursues.
+
+### What the process actually links
+
+The table below is the audited inventory as of this document's writing. "Validated module" asks only whether the code performing the operation is a FIPS-validated cryptographic module; the answer is uniformly no.
+
+| Layer | Where | Implementation | Primitives | Validated module |
+| --- | --- | --- | --- | --- |
+| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange | No — this is the ordinary `aws-lc-rs` build, not the `aws-lc-fips-sys`-backed FIPS variant |
+| Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No |
+| DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No |
+| Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No |
+| Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No |
+| JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS build | No |
+
+Two consequences follow directly from the table and are worth stating explicitly, because both are commonly assumed the other way:
+
+- **AWS-LC being present does not imply FIPS.** `aws-lc-rs` has a FIPS variant; the workspace does not enable it. Every `aws-lc-rs` dependency in the workspace is the default, non-FIPS build.
+- **The data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations. Swapping the TLS provider would not change that; see [route 1](#route-1-adopt-the-aws-lc-rs-fips-variant) for what would.
+
+## Terminology red lines for external material
+
+These rules apply to the README, CHANGELOG, release notes, marketing pages, sales decks, RFP responses, and security questionnaires. Claiming validation RustFS does not have is a false statement of fact with regulatory and contractual consequences, not a marketing overreach.
+
+### Never use
+
+- "FIPS validated", "FIPS certified", "FIPS 140-2/140-3 compliant", "FIPS compliant"
+- "FIPS mode", "runs in FIPS mode", "FIPS-enabled"
+- "NIST certified", "NIST approved", "CMVP certificate", any certificate number
+- "meets FIPS requirements", "satisfies FIPS", or any phrasing a reader would reasonably read as validation
+- The internal Cargo feature name `fips` as a product capability. It is a build-time algorithm selector (see below), and surfacing it as a feature name invites exactly the misreading this section exists to prevent.
+
+### Permitted, with the qualifier attached
+
+- **"FIPS-preferred algorithms"** — permitted only when accompanied, in the same paragraph or table cell, by an explicit non-validation statement. The defined meaning is: *the default algorithm selection is restricted to algorithms on the FIPS 140-3 approved list, implemented by software that has not been validated as a cryptographic module.*
+- Naming specific primitives factually ("AES-256-GCM", "ChaCha20-Poly1305", "PBKDF2-HMAC-SHA256") is always fine. Algorithm names carry no validation claim.
+
+Suggested boilerplate when the topic cannot be avoided:
+
+> RustFS encrypts object data with AES-256-GCM and supports ChaCha20-Poly1305. These are FIPS-approved algorithms, but the implementations are not FIPS 140-3 validated cryptographic modules and RustFS makes no FIPS validation claim.
+
+### Guard
+
+There is currently no FIPS-related wording anywhere in the repository's Markdown; that clean baseline is what a grep anchor test protects. Any future occurrence of the banned strings in shipped documentation should be treated as a defect and either removed or brought under the qualifier rule above.
+
+## The `rustfs-crypto` `fips` feature: what it actually does
+
+`crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way.
+
+| `fips` | Algorithm ID written | KDF | AEAD |
+| --- | --- | --- | --- |
+| enabled (default) | `ID::Pbkdf2AESGCM` (`0x02`) | PBKDF2-HMAC-SHA256, 8192 iterations | AES-256-GCM |
+| disabled | `ID::Argon2idAESGCM` (`0x00`) or `ID::Argon2idChaCHa20Poly1305` (`0x01`), chosen at runtime by CPU AES support | Argon2id (64 MiB, t=1, p=4) | AES-256-GCM or ChaCha20-Poly1305 |
+
+The selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src/encdec/stream_io.rs`; the algorithm identifiers and their KDF parameters live in `crates/crypto/src/encdec/id.rs`.
+
+Three properties matter for anyone reasoning about this feature:
+
+- **It affects writes only.** The decrypt path in `crates/crypto/src/encdec/id.rs` accepts all three identifiers unconditionally, and every ciphertext carries its identifier byte. Toggling the feature therefore never orphans existing data in either direction.
+- **It does not select a different implementation.** Both branches call RustCrypto. There is no validated module on either side of the switch, so the feature cannot move RustFS toward or away from validation.
+- **It is a trade-off, not an upgrade.** The FIPS-preferred branch uses PBKDF2-HMAC-SHA256 at 8192 iterations, a work factor well below current password-hashing guidance, whereas the non-FIPS branch uses memory-hard Argon2id. Against an attacker who has obtained an encrypted config or IAM blob and is attacking the passphrase offline, the default branch is the weaker of the two. Enabling the feature buys approved-algorithm alignment, not more resistance.
+
+### Rename recommendation
+
+The name `fips` states a compliance property the feature does not provide, and `rustfs-crypto` is published, so the name is visible to downstream consumers. Recommended direction:
+
+1. Introduce `fips-preferred-algs` as the real feature name, carrying the current behavior.
+2. Redefine `fips = ["fips-preferred-algs"]` so existing consumers keep building, and mark it deprecated in the crate documentation with a pointer to this document.
+3. Drop the `fips` alias after one release cycle.
+4. While renaming, raise the PBKDF2 iteration count or document the trade-off above at the feature definition, so the choice is explicit rather than inherited.
+
+This is a naming and documentation change only; no ciphertext format changes, because the identifier bytes stay as they are.
+
+## Routes to a stronger position, and what each costs
+
+### Route 1: adopt the `aws-lc-rs` FIPS variant
+
+Switch the whole process to `aws-lc-rs`'s FIPS build (backed by `aws-lc-fips-sys`) so cryptographic operations run inside a validated module boundary.
+
+**Scope.** The TLS provider swap is the small part — one feature flag plus the provider install sites. The substantial work is the data path: every AEAD call in `crates/kms/src/encryption/ciphers.rs`, `crates/kms/src/encryption/dek.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` would have to be re-implemented against `aws-lc-rs` primitives. Anything the validated module does not expose has to be dropped or moved out of the boundary: Argon2id has no FIPS status, so the Local backend's KDF and the non-FIPS branch of `rustfs-crypto` would need a compatibility story (read-only support for existing records, PBKDF2 for new ones), and ChaCha20-Poly1305 would become non-approved for new writes.
+
+**Build and platform cost.** `aws-lc-fips-sys` builds a pinned, validated source release and needs CMake, a C toolchain, and Go at build time; it supports a narrower target set than the ordinary crate. The platform matrix cost of plain AWS-LC is already documented and non-hypothetical: rustfs/backlog#883 records that the static musl release build compiles AWS-LC's `getentropy` entropy backend, which aborts on Linux kernels older than 3.17 (the Synology class of device), and that upstream considers this by design with no plan to fix it. The FIPS variant constrains the buildable matrix strictly harder than that, and pins upgrades to whatever the certified source revision allows.
+
+**What it would and would not buy.** Linking the validated module makes the accurate claim "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated". A product-level claim additionally requires a documented module boundary, approved-mode enforcement, power-on self-tests, key zeroization, and entropy-source documentation, plus the operational procedures to keep them true across releases.
+
+**Verdict.** Heavy, and it re-opens a platform-support question that is already an open problem. Justified only by a concrete customer or regulatory commitment that names FIPS as a requirement.
+
+### Route 2: let an externally validated KMS carry key operations
+
+Keep RustFS as-is and place key management inside someone else's validated boundary: the Vault Transit backend against a Vault deployment whose seal/HSM is validated, or an equivalent managed KMS.
+
+**Scope.** Mostly already built. The Transit backend (`VaultTransit`) never lets key-encryption key material leave Vault; RustFS only ever holds Transit ciphertext. What remains is configuration guidance, a supported-deployment statement, and the operational documentation that says which parts of the system are covered.
+
+**What it buys.** Master key generation, wrapping, unwrapping, and rotation happen inside the external module. That is a real, defensible partial answer to "where do keys live and who validated that": it covers the key operations, which is often the part an auditor actually asks about.
+
+**What it does not buy.** The object data path is untouched. DEKs are used for bulk AEAD by RustCrypto inside the RustFS process, and TLS still runs the non-FIPS AWS-LC build. The honest formulation is "key management operations are performed by an externally validated module; the object data path is not validated".
+
+**Verdict.** The nearest partial step, with no code rewrite and no platform-matrix risk. This is the route to point customers at when the requirement is about key custody rather than about a certificate covering the storage layer.
+
+### Route 3: make no validation claim (current default)
+
+Document the position, hold the terminology line, and revisit only when a requirement with a name attached shows up.
+
+**Cost.** This document plus the grep guard. Nothing else.
+
+**Verdict.** The current decision. FIPS 140-3 validation is explicitly not a roadmap target, and adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI.
+
+## Algorithm disablement and migration policy
+
+Retiring an algorithm from a storage system is not a code change; it is a data migration with a code change at each end. This section fixes the sequence so that no future deprecation removes a decrypt path while data still depends on it.
+
+### Every persisted artifact is self-describing
+
+The precondition for safe migration already holds: nothing relies on a global "current algorithm" setting to be decodable.
+
+- `rustfs-crypto` blobs carry the `ID` byte (`crates/crypto/src/encdec/id.rs`) immediately after the salt.
+- KMS ciphers are selected from the recorded `EncryptionAlgorithm` (`crates/kms/src/types.rs`).
+- DEK envelopes record which master key version wrapped them in `DataKeyEnvelope::master_key_version` (`crates/kms/src/encryption/dek.rs`).
+
+So for any stored object it is decidable, from the object alone, which algorithm and which key version it needs.
+
+### Deprecation classes
+
+Retirement moves an algorithm through these states, never skipping one:
+
+1. **Write-disabled, read-supported.** New writes select a replacement; existing data decrypts unchanged. This is the only step that is cheap and reversible.
+2. **Read-deprecated.** Reads still work but are counted and warned on, so the remaining population is measurable.
+3. **Read-removed.** The decrypt path is deleted. Permitted only once the remaining population is provably zero.
+
+### Sequencing rules
+
+- Never advance to read-removed on the strength of an argument that data "should have been" migrated. Removal requires evidence that nothing references the algorithm, not an elapsed-time policy.
+- A change to default algorithm selection is a compatibility event: it changes what new nodes write, which matters in a mixed-version cluster. Record it in the release notes and in the relevant crate's feature documentation, and check it against the [mixed-version constraints](kms-backend-security.md#mixed-version-clusters-during-a-rolling-upgrade).
+- Roll out write-disablement before the corresponding read change, and let the cluster fully converge in between. A build that cannot read what a peer is still writing is the failure mode to avoid.
+
+### Known gap
+
+Step 3 is currently unreachable for object data. There is no object rewrap or re-encryption capability, so there is no supported way to migrate already-written objects off an algorithm or off a master key version — the same gap that forces the rotation retention rule in [KMS backend security properties](kms-backend-security.md#retention-and-destruction-preconditions). Until a rewrap capability exists, treat every algorithm that has ever been written as permanently read-required, and confine deprecation to step 1.
@@ -12,9 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
+use std::alloc::{GlobalAlloc, Layout};
+
+#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
+#[derive(Default)]
+struct DefaultMiMalloc;
+
+#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
+// SAFETY: allocation and deallocation are forwarded unchanged to MiMalloc, so
+// MiMalloc's GlobalAlloc guarantees apply to every returned pointer and layout.
+#[allow(unsafe_code)]
+unsafe impl GlobalAlloc for DefaultMiMalloc {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        // SAFETY: the caller upholds GlobalAlloc's contract for layout.
+        unsafe { mimalloc::MiMalloc.alloc(layout) }
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+        // SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
+        unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
+    }
+}
+
 #[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
 #[global_allocator]
-static GLOBAL: hotpath::CountingAllocator = hotpath::CountingAllocator::new();
+static GLOBAL: hotpath::CountingAllocator<DefaultMiMalloc> = hotpath::CountingAllocator::new();

 #[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
 #[global_allocator]
@@ -25,3 +48,15 @@ fn main() {

    rustfs::startup_entrypoint::run_process();
 }
+
+#[cfg(all(test, feature = "hotpath", feature = "hotpath-alloc"))]
+mod tests {
+    #[test]
+    #[allow(unsafe_code)]
+    fn hotpath_allocator_uses_mimalloc() {
+        let allocation = Box::new([0_u8; 64]);
+
+        // SAFETY: the live Box pointer is valid to inspect for heap ownership.
+        assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
+    }
+}
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# ci.yml's pull_request paths-ignore and ci-docs-only.yml's paths must be equal.
+#
+# ci-docs-only.yml exists to report the required checks for pull requests that
+# ci.yml skips. The two lists are the complement of each other, so any drift
+# breaks one of two ways, both silent:
+#
+#   - an entry only in ci.yml's paths-ignore: a PR touching only those files
+#     triggers neither workflow, nobody reports "Test and Lint" or "Quick
+#     Checks", and the PR waits on a required check forever;
+#   - an entry only in ci-docs-only.yml's paths: both workflows run, which is
+#     merely wasteful — but it also means the lists no longer describe the same
+#     intent, and the next edit is made against a wrong assumption.
+#
+# The push paths-ignore in ci.yml is deliberately NOT compared: no required
+# check is reported for push events, so it does not have to pair with anything.
+#
+# Also asserts ci-docs-only.yml still declares both companion job names, since a
+# rename there produces exactly the permanent-pending failure above.
+#
+# Usage: scripts/check_ci_paths_sync.sh
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+CI=".github/workflows/ci.yml"
+DOCS=".github/workflows/ci-docs-only.yml"
+
+# Print the quoted list items that follow $2 within the block introduced by $1.
+# Both files keep these as a flat list of quoted scalars, so no YAML parser is
+# needed and the script stays dependency-free like its check_* siblings.
+extract() {
+    local file="$1" event="$2" key="$3"
+    awk -v event="$event" -v key="$key" '
+        $0 ~ "^  " event ":[[:space:]]*$" { in_event = 1; next }
+        in_event && /^  [a-z_]+:[[:space:]]*$/ { in_event = 0 }
+        in_event && $0 ~ "^    " key ":[[:space:]]*$" { in_list = 1; next }
+        in_list {
+            if ($0 ~ /^      - /) {
+                item = $0
+                sub(/^      - /, "", item)
+                gsub(/^"|"$/, "", item)
+                print item
+                next
+            }
+            if ($0 !~ /^[[:space:]]*#/ && $0 !~ /^[[:space:]]*$/) in_list = 0
+        }
+    ' "$file" | sort
+}
+
+ci_list="$(extract "$CI" "pull_request" "paths-ignore")"
+docs_list="$(extract "$DOCS" "pull_request" "paths")"
+
+if [ -z "$ci_list" ] || [ -z "$docs_list" ]; then
+    echo "ERROR: could not read one of the path lists — did the file structure change?" >&2
+    echo "  $CI pull_request.paths-ignore: $(printf '%s' "$ci_list" | grep -c . || true) entries" >&2
+    echo "  $DOCS pull_request.paths: $(printf '%s' "$docs_list" | grep -c . || true) entries" >&2
+    exit 1
+fi
+
+status=0
+
+if ! diff_out="$(diff <(printf '%s\n' "$ci_list") <(printf '%s\n' "$docs_list"))"; then
+    echo "ERROR: $CI pull_request paths-ignore and $DOCS paths have drifted." >&2
+    echo "  '<' is only in $CI, '>' is only in $DOCS:" >&2
+    printf '%s\n' "$diff_out" | sed 's/^/    /' >&2
+    status=1
+fi
+
+for job_name in "Test and Lint" "Quick Checks"; do
+    if ! grep -q "name: ${job_name}\$" "$DOCS"; then
+        echo "ERROR: $DOCS no longer declares a job named '${job_name}'." >&2
+        echo "  It is a required status check; without a companion job here, a" >&2
+        echo "  docs-only PR waits on it forever." >&2
+        status=1
+    fi
+done
+
+if [ "$status" -ne 0 ]; then
+    exit 1
+fi
+
+echo "OK: ci.yml and ci-docs-only.yml path lists agree ($(printf '%s\n' "$ci_list" | wc -l | tr -d ' ') entries)"
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# The io_uring CI lane runs `cargo test -p rustfs-ecstore --lib uring_`.
+#
+# `--lib` is there to avoid compiling and linking the 7 integration binaries
+# under crates/ecstore/tests/, which selected zero tests for this filter. That is
+# only safe while it stays true: add a matching test under tests/ and `--lib`
+# would skip it silently, with CI staying green — the failure mode is invisible,
+# so it is asserted here instead.
+#
+# The pattern must use the same `uring_` substring semantics as libtest, which
+# also matches names containing `during_`. Narrowing it to `io_uring` would make
+# this guard disagree with the filter it is protecting.
+#
+# Usage: scripts/check_uring_lane_lib_only.sh
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+TESTS_DIR="crates/ecstore/tests"
+
+if [ ! -d "$TESTS_DIR" ]; then
+    echo "OK: $TESTS_DIR does not exist; nothing for --lib to miss"
+    exit 0
+fi
+
+hits="$(grep -rEn '^[[:space:]]*(pub[[:space:]]+)?(async[[:space:]]+)?fn[[:space:]]+[A-Za-z0-9_]*uring_' \
+        "$TESTS_DIR" || true)"
+
+if [ -n "$hits" ]; then
+    echo "ERROR: test functions matching the 'uring_' substring found under $TESTS_DIR:" >&2
+    printf '%s\n' "$hits" | sed 's/^/  /' >&2
+    echo "" >&2
+    echo "The io_uring lane in .github/workflows/ci.yml uses --lib, so these would" >&2
+    echo "never run and CI would stay green. Pick one:" >&2
+    echo "  - move them into a #[cfg(test)] mod under crates/ecstore/src, or" >&2
+    echo "  - drop --lib from that step (and pay the link cost for all 7 binaries), or" >&2
+    echo "  - add an explicit --test <name> for the binary and update this script." >&2
+    exit 1
+fi
+
+echo "OK: no 'uring_'-matching test functions under $TESTS_DIR; --lib is safe"
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Background resource sampler for the Test and Lint lane (issue #5394).
+#
+# The post-mortem `pgrep` in ci.yml runs only after `timeout` has already TERM'd
+# the whole cargo process group, so it can never name a wedged process. This
+# samples every 60s instead; the last samples before a timeout show what was
+# stuck — rustc, the linker, a build script, memory pressure.
+#
+# WHY THE CGROUP READINGS MATTER
+#
+# The runners are ARC (Actions Runner Controller) pods, so /proc/loadavg,
+# /proc/pressure/*, `free` and `df` are all *node*-level: they include every
+# other runner pod scheduled on the same Kubernetes node. A sample showing high
+# I/O pressure says nothing about whether *this* job caused it. Measured
+# evidence: one sample reported loadavg 6.67 with 832 threads node-wide while
+# `ps` inside the pod showed about 10 processes.
+#
+# Only the /sys/fs/cgroup/* values and `ps` are attributable to this job, so
+# those are what any CARGO_BUILD_JOBS experiment must be judged on. The
+# node-level readings are kept — co-tenancy is itself a real cause of stalls,
+# and a 9m57s `git checkout` was traced to it — but labelled NODE-LEVEL so
+# nobody reads them as this pod's own load.
+#
+# Usage:
+#   scripts/ci/resource_sampler.sh start <phase>   # e.g. clippy, nextest, doctest
+#   scripts/ci/resource_sampler.sh stop
+#
+# Safe to call `stop` when nothing is running, and safe to `start` a new phase
+# while another is sampling — the previous one is stopped first. Never fails the
+# calling step: this is diagnostics, not a gate.
+set -uo pipefail
+
+# GNU date on the runners; the fallback keeps local smoke tests on macOS quiet.
+now() { date --utc --iso-8601=seconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%S+00:00; }
+
+LOG_DIR="artifacts/test-and-lint"
+LOG_FILE="${LOG_DIR}/sampler.log"
+PID_FILE="${LOG_DIR}/.sampler.pid"
+INTERVAL="${SAMPLER_INTERVAL_SECS:-60}"
+
+stop_sampler() {
+    [ -f "$PID_FILE" ] || return 0
+    local pid
+    pid="$(cat "$PID_FILE" 2>/dev/null || true)"
+    if [ -n "${pid:-}" ]; then
+        kill "$pid" 2>/dev/null || true
+    fi
+    rm -f "$PID_FILE"
+}
+
+sample_once() {
+    echo "=== $(now)"
+
+    # Attributable to this pod: cgroup v2 accounting and our own process table.
+    echo "--- POD cpu/mem limits"
+    grep -H . /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max \
+              /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current 2>/dev/null || true
+    echo "--- POD pressure (cgroup v2, this pod only)"
+    grep -H . /sys/fs/cgroup/cpu.pressure /sys/fs/cgroup/io.pressure \
+              /sys/fs/cgroup/memory.pressure 2>/dev/null || true
+    echo "--- POD nproc"; nproc 2>/dev/null || true
+
+    # NODE-LEVEL: shared with every other runner pod on this Kubernetes node.
+    # Useful for spotting co-tenancy, useless for attributing load to this job.
+    echo "--- NODE-LEVEL load (shared with co-tenant runners)"; cat /proc/loadavg 2>/dev/null || true
+    echo "--- NODE-LEVEL psi (shared with co-tenant runners)"
+    grep -H . /proc/pressure/* 2>/dev/null || true
+    echo "--- NODE-LEVEL mem (shared with co-tenant runners)"; free -m 2>/dev/null || true
+    echo "--- NODE-LEVEL disk (shared with co-tenant runners)"
+    df -h / /home/runner 2>/dev/null || df -h / 2>/dev/null || true
+
+    echo "--- top-rss"
+    ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss 2>/dev/null | head -15 || true
+    echo "--- build/test processes"
+    ps -eo pid,ppid,stat,etime,rss,pcpu,args 2>/dev/null \
+        | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
+    echo "--- d-state (uninterruptible IO)"
+    ps -eo pid,stat,etime,args 2>/dev/null | awk 'NR > 1 && $2 ~ /D/' || true
+    echo
+}
+
+case "${1:-}" in
+    start)
+        phase="${2:-unknown}"
+        mkdir -p "$LOG_DIR"
+        stop_sampler
+        {
+            echo "--- phase=${phase} started_at=$(now) runner=${RUNNER_NAME:-unknown}"
+        } >> "$LOG_FILE" 2>&1 || true
+        (
+            while true; do
+                sample_once >> "$LOG_FILE" 2>&1 || true
+                sleep "$INTERVAL"
+            done
+        ) &
+        echo $! > "$PID_FILE"
+        ;;
+    stop)
+        stop_sampler
+        ;;
+    *)
+        echo "usage: $0 start <phase> | stop" >&2
+        exit 2
+        ;;
+esac
+
+exit 0
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Every `uses: ./.github/actions/setup` must state `cache-save-if` explicitly.
+#
+# The composite action's default is "false" (fail-safe: a forgotten input costs
+# a cold cache, not a slice of the repository-wide 10GB Actions cache quota).
+# But a silent default in either direction is how audit.yml ended up saving a
+# ~843MB PR-scoped entry on every dependency change, evicting the main-scoped
+# lanes that the expensive jobs restore from. Requiring the input to be spelled
+# out keeps the decision visible in review.
+#
+# Usage: scripts/security/check_cache_save_if.sh
+set -euo pipefail
+
+cd "$(dirname "$0")/../.."
+
+status=0
+
+for file in .github/workflows/*.yml .github/workflows/*.yaml; do
+    [ -e "$file" ] || continue
+
+    # Walk the file, and for every `uses: ./.github/actions/setup` scan the rest
+    # of that step for cache-save-if. `uses:` and `with:` are siblings at the
+    # same indentation, so the step ends only at a line indented *less* than the
+    # `uses:` line (the next `- name:` item, or a dedent out of the steps list).
+    awk -v file="$file" '
+        function flush() {
+            if (pending && !found) {
+                printf "%s:%d: `uses: ./.github/actions/setup` without an explicit `cache-save-if:`\n", file, uses_line > "/dev/stderr"
+                bad++
+            }
+            pending = 0; found = 0
+        }
+        {
+            line = $0
+            sub(/[[:space:]]+$/, "", line)
+            if (line ~ /^[[:space:]]*#/ || line ~ /^[[:space:]]*$/) next
+
+            match(line, /^[[:space:]]*/)
+            indent = RLENGTH
+
+            if (pending && indent < uses_indent) flush()
+
+            if (line ~ /uses:[[:space:]]*\.\/\.github\/actions\/setup[[:space:]]*$/) {
+                flush()
+                pending = 1; found = 0
+                uses_indent = indent
+                uses_line = NR
+                next
+            }
+
+            if (pending && line ~ /^[[:space:]]*cache-save-if:/) found = 1
+        }
+        END { flush(); exit (bad > 0) }
+    ' "$file" || status=1
+done
+
+if [ "$status" -ne 0 ]; then
+    echo "" >&2
+    echo "Add an explicit cache-save-if to each call above, e.g." >&2
+    echo "    cache-save-if: \${{ github.ref == 'refs/heads/main' }}   # this lane owns the key" >&2
+    echo "    cache-save-if: 'false'                                   # this lane only reads it" >&2
+    echo "Exactly one lane per shared-key may save; see rustfs/backlog#1600." >&2
+    exit 1
+fi
+
+echo "OK: every ./.github/actions/setup call states cache-save-if explicitly"
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Every job that occupies a runner must declare timeout-minutes.
+#
+# GitHub's default is 360 minutes. This repository has a history of runners
+# stalling intermittently (#5394) and of a plain `git checkout` taking 9m57s
+# under node-level I/O contention, so an undeclared timeout means one wedged job
+# can hold a runner for six hours out of a pool of roughly 15-21.
+#
+# Only jobs with `runs-on` are checked: a job that calls a reusable workflow has
+# no runner of its own and cannot declare a timeout.
+#
+# Usage: scripts/security/check_job_timeouts.sh
+set -euo pipefail
+
+cd "$(dirname "$0")/../.."
+
+status=0
+
+for file in .github/workflows/*.yml .github/workflows/*.yaml; do
+    [ -e "$file" ] || continue
+
+    awk -v file="$file" '
+        function flush() {
+            if (job != "" && has_runs_on && !has_timeout) {
+                printf "%s:%d: job `%s` has runs-on but no timeout-minutes\n", file, job_line, job > "/dev/stderr"
+                bad++
+            }
+            job = ""; has_runs_on = 0; has_timeout = 0
+        }
+        /^jobs:[[:space:]]*$/ { in_jobs = 1; next }
+        {
+            line = $0
+            sub(/[[:space:]]+$/, "", line)
+            if (line ~ /^[[:space:]]*#/ || line ~ /^[[:space:]]*$/) next
+
+            # A non-indented key ends the jobs mapping.
+            if (in_jobs && line !~ /^[[:space:]]/) { flush(); in_jobs = 0 }
+            if (!in_jobs) next
+
+            if (line ~ /^  [a-zA-Z0-9_-]+:[[:space:]]*$/) {
+                flush()
+                job = line
+                sub(/^[[:space:]]*/, "", job)
+                sub(/:.*$/, "", job)
+                job_line = NR
+                next
+            }
+            if (job == "") next
+            if (line ~ /^    runs-on:/) has_runs_on = 1
+            if (line ~ /^    timeout-minutes:/) has_timeout = 1
+        }
+        END { flush(); exit (bad > 0) }
+    ' "$file" || status=1
+done
+
+if [ "$status" -ne 0 ]; then
+    echo "" >&2
+    echo "Add timeout-minutes to each job above. Rough budgets used in this repo:" >&2
+    echo "  10  echo-only and guard-script jobs" >&2
+    echo "  30  jobs that call the GitHub API, upload assets, or push over the network" >&2
+    echo "  90+ anything using ./.github/actions/setup (cold cache restore alone is 11-21 min)" >&2
+    exit 1
+fi
+
+echo "OK: every job with runs-on declares timeout-minutes"