fix(admin): protect notification target edits (#9048)

* fix(admin): protect notification target edits

Use a durable configuration revision and conditional mutation to preserve concurrent changes and inherited target settings.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(admin): use the gateway error boundary

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* refactor(admin): limit target detail health probes

Reuse normalized source information and probe only runtime targets matching the requested instance.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* refactor(admin): streamline target config validation

Preserve duplicate-key order when comparing snapshots, avoid copying secret values, and reuse the validated target configuration.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=a67a347f26be186c598b5beaf68e6bcbe1855350a35e1423dd5b8a81bcba06a5
-sha256-linux=ad4fabe811c6cca1aba89fad34ccd404b5832a6a955875c2826f6e0fa1ea4f1f
+sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
+sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=f065e7071aac9d48a7c1b301f328fa7cc14f11c4e7dc46da3e55057c4ea29bd5
+sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3

.config/nextest.toml

@@ -128,7 +128,7 @@ setup = 'ecstore-large-stack'
 # path on a real store; measured with RUST_MIN_STACK on 2026-10-15 they need
 # 2016-2040 KiB of the 2 MiB libtest default and overflowed on Linux CI.
 [[profile.default.scripts]]
-filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
+filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
 setup = 'ecstore-base-stack'
 
 [[profile.default.scripts]]
@@ -419,7 +419,7 @@ setup = 'ecstore-large-stack'
 
 # These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
 [[profile.ci.scripts]]
-filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
+filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
 setup = 'ecstore-base-stack'
 
 [[profile.ci.scripts]]

.github/workflows/README.md

@@ -16,7 +16,7 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
 | `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
 | `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
 | `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
-| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
+| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | Docker workloads with a measured resource or environment requirement beyond the hosted runner |
 | `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
 
 The self-hosted pods and the hosted image are both Ubuntu 24.04, so a binary built on one runs on the other; `ci.yml` builds the debug binary on `sm-standard-4` and runs it on both pools.
@@ -27,11 +27,15 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
 
 1. **Disk.** It restores or produces the workspace `target/`: `cargo build` or `cargo test` with `--all-targets`, `cargo nextest run --all`, or any reader of the `ci-dev` / `ci-feat-*` caches. The workspace test build leaves about 64 GB in `target/`; a hosted runner has about 14 GB.
 2. **Critical-path CPU.** It is a build that several downstream lanes wait on and would blow its timeout at 4 vCPU (`build-rustfs-debug-binary` gates the e2e and s3-tests lanes).
-3. **Hardware.** Native aarch64, Docker-in-Docker, or the lab machines.
+3. **Hardware or environment.** Native aarch64 or access to the lab machines. Docker alone is not a reason: hosted Ubuntu ships Docker and supports the QEMU and Buildx setup actions.
 4. **Cache ownership.** It reads a `rust-cache` key that a self-hosted job writes (see the cache rule below).
 
 Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
 
+The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
+
+Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
+
 A hosted runner is also the only choice for a few things the pods cannot do: `mount -o loop` and `mount -t tmpfs` fail inside the ARC pods, and a container runtime cannot guarantee that `io_uring_setup` is allowed. `uring-integration` and `e2e-distributed` are on `ubuntu-latest` for that reason, not for cost.
 
 ## Cache rule

.github/workflows/docker.yml

@@ -337,7 +337,7 @@ jobs:
     name: Build Docker Images
     needs: build-check
     if: needs.build-check.outputs.should_build == 'true'
-    runs-on: dind-sm-standard-2
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/runner-hygiene.yml

@@ -44,7 +44,7 @@ jobs:
   check-ephemerality:
     if: github.repository == 'rustfs/rustfs'
     name: Check runner ephemerality
-    runs-on: sm-standard-2
+    runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
       - name: Checkout repository

.github/workflows/rustfs-functional-chain.yml

@@ -42,7 +42,7 @@ concurrency:
 jobs:
   prepare:
     if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
-    runs-on: smoke-testing
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
       manifest: ${{ steps.candidate.outputs.manifest }}
@@ -188,7 +188,7 @@ jobs:
   complete-chain:
     needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, fault-tolerance, table, performance]
     if: ${{ always() && needs.prepare.result != 'skipped' }}
-    runs-on: smoke-testing
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/rustfs-performance-test.yml

@@ -90,7 +90,7 @@ env:
 jobs:
   runner-check:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    runs-on: sm-standard-2
+    runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/windows-filesystem.yml

@@ -1,259 +1,224 @@
-# Temporary, manual-only verification of the test-only isolation fixture.
-# The exporter is in memory; no OTLP wire serialization is exercised.
-name: TLS diagnostic async isolation preflight
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Windows Filesystem Tests
 
 on:
+  push:
+    branches: [ main ]
+    paths:
+      - "crates/ecstore/src/disk/**"
+      - "crates/ecstore/src/layout/**"
+      - "crates/ecstore/src/store/init_format.rs"
+      - "crates/ecstore/src/set_disk/ops/locking.rs"
+      - "crates/ecstore/src/object_api/write_commit_context.rs"
+      - "crates/ecstore/src/bucket/utils.rs"
+      - "crates/ecstore/Cargo.toml"
+      - "crates/utils/src/dunce.rs"
+      - "crates/utils/src/os/**"
+      - "crates/utils/Cargo.toml"
+      - "crates/utils/src/path.rs"
+      - "rustfs/build.rs"
+      - "rustfs/Cargo.toml"
+      - "rustfs/src/main.rs"
+      - "rustfs/src/server/runtime.rs"
+      - "rustfs/src/startup_*.rs"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - ".github/actions/setup/**"
+      - ".github/workflows/windows-filesystem.yml"
+  pull_request:
+    branches: [ main ]
+    paths:
+      - "crates/ecstore/src/disk/**"
+      - "crates/ecstore/src/layout/**"
+      - "crates/ecstore/src/store/init_format.rs"
+      - "crates/ecstore/src/set_disk/ops/locking.rs"
+      - "crates/ecstore/src/object_api/write_commit_context.rs"
+      - "crates/ecstore/src/bucket/utils.rs"
+      - "crates/ecstore/Cargo.toml"
+      - "crates/utils/src/dunce.rs"
+      - "crates/utils/src/os/**"
+      - "crates/utils/Cargo.toml"
+      - "crates/utils/src/path.rs"
+      - "rustfs/build.rs"
+      - "rustfs/Cargo.toml"
+      - "rustfs/src/main.rs"
+      - "rustfs/src/server/runtime.rs"
+      - "rustfs/src/startup_*.rs"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - ".github/actions/setup/**"
+      - ".github/workflows/windows-filesystem.yml"
   workflow_dispatch:
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+  RUST_BACKTRACE: 1
+
 jobs:
-  isolation:
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-      CARGO_INCREMENTAL: "0"
-      CARGO_BUILD_JOBS: "2"
-      CARGO_PROFILE_DEV_DEBUG: "0"
-      CARGO_PROFILE_TEST_DEBUG: "0"
-      CARGO_TERM_COLOR: never
-      RUST_BACKTRACE: "0"
-      CARGO_TARGET_DIR: ${{ github.workspace }}/target
-      EXPECTED_REVISION: 6010f8cd2d0dab27e52a98d305afd11926df56dd
-      EXPECTED_TREE: cf952d55d16f7c159fc6c7b0dfa2ec35a6df38a9
+  rename-safety:
+    name: Rename Safety
+    runs-on: windows-latest
+    timeout-minutes: 60
     steps:
-      - name: Checkout fixed source
-        id: source
+      - name: Checkout repository
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
-          ref: 6010f8cd2d0dab27e52a98d305afd11926df56dd
-          clean: false
           persist-credentials: false
 
-      - name: Verify source and prepare private output
-        id: identity
-        env:
-          WORKFLOW_REVISION: ${{ github.sha }}
-        run: |
-          set -euo pipefail
-          test "$(git rev-parse HEAD)" = "$EXPECTED_REVISION"
-          test "$(git rev-parse 'HEAD^{tree}')" = "$EXPECTED_TREE"
-          test -z "$(git status --porcelain=v1 --untracked-files=all)"
-          umask 077
-          mkdir "$RUNNER_TEMP/tls-async-private"
-          mkdir "$RUNNER_TEMP/tls-async-evidence"
-          python3 - <<'PY'
-          import hashlib
-          import json
-          import os
-          import sys
-          from pathlib import Path
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: build-x86_64-pc-windows-msvc
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+          install-test-tools: 'false'
 
-          sys.excepthook = lambda *_: print('Source identity unavailable; raw output withheld.', file=sys.stderr)
-          source = Path('crates/obs/src/telemetry/isolation_probe_tests.rs').read_bytes()
-          identity = {
-              'source_revision': os.environ['EXPECTED_REVISION'],
-              'source_tree': os.environ['EXPECTED_TREE'],
-              'workflow_revision': os.environ['WORKFLOW_REVISION'],
-              'test_sha256': hashlib.sha256(source).hexdigest(),
+      - name: Check production Windows dependencies
+        shell: pwsh
+        run: cargo check -p rustfs-ecstore --lib
+
+      - name: Test guarded rename publication
+        shell: pwsh
+        run: cargo test -p rustfs-ecstore --lib rename_all_ -- --nocapture
+
+      - name: Test multipart parent durability barriers
+        shell: pwsh
+        run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
+
+      - name: Test Windows path conversion
+        shell: pwsh
+        run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
+
+      - name: Test Windows volume and disk-space probes
+        shell: pwsh
+        run: |
+          # Map a drive letter to the temp drive's administrative share so the
+          # remote-volume probe also covers mapped network drives.
+          $drive = (Split-Path -Qualifier ([System.IO.Path]::GetTempPath())).TrimEnd(':')
+          net use R: "\\localhost\$drive`$" /persistent:no
+          if ($LASTEXITCODE -eq 0) {
+            $env:RUSTFS_TEST_WINDOWS_MAPPED_DRIVE = 'R:'
+          } else {
+            Write-Output "::warning::could not map R: to \\localhost\$drive`$; mapped-drive test will skip"
           }
-          output = Path(os.environ['RUNNER_TEMP'], 'tls-async-evidence', 'identity.json')
-          output.write_text(json.dumps(identity, sort_keys=True) + '\n')
-          PY
+          cargo test -p rustfs-utils --features os --lib os:: -- --nocapture
+          $status = $LASTEXITCODE
+          if ($env:RUSTFS_TEST_WINDOWS_MAPPED_DRIVE) { net use R: /delete /y | Out-Null }
+          exit $status
 
-      - name: Install system dependencies
-        id: dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential pkg-config libssl-dev ripgrep
+      - name: Test Windows handle guards
+        shell: pwsh
+        run: cargo test -p rustfs-ecstore --lib windows_ -- --nocapture
 
-      - name: Install protoc
-        id: protoc
-        uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
+      - name: Test endpoint path parsing
+        shell: pwsh
+        run: "cargo test -p rustfs-ecstore --lib layout::endpoint:: -- --nocapture"
+
+      - name: Test startup temporary-directory cleanup
+        shell: pwsh
+        run: cargo test -p rustfs-ecstore --lib cleanup_tmp_on_startup_ -- --nocapture
+
+      - name: Test object key Windows admission
+        shell: pwsh
+        run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
+
+      - name: Test Windows file-name rules
+        shell: pwsh
+        run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
+
+      - name: Test fresh format publication
+        shell: pwsh
+        run: cargo test -p rustfs-ecstore --lib fresh_format_load_initializes_all_disks -- --nocapture
+
+  # The native Windows main thread only gets the 1 MiB PE stack reserve, while
+  # Linux and macOS hand out 8 MiB. Startup runs the whole root future on the
+  # thread that calls `block_on`, so a debug server once overflowed before it
+  # listened on Windows only. Start a real server to keep that covered.
+  startup-smoke:
+    name: Startup Smoke
+    runs-on: windows-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
-          version: "35.1"
-          repo-token: ${{ github.token }}
+          persist-credentials: false
 
-      - name: Install flatc
-        id: flatc
-        uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
         with:
-          version: "25.12.19"
+          rust-version: stable
+          cache-shared-key: build-x86_64-pc-windows-msvc
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+          install-test-tools: 'false'
 
-      - name: Install Rust
-        id: rust
-        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-        with:
-          toolchain: stable
+      - name: Build debug server binary
+        shell: pwsh
+        run: cargo build --locked -p rustfs --bin rustfs
 
-      - name: Record toolchain identity
-        id: toolchain
+      - name: Print version
+        shell: pwsh
         run: |
-          python3 - <<'PY'
-          import json
-          import os
-          import re
-          import subprocess
-          import sys
-          from pathlib import Path
+          & .\target\debug\rustfs.exe --version
+          if ($LASTEXITCODE -ne 0) { throw "rustfs --version exited with $LASTEXITCODE" }
 
-          sys.excepthook = lambda *_: print('Toolchain identity unavailable; raw output withheld.', file=sys.stderr)
-          versions = {}
-          for program in ('rustc', 'cargo'):
-              version = subprocess.run([program, '--version'], check=True, capture_output=True, text=True).stdout.strip()
-              if not re.fullmatch(program + r' [a-zA-Z0-9+(). _-]{1,120}', version):
-                  raise SystemExit('Unexpected toolchain version; raw output withheld.')
-              versions[program] = version
-          output = Path(os.environ['RUNNER_TEMP'], 'tls-async-evidence', 'toolchain.json')
-          output.write_text(json.dumps(versions, sort_keys=True) + '\n')
-          PY
-
-      - name: Compile the library test binary
-        id: compile
-        run: |
-          set -euo pipefail
-          if ! cargo test --locked -p rustfs-obs --lib --no-run --message-format=json >"$RUNNER_TEMP/tls-async-private/build.jsonl" 2>"$RUNNER_TEMP/tls-async-private/build.log"; then
-            echo "The library test binary did not compile; raw output withheld."
-            exit 1
-          fi
-
-      - name: Run the six named isolation tests
-        id: assertions
-        run: |
-          set -euo pipefail
-          test "$(git rev-parse HEAD)" = "$EXPECTED_REVISION"
-          test "$(git rev-parse 'HEAD^{tree}')" = "$EXPECTED_TREE"
-          test -z "$(git status --porcelain=v1 --untracked-files=all)"
-          python3 - <<'PY'
-          import hashlib
-          import json
-          import os
-          import subprocess
-          import sys
-          from pathlib import Path
-
-          sys.excepthook = lambda *_: print('Preflight evidence unavailable; raw output withheld.', file=sys.stderr)
-          prefix = 'telemetry::isolation_probe_tests::async_isolation::'
-          cases = (
-              ('threads', prefix + 'separate_threads_keep_their_contexts_across_async_suspension'),
-              ('shared_runtime', prefix + 'shared_runtime_reenters_outer_context_after_each_await'),
-              ('one_step_control', prefix + 'either_isolation_step_alone_leaves_ambient_context'),
-              ('filters_and_resource', prefix + 'filters_and_resources_do_not_cross_dispatchers'),
-              ('cancel_drop', prefix + 'cancellation_drop_emits_in_isolation_and_restores_context'),
-              ('panic_restore', prefix + 'panic_during_sink_write_restores_the_outer_context'),
-          )
-          root = Path(os.environ['RUNNER_TEMP'])
-          private = root / 'tls-async-private'
-          evidence = root / 'tls-async-evidence'
-          build = private / 'build.jsonl'
-          if not build.is_file() or build.stat().st_size > 256 * 1024 * 1024:
-              raise SystemExit('Build output unavailable or too large; raw output withheld.')
-          binaries = set()
-          with build.open(encoding='utf-8') as lines:
-              for line in lines:
-                  if len(line) > 2 * 1024 * 1024:
-                      raise SystemExit('Build output record too large; raw output withheld.')
-                  try:
-                      value = json.loads(line)
-                  except ValueError:
-                      raise SystemExit('Build output was not valid JSON; raw output withheld.')
-                  if value.get('reason') != 'compiler-artifact':
-                      continue
-                  target = value.get('target', {})
-                  profile = value.get('profile', {})
-                  executable = value.get('executable')
-                  if (target.get('name') == 'rustfs_obs' and 'lib' in target.get('kind', [])
-                          and profile.get('test') is True and isinstance(executable, str)):
-                      binaries.add(executable)
-          if len(binaries) != 1:
-              raise SystemExit('Expected one library test binary; raw output withheld.')
-          binary = Path(next(iter(binaries))).resolve(strict=True)
-          target_root = Path(os.environ['CARGO_TARGET_DIR']).resolve(strict=True)
-          if not binary.is_file() or not binary.is_relative_to(target_root):
-              raise SystemExit('Test binary identity was unavailable; raw output withheld.')
-
-          def digest(path):
-              hasher = hashlib.sha256()
-              with path.open('rb') as handle:
-                  for block in iter(lambda: handle.read(1024 * 1024), b''):
-                      hasher.update(block)
-              return hasher.hexdigest()
-
-          binary_sha256 = digest(binary)
-          binary_size = binary.stat().st_size
-          results = []
-          for index, (label, name) in enumerate(cases):
-              output = private / f'test-{index}.log'
-              command = ['cargo', 'test', '--offline', '--locked', '-p', 'rustfs-obs', '--lib', name,
-                         '--', '--exact', '--color', 'never']
-              try:
-                  with output.open('wb') as handle:
-                      completed = subprocess.run(command, stdout=handle, stderr=subprocess.STDOUT, timeout=120, check=False)
-              except subprocess.TimeoutExpired:
-                  results.append({'case': label, 'state': 'timeout', 'exit_code': None, 'selected': False, 'passed': False})
-                  break
-              if not output.is_file():
-                  results.append({'case': label, 'state': 'missing', 'exit_code': completed.returncode, 'selected': False, 'passed': False})
-                  continue
-              if output.stat().st_size > 16 * 1024 * 1024:
-                  results.append({'case': label, 'state': 'truncated', 'exit_code': completed.returncode, 'selected': False, 'passed': False})
-                  continue
-              lines = output.read_text(encoding='utf-8', errors='replace').splitlines()
-              selected = lines.count('running 1 test') == 1
-              passed = lines.count(f'test {name} ... ok') == 1
-              results.append({'case': label, 'state': 'available', 'exit_code': completed.returncode,
-                              'selected': selected, 'passed': passed})
-          binary_unchanged = binary.is_file() and binary.stat().st_size == binary_size and digest(binary) == binary_sha256
-          accepted = (len(results) == len(cases) and binary_unchanged
-                      and all(item['state'] == 'available' and item['exit_code'] == 0
-                              and item['selected'] and item['passed'] for item in results))
-          report = {'binary_sha256': binary_sha256, 'binary_size': binary_size, 'binary_unchanged': binary_unchanged,
-                    'expected_cases': len(cases), 'cases': results, 'expectation_met': accepted}
-          (evidence / 'results.json').write_text(json.dumps(report, sort_keys=True) + '\n')
-          if not accepted:
-              raise SystemExit('Isolation assertions did not all pass; raw output withheld.')
-          print('All six exact isolation tests passed.')
-          PY
-          test "$(git rev-parse HEAD)" = "$EXPECTED_REVISION"
-          test "$(git rev-parse 'HEAD^{tree}')" = "$EXPECTED_TREE"
-          test -z "$(git status --porcelain=v1 --untracked-files=all)"
-
-      - name: Record stage outcomes
-        if: always()
+      - name: Start server and wait for readiness
+        shell: pwsh
         env:
-          SOURCE: ${{ steps.source.outcome }}
-          IDENTITY: ${{ steps.identity.outcome }}
-          DEPENDENCIES: ${{ steps.dependencies.outcome }}
-          PROTOC: ${{ steps.protoc.outcome }}
-          FLATC: ${{ steps.flatc.outcome }}
-          RUST: ${{ steps.rust.outcome }}
-          TOOLCHAIN: ${{ steps.toolchain.outcome }}
-          COMPILE: ${{ steps.compile.outcome }}
-          ASSERTIONS: ${{ steps.assertions.outcome }}
+          RUSTFS_ACCESS_KEY: rustfsadmin
+          RUSTFS_SECRET_KEY: rustfsadmin
         run: |
-          umask 077
-          mkdir -p "$RUNNER_TEMP/tls-async-evidence"
-          python3 - <<'PY'
-          import json
-          import os
-          import sys
-          from pathlib import Path
-
-          sys.excepthook = lambda *_: print('Stage evidence unavailable; raw output withheld.', file=sys.stderr)
-          names = ('SOURCE', 'IDENTITY', 'DEPENDENCIES', 'PROTOC', 'FLATC', 'RUST', 'TOOLCHAIN', 'COMPILE', 'ASSERTIONS')
-          allowed = {'success', 'failure', 'cancelled', 'skipped'}
-          stages = {name.lower(): os.environ.get(name) if os.environ.get(name) in allowed else 'missing' for name in names}
-          output = Path(os.environ['RUNNER_TEMP'], 'tls-async-evidence', 'stages.json')
-          output.write_text(json.dumps(stages, sort_keys=True) + '\n')
-          PY
-
-      - name: Upload sanitized evidence
-        if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: tls-async-isolation-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/tls-async-evidence/
-          retention-days: 7
-          if-no-files-found: error
+          $volume = Join-Path $env:RUNNER_TEMP 'rustfs-startup-smoke'
+          $stdout = Join-Path $env:RUNNER_TEMP 'rustfs-startup-smoke.out.log'
+          $stderr = Join-Path $env:RUNNER_TEMP 'rustfs-startup-smoke.err.log'
+          New-Item -ItemType Directory -Force -Path $volume | Out-Null
+          $server = Start-Process -FilePath .\target\debug\rustfs.exe `
+            -ArgumentList @('--address', '127.0.0.1:9000', "`"$volume`"") `
+            -RedirectStandardOutput $stdout -RedirectStandardError $stderr `
+            -NoNewWindow -PassThru
+          # Cache the process handle now; otherwise ExitCode reads back empty.
+          $null = $server.Handle
+          $ready = $false
+          try {
+            for ($i = 0; $i -lt 180; $i++) {
+              if ($server.HasExited) { break }
+              try {
+                $response = Invoke-WebRequest -Uri 'http://127.0.0.1:9000/health/ready' -UseBasicParsing -NoProxy -TimeoutSec 2
+                if ($response.StatusCode -eq 200) { $ready = $true; break }
+              } catch {}
+              Start-Sleep -Seconds 1
+            }
+          } finally {
+            if (-not $server.HasExited) { Stop-Process -Id $server.Id -Force }
+            $server.WaitForExit()
+            Write-Host '--- server stdout (tail) ---'
+            Get-Content $stdout -Tail 100 -ErrorAction SilentlyContinue
+            Write-Host '--- server stderr (tail) ---'
+            Get-Content $stderr -Tail 100 -ErrorAction SilentlyContinue
+          }
+          if (-not $ready) {
+            throw ("rustfs did not report ready on /health/ready (exit code: {0})" -f $server.ExitCode)
+          }

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ rustfs-io-core = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 
 # Async runtime
-tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "time"] }
 
 # Logging
 tracing = { workspace = true }

crates/config/README.md

@@ -100,6 +100,7 @@ when restoring `/rustfs/console` for a custom OEM build.
 ## S3 API environment variables
 
 - `RUSTFS_API_OBJECT_MAX_VERSIONS` caps the number of retained versions for a single object. It defaults to `9223372036854775807`, matching MinIO's practical-unlimited default. Set a positive integer to enforce a lower per-object metadata bound.
+- `RUSTFS_VEEAM_SOSAPI_ENABLE` serves Veeam SOSAPI `system.xml` and `capacity.xml` for an authenticated GET or HEAD of those exact bucket-root keys. It defaults to `true`. Explicit version reads use the normal object path. Capacity requests return `ServiceUnavailable` until the scanner publishes a complete snapshot certified for the current bucket incarnation and authoritative quota metadata can be read. Legacy or mixed-version snapshots without an incarnation proof cannot serve capacity until a compatible scanner publishes a certified snapshot. Capacity also returns `ServiceUnavailable` when cluster or disk observations are missing, stale, or degraded. Retry after scanner convergence and storage or metadata recovery. Capacity assumes shard paths have independent backing space; shared physical storage can make the estimate optimistic. Set the flag to `false` to restore the normal object path, including deletion of a real object previously stored at that key.
 - `MINIO_API_OBJECT_MAX_VERSIONS` is accepted as a compatibility alias when the canonical RustFS variable is not set.
 
 ## Distributed endpoint locality

crates/config/src/constants/app.rs

@@ -222,6 +222,16 @@ pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
 /// Environment variable for server KMS enable.
 pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
 
+/// Environment variable for Veeam SOSAPI virtual system objects.
+///
+/// When true (the default), authenticated GET and HEAD of the two Veeam
+/// SOSAPI keys return generated XML. When false, those keys use the normal
+/// object path so an operator can remove a real object stored there earlier.
+pub const ENV_VEEAM_SOSAPI_ENABLE: &str = "RUSTFS_VEEAM_SOSAPI_ENABLE";
+
+/// Default Veeam SOSAPI enablement.
+pub const DEFAULT_VEEAM_SOSAPI_ENABLE: bool = true;
+
 /// Default KMS enable for server-side encryption
 /// This is the default value for enabling KMS encryption for server-side encryption.
 /// Default value: false

crates/data-usage/src/data_usage.rs

@@ -466,6 +466,9 @@ pub struct BucketTargetUsageInfo {
 #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct BucketUsageInfo {
     pub size: u64,
+    /// Bucket identity certified by every set contributing to this snapshot.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub bucket_incarnation: Option<String>,
     // Following five fields suffixed with V1 are here for backward compatibility
     // Total Size for objects that have not yet been replicated
     pub replication_pending_size_v1: u64,
@@ -2316,6 +2319,9 @@ impl BucketUsageInfo {
     /// Add size summary to this bucket usage
     /// Merge another BucketUsageInfo into this one
     pub fn merge(&mut self, other: &BucketUsageInfo) {
+        if self.bucket_incarnation != other.bucket_incarnation {
+            self.bucket_incarnation = None;
+        }
         self.size += other.size;
         self.objects_count += other.objects_count;
         self.versions_count += other.versions_count;
@@ -3299,6 +3305,31 @@ mod tests {
         assert_eq!(usage1.versions_count, 15);
     }
 
+    #[test]
+    fn bucket_usage_merge_discards_incompatible_incarnation_proofs() {
+        let id = "00000000-0000-4000-8000-000000000001";
+        let mut same = BucketUsageInfo {
+            bucket_incarnation: Some(id.into()),
+            ..Default::default()
+        };
+        same.merge(&BucketUsageInfo {
+            bucket_incarnation: Some(id.into()),
+            ..Default::default()
+        });
+        assert_eq!(same.bucket_incarnation.as_deref(), Some(id));
+        same.merge(&BucketUsageInfo::default());
+        assert!(same.bucket_incarnation.is_none());
+        let mut different = BucketUsageInfo {
+            bucket_incarnation: Some(id.into()),
+            ..Default::default()
+        };
+        different.merge(&BucketUsageInfo {
+            bucket_incarnation: Some("00000000-0000-4000-8000-000000000002".into()),
+            ..Default::default()
+        });
+        assert!(different.bucket_incarnation.is_none());
+    }
+
     #[test]
     fn size_summary_add_saturates_instead_of_overflowing() {
         // The scanner folds one summary per object into a per-prefix total, so a

crates/e2e_test/src/distributed/multipart_fence_acceptance_test.rs

@@ -1129,9 +1129,198 @@ async fn phase_quorum_matrix(
     Ok(())
 }
 
+async fn list_parts_disk_loss_after_metadata_read(
+    cluster: &RustFSTestClusterEnvironment,
+    proxies: &[RpcFaultProxy],
+    clients: &[Client],
+    bucket: &str,
+) -> TestResult {
+    let key = "list-parts-after-metadata-loss";
+    let body = payload_for("list-parts-acknowledged", 4096);
+    let upload = clients[0].create_multipart_upload().bucket(bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload ID")?.to_owned();
+    let part = clients[0]
+        .upload_part()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(body.clone()))
+        .send()
+        .await?;
+    let etag = part.e_tag().ok_or("missing part ETag")?.to_owned();
+    let path = upload_path_scope(bucket, key, &upload_id)?;
+    let root = Path::new(&cluster.nodes[0].data_dirs[0])
+        .join(RUSTFS_META_MULTIPART_BUCKET)
+        .join(&path);
+    let data_dirs = std::fs::read_dir(&root)?
+        .filter_map(Result::ok)
+        .map(|entry| entry.path())
+        .filter(|entry| entry.join("part.1").is_file() && entry.join("part.1.meta").is_file())
+        .collect::<Vec<_>>();
+    assert_eq!(data_dirs.len(), 1, "the coordinator's actual part directory must be identified");
+    let data_dir = &data_dirs[0];
+    let dir_name = data_dir
+        .file_name()
+        .and_then(|name| name.to_str())
+        .ok_or("non-UTF8 data directory")?;
+    let scoped_path = format!("{path}/{dir_name}");
+    let held_dir = data_dir.with_extension("e2e-held");
+    std::fs::rename(data_dir, &held_dir)?;
+    let pauses = proxies[1..]
+        .iter()
+        .map(|proxy| proxy.pause(&["ListDir"], usize::MAX, Some(&scoped_path)))
+        .collect::<Result<Vec<_>, _>>()?;
+    let request = clients[0].list_parts().bucket(bucket).key(key).upload_id(&upload_id);
+    let task = tokio::spawn(async move { request.send().await });
+    // ListDir is dispatched only after the upload metadata read succeeded.
+    // Require all three scoped remote requests; unrelated failures do not count.
+    wait_for_faults(&pauses, &["ListDir"]).await?;
+    let result = tokio::time::timeout(DEADLINE, task).await??;
+    for pause in &pauses {
+        pause.discard_pending();
+    }
+    drop(pauses);
+    std::fs::rename(&held_dir, data_dir)?;
+    let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
+    let response = error.raw_response().expect("ListParts must return an S3 service response");
+    assert_eq!(response.status().as_u16(), 503);
+    assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
+    assert!(response.headers().get("x-rustfs-error-cause").is_none());
+    assert!(response.headers().get("retry-after").is_none());
+    assert!(
+        response
+            .headers()
+            .get("content-type")
+            .is_some_and(|value| value.starts_with("application/xml"))
+    );
+    assert!(
+        response
+            .headers()
+            .get("x-amz-request-id")
+            .is_some_and(|value| !value.is_empty())
+    );
+    let listed = recover_after_fault("ListParts after directory restoration", || {
+        clients[0].list_parts().bucket(bucket).key(key).upload_id(&upload_id).send()
+    })
+    .await??;
+    assert_eq!(listed.parts().len(), 1);
+    assert_eq!(listed.parts()[0].e_tag(), Some(etag.as_str()));
+    clients[0]
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(CompletedPart::builder().part_number(1).e_tag(&etag).build())
+                .build(),
+        )
+        .send()
+        .await?;
+    for client in clients {
+        assert_object_bytes_after_fault(client, bucket, key, &body).await?;
+    }
+    eprintln!(
+        "MPU list proof: metadata read completed, three scoped ListDir RPCs held, missing local directory rejected; restored part and all readers verified"
+    );
+    Ok(())
+}
+
+async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Client], bucket: &str) -> TestResult {
+    let key = "concurrent-capped-parts";
+    let upload = clients[0]
+        .create_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .customize()
+        .mutate_request(|request| {
+            let separator = if request.uri().contains('?') { "&" } else { "?" };
+            let uri = format!("{}{separator}x-rustfs-max-total-object-size=10", request.uri());
+            request
+                .set_uri(uri)
+                .expect("the signed multipart size query must be a valid URI");
+        })
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("missing capped upload ID")?.to_owned();
+    let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(3));
+    let mut tasks = Vec::new();
+    for (index, number, byte) in [(0_usize, 1_i32, 0x41_u8), (1, 2, 0x42)] {
+        let client = clients[index].clone();
+        let bucket = bucket.to_owned();
+        let upload_id = upload_id.clone();
+        let barrier = std::sync::Arc::clone(&barrier);
+        tasks.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let result = client
+                .upload_part()
+                .bucket(&bucket)
+                .key(key)
+                .upload_id(&upload_id)
+                .part_number(number)
+                .body(ByteStream::from(vec![byte; 6]))
+                .send()
+                .await;
+            (number, byte, result)
+        }));
+    }
+    barrier.wait().await;
+    let mut winner = None;
+    let mut rejected = 0;
+    for task in tasks {
+        let (number, byte, result) = tokio::time::timeout(DEADLINE, task).await??;
+        match result {
+            Ok(part) => {
+                assert!(winner.is_none(), "both six-byte parts cannot fit the ten-byte limit");
+                winner = Some((number, byte, part.e_tag().ok_or("missing accepted ETag")?.to_owned()));
+            }
+            Err(error) => {
+                assert_eq!(
+                    error.as_service_error().and_then(|err| err.meta().code()),
+                    Some("EntityTooLarge"),
+                    "{error:?}"
+                );
+                rejected += 1;
+            }
+        }
+    }
+    let (number, byte, etag) = winner.ok_or("no part was admitted")?;
+    assert_eq!(rejected, 1);
+    let listed = clients[0]
+        .list_parts()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .send()
+        .await?;
+    assert_eq!(listed.parts().len(), 1);
+    assert_eq!(listed.parts()[0].part_number(), Some(number));
+    assert_eq!(listed.parts()[0].e_tag(), Some(etag.as_str()));
+    clients[0]
+        .complete_multipart_upload()
+        .bucket(bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(CompletedPart::builder().part_number(number).e_tag(&etag).build())
+                .build(),
+        )
+        .send()
+        .await?;
+    for client in clients {
+        assert_object_bytes_after_fault(client, bucket, key, &[byte; 6]).await?;
+    }
+    eprintln!("MPU size limit: two coordinators raced; exactly one six-byte part admitted under ten-byte limit");
+    Ok(())
+}
+
 #[tokio::test]
 async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
     let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
+    list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
+    concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
     phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
 }
 
@@ -1214,6 +1403,22 @@ async fn mixed_version_failed_complete_requires_current_coordinator_for_abort()
     cluster.start_node(0).await?;
     crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness(&clients, &bucket, "mixed-mpu-current-writer", DEADLINE)
         .await?;
+    // Read-only ListParts remains available during the documented mutation pause.
+    // Compare the actual old and current coordinators on the same legacy upload.
+    for (version, client) in [("current", &clients[0]), ("previous", &clients[1])] {
+        let listed = client
+            .list_parts()
+            .bucket(&bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .send()
+            .await?;
+        assert_eq!(listed.parts().len(), 1, "{version} coordinator must list the legacy part");
+        assert_eq!(listed.parts()[0].part_number(), Some(1));
+        assert_eq!(listed.parts()[0].size(), Some(9 * 1024 * 1024));
+        assert_eq!(listed.parts()[0].e_tag(), part.e_tag());
+    }
+    eprintln!("MPU mixed version: old and current coordinators agree on legacy ListParts identity and size");
     let scope = upload_scope(upload_id)?;
     let observers = proxies[1..]
         .iter()
@@ -1344,7 +1549,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
         capability_sha256: Option<[u8; 32]>,
         abort: Option<AbortRef>,
     }
-    let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
+    let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
     let key = "partial-abort";
     let committed = payload_for("committed-before-abort", 4096);
     clients[0]
@@ -1475,6 +1680,48 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
             .raw_response()
             .and_then(|response| response.headers().get("x-amz-request-id"))
     );
+    assert!(
+        retained.values().all(|source| source.participant.0 != 3),
+        "the selected offline node must have no surviving root"
+    );
+    cluster.stop_node(3)?;
+    let offline = tokio::time::timeout(
+        DEADLINE,
+        clients[0]
+            .abort_multipart_upload()
+            .bucket(&bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .send(),
+    )
+    .await?
+    .expect_err("an unavailable original participant cannot be replaced by the certificate");
+    assert_unavailable(&offline, "storage_write_quorum", None);
+    assert!(
+        multipart_sources(&cluster, Some(&hash))? == retained,
+        "an unavailable participant must not authorize deletion or change retained evidence"
+    );
+    cluster.start_node(3).await?;
+    wait_for_storage_peer_recovery(&proxies, &clients[0], &bucket, upload_id, 4).await?;
+    eprintln!("MPU certified residual: unavailable original returned storage_write_quorum; all surviving bytes retained");
+
+    // Kill the coordinator and a node owning a real surviving source. The
+    // retained bytes and full-N certificate must survive both process restarts
+    // before a fresh coordinator can authorize the retry.
+    let survivor_node = retained.values().next().ok_or("missing surviving source")?.participant.0;
+    let mut restarted = vec![0, survivor_node];
+    restarted.sort_unstable();
+    restarted.dedup();
+    for node in restarted {
+        cluster.stop_node(node)?;
+        cluster.start_node(node).await?;
+        wait_for_storage_peer_recovery(&proxies, &clients[0], &bucket, upload_id, 4).await?;
+        assert!(
+            multipart_sources(&cluster, Some(&hash))? == retained,
+            "restart must preserve every surviving source and certificate byte"
+        );
+    }
+    eprintln!("MPU certified residual: coordinator and surviving owner restarted; all retained bytes unchanged");
     let delete_observers = proxies
         .iter()
         .map(|proxy| proxy.pause(&["RecoverMultipartSource"], usize::MAX, Some(&path)))

crates/e2e_test/src/distributed/upgrade_test.rs

@@ -47,6 +47,9 @@ struct UpgradeSeed {
     history_bucket: String,
     history_key: &'static str,
     history_body: Vec<u8>,
+    multipart_upload: String,
+    multipart_etag: String,
+    multipart_body: Vec<u8>,
     versioned_bucket: String,
     versioned_key: &'static str,
     version1: String,
@@ -241,6 +244,25 @@ async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
     let history_body = b"written by the previous 4-node release".to_vec();
     put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
 
+    let multipart_body = vec![0x5a; 4096];
+    let multipart = root
+        .create_multipart_upload()
+        .bucket(&history_bucket)
+        .key("legacy-multipart.bin")
+        .send()
+        .await?;
+    let multipart_upload = multipart.upload_id().ok_or("legacy multipart upload ID missing")?.to_owned();
+    let part = root
+        .upload_part()
+        .bucket(&history_bucket)
+        .key("legacy-multipart.bin")
+        .upload_id(&multipart_upload)
+        .part_number(1)
+        .body(aws_sdk_s3::primitives::ByteStream::from(multipart_body.clone()))
+        .send()
+        .await?;
+    let multipart_etag = part.e_tag().ok_or("legacy multipart part ETag missing")?.to_owned();
+
     let versioned_key = "versioned-history.txt";
     let version1_body = b"version-one-before-upgrade".to_vec();
     let version1 = root
@@ -278,6 +300,9 @@ async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
         history_bucket,
         history_key,
         history_body,
+        multipart_upload,
+        multipart_etag,
+        multipart_body,
         versioned_bucket,
         versioned_key,
         version1,
@@ -304,6 +329,21 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
     )
     .await?;
     assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+    // During mixed-version operation this is read-only; mutation resumes only
+    // after the whole fleet is upgraded.
+    for client in [&root_a, &root_b] {
+        let listed = client
+            .list_parts()
+            .bucket(&seed.history_bucket)
+            .key("legacy-multipart.bin")
+            .upload_id(&seed.multipart_upload)
+            .send()
+            .await?;
+        assert_eq!(listed.parts().len(), 1, "{context}: legacy part listing");
+        assert_eq!(listed.parts()[0].part_number(), Some(1));
+        assert_eq!(listed.parts()[0].size(), Some(4096));
+        assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
+    }
 
     let v1 = root_b
         .get_object()
@@ -387,17 +427,146 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
     Ok(())
 }
 
+async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
+    let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
+    let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
+    let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
+    let root = format!("{hash}/{upload_dir}");
+    let mut held = Vec::new();
+    let mut original_files = Vec::new();
+    for (index, node) in dist.cluster.nodes.iter().enumerate() {
+        let path = Path::new(&node.data_dirs[0])
+            .join(crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET)
+            .join(&root);
+        let entries = std::fs::read_dir(path)?.collect::<Result<Vec<_>, _>>()?;
+        let dirs = entries
+            .into_iter()
+            .map(|entry| entry.path())
+            .filter(|entry| entry.join("part.1").is_file() && entry.join("part.1.meta").is_file())
+            .collect::<Vec<_>>();
+        assert_eq!(dirs.len(), 1, "each configured disk must hold the original part before injection");
+        let source_root = dirs[0].parent().ok_or("part directory has no upload root")?;
+        for path in [
+            source_root.join("xl.meta"),
+            dirs[0].join("part.1"),
+            dirs[0].join("part.1.meta"),
+        ] {
+            original_files.push((path.clone(), std::fs::read(path)?));
+        }
+        if index < 3 {
+            held.push((dirs[0].clone(), dirs[0].with_extension("held-for-mixed-test")));
+        }
+    }
+    dist.cluster.stop_node(3)?;
+    for (original, temporary) in &held {
+        std::fs::rename(original, temporary)?;
+    }
+
+    let clients = dist.clients()?;
+    let current = tokio::time::timeout(
+        CREDENTIAL_TIMEOUT,
+        clients[0]
+            .list_parts()
+            .bucket(&seed.history_bucket)
+            .key("legacy-multipart.bin")
+            .upload_id(&seed.multipart_upload)
+            .send(),
+    )
+    .await?;
+    let legacy = tokio::time::timeout(
+        CREDENTIAL_TIMEOUT,
+        clients[1]
+            .list_parts()
+            .bucket(&seed.history_bucket)
+            .key("legacy-multipart.bin")
+            .upload_id(&seed.multipart_upload)
+            .send(),
+    )
+    .await?;
+    for (original, temporary) in &held {
+        std::fs::rename(temporary, original)?;
+    }
+    // Three online disks retain metadata quorum but have no part directory;
+    // the fourth disk is offline. A legacy empty response is not a proof that
+    // the unavailable participant has no part.
+    let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
+    assert!(legacy.parts().is_empty());
+    let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
+    assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
+    assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
+    dist.cluster.start_node_from_binary(3, previous).await?;
+    wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
+    for (path, before) in original_files {
+        assert_eq!(
+            std::fs::read(&path)?,
+            before,
+            "the fault must preserve every original source file: {path:?}"
+        );
+    }
+    for client in [&clients[0], &clients[1]] {
+        let listed = client
+            .list_parts()
+            .bucket(&seed.history_bucket)
+            .key("legacy-multipart.bin")
+            .upload_id(&seed.multipart_upload)
+            .send()
+            .await?;
+        assert_eq!(listed.parts().len(), 1);
+        assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
+    }
+    eprintln!(
+        "MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
+    );
+    Ok(())
+}
+
+async fn complete_legacy_multipart(dist: &DistCluster, seed: &UpgradeSeed) -> TestResult {
+    // Native legacy-source upgrade needs every original participant, not just
+    // the ordinary three-of-four write quorum proven by the earlier probes.
+    let clients = dist.clients()?;
+    crate::upgrade_compatibility_test::wait_for_drive_readmission(
+        &clients,
+        &seed.history_bucket,
+        "legacy-multipart-all-drives",
+        CREDENTIAL_TIMEOUT,
+    )
+    .await?;
+    let root = &clients[0];
+    root.complete_multipart_upload()
+        .bucket(&seed.history_bucket)
+        .key("legacy-multipart.bin")
+        .upload_id(&seed.multipart_upload)
+        .multipart_upload(
+            aws_sdk_s3::types::CompletedMultipartUpload::builder()
+                .parts(
+                    aws_sdk_s3::types::CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(&seed.multipart_etag)
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+    for client in &clients {
+        assert_object_bytes(client, &seed.history_bucket, "legacy-multipart.bin", &seed.multipart_body).await?;
+    }
+    Ok(())
+}
+
 #[tokio::test]
 async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
     init_logging();
     let previous = source_binary()?;
     let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
     capture_upgrade_logs(&mut dist, "direct-upgrade")?;
     dist.start_from_binary(&previous).await?;
 
     let seed = seed_history_and_iam(&dist).await?;
     dist.restart_with_current_binary().await?;
     assert_history_and_iam(&dist, &seed, "direct").await?;
+    complete_legacy_multipart(&dist, &seed).await?;
     Ok(())
 }
 
@@ -419,6 +588,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
     init_logging();
     let previous = source_binary()?;
     let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
     capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
     dist.start_from_binary(&previous).await?;
 
@@ -426,6 +596,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
 
     replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
     assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+    compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
 
     for node_idx in [1, 2] {
         replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
@@ -434,5 +605,6 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
 
     replace_node_and_wait_for_write_readiness(&mut dist, &seed, 3).await?;
     assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    complete_legacy_multipart(&dist, &seed).await?;
     Ok(())
 }

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -1716,6 +1716,15 @@ mod tests {
         cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", background_enabled.to_string());
         cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", background_enabled.to_string());
         cluster.set_env("RUSTFS_SCANNER_ENABLED", background_enabled.to_string());
+        let bucket = "heal-restart-during-rebuild";
+        let outage_key_prefix = "cluster/written-while-node-down-";
+        if !background_enabled {
+            // With MRF disabled, an outage PUT still hands its missing shard to a
+            // best-effort repair on the PUT coordinator. That repair retries until
+            // the target rejoins, so suppress it to keep the explicit root heal
+            // the only owner of the outage object.
+            cluster.set_env("RUSTFS_E2E_SUPPRESS_RENAME_TAIL_HEAL_PREFIX", format!("{bucket}/{outage_key_prefix}"));
+        }
         if background_enabled {
             // Only the scanner cadence is accelerated. Keep normal Heal
             // concurrency and every automatic recovery owner enabled.
@@ -1748,7 +1757,6 @@ mod tests {
         cluster.start_with_binary(&server_binary).await?;
         let clients = cluster.create_all_clients()?;
 
-        let bucket = "heal-restart-during-rebuild";
         clients[0].create_bucket().bucket(bucket).send().await?;
 
         let ReplacementDriveSelection {
@@ -1876,7 +1884,7 @@ mod tests {
         let mut replacement_outage_erasure_index = None;
         let mut rejected_outage_keys = Vec::new();
         for attempt in 0..max_outage_write_attempts {
-            let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin");
+            let candidate_key = format!("{outage_key_prefix}{attempt:04}.bin");
             let put_result = timeout(
                 Duration::from_secs(30),
                 clients[2]
@@ -1986,25 +1994,6 @@ mod tests {
         }
 
         let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
-        if !background_enabled {
-            // The PUT path may have admitted a direct Internal object repair while
-            // node 1 was offline. Cancel the isolated bucket path before the target
-            // returns; otherwise it could rebuild the outage object and invalidate
-            // the explicit-root ownership assertion below.
-            let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
-            let (cancel_status, cancel_body) = admin_request(
-                &cluster.nodes[0].url,
-                Method::POST,
-                &cancel_outage_heal_path,
-                Some(heal_body.to_string()),
-                &cluster.access_key,
-                &cluster.secret_key,
-            )
-            .await?;
-            if !cancel_status.is_success() {
-                return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
-            }
-        }
 
         // Keep the partial-repair checkpoint stable across readiness and admin
         // requests. Endpoint-blackhole tests must prove their own network stall.

crates/e2e_test/src/list_objects_v2_pagination_test.rs

@@ -38,6 +38,8 @@ mod tests {
     use tracing::info;
 
     const PAGINATION_TIMEOUT: Duration = Duration::from_secs(120);
+    // Bound queued durable commits on the single fixture disk under CI I/O pressure.
+    const FIXTURE_PUT_CONCURRENCY: usize = 4;
 
     // Fixture population can require thousands of durable PUTs. Bound the
     // listing traversal itself with one deadline shared by every page.
@@ -564,7 +566,7 @@ mod tests {
         let object_count = 1002;
         // Finish every durable PUT before listing, with the same bounded fanout as the delimiter fixtures.
         stream::iter(0..object_count)
-            .for_each_concurrent(16, |i| {
+            .for_each_concurrent(FIXTURE_PUT_CONCURRENCY, |i| {
                 let client = &client;
                 async move {
                     let key = format!("object{i:04}.txt");
@@ -575,7 +577,7 @@ mod tests {
                         .body(ByteStream::from_static(b"test content"))
                         .send()
                         .await
-                        .unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
+                        .unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err:?}"));
                 }
             })
             .await;
@@ -822,7 +824,7 @@ mod tests {
             .flat_map(|dir| (0..10).map(move |file| format!("dir-{dir:03}/file{file:02}.txt")))
             .collect();
         stream::iter(&all_keys)
-            .for_each_concurrent(16, |key| {
+            .for_each_concurrent(FIXTURE_PUT_CONCURRENCY, |key| {
                 let client = &client;
                 async move {
                     client
@@ -832,7 +834,7 @@ mod tests {
                         .body(ByteStream::from_static(b"x"))
                         .send()
                         .await
-                        .unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
+                        .unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err:?}"));
                 }
             })
             .await;
@@ -1055,7 +1057,7 @@ mod tests {
         let files_per_dir = 100;
         let fixture_started = std::time::Instant::now();
         stream::iter((0..dir_count).flat_map(|d| (0..files_per_dir).map(move |f| (d, f))))
-            .for_each_concurrent(16, |(d, f)| {
+            .for_each_concurrent(FIXTURE_PUT_CONCURRENCY, |(d, f)| {
                 let client = &client;
                 async move {
                     let key = format!("dir-{d:02}/file{f:03}.txt");
@@ -1066,7 +1068,7 @@ mod tests {
                         .body(ByteStream::from_static(b"x"))
                         .send()
                         .await
-                        .unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err}"));
+                        .unwrap_or_else(|err| panic!("Failed to put fixture object {key}: {err:?}"));
                 }
             })
             .await;

crates/e2e_test/src/snowball_auto_extract_test.rs

@@ -261,6 +261,61 @@ mod tests {
         archive
     }
 
+    #[tokio::test]
+    async fn snowball_auto_extract_rejects_veeam_virtual_members() -> Result<(), Box<dyn Error + Send + Sync>> {
+        init_logging();
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server_with_env(vec![], &[("RUSTFS_VEEAM_SOSAPI_ENABLE", "true")])
+            .await?;
+        let client = env.create_s3_client();
+        let bucket = "snowball-veeam-reserved";
+        client.create_bucket().bucket(bucket).send().await?;
+        let prefix = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c";
+        for (member, destination, use_prefix) in [
+            (format!("{prefix}/system.xml"), format!("{prefix}/system.xml"), false),
+            ("capacity.xml".to_string(), format!("{prefix}/capacity.xml"), true),
+        ] {
+            let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+            append_typed_entry(&mut builder, &member, tokio_tar::EntryType::Regular, b"blocked").await?;
+            let archive = builder.into_inner().await?.into_inner();
+            let mut request = client
+                .put_object()
+                .bucket(bucket)
+                .key("fixture.tar")
+                .metadata("Snowball-Auto-Extract", "true")
+                .body(ByteStream::from(archive));
+            if use_prefix {
+                request = request
+                    .metadata("Minio-Snowball-Prefix", prefix)
+                    .metadata("Minio-Snowball-Ignore-Errors", "true");
+            }
+            let error = request.send().await.expect_err("virtual archive members must be rejected");
+            assert_eq!(error.code(), Some("MethodNotAllowed"));
+            let listed = client.list_objects_v2().bucket(bucket).prefix(&destination).send().await?;
+            assert!(listed.contents().is_empty(), "rejected virtual member must not be stored");
+        }
+        let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
+        append_typed_entry(&mut builder, "ordinary.xml", tokio_tar::EntryType::Regular, b"ordinary").await?;
+        client
+            .put_object()
+            .bucket(bucket)
+            .key("fixture.tar")
+            .metadata("Snowball-Auto-Extract", "true")
+            .metadata("Minio-Snowball-Prefix", prefix)
+            .body(ByteStream::from(builder.into_inner().await?.into_inner()))
+            .send()
+            .await?;
+        let ordinary = client
+            .get_object()
+            .bucket(bucket)
+            .key(format!("{prefix}/ordinary.xml"))
+            .send()
+            .await?;
+        assert_eq!(ordinary.body.collect().await?.into_bytes().as_ref(), b"ordinary");
+        env.stop_server();
+        Ok(())
+    }
+
     #[tokio::test]
     async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box<dyn Error + Send + Sync>> {
         init_logging();

crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs

@@ -13,9 +13,13 @@
 // limitations under the License.
 
 use crate::common::{RustFSTestClusterEnvironment, init_logging};
+use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
 use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::primitives::ByteStream;
-use tokio::time::{Duration, sleep};
+use sha2::{Digest, Sha256};
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use tokio::time::{Duration, Instant, sleep};
 use tracing::info;
 use uuid::Uuid;
 
@@ -84,6 +88,126 @@ async fn wait_for_cleanup_on_all_nodes(
     Err("stale multipart upload was not cleaned up on all nodes within timeout".into())
 }
 
+type PhysicalTree = BTreeMap<PathBuf, Option<Vec<u8>>>;
+
+fn physical_tree(root: &Path) -> Result<PhysicalTree, Box<dyn std::error::Error + Send + Sync>> {
+    let mut result = BTreeMap::new();
+    let mut pending = vec![root.to_path_buf()];
+    while let Some(directory) = pending.pop() {
+        for entry in std::fs::read_dir(directory)? {
+            let entry = entry?;
+            let path = entry.path();
+            let relative = path.strip_prefix(root)?.to_path_buf();
+            let kind = entry.file_type()?;
+            if kind.is_dir() {
+                result.insert(relative, None);
+                pending.push(path);
+            } else if kind.is_file() {
+                result.insert(relative, Some(std::fs::read(path)?));
+            } else {
+                return Err(format!("unexpected staging entry type: {relative:?}").into());
+            }
+        }
+    }
+    Ok(result)
+}
+
+async fn assert_low_quorum_residuals_are_diagnosed_and_preserved(
+    cluster: &RustFSTestClusterEnvironment,
+    clients: &[aws_sdk_s3::Client],
+    log_paths: &[PathBuf],
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let mut residuals = Vec::new();
+    for copies in [1_usize, 2] {
+        let key = format!("multipart/partial-{copies}-{}", Uuid::new_v4().simple());
+        let upload = clients[0]
+            .create_multipart_upload()
+            .bucket(CLEANUP_BUCKET)
+            .key(&key)
+            .send()
+            .await?;
+        let upload_id = upload.upload_id().ok_or("create response missing upload ID")?;
+        clients[0]
+            .upload_part()
+            .bucket(CLEANUP_BUCKET)
+            .key(&key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from(vec![0x41; 4096]))
+            .send()
+            .await?;
+        let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes())?)?;
+        let (_, upload_dir) = decoded.split_once('.').ok_or("upload ID lacks deployment delimiter")?;
+        let digest = Sha256::digest(format!("{CLEANUP_BUCKET}/{key}").as_bytes());
+        let hash = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
+        let relative = format!("{hash}/{upload_dir}");
+        let roots = cluster
+            .nodes
+            .iter()
+            .map(|node| {
+                Path::new(&node.data_dirs[0])
+                    .join(RUSTFS_META_MULTIPART_BUCKET)
+                    .join(&relative)
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(roots.len(), 4);
+        let mut preserved = Vec::new();
+        for root in &roots {
+            assert!(
+                root.join("xl.meta").is_file(),
+                "all four original metadata files must exist before injection"
+            );
+            preserved.push(physical_tree(root)?);
+        }
+        for root in roots.iter().skip(copies) {
+            std::fs::remove_dir_all(root)?;
+        }
+        residuals.push((relative, roots, preserved, copies));
+    }
+
+    let deadline = Instant::now() + Duration::from_secs(45);
+    loop {
+        let logs = log_paths
+            .iter()
+            .filter_map(|path| std::fs::read_to_string(path).ok())
+            .collect::<Vec<_>>();
+        let diagnosed = residuals.iter().all(|(path, _, _, _)| {
+            logs.iter().any(|log| {
+                log.lines()
+                    .any(|line| line.contains(path) && line.contains("multipart_cleanup_lock_or_recheck_failed"))
+            })
+        });
+        if diagnosed {
+            break;
+        }
+        if Instant::now() >= deadline {
+            return Err("stale low-quorum residuals were not diagnosed before the deadline".into());
+        }
+        sleep(Duration::from_millis(200)).await;
+    }
+    for (path, roots, preserved, copies) in &residuals {
+        for slot in 0..*copies {
+            assert_eq!(
+                physical_tree(&roots[slot])?,
+                preserved[slot],
+                "retained source evidence changed: {path}, slot {slot}"
+            );
+        }
+        for root in roots.iter().skip(*copies) {
+            assert!(!root.exists(), "cleanup must not recreate missing source roots: {path}");
+        }
+    }
+    for path in log_paths {
+        let log = std::fs::read_to_string(path)?;
+        assert!(
+            !log.contains("op=\"upload_part\"") && !log.contains("\"op\":\"upload_part\""),
+            "background cleanup must not be logged as a client UploadPart"
+        );
+    }
+    eprintln!("Stale MPU low-quorum: EC:2 one/two-copy roots diagnosed; full surviving file inventories preserved");
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
 -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -92,6 +216,22 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
     let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
     cluster.set_env("RUSTFS_API_STALE_UPLOADS_EXPIRY", "5s");
     cluster.set_env("RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL", "1s");
+    cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
+    cluster.set_env(
+        "RUST_LOG",
+        concat!("warn,rustfs_ecstore", "::bucket::lifecycle::bucket_lifecycle_ops=debug"),
+    );
+    let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")
+        .map(PathBuf::from)
+        .map(|root| root.join(format!("stale-multipart-{}", Uuid::new_v4().simple())))
+        .unwrap_or_else(|| PathBuf::from(&cluster.temp_dir));
+    std::fs::create_dir_all(&log_dir)?;
+    let log_paths = (0..cluster.nodes.len())
+        .map(|index| log_dir.join(format!("stale-node-{index}.log")))
+        .collect::<Vec<_>>();
+    for (index, path) in log_paths.iter().enumerate() {
+        cluster.set_node_capture_log_path(index, path.to_string_lossy().into_owned())?;
+    }
     cluster.start().await?;
     cluster.create_test_bucket(CLEANUP_BUCKET).await?;
 
@@ -137,6 +277,7 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
     );
 
     wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
+    assert_low_quorum_residuals_are_diagnosed_and_preserved(&cluster, &clients, &log_paths).await?;
 
     Ok(())
 }

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -1066,7 +1066,7 @@ pub(crate) async fn wait_for_upgrade_write_readiness(
 /// reduced-redundancy class then fails with 503 `ServiceUnavailable` until the
 /// serving node writes through every drive, on the previous release and on
 /// the current build alike.
-async fn wait_for_drive_readmission(clients: &[Client], bucket: &str, context: &str, budget: Duration) -> TestResult {
+pub(crate) async fn wait_for_drive_readmission(clients: &[Client], bucket: &str, context: &str, budget: Duration) -> TestResult {
     wait_for_upgrade_writes(
         clients,
         bucket,

crates/e2e_test/src/version_id_regression_test.rs

@@ -600,6 +600,154 @@ mod tests {
         info!("✅ PASSED: Veeam backup workflow simulation completed successfully");
     }
 
+    /// Veeam B&R discovers SOSAPI by GET/HEAD of two well-known keys.
+    #[tokio::test]
+    async fn test_veeam_sosapi_system_and_capacity() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        use crate::common::local_http_client;
+
+        const SYSTEM_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml";
+        const CAPACITY_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml";
+
+        init_logging();
+        info!("🧪 TEST: Veeam SOSAPI system.xml and capacity.xml");
+
+        let mut env = RustFSTestEnvironment::new().await?;
+        env.start_rustfs_server_with_env(vec![], crate::common::FAST_DATA_USAGE_SCANNER_ENV)
+            .await?;
+        let client = create_s3_client(&env);
+        let bucket = "veeam-sosapi";
+        create_bucket(&client, bucket).await?;
+
+        let system = client.get_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
+        let system_etag = system.e_tag().map(str::to_string);
+        let system_len = system.content_length();
+        assert_eq!(system.content_type(), Some("application/xml"));
+        let system_body = system.body.collect().await?.into_bytes();
+        let system_xml = std::str::from_utf8(&system_body)?;
+        assert!(system_xml.contains("<ProtocolVersion>\"1.0\"</ProtocolVersion>"), "{system_xml}");
+        assert!(
+            system_xml.contains("<ModelName>\"RustFS ") && system_xml.contains("\"</ModelName>"),
+            "{system_xml}"
+        );
+        assert!(system_xml.contains("<CapacityInfo>true</CapacityInfo>"));
+        assert!(system_xml.contains("<UploadSessions>false</UploadSessions>"));
+        assert!(system_xml.contains("<IAMSTS>false</IAMSTS>"));
+        assert!(!system_xml.contains("SystemRecommendations"));
+        assert!(!system_xml.contains("APIEndpoints"));
+        assert!(!system_xml.contains("&quot;"));
+
+        let head = client.head_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
+        assert_eq!(head.content_type(), Some("application/xml"));
+        assert_eq!(head.content_length(), system_len);
+        assert_eq!(head.e_tag().map(str::to_string), system_etag);
+        assert!(head.last_modified().is_some());
+
+        let ranged = client
+            .get_object()
+            .bucket(bucket)
+            .key(SYSTEM_KEY)
+            .range("bytes=0-3")
+            .send()
+            .await?;
+        assert_eq!(ranged.content_length(), Some(4));
+        let ranged_header = ranged.content_range().unwrap_or("");
+        assert!(ranged_header.starts_with("bytes 0-3/"), "{ranged_header}");
+        let ranged_body = ranged.body.collect().await?.into_bytes();
+        assert_eq!(&ranged_body[..], &system_body[..4]);
+
+        let capacity = tokio::time::timeout(std::time::Duration::from_secs(180), async {
+            loop {
+                match client.get_object().bucket(bucket).key(CAPACITY_KEY).send().await {
+                    Ok(capacity) => break Ok(capacity),
+                    Err(err) if err.code() == Some("ServiceUnavailable") => {
+                        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+                    }
+                    Err(err) => break Err(err),
+                }
+            }
+        })
+        .await??;
+        assert_eq!(capacity.content_type(), Some("application/xml"));
+        let capacity_body = capacity.body.collect().await?.into_bytes();
+        let capacity_xml = std::str::from_utf8(&capacity_body)?;
+        let capacity_value = xml_u64(capacity_xml, "Capacity")?;
+        let available_value = xml_u64(capacity_xml, "Available")?;
+        let used_value = xml_u64(capacity_xml, "Used")?;
+        assert!(used_value <= capacity_value, "{capacity_xml}");
+        assert!(available_value <= capacity_value.saturating_sub(used_value), "{capacity_xml}");
+
+        let put_err = client
+            .put_object()
+            .bucket(bucket)
+            .key(SYSTEM_KEY)
+            .body(ByteStream::from_static(b"overwrite"))
+            .send()
+            .await
+            .expect_err("SOSAPI put must fail");
+        assert_eq!(put_err.code(), Some("MethodNotAllowed"));
+        let delete_err = client
+            .delete_object()
+            .bucket(bucket)
+            .key(CAPACITY_KEY)
+            .send()
+            .await
+            .expect_err("SOSAPI delete must fail");
+        assert_eq!(delete_err.code(), Some("MethodNotAllowed"));
+
+        let normal_key = "Veeam/Backup/job.dat";
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(normal_key)
+            .body(ByteStream::from_static(b"backup-bytes"))
+            .send()
+            .await?;
+        let normal = client.get_object().bucket(bucket).key(normal_key).send().await?;
+        let normal_body = normal.body.collect().await?.into_bytes();
+        assert_eq!(&normal_body[..], b"backup-bytes");
+
+        let anon = local_http_client()
+            .get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
+            .send()
+            .await?;
+        assert_eq!(anon.status(), reqwest::StatusCode::FORBIDDEN);
+        let anon_body = anon.text().await?;
+        assert!(!anon_body.contains("ProtocolVersion"), "{anon_body}");
+
+        let now = time::OffsetDateTime::now_utc();
+        let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
+        let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
+        let bad = local_http_client()
+            .get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
+            .header("x-amz-date", &amz_date)
+            .header("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
+            .header(
+                "authorization",
+                format!(
+                    "AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
+                ),
+            )
+            .send()
+            .await?;
+        assert_eq!(bad.status(), reqwest::StatusCode::FORBIDDEN);
+        let bad_body = bad.text().await?;
+        assert!(!bad_body.contains("ProtocolVersion"), "{bad_body}");
+
+        info!("✅ PASSED: Veeam SOSAPI system.xml and capacity.xml");
+        Ok(())
+    }
+
+    fn xml_u64(xml: &str, element: &str) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
+        let start = format!("<{element}>");
+        let end = format!("</{element}>");
+        let start_idx = xml.find(&start).ok_or_else(|| format!("missing {element} in {xml}"))? + start.len();
+        let end_idx = xml[start_idx..]
+            .find(&end)
+            .ok_or_else(|| format!("unclosed {element} in {xml}"))?
+            + start_idx;
+        Ok(xml[start_idx..end_idx].parse()?)
+    }
+
     #[tokio::test]
     async fn test_terraform_put_after_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();

crates/ecstore/src/api/mod.rs

@@ -252,7 +252,7 @@ pub mod bucket {
     pub mod quota {
         pub use crate::bucket::quota::{
             BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
-            QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration,
+            QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration, QuotaType,
         };
 
         pub mod checker {
@@ -371,15 +371,24 @@ pub mod compression {
 
 pub mod config {
     pub mod com {
+        /// Preserve empty existing objects so callers can distinguish corruption from absence.
+        pub async fn read_config_preserve_empty_with_metadata_opts(
+            api: std::sync::Arc<crate::store::ECStore>,
+            file: &str,
+            opts: &crate::object_api::ObjectOptions,
+        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
+            crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
+        }
+
         pub use crate::config::com::{
             COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
             ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
             is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
             read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
-            read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_config_with_opts_quiet,
-            save_server_config, save_server_config_no_lock, save_server_config_snapshot,
-            save_server_config_snapshot_with_generation, server_config_path, try_migrate_server_config,
-            with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
+            read_server_config_snapshot, read_server_config_with_generation, save_config, save_config_no_lock,
+            save_config_with_opts, save_config_with_opts_quiet, save_server_config, save_server_config_no_lock,
+            save_server_config_snapshot, save_server_config_snapshot_with_generation, server_config_path,
+            try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock, with_server_config_read_lock,
             with_server_config_write_lock,
         };
     }
@@ -416,9 +425,10 @@ pub mod data_usage {
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
         init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
         invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
-        load_compression_total_from_memory, load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size,
-        record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
-        record_bucket_object_write_memory, record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
+        load_bucket_usage_from_backend_cached, load_compression_total_from_memory, load_data_usage_from_backend,
+        load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
+        record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
+        record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
         refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
         store_data_usage_in_backend,

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -63,6 +63,7 @@ use crate::services::tier::{
 };
 use crate::set_disk::{
     MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks, get_lock_acquire_timeout,
+    multipart_part_logical_size, should_persist_encryption_original_size,
 };
 use crate::storage_api_contracts::{
     lifecycle::ExpirationOptions,
@@ -119,6 +120,7 @@ const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
 const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
 const EVENT_LIFECYCLE_TRANSITION_COMPENSATION: &str = "lifecycle_transition_compensation";
 const EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP: &str = "lifecycle_stale_multipart_cleanup";
+const METRIC_STALE_MULTIPART_CLEANUP_BLOCKED_TOTAL: &str = "rustfs_stale_multipart_cleanup_blocked_total";
 const EVENT_LIFECYCLE_SCAN_SKIPPED: &str = "lifecycle_scan_skipped";
 const EVENT_LIFECYCLE_EVALUATION_FAILED: &str = "lifecycle_evaluation_failed";
 const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
@@ -1086,8 +1088,9 @@ impl ExpiryState {
         }))
     }
 
-    #[cfg(test)]
-    fn new_with_unconsumed_worker_channel(capacity: usize) -> Arc<RwLock<Self>> {
+    #[cfg(any(test, feature = "test-util"))]
+    #[doc(hidden)]
+    pub fn new_with_unconsumed_worker_channel(capacity: usize) -> Arc<RwLock<Self>> {
         let (tx, rx) = mpsc::channel(capacity);
         Arc::new(RwLock::new(Self {
             tasks_tx: vec![tx],
@@ -3510,19 +3513,29 @@ async fn stale_upload_current_size_with_opts(
             &ObjectOptions {
                 data_movement,
                 no_lock,
+                strict_multipart_part_sizes: true,
                 ..Default::default()
             },
         )
         .await
         .ok()?;
 
-    Some(
-        parts
-            .parts
-            .iter()
-            .map(|part| part.actual_size.max(part.size as i64).max(0) as usize)
-            .sum(),
-    )
+    stale_upload_size_from_parts(&parts)
+}
+
+fn stale_upload_size_from_parts(parts: &crate::storage_api_contracts::multipart::ListPartsInfo) -> Option<usize> {
+    if parts.is_truncated {
+        return None;
+    }
+    let transformed = rustfs_utils::http::contains_key_str(&parts.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION)
+        || should_persist_encryption_original_size(&parts.user_defined);
+    parts.parts.iter().try_fold(0usize, |total, part| {
+        // ListParts represents a quorum-confirmed missing part with an empty ETag.
+        if part.etag.as_deref().is_none_or(str::is_empty) {
+            return None;
+        }
+        total.checked_add(multipart_part_logical_size(part.size, part.actual_size, transformed)?)
+    })
 }
 
 async fn stale_upload_lifecycle_due(
@@ -3539,18 +3552,43 @@ async fn stale_upload_lifecycle_due(
     let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
     let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
 
-    let lifecycle = match metadata_boundary::get_lifecycle_config(bucket).await {
+    let mut lifecycle = match metadata_boundary::get_lifecycle_config(bucket).await {
         Ok((lifecycle, _)) => lifecycle,
         Err(_) => return None,
     };
 
+    let has_size_filter = |filter: &s3s::dto::LifecycleRuleFilter| {
+        filter.object_size_greater_than.is_some()
+            || filter.object_size_less_than.is_some()
+            || filter
+                .and
+                .as_ref()
+                .is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
+    };
+    let needs_size = lifecycle
+        .rules
+        .iter()
+        .any(|rule| rule.abort_incomplete_multipart_upload.is_some() && rule.filter.as_ref().is_some_and(has_size_filter));
+    let size = if needs_size {
+        match stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock).await {
+            Some(size) => size,
+            None => {
+                // An unknown size cannot authorize a size-filtered abort.
+                // Unrelated rules can still be evaluated independently.
+                lifecycle
+                    .rules
+                    .retain(|rule| !rule.filter.as_ref().is_some_and(has_size_filter));
+                0
+            }
+        }
+    } else {
+        0
+    };
     let object_opts = ObjectOpts {
         name: object.clone(),
         user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
         mod_time: Some(initiated),
-        size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock)
-            .await
-            .unwrap_or_default(),
+        size,
         is_latest: true,
         delete_marker: false,
         user_defined: metadata.clone(),
@@ -3707,6 +3745,23 @@ async fn cleanup_empty_multipart_sha_dirs_on_local_disks(set: &Arc<SetDisks>) {
     }
 }
 
+fn record_stale_multipart_cleanup_blocked(stage: &'static str) {
+    metrics::counter!(METRIC_STALE_MULTIPART_CLEANUP_BLOCKED_TOTAL, "stage" => stage).increment(1);
+}
+
+fn record_stale_multipart_cleanup_lock_lost(path: &str) {
+    record_stale_multipart_cleanup_blocked("lock_lost");
+    debug!(
+        event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
+        component = LOG_COMPONENT_ECSTORE,
+        subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+        state = "skipped",
+        path = %path,
+        reason = "multipart_cleanup_lock_lost",
+        "Skipped stale multipart cleanup"
+    );
+}
+
 async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: OffsetDateTime, default_expiry: StdDuration) -> usize {
     let mut deleted = 0usize;
     let mut candidates = HashMap::new();
@@ -3803,7 +3858,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
     for candidate in candidates.into_values() {
         let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
         let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
-        if let Some(metadata) = candidate.metadata.as_ref()
+        if now < due
+            && let Some(metadata) = candidate.metadata.as_ref()
             && let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await
             && lifecycle_due < due
         {
@@ -3817,6 +3873,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
         let cleanup_guard = match set.lock_stale_multipart_cleanup(&candidate.path).await {
             Ok(guard) => guard,
             Err(err) => {
+                record_stale_multipart_cleanup_blocked("recheck");
                 debug!(
                     event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
                     component = LOG_COMPONENT_ECSTORE,
@@ -3832,13 +3889,18 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
         let current_metadata = cleanup_guard.file_info().metadata.clone();
         let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time);
         let mut current_due = stale_upload_default_due(current_initiated, default_expiry);
-        if let Some(lifecycle_due) =
-            stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
+        if now < current_due
+            && let Some(lifecycle_due) =
+                stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
             && lifecycle_due < current_due
         {
             current_due = lifecycle_due;
         }
-        if now < current_due || cleanup_guard.is_lock_lost() {
+        if now < current_due {
+            continue;
+        }
+        if cleanup_guard.is_lock_lost() {
+            record_stale_multipart_cleanup_lock_lost(&candidate.path);
             continue;
         }
 
@@ -3858,15 +3920,18 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
                     "Removed stale multipart upload"
                 );
             }
-            Err(err) => debug!(
-                event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_LIFECYCLE,
-                path = %candidate.path,
-                error = ?err,
-                reason = "multipart_remove_failed",
-                "Failed to remove stale multipart upload"
-            ),
+            Err(err) => {
+                record_stale_multipart_cleanup_blocked("delete");
+                debug!(
+                    event = EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
+                    path = %candidate.path,
+                    error = ?err,
+                    reason = "multipart_remove_failed",
+                    "Failed to remove stale multipart upload"
+                );
+            }
         }
     }
 
@@ -15868,6 +15933,530 @@ mod tests {
         assert!(ecstore.is_ok(), "fresh local ECStore boot should succeed, got {ecstore:?}");
     }
 
+    #[tokio::test(flavor = "current_thread")]
+    #[serial]
+    async fn stale_multipart_cleanup_preserves_partial_metadata_without_upload_part_warning() {
+        use tracing::instrument::WithSubscriber as _;
+        #[derive(Clone, Default)]
+        struct CapturedLogs(Arc<StdMutex<Vec<u8>>>);
+        struct LogWriter(Arc<StdMutex<Vec<u8>>>);
+        impl std::io::Write for LogWriter {
+            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+                self.0.lock().unwrap().extend_from_slice(bytes);
+                Ok(bytes.len())
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Ok(())
+            }
+        }
+        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
+            type Writer = LogWriter;
+            fn make_writer(&'a self) -> Self::Writer {
+                LogWriter(self.0.clone())
+            }
+        }
+
+        let (paths, ecstore) = setup_test_env().await;
+        let bucket = format!("stale-partial-{}", Uuid::new_v4().simple());
+        create_test_bucket(&ecstore, &bucket).await;
+        for copies in [1, 2] {
+            let object = format!("partial-{copies}.bin");
+            let initiated = OffsetDateTime::now_utc() - time::Duration::hours(30);
+            let upload = ecstore
+                .new_multipart_upload(
+                    &bucket,
+                    &object,
+                    &ObjectOptions {
+                        mod_time: Some(initiated),
+                        max_parity: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("multipart upload should be created");
+            let decoded = base64_simd::URL_SAFE_NO_PAD
+                .decode_to_vec(upload.upload_id.as_bytes())
+                .unwrap();
+            let decoded = String::from_utf8(decoded).unwrap();
+            let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload id");
+            let upload_path = format!("{}/{upload_dir}", multipart_sha_dir(&bucket, &object));
+            let physical_paths: Vec<_> = paths
+                .iter()
+                .map(|path| path.join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path))
+                .collect();
+            let original: Vec<_> = physical_paths
+                .iter()
+                .take(copies)
+                .map(|path| std::fs::read(path.join(STORAGE_FORMAT_FILE)).unwrap())
+                .collect();
+            let info = super::get_file_info(
+                &original[0],
+                RUSTFS_META_MULTIPART_BUCKET,
+                &upload_path,
+                "",
+                super::FileInfoOpts {
+                    data: false,
+                    include_free_versions: false,
+                    include_part_checksums: false,
+                },
+            )
+            .unwrap();
+            assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (2, 2));
+            for path in physical_paths.iter().skip(copies) {
+                fs::remove_dir_all(path).await.expect("remove selected metadata copies");
+            }
+            if copies == 2 {
+                ecstore
+                    .get_multipart_info(&bucket, &object, &upload.upload_id, &ObjectOptions::default())
+                    .await
+                    .expect("two metadata copies must meet read quorum");
+            }
+
+            let logs = CapturedLogs::default();
+            let subscriber = tracing_subscriber::fmt()
+                .with_max_level(tracing::Level::DEBUG)
+                .with_writer(logs.clone())
+                .with_ansi(false)
+                .without_time()
+                .finish();
+            let recorder = crate::test_metrics::CapturingRecorder::default();
+            let recorder_guard = metrics::set_default_local_recorder(&recorder);
+            cleanup_stale_multipart_uploads_once_at(
+                ecstore.clone(),
+                OffsetDateTime::now_utc(),
+                StdDuration::from_secs(24 * 60 * 60),
+            )
+            .with_subscriber(subscriber)
+            .await;
+            drop(recorder_guard);
+            for (path, expected) in physical_paths.iter().zip(&original) {
+                assert_eq!(fs::read(path.join(STORAGE_FORMAT_FILE)).await.unwrap(), *expected);
+            }
+            let log_output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
+            let rechecks = log_output
+                .lines()
+                .filter(|line| line.contains("multipart_cleanup_lock_or_recheck_failed"));
+            let matching_path = rechecks.clone().any(|line| line.contains(&upload_path));
+            assert!(
+                matching_path,
+                "missing path-specific background diagnostic for {upload_path}: {log_output}"
+            );
+            assert_eq!(
+                recorder.counter_value(super::METRIC_STALE_MULTIPART_CLEANUP_BLOCKED_TOTAL, &[("stage", "recheck")]),
+                u64::try_from(rechecks.count()).expect("diagnostic count fits u64"),
+                "each observed recheck failure must increment the metric exactly once"
+            );
+            assert!(!log_output.contains("op=\"upload_part\""), "unexpected upload-part warning: {log_output}");
+            for path in physical_paths.iter().take(copies) {
+                fs::remove_dir_all(path).await.unwrap();
+            }
+        }
+
+        let logs = CapturedLogs::default();
+        let subscriber = tracing_subscriber::fmt()
+            .with_max_level(tracing::Level::DEBUG)
+            .with_writer(logs.clone())
+            .with_ansi(false)
+            .without_time()
+            .finish();
+        let recorder = crate::test_metrics::CapturingRecorder::default();
+        metrics::with_local_recorder(&recorder, || {
+            tracing::subscriber::with_default(subscriber, || {
+                super::record_stale_multipart_cleanup_lock_lost("test-hash/test-upload");
+            })
+        });
+        let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
+        assert!(
+            output.contains("multipart_cleanup_lock_lost") && output.contains("test-hash/test-upload"),
+            "{output}"
+        );
+        assert!(!output.contains("op=\"upload_part\""), "{output}");
+        assert_eq!(
+            recorder.counter_value(super::METRIC_STALE_MULTIPART_CLEANUP_BLOCKED_TOTAL, &[("stage", "lock_lost")]),
+            1
+        );
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    #[serial]
+    async fn stale_multipart_cleanup_lists_parts_only_when_size_is_needed() {
+        use std::sync::atomic::{AtomicUsize, Ordering};
+        use tracing::instrument::WithSubscriber as _;
+        use tracing_subscriber::prelude::*;
+
+        struct PartListCounter {
+            calls: Arc<AtomicUsize>,
+            bucket: String,
+            object: &'static str,
+        }
+        struct PartListFields<'a> {
+            bucket: &'a str,
+            object: &'a str,
+            matching_bucket: bool,
+            matching_object: bool,
+        }
+        impl tracing::field::Visit for PartListFields<'_> {
+            fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+                match field.name() {
+                    "bucket" => self.matching_bucket = value == self.bucket,
+                    "object" => self.matching_object = value == self.object,
+                    _ => {}
+                }
+            }
+            fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
+                let value = format!("{value:?}");
+                match field.name() {
+                    "bucket" => self.matching_bucket = value == format!("{:?}", self.bucket),
+                    "object" => self.matching_object = value == format!("{:?}", self.object),
+                    _ => {}
+                }
+            }
+        }
+        impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for PartListCounter {
+            fn on_new_span(
+                &self,
+                attrs: &tracing::span::Attributes<'_>,
+                _id: &tracing::span::Id,
+                _ctx: tracing_subscriber::layer::Context<'_, S>,
+            ) {
+                if attrs.metadata().name() != "list_object_parts" {
+                    return;
+                }
+                let mut fields = PartListFields {
+                    bucket: &self.bucket,
+                    object: self.object,
+                    matching_bucket: false,
+                    matching_object: false,
+                };
+                attrs.record(&mut fields);
+                if fields.matching_bucket && fields.matching_object {
+                    self.calls.fetch_add(1, Ordering::Relaxed);
+                }
+            }
+        }
+
+        let (_, ecstore) = setup_test_env().await;
+        let bucket = format!("stale-list-parts-{}", Uuid::new_v4().simple());
+        create_test_bucket(&ecstore, &bucket).await;
+        set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await;
+        let now = OffsetDateTime::now_utc();
+        for (object, age, size_filter, should_list) in [
+            ("logs/default-expired", time::Duration::hours(30), false, false),
+            ("logs/lifecycle-only", time::Duration::minutes(5), false, false),
+            ("logs/size-filter", time::Duration::minutes(5), true, true),
+        ] {
+            if size_filter {
+                set_abort_incomplete_lifecycle_with_size(&bucket, "logs/", 0, 0).await;
+            }
+            let upload = ecstore
+                .new_multipart_upload(
+                    &bucket,
+                    object,
+                    &ObjectOptions {
+                        mod_time: Some(now - age),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("create multipart upload");
+            if size_filter {
+                let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]);
+                ecstore
+                    .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
+                    .await
+                    .expect("upload a part for size filtering");
+            }
+            let calls = Arc::new(AtomicUsize::new(0));
+            let subscriber = tracing_subscriber::registry().with(PartListCounter {
+                calls: Arc::clone(&calls),
+                bucket: bucket.clone(),
+                object,
+            });
+            cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60))
+                .with_subscriber(subscriber)
+                .await;
+            let listed = calls.load(Ordering::Relaxed);
+            assert_eq!(listed > 0, should_list, "unexpected ListParts calls for {object}: {listed}");
+            let result = ecstore
+                .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
+                .await;
+            assert!(
+                matches!(result, Err(ref err) if is_err_invalid_upload_id(err)),
+                "upload must be removed: {result:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn stale_multipart_cleanup_rejects_incomplete_or_overflowing_part_sizes() {
+        use crate::storage_api_contracts::multipart::{ListPartsInfo, PartInfo};
+
+        let part = |size, actual_size, etag: &str| PartInfo {
+            size,
+            actual_size,
+            etag: Some(etag.to_owned()),
+            ..Default::default()
+        };
+        let mut parts = ListPartsInfo {
+            parts: vec![part(6, 6, "valid")],
+            ..Default::default()
+        };
+        assert_eq!(super::stale_upload_size_from_parts(&parts), Some(6));
+        parts.is_truncated = true;
+        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
+        parts.is_truncated = false;
+        parts.parts = vec![part(0, 0, "")];
+        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
+        parts.parts = vec![part(1, -1, "valid")];
+        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
+        parts.parts = vec![part(8, 6, "valid")];
+        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
+        parts.parts = vec![part(4, 6, "valid")];
+        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
+        rustfs_utils::http::insert_str(&mut parts.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION, "test".to_owned());
+        assert_eq!(super::stale_upload_size_from_parts(&parts), Some(6));
+        parts.parts = vec![part(8, 6, "valid")];
+        assert_eq!(super::stale_upload_size_from_parts(&parts), Some(6));
+        rustfs_utils::http::remove_str(&mut parts.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
+        parts
+            .user_defined
+            .insert("x-amz-server-side-encryption".to_owned(), "AES256".to_owned());
+        assert_eq!(super::stale_upload_size_from_parts(&parts), Some(6));
+        parts.user_defined.clear();
+        parts.parts = vec![part(usize::MAX, 0, "valid"), part(1, 1, "valid")];
+        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn stale_multipart_cleanup_handles_unknown_size_without_false_abort() {
+        let (_, ecstore) = setup_test_env().await;
+        let bucket = format!("stale-unknown-size-{}", Uuid::new_v4().simple());
+        let object = "logs/object";
+        create_test_bucket(&ecstore, &bucket).await;
+        let size_rule = r#"<Rule><ID>size</ID><Status>Enabled</Status><Filter><ObjectSizeLessThan>5</ObjectSizeLessThan></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule>"#;
+        let independent_rule = r#"<Rule><ID>prefix</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule>"#;
+        let lifecycle = |rules: &str| format!("<LifecycleConfiguration>{rules}</LifecycleConfiguration>").into_bytes();
+        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle(size_rule))
+            .await
+            .expect("install upper-size lifecycle rule");
+        let metadata = HashMap::from([
+            (super::RUSTFS_MULTIPART_BUCKET_KEY.to_owned(), bucket.clone()),
+            (super::RUSTFS_MULTIPART_OBJECT_KEY.to_owned(), object.to_owned()),
+        ]);
+        let initiated = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        let set = &ecstore.pools[0].disk_set[0];
+        let upload = ecstore
+            .new_multipart_upload(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect("create empty multipart upload");
+        let decoded = base64_simd::URL_SAFE_NO_PAD
+            .decode_to_vec(upload.upload_id.as_bytes())
+            .expect("decode upload ID");
+        let decoded = String::from_utf8(decoded).expect("upload ID must be UTF-8");
+        let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload ID");
+        assert!(
+            super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
+                .await
+                .is_some(),
+            "a known empty upload must match the upper-size rule"
+        );
+        let missing_upload = format!("{}x{}", Uuid::new_v4(), initiated.unix_timestamp_nanos());
+        assert!(
+            super::stale_upload_lifecycle_due(set, &metadata, initiated, &missing_upload, false)
+                .await
+                .is_none(),
+            "an unreadable size must not match the upper-size rule"
+        );
+        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle(&format!("{size_rule}{independent_rule}")))
+            .await
+            .expect("install size-independent lifecycle rule");
+        assert!(
+            super::stale_upload_lifecycle_due(set, &metadata, initiated, &missing_upload, false)
+                .await
+                .is_some(),
+            "a size-independent rule remains eligible when size is unknown"
+        );
+        ecstore
+            .abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
+            .await
+            .expect("remove this test's remaining upload from the shared fixture");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn strict_multipart_lifecycle_size_skips_unavailable_participant_and_recovers() {
+        let (_, ecstore) = setup_test_env().await;
+        let bucket = format!("stale-strict-size-{}", Uuid::new_v4().simple());
+        let object = "logs/object";
+        create_test_bucket(&ecstore, &bucket).await;
+        let rule = br#"<LifecycleConfiguration><Rule><ID>size</ID><Status>Enabled</Status><Filter><ObjectSizeLessThan>5</ObjectSizeLessThan></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule></LifecycleConfiguration>"#;
+        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, rule.to_vec())
+            .await
+            .expect("install upper-size rule");
+        let upload = ecstore
+            .new_multipart_upload(&bucket, object, &ObjectOptions::default())
+            .await
+            .expect("create empty upload");
+        let decoded = base64_simd::URL_SAFE_NO_PAD
+            .decode_to_vec(upload.upload_id.as_bytes())
+            .expect("decode upload ID");
+        let decoded = String::from_utf8(decoded).expect("upload ID is UTF-8");
+        let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload ID");
+        let metadata = HashMap::from([
+            (super::RUSTFS_MULTIPART_BUCKET_KEY.to_owned(), bucket.clone()),
+            (super::RUSTFS_MULTIPART_OBJECT_KEY.to_owned(), object.to_owned()),
+        ]);
+        let initiated = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        let set = &ecstore.pools[0].disk_set[0];
+        assert!(
+            super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
+                .await
+                .is_some()
+        );
+        let disk = set.disks.write().await[3].take();
+        assert!(disk.is_some());
+        let due = super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false).await;
+        set.disks.write().await[3] = disk;
+        assert!(due.is_none(), "unknown size must not match an upper-size rule");
+        assert!(
+            super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
+                .await
+                .is_some()
+        );
+        ecstore
+            .abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
+            .await
+            .expect("remove this test's remaining upload from the shared fixture");
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn strict_multipart_lifecycle_does_not_treat_a_minority_part_as_empty() {
+        let (paths, ecstore) = setup_test_env().await;
+        let bucket = format!("stale-part-minority-{}", Uuid::new_v4().simple());
+        let object = "logs/object";
+        create_test_bucket(&ecstore, &bucket).await;
+        let initiated = OffsetDateTime::now_utc() - time::Duration::hours(1);
+        let upload = ecstore
+            .new_multipart_upload(
+                &bucket,
+                object,
+                &ObjectOptions {
+                    mod_time: Some(initiated),
+                    max_parity: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("create upload");
+        let mut body = PutObjReader::from_vec(vec![7; 6]);
+        ecstore
+            .put_object_part(&bucket, object, &upload.upload_id, 1, &mut body, &ObjectOptions::default())
+            .await
+            .expect("write a known six-byte part");
+        let decoded = base64_simd::URL_SAFE_NO_PAD
+            .decode_to_vec(upload.upload_id.as_bytes())
+            .expect("decode upload ID");
+        let decoded = String::from_utf8(decoded).expect("upload ID is UTF-8");
+        let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload ID");
+        let upload_path = format!("{}/{upload_dir}", multipart_sha_dir(&bucket, object));
+        let root = paths[0].join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path);
+        let bytes = fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("read source metadata");
+        let info = super::get_file_info(
+            &bytes,
+            RUSTFS_META_MULTIPART_BUCKET,
+            &upload_path,
+            "",
+            super::FileInfoOpts {
+                data: false,
+                include_free_versions: false,
+                include_part_checksums: false,
+            },
+        )
+        .expect("decode source metadata");
+        let data_dir = info.data_dir.expect("upload data directory").to_string();
+        let metadata = HashMap::from([
+            (super::RUSTFS_MULTIPART_BUCKET_KEY.to_owned(), bucket.clone()),
+            (super::RUSTFS_MULTIPART_OBJECT_KEY.to_owned(), object.to_owned()),
+        ]);
+        let set = &ecstore.pools[0].disk_set[0];
+        let rule = |filter: &str| {
+            format!("<LifecycleConfiguration><Rule><ID>size</ID><Status>Enabled</Status><Filter>{filter}</Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule></LifecycleConfiguration>").into_bytes()
+        };
+        let filters = [
+            "<ObjectSizeLessThan>10</ObjectSizeLessThan>",
+            "<ObjectSizeGreaterThan>5</ObjectSizeGreaterThan>",
+        ];
+        for filter in filters {
+            metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, rule(filter))
+                .await
+                .expect("install baseline size filter");
+            assert!(
+                super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
+                    .await
+                    .is_some(),
+                "known six-byte part must satisfy {filter} before the fault"
+            );
+        }
+        let mut saved = Vec::new();
+        for path in paths.iter().take(3) {
+            for file in ["part.1", "part.1.meta"] {
+                let path = path
+                    .join(RUSTFS_META_MULTIPART_BUCKET)
+                    .join(&upload_path)
+                    .join(&data_dir)
+                    .join(file);
+                let bytes = fs::read(&path).await.expect("snapshot original part");
+                fs::remove_file(&path).await.expect("simulate a missing part on this disk");
+                saved.push((path, bytes));
+            }
+        }
+        for filter in filters {
+            metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, rule(filter))
+                .await
+                .expect("install size filter");
+            super::metadata_boundary::get_lifecycle_config(&bucket)
+                .await
+                .expect("size filter must be readable during the fault");
+            assert!(
+                super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
+                    .await
+                    .is_none(),
+                "a minority part must not satisfy {filter}"
+            );
+        }
+        cleanup_stale_multipart_uploads_once_at(
+            ecstore.clone(),
+            OffsetDateTime::now_utc(),
+            StdDuration::from_secs(7 * 24 * 60 * 60),
+        )
+        .await;
+        ecstore
+            .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
+            .await
+            .expect("unknown size must preserve the upload during cleanup");
+        for (path, bytes) in saved {
+            fs::write(path, bytes).await.expect("restore original part");
+        }
+        let size = super::stale_upload_current_size_with_opts(set, &metadata, upload_dir, false).await;
+        assert_eq!(size, Some(6), "restored part must have its original logical size");
+        for filter in filters {
+            metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, rule(filter))
+                .await
+                .expect("restore size filter");
+            let config = super::metadata_boundary::get_lifecycle_config(&bucket)
+                .await
+                .expect("read lifecycle");
+            let due = super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false).await;
+            assert!(due.is_some(), "restored part must satisfy {filter}; size={size:?}, config={config:?}");
+        }
+        ecstore
+            .abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
+            .await
+            .expect("remove the restored test upload from the shared fixture");
+    }
+
     #[tokio::test]
     #[serial]
     async fn stale_multipart_cleanup_uses_default_expiry_without_lifecycle() {

crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs

@@ -63,14 +63,7 @@ pub(crate) async fn get_expiry_configs_for_options(
     bucket: &str,
     opts: &crate::object_api::ObjectOptions,
 ) -> Result<LifecycleExpiryConfigs> {
-    let bucket_incarnation_id = match opts
-        .object_lock_config_snapshot
-        .as_ref()
-        .and_then(|snapshot| snapshot.validated_bucket_incarnation_for(api.id, bucket, opts.expected_bucket_incarnation_id))
-    {
-        Some(incarnation_id) => incarnation_id,
-        None => metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?,
-    };
+    let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
     let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
     let table_bucket_enabled = metadata.table_bucket_enabled();
 

crates/ecstore/src/bucket/metadata_sys.rs

@@ -564,12 +564,39 @@ pub(crate) async fn validate_bucket_reconstruction_in(
     Ok(())
 }
 
-/// Validate against disk while retaining an already-held Object Lock fence.
+/// Resolve the authoritative bucket incarnation id for `opts`:
+/// 1. When namespace locking is enabled and `opts.object_lock_config_snapshot` is
+///    present, validate and return the snapshot's incarnation under its held
+///    lifecycle and metadata-transaction fences.
+/// 2. When namespace locking is enabled and `opts.bucket_lifecycle_lock_fence` is
+///    present and not lost, reuse the store's `BucketFenceRegistry` memo if a
+///    registered lifecycle read guard is currently held without lock loss.
+/// 3. Otherwise, acquire the bucket metadata transaction read lock (unless
+///    already held on `opts`) and read the incarnation from disk.
 pub(crate) async fn get_bucket_incarnation_id_for_options_in(
     ctx: &crate::runtime::instance::InstanceContext,
     bucket: &str,
     opts: &ObjectOptions,
 ) -> Result<Uuid> {
+    if !ctx.lock_manager().is_disabled() {
+        if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
+            let store = object_store_in(ctx).await?;
+            return snapshot
+                .validated_bucket_incarnation_for(store.id, bucket, opts.expected_bucket_incarnation_id)
+                .ok_or_else(|| {
+                    Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
+                });
+        }
+        if opts
+            .bucket_lifecycle_lock_fence
+            .as_ref()
+            .is_some_and(|fence| !fence.is_lock_lost())
+            && let Some(store) = object_store_if_initialized_in(ctx).await
+            && let Some(incarnation) = store.bucket_fence_registry.validated_incarnation(bucket)
+        {
+            return Ok(incarnation);
+        }
+    }
     let sys = bucket_metadata_sys_of(ctx)?;
     let sys = sys.read().await.clone();
     let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;

crates/ecstore/src/config/com.rs

@@ -488,6 +488,18 @@ where
     Ok(data)
 }
 
+/// Read config bytes and metadata while preserving an existing empty payload.
+pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
+    api: Arc<S>,
+    file: &str,
+    opts: &ObjectOptions,
+) -> Result<(Vec<u8>, ObjectInfo)>
+where
+    S: EcstoreObjectIO,
+{
+    read_config_with_metadata_inner(api, file, opts, true, None).await
+}
+
 pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
 where
     S: ObjectIO<
@@ -2605,6 +2617,15 @@ where
     save_server_config_snapshot(api, cfg, &snapshot).await.map(|_| ())
 }
 
+/// Read config and generation from the same object without taking the transaction write lock.
+pub async fn read_server_config_with_generation(api: Arc<ECStore>) -> Result<(Config, Option<Uuid>)> {
+    match read_config_with_metadata_inner(api.clone(), &server_config_path(), &ObjectOptions::default(), false, None).await {
+        Ok((raw, info)) => Ok((decode_persisted_server_config(&raw)?.merge(), info.data_dir.filter(|id| !id.is_nil()))),
+        Err(Error::ConfigNotFound) => read_config_without_migrate(api).await.map(|config| (config, None)),
+        Err(error) => Err(error),
+    }
+}
+
 /// One serialized read of the persisted server config used as the seed and
 /// compare-and-swap fence for an admin update.
 #[derive(Debug)]

crates/ecstore/src/data_usage/mod.rs

@@ -1493,6 +1493,48 @@ pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<
     project_data_usage_from_backend_cached(store, DataUsageInfo::clone).await
 }
 
+/// Read one bucket's persisted usage without cloning the complete snapshot.
+pub async fn load_bucket_usage_from_backend_cached(
+    store: Arc<ECStore>,
+    bucket: &str,
+    incarnation: uuid::Uuid,
+) -> Result<Option<u64>, Error> {
+    project_data_usage_from_backend_cached(store, |info| certified_bucket_usage(info, bucket, incarnation)).await
+}
+
+fn certified_bucket_usage(info: &DataUsageInfo, bucket: &str, incarnation: uuid::Uuid) -> Option<u64> {
+    if incarnation.is_nil()
+        || !info.is_complete_bucket_usage_snapshot()
+        || info.usage_snapshot_observational
+        || info.scanner_cycle.is_none()
+        || info.scanner_epoch.is_none()
+    {
+        return None;
+    }
+    let pending_count = u64::try_from(info.usage_snapshot_pending_buckets.len()).ok()?;
+    if pending_count != info.usage_snapshot_pending_bucket_count {
+        return None;
+    }
+    match info.usage_snapshot_converged {
+        Some(true) if pending_count == 0 => {}
+        Some(false)
+            if pending_count > 0
+                && info.usage_snapshot_pending_buckets.windows(2).all(|pair| pair[0] < pair[1])
+                && info
+                    .usage_snapshot_pending_buckets
+                    .binary_search_by(|pending| pending.as_str().cmp(bucket))
+                    .is_err() => {}
+        _ => return None,
+    }
+    let usage = info.buckets_usage.get(bucket)?;
+    let certified = usage
+        .bucket_incarnation
+        .as_deref()
+        .and_then(|id| uuid::Uuid::parse_str(id).ok())
+        .filter(|id| !id.is_nil())?;
+    (certified == incarnation).then_some(usage.size)
+}
+
 // Project under the cache guard so scalar readers never copy the snapshot's
 // bucket maps. The owned API shares the same TTL, refresh and invalidation path.
 async fn project_data_usage_from_backend_cached<T>(

crates/ecstore/src/data_usage/tests/snapshot_projection.rs

@@ -129,6 +129,41 @@ fn projected_publication_moves_bucket_maps_into_cache() {
     );
 }
 
+#[test]
+fn certified_bucket_usage_rejects_unproven_and_pending_data() {
+    let id = uuid::Uuid::from_u128(7);
+    let mut info = data_usage_info_for_test("known", 1, 42, SystemTime::UNIX_EPOCH);
+    info.usage_snapshot_converged = Some(true);
+    info.scanner_cycle = Some(1);
+    info.scanner_epoch = Some(1);
+    info.buckets_usage.get_mut("known").unwrap().bucket_incarnation = Some(id.to_string());
+    assert_eq!(certified_bucket_usage(&info, "known", id), Some(42));
+    assert_eq!(certified_bucket_usage(&info, "known", uuid::Uuid::from_u128(8)), None);
+    assert_eq!(certified_bucket_usage(&info, "known", uuid::Uuid::nil()), None);
+    let mut legacy = info.clone();
+    legacy.buckets_usage.get_mut("known").unwrap().bucket_incarnation = None;
+    assert_eq!(certified_bucket_usage(&legacy, "known", id), None);
+    legacy.buckets_usage.get_mut("known").unwrap().bucket_incarnation = Some("corrupt".into());
+    assert_eq!(certified_bucket_usage(&legacy, "known", id), None);
+    let mut pending = info.clone();
+    pending.usage_snapshot_converged = Some(false);
+    pending.usage_snapshot_pending_buckets = vec!["other".into()];
+    pending.usage_snapshot_pending_bucket_count = 1;
+    assert_eq!(certified_bucket_usage(&pending, "known", id), Some(42));
+    pending.usage_snapshot_pending_buckets = vec!["known".into()];
+    assert_eq!(certified_bucket_usage(&pending, "known", id), None);
+    pending.usage_snapshot_pending_bucket_count = 0;
+    assert_eq!(certified_bucket_usage(&pending, "known", id), None);
+    pending.usage_snapshot_pending_buckets = vec!["z".into(), "a".into()];
+    pending.usage_snapshot_pending_bucket_count = 2;
+    assert_eq!(certified_bucket_usage(&pending, "known", id), None);
+    let mut observational = info.clone();
+    observational.usage_snapshot_observational = true;
+    assert_eq!(certified_bucket_usage(&observational, "known", id), None);
+    info.usage_snapshot_converged = None;
+    assert_eq!(certified_bucket_usage(&info, "known", id), None);
+}
+
 #[tokio::test]
 #[serial]
 async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
@@ -140,6 +175,13 @@ async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
     info.buckets_usage.insert("empty".to_string(), BucketUsageInfo::default());
     info.bucket_sizes.insert("empty".to_string(), 0);
     info.buckets_count = 2;
+    info.usage_snapshot_converged = Some(true);
+    info.scanner_cycle = Some(1);
+    info.scanner_epoch = Some(1);
+    let incarnation = uuid::Uuid::from_u128(7);
+    for usage in info.buckets_usage.values_mut() {
+        usage.bucket_incarnation = Some(incarnation.to_string());
+    }
     save_config(
         store.clone(),
         DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -151,11 +193,29 @@ async fn persisted_scalar_projection_shares_cache_refresh_and_invalidation() {
 
     for (bucket, expected) in [("known", Some(42)), ("empty", Some(0)), ("missing", None)] {
         assert_eq!(persisted_bucket_usage(store.clone(), bucket).await, expected);
+        assert_eq!(
+            load_bucket_usage_from_backend_cached(store.clone(), bucket, incarnation)
+                .await
+                .expect("project bucket usage"),
+            expected
+        );
         let owned = load_data_usage_from_backend_cached(store.clone())
             .await
             .expect("owned cached snapshot");
         assert_eq!(expected, owned.buckets_usage.get(bucket).map(|usage| usage.size));
     }
+    assert_eq!(
+        load_bucket_usage_from_backend_cached(store.clone(), "known", uuid::Uuid::from_u128(8))
+            .await
+            .unwrap(),
+        None
+    );
+    assert_eq!(
+        load_bucket_usage_from_backend_cached(store.clone(), "known", uuid::Uuid::nil())
+            .await
+            .unwrap(),
+        None
+    );
     info.buckets_usage.get_mut("known").expect("known bucket").size = 84;
     info.bucket_sizes.insert("known".to_string(), 84);
     save_config(

crates/ecstore/src/disk/local.rs

@@ -8450,6 +8450,23 @@ impl LocalDisk {
                     return Err(std::io::Error::other("metadata read worker cancelled"));
                 }
                 let _io_permit = io_permit;
+                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
+                {
+                    let opened = root.open_readonly(&path);
+                    if opened.is_ok() {
+                        os::prepared_publication_test_hooks::run(
+                            os::prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
+                            &path,
+                        );
+                    }
+                    let result = read_fn(opened);
+                    os::prepared_publication_test_hooks::run(
+                        os::prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
+                        &path,
+                    );
+                    Ok(result)
+                }
+                #[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
                 Ok(read_fn(root.open_readonly(&path)))
             })
             .await
@@ -11313,7 +11330,7 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize,
     let submitted = metrics.as_ref().map(|_| std::time::Instant::now());
     tokio::task::spawn_blocking(move || {
         use rustfs_io_metrics::LocalBatchReadStage;
-        use std::os::unix::fs::FileExt;
+        use std::os::unix::fs::{FileExt, OpenOptionsExt};
 
         let execution_started = metrics.as_ref().map(|_| std::time::Instant::now());
         if let (Some(metrics), Some(submitted)) = (&metrics, submitted) {
@@ -11324,17 +11341,24 @@ pub(crate) async fn batch_shard_pread(requests: Vec<(std::path::PathBuf, usize,
         for (file_path, offset, length) in requests {
             let mut bytes_read = 0;
             let r = (|| -> Result<Bytes> {
-                let meta = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Metadata, || {
-                    std::fs::metadata(&file_path).map_err(DiskError::from)
-                })?;
                 let end = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
-                if meta.len() < u64::try_from(end).unwrap_or(u64::MAX) {
+                let end = u64::try_from(end).map_err(|_| DiskError::FileCorrupt)?;
+                // Bind validation and reads to one descriptor. Nonblocking open
+                // lets corrupt non-regular entries fail without waiting on a FIFO.
+                let file = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Open, || {
+                    std::fs::OpenOptions::new()
+                        .read(true)
+                        .custom_flags(libc::O_NONBLOCK)
+                        .open(&file_path)
+                        .map_err(DiskError::from)
+                })?;
+                let meta = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Metadata, || {
+                    file.metadata().map_err(DiskError::from)
+                })?;
+                if !meta.is_file() || meta.len() < end {
                     return Err(DiskError::FileCorrupt);
                 }
 
-                let file = measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Open, || {
-                    std::fs::File::open(&file_path).map_err(DiskError::from)
-                })?;
                 let mut buf = vec![0u8; length];
                 if length > 0 {
                     measure_local_batch_stage(metrics.as_deref(), LocalBatchReadStage::Read, || {
@@ -31028,7 +31052,7 @@ mod test {
         assert_eq!(recorder.counter_value(counter, &[("outcome", "error")]), 2);
         assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 9);
         let stages = "rustfs_io_get_object_local_batch_stage_seconds";
-        for (stage, success, errors) in [("metadata", 3, 1), ("open", 2, 0), ("read", 1, 0), ("execution", 0, 1)] {
+        for (stage, success, errors) in [("open", 3, 1), ("metadata", 3, 0), ("read", 1, 0), ("execution", 0, 1)] {
             assert_eq!(
                 recorder
                     .histogram_values(stages, &[("stage", stage), ("outcome", "success")])
@@ -31097,6 +31121,70 @@ mod test {
         assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 4);
     }
 
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn test_batch_shard_pread_rejects_nonregular_files_without_blocking() {
+        use std::os::unix::fs::OpenOptionsExt;
+
+        let dir = tempfile::tempdir().expect("nonregular shard fixture");
+        let fifo = dir.path().join("fifo");
+        assert!(
+            std::process::Command::new("mkfifo")
+                .arg(&fifo)
+                .status()
+                .expect("run POSIX mkfifo for the shard fixture")
+                .success()
+        );
+        let good = dir.path().join("good");
+        std::fs::write(&good, b"data").expect("write regular shard");
+        let mut batch = Box::pin(batch_shard_pread(vec![
+            (dir.path().to_owned(), 0, 0),
+            (fifo.clone(), 0, 0),
+            (fifo.clone(), 0, 4),
+            (good, 0, 4),
+        ]));
+        let results = match tokio::time::timeout(Duration::from_secs(2), &mut batch).await {
+            Ok(results) => results,
+            Err(_) => {
+                // Release a regressed blocking open before failing, so runtime
+                // shutdown does not leave the test waiting on its worker.
+                let _release = std::fs::OpenOptions::new()
+                    .read(true)
+                    .write(true)
+                    .custom_flags(libc::O_NONBLOCK)
+                    .open(&fifo)
+                    .expect("release blocked FIFO reader");
+                batch.await;
+                panic!("a nonregular shard must not block its batch");
+            }
+        };
+        for result in &results[..3] {
+            assert!(matches!(result, Err(DiskError::FileCorrupt)), "nonregular shard: {result:?}");
+        }
+        assert_eq!(results[3].as_ref().expect("later regular shard remains readable").as_ref(), b"data");
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn test_batch_shard_pread_validates_descriptor_ranges() {
+        let dir = tempfile::tempdir().expect("range validation fixture");
+        let path = dir.path().join("shard");
+        std::fs::write(&path, b"01234567").expect("write range fixture");
+        let results = batch_shard_pread(vec![
+            (path.clone(), 3, 5),
+            (path.clone(), 8, 0),
+            (path.clone(), 8, 1),
+            (path.clone(), 9, 0),
+            (path, usize::MAX, 1),
+        ])
+        .await;
+        assert_eq!(results[0].as_ref().expect("range ending at EOF").as_ref(), b"34567");
+        assert!(results[1].as_ref().expect("empty range at EOF").is_empty());
+        for result in &results[2..] {
+            assert!(matches!(result, Err(DiskError::FileCorrupt)), "invalid range: {result:?}");
+        }
+    }
+
     #[cfg(unix)]
     #[tokio::test]
     async fn test_batch_shard_pread_basic() {

crates/ecstore/src/disk/os.rs

@@ -333,6 +333,8 @@ pub(crate) mod prepared_publication_test_hooks {
         PreparedRename,
         Rename,
         Remove,
+        MetadataReadAfterOpen,
+        MetadataReadAfterRead,
         #[cfg(test)]
         Rollback,
         #[cfg(test)]
@@ -457,6 +459,8 @@ pub enum LocalPublicationStage {
     PreparedRename,
     Rename,
     Remove,
+    MetadataReadAfterOpen,
+    MetadataReadAfterRead,
 }
 
 #[cfg(all(feature = "test-util", not(windows)))]
@@ -469,6 +473,8 @@ impl LocalPublicationPause {
             LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
             LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
             LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
+            LocalPublicationStage::MetadataReadAfterOpen => prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
+            LocalPublicationStage::MetadataReadAfterRead => prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
         };
         let (entered_tx, entered) = oneshot::channel();
         let (release, release_rx) = std::sync::mpsc::channel::<()>();

crates/ecstore/src/object_api/types.rs

@@ -959,6 +959,10 @@ pub struct ObjectOptions {
     /// Persisted bucket incarnation observed before authorization.
     pub expected_bucket_incarnation_id: Option<Uuid>,
     pub no_lock: bool,
+    /// Require complete participant observations for internal multipart size decisions.
+    /// This transient flag is never derived from client input or stored on disk.
+    #[doc(hidden)]
+    pub strict_multipart_part_sizes: bool,
     /// A genuine target write guard; loss fences alone do not authorize a borrowed write.
     #[doc(hidden)]
     pub write_lock_context: Option<WriteLockContext>,

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -280,7 +280,7 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
             continue;
         }
 
-        if !parts[part_idx].etag.is_empty() {
+        if !parts[part_idx].etag.is_empty() && parts[part_idx].error.is_none() {
             present_count += 1;
             let part = &parts[part_idx];
             let key = (part.etag.as_str(), part.number, part.size, part.actual_size, part.integrity.as_ref());
@@ -1118,9 +1118,9 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
     // offset can be advanced to the failing stripe. A parity reader that was
     // opened eagerly during setup is pinned at the stripe-0 stream position
     // and could never be engaged mid-object, so swap it for an unopened
-    // deferred reader carrying a stripe handle. The eager open already proved
-    // the shard is reachable; no shard bytes were read from it, and the
-    // ready/error bookkeeping that quorum decisions rely on is left untouched.
+    // deferred reader carrying a stripe handle. The eager setup already proved
+    // the shard is reachable; the ready/error bookkeeping that quorum decisions
+    // rely on is left untouched.
     // Gate off (default): keep the eagerly opened parity readers exactly as
     // before — the lockstep path reads them on every stripe.
     if !demand_bound_lockstep {
@@ -1217,8 +1217,8 @@ pub(in crate::set_disk) fn record_bitrot_reader_setup_strategy(
 }
 
 /// When all online shards are local and mmap-read is enabled, materialize
-/// shard bytes with one `batch_shard_pread` instead of per-shard
-/// `spawn_blocking` via `open_disk_reader`.
+/// shard bytes in batches instead of per-shard `spawn_blocking` via
+/// `open_disk_reader`. Demand-bound reads add parity batches only as needed.
 #[cfg(unix)]
 #[allow(clippy::too_many_arguments)]
 async fn try_create_bitrot_readers_via_batch_pread(
@@ -1232,6 +1232,10 @@ async fn try_create_bitrot_readers_via_batch_pread(
     shard_size: usize,
     checksum_algo: HashAlgorithm,
     skip_verify_bitrot: bool,
+    data_shards: usize,
+    parity_shards: usize,
+    mode: BitrotReaderSetupMode,
+    strategy: BitrotReaderSetupStrategy,
 ) -> Option<BitrotReaderSetup> {
     use crate::disk::local::batch_shard_pread;
     use rustfs_io_metrics::{LocalBatchReadDecision, record_local_batch_read_decision};
@@ -1275,42 +1279,62 @@ async fn try_create_bitrot_readers_via_batch_pread(
         return None;
     }
 
-    let requests: Vec<_> = batch_items.iter().map(|(_, p, off, len)| (p.clone(), *off, *len)).collect();
-    #[cfg(test)]
-    for (disk_index, _, _, _) in &batch_items {
-        disk_call_counters::record(object, disk_call_counters::KIND_DISK_SHARD_OPEN, *disk_index);
-    }
-    record_decision(LocalBatchReadDecision::Selected);
-    let batch_results = batch_shard_pread(requests).await;
-
+    // Legacy lockstep consumes every shard, so keep its coalesced batch even
+    // when setup prefers data blocks. Only demand-bound decode can leave the
+    // parity readers unopened; preading them here would discard their bytes
+    // when fill_deferred_bitrot_readers installs stripe-aligned reserves.
+    let data_first = strategy != BitrotReaderSetupStrategy::AllShards
+        && crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled();
     let mut setup = BitrotReaderSetup::new(disks.len());
-    for (i, (idx, _, _, _)) in batch_items.iter().enumerate() {
-        setup.mark_scheduled(*idx);
-        match &batch_results[i] {
-            Ok(bytes) => {
-                let reader = BitrotReader::new(
-                    ShardReader::InMemory(Cursor::new(bytes.clone())),
-                    shard_size,
-                    checksum_algo.clone(),
-                    skip_verify_bitrot,
-                );
-                setup.apply_reader_result(*idx, Ok(Some(reader)));
-            }
-            Err(e) => {
-                setup.apply_reader_result(*idx, Err(e.clone()));
-            }
-        }
-    }
-
-    for (idx, disk_op) in disks.iter().enumerate() {
-        if setup.scheduled[idx] {
-            continue;
-        }
-        setup.mark_scheduled(idx);
-        if disk_op.is_none() {
+    for (idx, disk) in disks.iter().enumerate() {
+        if disk.is_none() {
+            setup.mark_scheduled(idx);
             setup.apply_reader_result(idx, Ok(None));
         }
     }
+    let mut batch_size = if data_first {
+        batch_items.iter().take_while(|(idx, _, _, _)| *idx < data_shards).count()
+    } else {
+        batch_items.len()
+    };
+    let mut pending = batch_items.into_iter();
+    record_decision(LocalBatchReadDecision::Selected);
+    loop {
+        let mut indices = Vec::with_capacity(batch_size.min(pending.len()));
+        let requests: Vec<_> = pending
+            .by_ref()
+            .take(batch_size)
+            .map(|(idx, path, offset, length)| {
+                setup.mark_scheduled(idx);
+                indices.push(idx);
+                #[cfg(test)]
+                disk_call_counters::record(object, disk_call_counters::KIND_DISK_SHARD_OPEN, idx);
+                (path, offset, length)
+            })
+            .collect();
+        if !requests.is_empty() {
+            for (idx, result) in indices.into_iter().zip(batch_shard_pread(requests).await) {
+                let result = result.map(|bytes| {
+                    Some(BitrotReader::new(
+                        ShardReader::InMemory(Cursor::new(bytes)),
+                        shard_size,
+                        checksum_algo.clone(),
+                        skip_verify_bitrot,
+                    ))
+                });
+                setup.apply_reader_result(idx, result);
+            }
+        }
+        if !data_first || setup.has_setup_quorum(data_shards, parity_shards, mode) || pending.len() == 0 {
+            break;
+        }
+        // A missing/corrupt data file needs replacement parity. Reuse the
+        // canonical target, including the extra verification source required
+        // for reconstruction, and retry only the outstanding deficit.
+        batch_size = setup
+            .scheduling_target(data_shards, parity_shards, mode)
+            .saturating_sub(setup.available_shards());
+    }
 
     Some(setup)
 }
@@ -1328,6 +1352,10 @@ async fn try_create_bitrot_readers_via_batch_pread(
     _shard_size: usize,
     _checksum_algo: HashAlgorithm,
     _skip_verify_bitrot: bool,
+    _data_shards: usize,
+    _parity_shards: usize,
+    _mode: BitrotReaderSetupMode,
+    _strategy: BitrotReaderSetupStrategy,
 ) -> Option<BitrotReaderSetup> {
     rustfs_io_metrics::record_local_batch_read_decision(rustfs_io_metrics::LocalBatchReadDecision::UnsupportedPlatform, None);
     None
@@ -1517,6 +1545,10 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_with_prefere
             shard_size,
             checksum_algo.clone(),
             skip_verify_bitrot,
+            data_shards,
+            parity_shards,
+            mode,
+            strategy,
         )
         .await
     {
@@ -7889,6 +7921,329 @@ mod tests {
         (dirs, disks)
     }
 
+    #[cfg(unix)]
+    async fn local_batch_quorum_fixture(
+        bucket: &str,
+        object: &str,
+    ) -> (Vec<TempDir>, Vec<Option<DiskStore>>, Vec<FileInfo>, Vec<u8>) {
+        let payload: Vec<_> = (0..1024 * 1024)
+            .map(|index| u8::try_from(index % 251).expect("bounded byte"))
+            .collect();
+        let (dirs, disks) = call_counter_local_disks(bucket, 8).await;
+        let mut files = inline_metadata_fanout_fileinfos_with_geometry(bucket, object, &payload, false, 4, 4).await;
+        for (file, disk) in files.iter_mut().zip(&disks) {
+            let path = format!("{object}/{}/part.1", file.data_dir.expect("fixture data directory"));
+            disk.as_ref()
+                .expect("fixture disk")
+                .write_all(bucket, &path, file.data.take().expect("encoded fixture shard"))
+                .await
+                .expect("external shard fixture should write");
+        }
+        (dirs, disks, files, payload)
+    }
+
+    #[cfg(unix)]
+    async fn local_batch_quorum_setup(
+        files: &[FileInfo],
+        disks: &[Option<DiskStore>],
+        bucket: &str,
+        object: &str,
+        mode: BitrotReaderSetupMode,
+    ) -> BitrotReaderSetup {
+        let erasure = coding::Erasure::new_with_options(4, 4, files[0].erasure.block_size, false);
+        create_bitrot_readers_until_quorum_with_preference(
+            files,
+            disks,
+            bucket,
+            object,
+            1,
+            0,
+            erasure.shard_file_offset(0, 1024 * 1024, 1024 * 1024),
+            erasure.shard_size(),
+            HashAlgorithm::HighwayHash256S,
+            false,
+            true,
+            4,
+            4,
+            mode,
+            false,
+            None,
+            None,
+        )
+        .await
+    }
+
+    #[cfg(unix)]
+    async fn local_batch_quorum_decode(setup: BitrotReaderSetup, block_size: usize) -> (Vec<u8>, Option<std::io::Error>) {
+        let erasure = coding::Erasure::new_with_options(4, 4, block_size, false);
+        let mut output = Vec::new();
+        let (_, error) = erasure
+            .decode_with_stripe_handles_and_reopeners(
+                &mut output,
+                setup.readers,
+                0,
+                1024 * 1024,
+                1024 * 1024,
+                None,
+                setup.deferred_stripe_handles,
+                setup.deferred_reopeners,
+            )
+            .await;
+        (output, error)
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_batch_quorum_healthy_reads_only_data_when_demand_bound() {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
+                ("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("false")),
+            ],
+            crate::erasure::coding::decode::with_decode_read_policy(
+                crate::erasure::coding::decode::DecodeReadPolicy::DemandBound,
+                async {
+                    let bucket = "batch-data-only";
+                    let object = "healthy-one-mib";
+                    let (_dirs, disks, files, payload) = local_batch_quorum_fixture(bucket, object).await;
+                    let calls = disk_call_counters::observe(object);
+                    let setup = local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::ReadQuorum).await;
+                    assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
+                    assert_eq!(setup.available_shards(), 4);
+                    assert_eq!(setup.deferred_shards(), 4);
+                    for index in 4..8 {
+                        assert_eq!(calls.for_disk(disk_call_counters::KIND_DISK_SHARD_OPEN, index), 0);
+                        assert!(!setup.attempted[index], "parity must stay unopened until needed");
+                    }
+                    let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
+                    assert!(error.is_none(), "healthy decode failed: {error:?}");
+                    assert_eq!(output, payload);
+                    assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
+                },
+            ),
+        )
+        .await;
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_batch_quorum_preserves_legacy_and_all_shards_override() {
+        for (data_first, demand_bound) in [("true", "false"), ("false", "true")] {
+            temp_env::async_with_vars(
+                [
+                    (ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some(data_first)),
+                    ("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some(demand_bound)),
+                ],
+                async {
+                    let bucket = "batch-legacy";
+                    let object = "all-shards";
+                    let (_dirs, disks, files, payload) = local_batch_quorum_fixture(bucket, object).await;
+                    let calls = disk_call_counters::observe(object);
+                    let setup = local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::ReadQuorum).await;
+                    assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 8);
+                    assert_eq!(setup.available_shards(), 8);
+                    let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
+                    assert!(error.is_none(), "legacy decode failed: {error:?}");
+                    assert_eq!(output, payload);
+                },
+            )
+            .await;
+        }
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_batch_quorum_recovers_missing_truncated_and_bitrot_data() {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
+                ("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
+            ],
+            async {
+                for fault in ["missing", "truncated", "bitrot"] {
+                    let bucket = "batch-degraded";
+                    let object = format!("degraded-{fault}");
+                    let (dirs, disks, files, payload) = local_batch_quorum_fixture(bucket, &object).await;
+                    let path = dirs[0]
+                        .path()
+                        .join(bucket)
+                        .join(&object)
+                        .join(files[0].data_dir.expect("fixture directory").to_string())
+                        .join("part.1");
+                    match fault {
+                        "missing" => tokio::fs::remove_file(&path).await.expect("remove data shard"),
+                        "truncated" => tokio::fs::write(&path, b"short").await.expect("truncate data shard"),
+                        _ => {
+                            let mut bytes = tokio::fs::read(&path).await.expect("read bitrot fixture");
+                            bytes[32] ^= 0xff;
+                            tokio::fs::write(&path, bytes)
+                                .await
+                                .expect("corrupt data without changing length");
+                        }
+                    }
+                    let setup =
+                        local_batch_quorum_setup(&files, &disks, bucket, &object, BitrotReaderSetupMode::ReadQuorum).await;
+                    assert_eq!(setup.available_shards(), 4);
+                    if fault == "bitrot" {
+                        assert_eq!(setup.attempted_shards(), 4, "bitrot is detected on decode, not pread");
+                    } else {
+                        assert_eq!(setup.attempted_shards(), 5, "setup reads only one replacement parity shard");
+                    }
+                    let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
+                    assert!(error.is_none(), "{fault} should recover: {error:?}");
+                    assert_eq!(output, payload, "{fault} must reconstruct the exact body");
+                }
+            },
+        )
+        .await;
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_batch_quorum_keeps_extra_reconstruction_verification_source() {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
+                ("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
+            ],
+            async {
+                let bucket = "batch-verification";
+                let object = "missing-data";
+                let (_dirs, mut disks, files, payload) = local_batch_quorum_fixture(bucket, object).await;
+                disks[0] = None;
+                let calls = disk_call_counters::observe(object);
+                let setup =
+                    local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::VerifyReconstruction).await;
+                assert_eq!(setup.available_shards(), 5, "degraded reconstruction needs quorum plus verification");
+                assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 5);
+                assert!(setup.ready[4] && setup.ready[5], "two parity sources must be available");
+                assert!(!setup.attempted[6] && !setup.attempted[7], "unneeded parity stays deferred");
+                let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
+                assert!(error.is_none(), "verified reconstruction failed: {error:?}");
+                assert_eq!(output, payload);
+            },
+        )
+        .await;
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_batch_quorum_fails_without_read_quorum() {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
+                ("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
+            ],
+            async {
+                let bucket = "batch-no-quorum";
+                let object = "all-data-offline";
+                let (_dirs, mut disks, files, _payload) = local_batch_quorum_fixture(bucket, object).await;
+                disks[..5].fill(None);
+                let setup = local_batch_quorum_setup(&files, &disks, bucket, object, BitrotReaderSetupMode::ReadQuorum).await;
+                assert_eq!(setup.available_shards(), 3);
+                assert!(!setup.has_setup_quorum(4, 4, BitrotReaderSetupMode::ReadQuorum));
+                assert_eq!(setup.deferred_shards(), 0);
+                let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
+                assert!(error.is_some(), "insufficient shards must fail closed");
+                assert!(output.is_empty(), "no unverified bytes may be emitted");
+            },
+        )
+        .await;
+    }
+
+    #[cfg(unix)]
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn local_batch_quorum_recovers_late_bitrot_after_range_offset() {
+        temp_env::async_with_vars(
+            [
+                (ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP, Some("true")),
+                ("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true")),
+            ],
+            async {
+                let bucket = "batch-ranged";
+                let object = "late-bitrot";
+                let (_dirs, disks) = call_counter_local_disks(bucket, 8).await;
+                let erasure = coding::Erasure::new_with_options(4, 4, 64, false);
+                let payload: Vec<u8> = (0..192).collect();
+                let mut shards = vec![Vec::new(); 8];
+                for block in payload.chunks(64) {
+                    for (index, shard) in erasure
+                        .encode_data(block)
+                        .expect("encode range stripe")
+                        .into_iter()
+                        .enumerate()
+                    {
+                        let mut writer = coding::BitrotWriterWrapper::new(
+                            coding::CustomWriter::new_inline_buffer(),
+                            erasure.shard_size(),
+                            HashAlgorithm::HighwayHash256S,
+                        );
+                        writer.write(&shard).await.expect("write range stripe");
+                        writer.shutdown().await.expect("finish range stripe");
+                        shards[index].extend(writer.into_inline_data().expect("range stripe bytes"));
+                    }
+                }
+                // Corrupt the third stripe; the first requested stripe is healthy.
+                shards[0][2 * (32 + erasure.shard_size()) + 32] ^= 0xff;
+                let mut files = vec![FileInfo::new(object, 4, 4); 8];
+                let data_dir = Uuid::new_v4();
+                for ((file, disk), bytes) in files.iter_mut().zip(&disks).zip(shards) {
+                    file.data_dir = Some(data_dir);
+                    disk.as_ref()
+                        .expect("fixture disk")
+                        .write_all(bucket, &format!("{object}/{data_dir}/part.1"), Bytes::from(bytes))
+                        .await
+                        .expect("write multi-stripe shard");
+                }
+                let calls = disk_call_counters::observe(object);
+                let setup = create_bitrot_readers_until_quorum_with_preference(
+                    &files,
+                    &disks,
+                    bucket,
+                    object,
+                    1,
+                    erasure.shard_size(),
+                    2 * erasure.shard_size(),
+                    erasure.shard_size(),
+                    HashAlgorithm::HighwayHash256S,
+                    false,
+                    true,
+                    4,
+                    4,
+                    BitrotReaderSetupMode::ReadQuorum,
+                    false,
+                    None,
+                    None,
+                )
+                .await;
+                assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
+                let mut output = Vec::new();
+                let (written, error) = erasure
+                    .decode_with_stripe_handles_and_reopeners(
+                        &mut output,
+                        setup.readers,
+                        64,
+                        128,
+                        payload.len(),
+                        None,
+                        setup.deferred_stripe_handles,
+                        setup.deferred_reopeners,
+                    )
+                    .await;
+                assert!(error.is_none(), "late bitrot must recover from stripe-aligned parity: {error:?}");
+                assert_eq!(written, 128);
+                assert_eq!(output, payload[64..]);
+            },
+        )
+        .await;
+    }
+
     #[cfg(unix)]
     #[test]
     fn local_batch_read_decisions_follow_the_actual_reader_setup_entry() {
@@ -7957,6 +8312,10 @@ mod tests {
                             4,
                             HashAlgorithm::None,
                             false,
+                            1,
+                            1,
+                            BitrotReaderSetupMode::ReadQuorum,
+                            BitrotReaderSetupStrategy::AllShards,
                         )
                         .await
                         .is_none(),
@@ -7974,6 +8333,10 @@ mod tests {
                             4,
                             HashAlgorithm::HighwayHash256S,
                             false,
+                            1,
+                            1,
+                            BitrotReaderSetupMode::ReadQuorum,
+                            BitrotReaderSetupStrategy::AllShards,
                         )
                         .await
                         .is_none(),
@@ -7993,6 +8356,10 @@ mod tests {
                             4,
                             HashAlgorithm::None,
                             false,
+                            1,
+                            1,
+                            BitrotReaderSetupMode::ReadQuorum,
+                            BitrotReaderSetupStrategy::AllShards,
                         )
                         .await
                         .is_none()
@@ -8020,6 +8387,10 @@ mod tests {
                             4,
                             HashAlgorithm::None,
                             false,
+                            1,
+                            1,
+                            BitrotReaderSetupMode::ReadQuorum,
+                            BitrotReaderSetupStrategy::AllShards,
                         )
                         .await
                         .is_none(),
@@ -8037,6 +8408,10 @@ mod tests {
                             4,
                             HashAlgorithm::None,
                             false,
+                            1,
+                            1,
+                            BitrotReaderSetupMode::ReadQuorum,
+                            BitrotReaderSetupStrategy::AllShards,
                         )
                         .await
                         .is_none()
@@ -8054,6 +8429,10 @@ mod tests {
                             4,
                             HashAlgorithm::None,
                             false,
+                            1,
+                            1,
+                            BitrotReaderSetupMode::ReadQuorum,
+                            BitrotReaderSetupStrategy::AllShards,
                         )
                         .await
                         .is_none()
@@ -12823,6 +13202,20 @@ mod tests {
         assert_eq!(err, DiskError::ErasureReadQuorum);
     }
 
+    #[test]
+    fn resolve_read_part_does_not_count_error_response_with_etag_as_a_vote() {
+        let healthy = read_part_test_part(1, "same-etag");
+        let mut failed = healthy.clone();
+        failed.error = Some("metadata read failed".to_owned());
+        let responses = vec![Some(vec![healthy.clone()]), Some(vec![healthy]), Some(vec![failed])];
+        let err = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 3)
+            .expect_err("an error-bearing response must not supply the third vote");
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+        let part = resolve_read_part_from_responses("bucket", "upload/part.1.meta", 1, 0, 1, &responses, 2)
+            .expect("the two healthy matching responses remain a valid quorum");
+        assert!(part.error.is_none());
+    }
+
     #[test]
     fn resolve_read_part_accepts_alternate_missing_error_markers() {
         let responses = vec![

crates/ecstore/src/set_disk/core/multipart_source.rs

@@ -338,17 +338,26 @@ fn abort_slot(
     Ok(())
 }
 
-fn abort_certificate_cohort(
+pub(in crate::set_disk) fn abort_certificate_cohort(
     certificate: &MultipartAbortCertificate,
     ids: &[Option<Uuid>],
     authority: &ProofVerificationContext,
 ) -> Result<()> {
     certificate.verify_with_authority(authority)?;
-    if ids.len() != certificate.disk_ids.len()
-        || ids.iter().copied().collect::<Option<Vec<_>>>().as_ref() != Some(&certificate.disk_ids)
-    {
+    if ids.len() != certificate.disk_ids.len() {
         return Err(DiskError::FileCorrupt);
     }
+    let mut missing = false;
+    for (actual, expected) in ids.iter().zip(&certificate.disk_ids) {
+        match actual {
+            Some(actual) if actual != expected => return Err(DiskError::FileCorrupt),
+            Some(_) => {}
+            None => missing = true,
+        }
+    }
+    if missing {
+        return Err(DiskError::ErasureWriteQuorum);
+    }
     Ok(())
 }
 

crates/ecstore/src/set_disk/mod.rs

@@ -283,6 +283,23 @@ fn ensure_decommission_tier_free_version_commit_fence(bucket: &str, object: &str
     Ok(())
 }
 
+/// Names a `bucket/object` key prefix whose best-effort rename-tail repairs
+/// are not submitted. A root-heal fixture arms it so the explicit admin heal
+/// remains the only owner of objects written while their target was offline.
+#[cfg(feature = "e2e-test-hooks")]
+const RENAME_TAIL_HEAL_SUPPRESS_PREFIX_ENV: &str = "RUSTFS_E2E_SUPPRESS_RENAME_TAIL_HEAL_PREFIX";
+
+#[cfg(feature = "e2e-test-hooks")]
+fn rename_tail_heal_suppressed_for_e2e(bucket: &str, object: Option<&str>) -> bool {
+    let Some(prefix) = std::env::var(RENAME_TAIL_HEAL_SUPPRESS_PREFIX_ENV)
+        .ok()
+        .filter(|prefix| !prefix.is_empty())
+    else {
+        return false;
+    };
+    object.is_some_and(|object| format!("{bucket}/{object}").starts_with(&prefix))
+}
+
 impl SetDisks {
     pub(super) async fn require_current_restore_operation_id(
         &self,
@@ -486,10 +503,16 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
     rustfs_utils::http::metadata_compat::remove_str(metadata, SUFFIX_BUCKET_INCARNATION_ID);
 }
 
-fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
+pub(crate) fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
     metadata.keys().any(|key| is_object_encryption_marker(key))
 }
 
+/// Reject ambiguous part sizes unless the upload records a transformation.
+pub(crate) fn multipart_part_logical_size(stored_size: usize, actual_size: i64, transformed: bool) -> Option<usize> {
+    let actual_size = usize::try_from(actual_size).ok()?;
+    (transformed || actual_size == stored_size).then_some(actual_size)
+}
+
 /// Per-set memoized capacity dirty scope.
 ///
 /// The disk endpoint/path identity of each slot in a set is immutable for the
@@ -4245,14 +4268,23 @@ pub(super) struct MultipartWriteQuorumContext<'a> {
     part_number: Option<usize>,
 }
 
+fn record_multipart_write_quorum_failure(
+    stage: &'static str,
+    errs: &[Option<DiskError>],
+    write_quorum: usize,
+) -> crate::disk::error_reduce::WriteQuorumFailureSummary {
+    let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
+    runtime_sources::record_erasure_write_quorum_failure(stage, summary.dominant_error_label);
+    summary
+}
+
 fn log_multipart_write_quorum_failure(
     context: MultipartWriteQuorumContext<'_>,
     errs: &[Option<DiskError>],
     write_quorum: usize,
     returned_error: &DiskError,
 ) {
-    let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
-    runtime_sources::record_erasure_write_quorum_failure(context.stage, summary.dominant_error_label);
+    let summary = record_multipart_write_quorum_failure(context.stage, errs, write_quorum);
     warn!(
         target: "rustfs_ecstore::set_disk",
         event = EVENT_SET_DISK_MULTIPART,
@@ -5134,6 +5166,11 @@ impl SetDisks {
             }
         }
 
+        #[cfg(feature = "e2e-test-hooks")]
+        if rename_tail_heal_suppressed_for_e2e(&request.bucket, request.object_prefix.as_deref()) {
+            return Ok(());
+        }
+
         let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
         Ok(())
     }

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -40,9 +40,35 @@ pub(crate) mod test_hooks {
     use std::sync::{Arc, Mutex};
 
     type Hook = Arc<dyn Fn() -> BoxFuture<'static, ()> + Send + Sync>;
+    type TransactionLockHook = Arc<dyn Fn(&rustfs_lock::NamespaceLockGuard) + Send + Sync>;
 
     static BEFORE_GC_FINALIZE: Mutex<Vec<(String, String, Hook)>> = Mutex::new(Vec::new());
     static CLEANUP_ATTEMPTS: Mutex<Vec<(String, String)>> = Mutex::new(Vec::new());
+    static AFTER_TRANSACTION_LOCK: Mutex<Vec<(String, String, TransactionLockHook)>> = Mutex::new(Vec::new());
+
+    pub(crate) fn after_transaction_lock_acquired(bucket: &str, object: &str, hook: TransactionLockHook) {
+        AFTER_TRANSACTION_LOCK
+            .lock()
+            .expect("transaction lock hooks")
+            .push((bucket.to_owned(), object.to_owned(), hook));
+    }
+
+    pub(in crate::set_disk) fn run_after_transaction_lock_acquired(
+        bucket: &str,
+        object: &str,
+        guard: &rustfs_lock::NamespaceLockGuard,
+    ) {
+        let hook = {
+            let mut hooks = AFTER_TRANSACTION_LOCK.lock().expect("transaction lock hooks");
+            hooks
+                .iter()
+                .position(|(hook_bucket, hook_object, _)| hook_bucket == bucket && hook_object == object)
+                .map(|index| hooks.remove(index).2)
+        };
+        if let Some(hook) = hook {
+            hook(guard);
+        }
+    }
 
     pub(super) fn record_cleanup_attempt(bucket: &str, object: &str) {
         CLEANUP_ATTEMPTS
@@ -112,6 +138,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
     /// Only the locked heal path may repair a live marker-creation source.
     pub repair_marker_source: bool,
     pub namespace_owner: Option<Arc<dyn Send + Sync>>,
+    pub transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
     pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
     pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
     pub can_commit: &'a (dyn Fn() -> bool + Sync),
@@ -125,6 +152,7 @@ impl<'a> DeleteRecoveryContext<'a> {
             explicit_delete: false,
             repair_marker_source: false,
             namespace_owner: None,
+            transaction_guard: None,
             recovery_ticket: None,
             published_hint: None,
             can_commit,
@@ -373,9 +401,22 @@ impl SetDisks {
         write_quorum: usize,
         mut context: DeleteRecoveryContext<'_>,
     ) -> Result<()> {
-        if context.namespace_owner.is_none() {
-            context.namespace_owner = Some(self.ctx.begin_namespace_commit());
-        }
+        let transaction_guard = match context.transaction_guard.take() {
+            Some(guard) => guard,
+            None => self
+                .acquire_delete_transaction_lock(bucket, object)
+                .await
+                .map_err(DiskError::other)?,
+        };
+        let outer_can_commit = context.can_commit;
+        let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
+        context.can_commit = &can_commit;
+        context.transaction_guard = Some(Arc::clone(&transaction_guard));
+        let owner = context
+            .namespace_owner
+            .take()
+            .unwrap_or_else(|| self.ctx.begin_namespace_commit());
+        context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
         let can_commit = context.can_commit;
         let states = inspect(disks, bucket, object, transaction).await;
         let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
@@ -547,8 +588,25 @@ impl SetDisks {
         &self,
         bucket: &str,
         object: &str,
-        context: DeleteRecoveryContext<'_>,
+        mut context: DeleteRecoveryContext<'_>,
     ) -> Result<DeleteRecoverySummary> {
+        let transaction_guard = match context.transaction_guard.take() {
+            Some(guard) => guard,
+            None => self
+                .acquire_delete_transaction_lock(bucket, object)
+                .await
+                .map_err(DiskError::other)?,
+        };
+        let outer_can_commit = context.can_commit;
+        let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
+        context.can_commit = &can_commit;
+        if !context.dry_run {
+            let owner = context
+                .namespace_owner
+                .take()
+                .unwrap_or_else(|| self.ctx.begin_namespace_commit());
+            context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
+        }
         let DeleteRecoveryContext {
             incarnation,
             dry_run,
@@ -558,6 +616,7 @@ impl SetDisks {
             published_hint,
             can_commit,
             recovery_ticket: _,
+            transaction_guard: _,
         } = context;
         let disks = self.disk_inventory().await;
         let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
@@ -1257,6 +1316,135 @@ mod tests {
         assert_eq!(body(&store, bucket, None).await, original);
     }
 
+    #[tokio::test]
+    async fn delete_recovery_deferred_batch_discards_released_alias_fence_and_preserves_caller_fence() {
+        use crate::bucket::metadata_sys::ObjectLockConfigState;
+        use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence, ObjectLockConfigSnapshot};
+        use crate::storage_api_contracts::object::ObjectToDelete;
+        use std::sync::Mutex;
+        use std::sync::atomic::{AtomicBool, Ordering};
+
+        for lose_caller in [false, true] {
+            let bucket = if lose_caller {
+                "deferred-batch-lost-caller"
+            } else {
+                "deferred-batch-released-alias"
+            };
+            let (_dirs, store, disks) = fixture(bucket).await;
+            put(&store, bucket, &[27; 4096], false).await;
+            let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
+            let before = join_all(
+                disks
+                    .iter()
+                    .map(|disk| disk.inspect_delete_transaction(bucket, "object", transaction)),
+            )
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()
+            .expect("capture pending recovery evidence");
+            let mut set = (*store.pools[0].disk_set[0]).clone();
+            set.lockers = vec![Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(
+                rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::FastObjectLockManager::new())),
+            )))];
+            set.ctx
+                .update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
+                .await;
+
+            let (caller_fence, caller_lost) = NamespaceLockFence::loss_handle_for_test();
+            let released_alias_lost = Arc::new(AtomicBool::new(false));
+            let signal_fence = Arc::new(Mutex::new(None));
+            let captured_signal_fence = Arc::clone(&signal_fence);
+            let captured_alias_lost = Arc::clone(&released_alias_lost);
+            test_hooks::after_transaction_lock_acquired(
+                bucket,
+                "object",
+                Arc::new(move |guard| {
+                    let signal = guard.lock_lost_signal().expect("distributed alias signal");
+                    let fence = NamespaceLockSignalTestFence::install_with_loss_handle(&signal, Arc::clone(&captured_alias_lost));
+                    // Retain the signal so a later allocation cannot reuse its
+                    // registry identity after the original guard is dropped.
+                    *captured_signal_fence.lock().expect("captured alias fence") = Some((fence, signal));
+                }),
+            );
+            let replay_started = Arc::new(AtomicBool::new(false));
+            let captured_replay_started = Arc::clone(&replay_started);
+            let expired_alias = Arc::clone(&released_alias_lost);
+            test_hooks::after_transaction_lock_acquired(
+                bucket,
+                "object",
+                Arc::new(move |_| {
+                    // Acquiring the same transaction key again proves the first
+                    // lease was released. Only its copied fence becomes stale.
+                    expired_alias.store(true, Ordering::Release);
+                    caller_lost.store(lose_caller, Ordering::Release);
+                    captured_replay_started.store(true, Ordering::Release);
+                }),
+            );
+
+            let (deleted, errors) = set
+                .delete_objects(
+                    bucket,
+                    vec![ObjectToDelete {
+                        object_name: "object".into(),
+                        ..Default::default()
+                    }],
+                    ObjectOptions {
+                        namespace_lock_fence: Some(caller_fence),
+                        object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
+                            ObjectLockConfigState::ConfirmedAbsent,
+                        ))),
+                        ..Default::default()
+                    },
+                )
+                .await;
+            assert!(
+                replay_started.load(Ordering::Acquire),
+                "missing metadata must take the deferred replay path"
+            );
+            assert!(
+                released_alias_lost.load(Ordering::Acquire),
+                "the completed batch alias must be marked lost"
+            );
+            if lose_caller {
+                assert!(errors[0].is_some(), "losing the original caller fence must reject deferred replay");
+                for (disk, original) in disks.iter().zip(before) {
+                    let after = disk
+                        .inspect_delete_transaction(bucket, "object", transaction)
+                        .await
+                        .expect("retained proof");
+                    assert_eq!(
+                        after.fingerprint, original.fingerprint,
+                        "lost caller authority must preserve every prepared journal"
+                    );
+                }
+            } else {
+                assert!(
+                    errors.iter().all(Option::is_none),
+                    "released alias loss must not reject fresh recovery: {errors:?}"
+                );
+                assert!(
+                    !deleted[0].found,
+                    "fresh unversioned recovery commits absence before the idempotent batch retry"
+                );
+                for disk in &disks {
+                    assert!(
+                        discover(disk, bucket, Some("object"))
+                            .await
+                            .expect("final recovery evidence")
+                            .is_empty()
+                    );
+                    assert_eq!(
+                        disk.read_version("", bucket, "object", "", &ReadOptions::default())
+                            .await
+                            .expect_err("replayed delete must remove the source"),
+                        DiskError::FileNotFound
+                    );
+                }
+            }
+            drop(signal_fence);
+        }
+    }
+
     #[tokio::test]
     async fn delete_recovery_corrupt_bucket_does_not_prevent_other_bucket_rollback() {
         let bad_bucket = "recover-corrupt-bucket";

crates/ecstore/src/set_disk/ops/heal.rs

@@ -3634,6 +3634,18 @@ impl SetDisks {
             return Ok((result, err.map(|e| e.into())));
         }
 
+        // Keep the transaction key through both replay and the metadata heal
+        // based on that replay. The raw target key always precedes this key.
+        let transaction_guard =
+            if _write_lock_guard.is_some() && !opts.read_repair && !crate::bucket::utils::is_meta_bucketname(bucket) {
+                Some(self.acquire_delete_transaction_lock(bucket, object).await?)
+            } else {
+                None
+            };
+        let locks_lost = || {
+            _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
+                || transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
+        };
         let mut recovery_removed_version = false;
         if !opts.read_repair
             && _write_lock_guard.is_some()
@@ -3655,12 +3667,17 @@ impl SetDisks {
                         dry_run: opts.dry_run,
                         explicit_delete,
                         repair_marker_source: true,
+                        transaction_guard: transaction_guard.clone(),
                         namespace_owner: (!opts.dry_run).then(|| {
-                            Arc::new((scope.clone(), self.ctx.begin_namespace_commit(), _write_lock_guard.clone()))
-                                as Arc<dyn Send + Sync>
+                            Arc::new((
+                                scope.clone(),
+                                self.ctx.begin_namespace_commit(),
+                                _write_lock_guard.clone(),
+                                transaction_guard.clone(),
+                            )) as Arc<dyn Send + Sync>
                         }),
                         ..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &|| {
-                            scope.check().is_ok() && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
+                            scope.check().is_ok() && !locks_lost()
                         })
                     },
                 )
@@ -3752,7 +3769,7 @@ impl SetDisks {
                 && errs
                     .iter()
                     .all(|err| matches!(err, Some(DiskError::FileNotFound | DiskError::FileVersionNotFound)))
-                && !_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
+                && !locks_lost()
             {
                 *absence = Some(HealedObjectAbsence {
                     pool_index: self.pool_index,
@@ -3767,8 +3784,10 @@ impl SetDisks {
         // Pass no_lock=true since we already obtained write lock (or are already called with no_lock=true)
         let mut inner_opts = *opts;
         inner_opts.no_lock = true;
-        let commit_fence = || !_write_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost());
-        let namespace_owner = _write_lock_guard.as_ref().map(|guard| guard.clone() as Arc<dyn Send + Sync>);
+        let commit_fence = || !locks_lost();
+        let namespace_owner = _write_lock_guard
+            .as_ref()
+            .map(|guard| Arc::new((Arc::clone(guard), transaction_guard.clone())) as Arc<dyn Send + Sync>);
         let (result, err) = pin_object_heal_future(|| {
             self.heal_object_with_explicit_version_regen(
                 bucket,
@@ -3815,7 +3834,7 @@ impl SetDisks {
                         result,
                         err.map(Into::into),
                         absence,
-                        _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()),
+                        locks_lost(),
                         bucket,
                         object,
                     ));
@@ -3827,7 +3846,7 @@ impl SetDisks {
             result,
             err.map(Into::into),
             absence,
-            _write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost()),
+            locks_lost(),
             bucket,
             object,
         ))

crates/ecstore/src/set_disk/ops/locking.rs

@@ -44,6 +44,21 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
 
     #[tracing::instrument(level = "trace", skip(self))]
     async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
+        // Lock the file the disk resolves, not the spelling: on Windows `a\b`
+        // and `a/b` share one xl.meta, so they must share one lock. The key is
+        // folded by the requesting node only; lock servers compare it verbatim.
+        let resource = ObjectKey {
+            bucket: Arc::from(bucket),
+            object: Arc::from(object_name_disk_identity(object).as_ref()),
+            version: None,
+        };
+
+        Ok(self.namespace_lock_for_key(resource).await)
+    }
+}
+
+impl SetDisks {
+    async fn namespace_lock_for_key(&self, resource: ObjectKey) -> NamespaceLockWrapper {
         // Resolved from this set's own instance context (backlog#1052), not the
         // ambient facade: the facade tracks whichever context is currently
         // published process-wide, so a second instance (or, in tests, another
@@ -69,20 +84,33 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
             NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
         };
 
-        // Lock the file the disk resolves, not the spelling: on Windows `a\b`
-        // and `a/b` share one xl.meta, so they must share one lock. The key is
-        // folded by the requesting node only; lock servers compare it verbatim.
-        let resource = ObjectKey {
-            bucket: Arc::from(bucket),
-            object: Arc::from(object_name_disk_identity(object).as_ref()),
-            version: None,
-        };
-
-        Ok(NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone()))
+        NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone())
+    }
+
+    /// Raw object locks precede this set-local transaction lock. Its versioned
+    /// resource cannot collide with normal namespace keys, which use None.
+    /// Keeping the target set's clients avoids nesting overlapping raw quorums.
+    pub(in crate::set_disk) async fn acquire_delete_transaction_lock(
+        &self,
+        bucket: &str,
+        object: &str,
+    ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
+        let resource = ObjectKey::with_version(
+            bucket,
+            object_name_disk_identity(object).as_ref(),
+            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
+        );
+        let guard = self
+            .namespace_lock_for_key(resource)
+            .await
+            .get_write_lock(super::super::get_lock_acquire_timeout())
+            .await
+            .map_err(|error| self.map_namespace_lock_error(bucket, object, "delete_transaction", error))?;
+        #[cfg(test)]
+        super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
+        Ok(Arc::new(guard))
     }
-}
 
-impl SetDisks {
     #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
     pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
         match err {

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -43,8 +43,9 @@ use super::super::{
     complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
     create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
     get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
-    log_multipart_write_quorum_failure, parts_after_marker, path_join_buf, prepare_complete_multipart_parts,
-    record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
+    log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
+    prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
+    reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
     restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
     to_object_err, warn,
 };
@@ -233,6 +234,19 @@ fn admitted_multipart_size(current: u64, candidate: u64, limit: u64) -> Result<u
     Ok(total)
 }
 
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum MultipartMetadataQuorum {
+    Read,
+    Write,
+    Cleanup,
+}
+
+impl From<bool> for MultipartMetadataQuorum {
+    fn from(write: bool) -> Self {
+        if write { Self::Write } else { Self::Read }
+    }
+}
+
 pub(crate) struct StaleMultipartCleanupGuard {
     file_info: FileInfo,
     upload_path: String,
@@ -410,6 +424,8 @@ impl StaleMultipartCleanupGuard {
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum MultipartCommitPause {
     NewUploadBeforeLockLost,
+    #[cfg(test)]
+    ListPartsAfterMetadata,
     PutPartBeforeLockAcquire,
     PutPartBeforeLockLost,
     PutPartAfterCapacityAdmission,
@@ -824,7 +840,8 @@ where
     }
 
     if successful_responses < read_quorum {
-        if empty_upload_fallback_possible(successful_responses, &errs) {
+        // A single missing directory cannot speak for unavailable disks.
+        if !errs.is_empty() && errs.iter().all(|err| matches!(err, Some(DiskError::FileNotFound))) {
             return Err(DiskError::FileNotFound);
         }
 
@@ -834,6 +851,51 @@ where
     Ok((errs, object_parts))
 }
 
+// Size-dependent decisions must not discard a part merely because only a
+// minority of its files was observed. All participants must answer, and each
+// observed part must have a complete file/metadata pair on a read quorum.
+fn validate_part_size_observations(
+    errs: &[Option<DiskError>],
+    object_parts: &[Vec<String>],
+    read_quorum: usize,
+) -> disk::error::Result<Vec<usize>> {
+    if errs.iter().any(Option::is_some) {
+        return Err(DiskError::ErasureReadQuorum);
+    }
+    let mut counts: HashMap<usize, usize> = HashMap::new();
+    for entries in object_parts {
+        let mut observed: HashMap<usize, u8> = HashMap::new();
+        for entry in entries {
+            let Some(name) = entry.strip_prefix("part.") else {
+                continue;
+            };
+            let (number, flag) = match name.strip_suffix(".meta") {
+                Some(number) => (number, 2),
+                None => (name, 1),
+            };
+            let parsed = number.parse::<usize>().map_err(|_| DiskError::FileCorrupt)?;
+            if !(1..=MAX_PARTS_COUNT).contains(&parsed)
+                || !number.as_bytes().first().is_some_and(|digit| (b'1'..=b'9').contains(digit))
+            {
+                return Err(DiskError::FileCorrupt);
+            }
+            *observed.entry(parsed).or_default() |= flag;
+        }
+        for (number, flags) in observed {
+            if flags != 3 {
+                return Err(DiskError::FileCorrupt);
+            }
+            *counts.entry(number).or_default() += 1;
+        }
+    }
+    if counts.values().any(|&count| count < read_quorum) {
+        return Err(DiskError::ErasureReadQuorum);
+    }
+    let mut part_numbers: Vec<_> = counts.into_keys().collect();
+    part_numbers.sort_unstable();
+    Ok(part_numbers)
+}
+
 fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
     let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
 
@@ -1004,7 +1066,7 @@ impl SetDisks {
             ]),
             SLASH_SEPARATOR
         );
-        let part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
+        let part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum, true).await {
             Ok(parts) => parts,
             Err(DiskError::FileNotFound) => return Ok(0),
             Err(err) => return Err(to_object_err(err.into(), vec![bucket, object, upload_id])),
@@ -1016,28 +1078,45 @@ impl SetDisks {
             .iter()
             .map(|number| format!("{part_path}part.{number}.meta"))
             .collect::<Vec<_>>();
-        let existing_parts =
-            Self::read_parts(&online_disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
-                .await
-                .map_err(|err| to_object_err(err.into(), vec![bucket, object, upload_id]))?;
+        let existing_parts = Self::read_parts(
+            &online_disks,
+            RUSTFS_META_MULTIPART_BUCKET,
+            &part_meta_paths,
+            &part_numbers,
+            fi.write_quorum(self.default_write_quorum()),
+        )
+        .await
+        .map_err(|err| to_object_err(err.into(), vec![bucket, object, upload_id]))?;
 
-        existing_parts.into_iter().try_fold(0_u64, |total, part| {
-            if part.error.is_some() || part.number == replacing_part {
-                return if part.error.is_some() {
-                    Err(Error::PartMissingOrCorrupt)
-                } else {
-                    Ok(total)
-                };
-            }
-            let part_size = u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
-            total.checked_add(part_size).ok_or_else(|| {
-                Error::InvalidArgument(
-                    "multipart upload".to_string(),
-                    SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
-                    "logical size overflow".to_string(),
-                )
+        if existing_parts.len() != part_numbers.len() {
+            return Err(Error::PartMissingOrCorrupt);
+        }
+        let transformed = fi.is_compressed() || should_persist_encryption_original_size(&fi.metadata);
+        existing_parts
+            .into_iter()
+            .zip(part_numbers)
+            .try_fold(0_u64, |total, (part, number)| {
+                if part.number != number {
+                    return Err(Error::PartMissingOrCorrupt);
+                }
+                if part.error.is_some() || part.number == replacing_part {
+                    return if part.error.is_some() {
+                        Err(Error::PartMissingOrCorrupt)
+                    } else {
+                        Ok(total)
+                    };
+                }
+                let part_size = multipart_part_logical_size(part.size, part.actual_size, transformed)
+                    .and_then(|size| u64::try_from(size).ok())
+                    .ok_or(Error::PartMissingOrCorrupt)?;
+                total.checked_add(part_size).ok_or_else(|| {
+                    Error::InvalidArgument(
+                        "multipart upload".to_string(),
+                        SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
+                        "logical size overflow".to_string(),
+                    )
+                })
             })
-        })
     }
 
     async fn discover_multipart_upload_paths(
@@ -1139,7 +1218,7 @@ impl SetDisks {
             }
             let upload_id = runtime_sources::deployment_upload_id(raw_upload_id);
             let file_info = match self
-                .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, false, true)
+                .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, MultipartMetadataQuorum::Read, true)
                 .await
             {
                 Ok((file_info, _)) => file_info,
@@ -1226,7 +1305,7 @@ impl SetDisks {
             .acquire_write_lock_diag("abort_data_movement_multipart", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
             .await?;
         let file_info = match self
-            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, true, true)
+            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, MultipartMetadataQuorum::Write, true)
             .await
         {
             Ok((file_info, _)) => file_info,
@@ -1269,6 +1348,7 @@ impl SetDisks {
         disks: &[Option<DiskStore>],
         part_path: &str,
         read_quorum: usize,
+        strict_size: bool,
     ) -> disk::error::Result<Vec<usize>> {
         let mut futures = Vec::with_capacity(disks.len());
         let part_path = part_path.to_string();
@@ -1277,21 +1357,24 @@ impl SetDisks {
             let part_path = part_path.clone();
             futures.push(async move {
                 if let Some(disk) = disk {
-                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
+                    match disk
+                        .list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
                         .await
+                    {
+                        Err(DiskError::FileNotFound) if strict_size => Ok(Vec::new()),
+                        result => result,
+                    }
                 } else {
                     Err(DiskError::DiskNotFound)
                 }
             });
         }
 
-        let mut errs = Vec::with_capacity(disks.len());
-        let mut object_parts = Vec::with_capacity(disks.len());
-
-        let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
-        errs.extend(collected_errs);
-        object_parts.extend(collected_parts);
+        let (errs, object_parts) = collect_list_parts_results(futures, read_quorum).await?;
 
+        if strict_size {
+            return validate_part_size_observations(&errs, &object_parts, read_quorum);
+        }
         if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
             return Err(err);
         }
@@ -1333,7 +1416,7 @@ impl SetDisks {
         data_movement: bool,
     ) -> Result<(FileInfo, Vec<FileInfo>)> {
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, data_movement)?;
-        self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write, false)
+        self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write.into(), false)
             .await
     }
 
@@ -1373,7 +1456,7 @@ impl SetDisks {
         object: &str,
         upload_id: &str,
         upload_id_path: &str,
-        write: bool,
+        quorum_mode: MultipartMetadataQuorum,
         include_recovery: bool,
     ) -> Result<(FileInfo, Vec<FileInfo>)> {
         // Slow metadata I/O must not hold the topology registry's read lock.
@@ -1489,22 +1572,26 @@ impl SetDisks {
         }
 
         let mut quorum = read_quorum as usize;
-        if write {
+        if quorum_mode != MultipartMetadataQuorum::Read {
             quorum = write_quorum as usize;
 
             if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
-                log_multipart_write_quorum_failure(
-                    MultipartWriteQuorumContext {
-                        stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
-                        bucket,
-                        object,
-                        upload_id,
-                        part_number: None,
-                    },
-                    &errs,
-                    quorum,
-                    &err,
-                );
+                if quorum_mode == MultipartMetadataQuorum::Write {
+                    log_multipart_write_quorum_failure(
+                        MultipartWriteQuorumContext {
+                            stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
+                            bucket,
+                            object,
+                            upload_id,
+                            part_number: None,
+                        },
+                        &errs,
+                        quorum,
+                        &err,
+                    );
+                } else {
+                    record_multipart_write_quorum_failure(MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, &errs, quorum);
+                }
                 return Err(map_error(err));
             }
         } else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
@@ -1523,7 +1610,7 @@ impl SetDisks {
             .acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path)
             .await?;
         let (file_info, _) = self
-            .check_multipart_upload_path_exists("", "", "", upload_path, true, true)
+            .check_multipart_upload_path_exists("", "", "", upload_path, MultipartMetadataQuorum::Cleanup, true)
             .await?;
         let write_quorum = file_info.write_quorum(self.default_write_quorum());
         Ok(StaleMultipartCleanupGuard {
@@ -2634,6 +2721,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
         ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
             .await?;
+        #[cfg(test)]
+        pause_multipart_commit(bucket, object, MultipartCommitPause::ListPartsAfterMetadata).await;
 
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
 
@@ -2682,16 +2771,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             SLASH_SEPARATOR
         );
 
-        let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
-            Ok(parts) => parts,
-            Err(err) => {
-                if err == DiskError::FileNotFound {
-                    return Ok(ret);
-                }
+        let mut part_numbers =
+            match Self::list_parts(&online_disks, &part_path, read_quorum, opts.strict_multipart_part_sizes).await {
+                Ok(parts) => parts,
+                Err(err) => {
+                    if err == DiskError::FileNotFound {
+                        return Ok(ret);
+                    }
 
-                return Err(to_object_err(err.into(), vec![bucket, object]));
-            }
-        };
+                    return Err(to_object_err(err.into(), vec![bucket, object]));
+                }
+            };
 
         if part_numbers.is_empty() {
             return Ok(ret);
@@ -2708,11 +2798,29 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             .map(|v| format!("{part_path}part.{v}.meta"))
             .collect::<Vec<String>>();
 
-        let object_parts =
-            Self::read_parts(&online_disks, RUSTFS_META_MULTIPART_BUCKET, &part_meta_paths, &part_numbers, read_quorum)
-                .await
-                .map_err(|e| to_object_err(e.into(), vec![bucket, object, upload_id]))?;
+        let object_parts = Self::read_parts(
+            &online_disks,
+            RUSTFS_META_MULTIPART_BUCKET,
+            &part_meta_paths,
+            &part_numbers,
+            if opts.strict_multipart_part_sizes {
+                fi.write_quorum(self.default_write_quorum())
+            } else {
+                read_quorum
+            },
+        )
+        .await
+        .map_err(|e| to_object_err(e.into(), vec![bucket, object, upload_id]))?;
 
+        if opts.strict_multipart_part_sizes
+            && (object_parts.len() != part_numbers.len()
+                || object_parts
+                    .iter()
+                    .zip(&part_numbers)
+                    .any(|(part, number)| part.number != *number || part.error.is_some() || part.etag.is_empty()))
+        {
+            return Err(Error::PartMissingOrCorrupt);
+        }
         let mut count = max_parts;
 
         for part in object_parts.iter() {
@@ -3032,7 +3140,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         // its full write quorum and the upload's incarnation/lifecycle fences.
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
         let (fi, _) = self
-            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
+            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, MultipartMetadataQuorum::Read, true)
             .await?;
         ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
         ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
@@ -3192,7 +3300,14 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     }
                     SourceRecovery::Missing => {
                         return match self
-                            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
+                            .check_multipart_upload_path_exists(
+                                bucket,
+                                object,
+                                upload_id,
+                                &upload_id_path,
+                                MultipartMetadataQuorum::Read,
+                                true,
+                            )
                             .await
                         {
                             Ok(_) => Err(Error::ErasureWriteQuorum),
@@ -7492,7 +7607,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn collect_list_parts_results_returns_file_not_found_for_empty_upload_dirs() {
+    async fn collect_list_parts_results_rejects_unavailable_empty_upload_dirs() {
         let tasks: Vec<_> = vec![
             (5_u64, Err(DiskError::FileNotFound)),
             (10, Err(DiskError::DiskNotFound)),
@@ -7507,11 +7622,372 @@ mod tests {
 
         let err = collect_list_parts_results(tasks, 2)
             .await
-            .expect_err("missing multipart directories should be treated as empty uploads");
+            .expect_err("an unavailable disk cannot prove an empty upload");
 
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn list_parts_rejects_disk_loss_after_upload_metadata_was_read() {
+        let (dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "multipart-list-loss-after-metadata";
+        let object = "object";
+        make_bucket_on_all(&disks, bucket).await;
+        let opts = ObjectOptions {
+            max_parity: true,
+            ..Default::default()
+        };
+        let (upload_id, _) = stage_upload_with_create_opts(&set, bucket, object, &[1; 32], &opts).await;
+        let (fi, _) = set
+            .check_upload_id_exists(bucket, object, &upload_id, false)
+            .await
+            .expect("read initial metadata");
+        let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
+        let part_dir = dirs[0]
+            .path()
+            .join(RUSTFS_META_MULTIPART_BUCKET)
+            .join(upload_path)
+            .join(fi.data_dir.expect("upload data directory").to_string());
+        let held_dir = part_dir.with_extension("held-for-test");
+        let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::ListPartsAfterMetadata);
+        let query_set = Arc::clone(&set);
+        let query_id = upload_id.clone();
+        let query = tokio::spawn(async move {
+            query_set
+                .list_object_parts(bucket, object, &query_id, None, 100, &ObjectOptions::default())
+                .await
+        });
+        barrier.wait_until_paused().await;
+        tokio::fs::rename(&part_dir, &held_dir)
+            .await
+            .expect("simulate the surviving disk missing the part directory");
+        {
+            let mut live = set.disks.write().await;
+            for slot in 1..4 {
+                live[slot] = None;
+            }
+        }
+        barrier.release();
+        let result = query.await.expect("list task must finish");
+        drop(barrier);
+        tokio::fs::rename(&held_dir, &part_dir)
+            .await
+            .expect("restore the original directory");
+        {
+            let mut live = set.disks.write().await;
+            for slot in 1..4 {
+                live[slot] = Some(disks[slot].clone());
+            }
+        }
+        assert!(
+            matches!(result, Err(StorageError::InsufficientReadQuorum(..))),
+            "disk loss must fail rather than return an empty part list: {result:?}"
+        );
+        let recovered = set
+            .list_object_parts(bucket, object, &upload_id, None, 100, &ObjectOptions::default())
+            .await
+            .expect("listing resumes after the original disks return");
+        assert_eq!(recovered.parts.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn collect_list_parts_results_proves_all_directories_missing() {
+        let tasks = (0..4)
+            .map(|_| async { Err::<Vec<String>, _>(DiskError::FileNotFound) })
+            .collect();
+        let err = collect_list_parts_results(tasks, 2)
+            .await
+            .expect_err("all missing directories are empty");
         assert_eq!(err, DiskError::FileNotFound);
     }
 
+    #[test]
+    fn strict_multipart_part_sizes_reject_incomplete_observations() {
+        let pair = || vec!["part.1".to_string(), "part.1.meta".to_string()];
+        let all_online = vec![None; 4];
+        assert!(validate_part_size_observations(&all_online, &[vec![], vec![], vec![], vec![]], 2).is_ok());
+        assert_eq!(
+            validate_part_size_observations(&all_online, &[pair(), pair(), vec![], vec![]], 2),
+            Ok(vec![1])
+        );
+        let unordered = vec!["part.9.meta", "part.1", "part.9", "part.1.meta", "part.1"]
+            .into_iter()
+            .map(str::to_owned)
+            .collect::<Vec<_>>();
+        assert_eq!(
+            validate_part_size_observations(&all_online, &[unordered.clone(), unordered, vec![], vec![]], 2),
+            Ok(vec![1, 9])
+        );
+        assert_eq!(
+            validate_part_size_observations(&all_online, &[pair(), vec![], vec![], vec![]], 2),
+            Err(DiskError::ErasureReadQuorum)
+        );
+        assert_eq!(
+            validate_part_size_observations(&all_online, &[vec!["part.1.meta".to_string()], pair(), pair(), vec![]], 2),
+            Err(DiskError::FileCorrupt)
+        );
+        for invalid in ["part.0", "part.10001", "part.01", "part.+1", "part.1x"] {
+            let entries = vec![invalid.to_string(), format!("{invalid}.meta")];
+            assert_eq!(
+                validate_part_size_observations(&all_online, &[entries.clone(), entries.clone(), entries.clone(), entries], 2),
+                Err(DiskError::FileCorrupt),
+                "noncanonical part names are not valid size evidence even on every disk"
+            );
+        }
+        assert_eq!(
+            validate_part_size_observations(
+                &[None, None, None, Some(DiskError::DiskNotFound)],
+                &[pair(), pair(), pair(), vec![]],
+                2
+            ),
+            Err(DiskError::ErasureReadQuorum)
+        );
+    }
+
+    #[tokio::test]
+    async fn strict_multipart_capped_upload_rejects_unavailable_participant_and_recovers() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "strict-multipart-size-bucket";
+        let object = "object";
+        make_bucket_on_all(&disks, bucket).await;
+        let mut metadata = HashMap::new();
+        insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "10".to_string());
+        let opts = ObjectOptions {
+            max_parity: true,
+            user_defined: metadata,
+            ..Default::default()
+        };
+        let (upload_id, _) = stage_upload_with_create_opts(&set, bucket, object, &[1; 6], &opts).await;
+
+        set.disks.write().await[3] = None;
+        let mut part = PutObjReader::from_vec(vec![2; 4]);
+        let err = set
+            .put_object_part(bucket, object, &upload_id, 2, &mut part, &ObjectOptions::default())
+            .await;
+        assert!(err.is_err(), "an unavailable disk must not produce a size admission: {err:?}");
+        set.disks.write().await[3] = Some(disks[3].clone());
+
+        // Two distinct part identities at read quorum cannot prove which size
+        // belongs to the acknowledged write; require the write quorum instead.
+        let (other_id, _) = stage_upload_with_create_opts(&set, bucket, "other", &[9; 2], &ObjectOptions::default()).await;
+        let (fi, _) = set
+            .check_upload_id_exists(bucket, object, &upload_id, false)
+            .await
+            .expect("read capped upload metadata");
+        let (other_fi, _) = set
+            .check_upload_id_exists(bucket, "other", &other_id, false)
+            .await
+            .expect("read other upload metadata");
+        let path = format!(
+            "{}/{}/part.1.meta",
+            SetDisks::get_upload_id_dir(bucket, object, &upload_id),
+            fi.data_dir.expect("data dir")
+        );
+        let other_path = format!(
+            "{}/{}/part.1.meta",
+            SetDisks::get_upload_id_dir(bucket, "other", &other_id),
+            other_fi.data_dir.expect("data dir")
+        );
+        let mut originals = Vec::new();
+        for disk in disks.iter().take(2) {
+            originals.push(
+                disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
+                    .await
+                    .expect("save original part metadata"),
+            );
+            let conflicting = disk
+                .read_all(RUSTFS_META_MULTIPART_BUCKET, &other_path)
+                .await
+                .expect("read conflicting part metadata");
+            disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, conflicting)
+                .await
+                .expect("install split part metadata");
+        }
+        let mut split = PutObjReader::from_vec(vec![5; 4]);
+        let split_result = set
+            .put_object_part(bucket, object, &upload_id, 2, &mut split, &ObjectOptions::default())
+            .await;
+        for (disk, original) in disks.iter().zip(originals) {
+            disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, original)
+                .await
+                .expect("restore original part metadata");
+        }
+        assert!(
+            split_result.is_err(),
+            "split part identities must not authorize a size decision: {split_result:?}"
+        );
+
+        let mut originals = Vec::new();
+        for disk in disks.iter().take(3) {
+            let original = disk
+                .read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
+                .await
+                .expect("save original part metadata");
+            let mut stale = ObjectPartInfo::unmarshal(&original).expect("decode original part metadata");
+            stale.actual_size = 1;
+            disk.write_all(
+                RUSTFS_META_MULTIPART_BUCKET,
+                &path,
+                Bytes::from(stale.marshal_msg().expect("encode stale part")),
+            )
+            .await
+            .expect("install stale size on write quorum");
+            originals.push(original);
+        }
+        let mut undercounted = PutObjReader::from_vec(vec![6; 4]);
+        let result = set
+            .put_object_part(bucket, object, &upload_id, 2, &mut undercounted, &ObjectOptions::default())
+            .await;
+        for (disk, original) in disks.iter().zip(originals) {
+            disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, original)
+                .await
+                .expect("restore original part metadata");
+        }
+        assert!(
+            matches!(result, Err(StorageError::PartMissingOrCorrupt)),
+            "smaller logical size must fail closed: {result:?}"
+        );
+
+        let mut originals = Vec::new();
+        for disk in disks.iter().take(3) {
+            let original = disk
+                .read_all(RUSTFS_META_MULTIPART_BUCKET, &path)
+                .await
+                .expect("save original part metadata");
+            let mut wrong_identity = ObjectPartInfo::unmarshal(&original).expect("decode original part metadata");
+            wrong_identity.number = 2;
+            disk.write_all(
+                RUSTFS_META_MULTIPART_BUCKET,
+                &path,
+                Bytes::from(wrong_identity.marshal_msg().expect("encode wrong identity")),
+            )
+            .await
+            .expect("install mismatched part identity on write quorum");
+            originals.push(original);
+        }
+        let list_result = set
+            .list_object_parts(
+                bucket,
+                object,
+                &upload_id,
+                None,
+                100,
+                &ObjectOptions {
+                    strict_multipart_part_sizes: true,
+                    ..Default::default()
+                },
+            )
+            .await;
+        let mut replacement = PutObjReader::from_vec(vec![7; 4]);
+        let result = set
+            .put_object_part(bucket, object, &upload_id, 2, &mut replacement, &ObjectOptions::default())
+            .await;
+        for (disk, original) in disks.iter().zip(originals) {
+            disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &path, original)
+                .await
+                .expect("restore original part metadata");
+        }
+        assert!(
+            matches!(list_result, Err(StorageError::PartMissingOrCorrupt)),
+            "strict listing must reject mismatched part identity: {list_result:?}"
+        );
+        assert!(
+            matches!(result, Err(StorageError::PartMissingOrCorrupt)),
+            "mismatched part identity must not bypass the cap: {result:?}"
+        );
+
+        let mut too_large = PutObjReader::from_vec(vec![3; 5]);
+        let err = set
+            .put_object_part(bucket, object, &upload_id, 2, &mut too_large, &ObjectOptions::default())
+            .await;
+        assert!(
+            matches!(err, Err(StorageError::EntityTooLarge(11, 10))),
+            "existing part must count: {err:?}"
+        );
+        let mut fits = PutObjReader::from_vec(vec![4; 4]);
+        set.put_object_part(bucket, object, &upload_id, 2, &mut fits, &ObjectOptions::default())
+            .await
+            .expect("the upload should resume at the exact limit after all disks recover");
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn capped_multipart_concurrent_parts_cannot_exceed_logical_limit() {
+        let (_dirs, disks, set) = hermetic_set_disks(4).await;
+        let bucket = "concurrent-capped-multipart";
+        let object = "object";
+        make_bucket_on_all(&disks, bucket).await;
+        let mut metadata = HashMap::new();
+        insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "10".to_owned());
+        let opts = ObjectOptions {
+            max_parity: true,
+            user_defined: metadata,
+            ..Default::default()
+        };
+        let upload = set
+            .new_multipart_upload(bucket, object, &opts)
+            .await
+            .expect("create capped upload");
+        let barrier = Arc::new(tokio::sync::Barrier::new(3));
+        let mut tasks = Vec::new();
+        for (number, byte) in [(1_usize, 0x41_u8), (2, 0x42)] {
+            let set = Arc::clone(&set);
+            let barrier = Arc::clone(&barrier);
+            let upload_id = upload.upload_id.clone();
+            tasks.push(tokio::spawn(async move {
+                let mut body = PutObjReader::from_vec(vec![byte; 6]);
+                barrier.wait().await;
+                set.put_object_part(bucket, object, &upload_id, number, &mut body, &ObjectOptions::default())
+                    .await
+            }));
+        }
+        barrier.wait().await;
+        let mut accepted = Vec::new();
+        let mut rejected = 0;
+        for task in tasks {
+            match tokio::time::timeout(Duration::from_secs(30), task)
+                .await
+                .expect("concurrent part deadline")
+                .expect("part task")
+            {
+                Ok(part) => accepted.push(part),
+                Err(StorageError::EntityTooLarge(12, 10)) => rejected += 1,
+                Err(other) => panic!("unexpected concurrent admission result: {other:?}"),
+            }
+        }
+        assert_eq!(accepted.len(), 1, "only one six-byte part fits the ten-byte limit");
+        assert_eq!(rejected, 1);
+        let listed = set
+            .list_object_parts(bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
+            .await
+            .expect("read final part listing");
+        assert_eq!(listed.parts.len(), 1);
+        assert_eq!(listed.parts[0].part_num, accepted[0].part_num);
+        let winner = accepted.remove(0);
+        let part_number = winner.part_num;
+        set.clone()
+            .complete_multipart_upload(
+                bucket,
+                object,
+                &upload.upload_id,
+                vec![CompletePart {
+                    part_num: part_number,
+                    etag: winner.etag,
+                    ..Default::default()
+                }],
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("complete the admitted part");
+        let mut reader = set
+            .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read completed object");
+        let mut actual = Vec::new();
+        reader.stream.read_to_end(&mut actual).await.expect("read whole object");
+        assert_eq!(actual, vec![if part_number == 1 { 0x41 } else { 0x42 }; 6]);
+    }
+
     #[tokio::test]
     async fn collect_list_parts_results_fails_early_when_file_not_found_fallback_is_impossible() {
         let started = Instant::now();
@@ -8131,9 +8607,7 @@ mod tests {
         assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part");
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn put_object_part_fences_upload_lock_loss_before_rename() {
+    async fn put_object_part_fences_upload_lock_loss_before_rename_case(capped: bool) {
         let target = Arc::new(std::sync::RwLock::new(None));
         let refresh_calls = Arc::new(AtomicUsize::new(0));
         let lockers: Vec<Arc<dyn LockClient>> = (0..4)
@@ -8145,10 +8619,12 @@ mod tests {
         let bucket = "multipart-put-part-lock-loss-bucket";
         let object = "object";
         make_bucket_on_all(&disk_stores, bucket).await;
+        let mut create_opts = ObjectOptions::default();
+        if capped {
+            insert_str(&mut create_opts.user_defined, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "8192".to_owned());
+        }
         let upload = temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
-            set_disks
-                .new_multipart_upload(bucket, object, &ObjectOptions::default())
-                .await
+            set_disks.new_multipart_upload(bucket, object, &create_opts).await
         })
         .await
         .expect("multipart upload should be created");
@@ -8199,6 +8675,18 @@ mod tests {
         assert!(listed.parts.is_empty(), "lock loss before rename must not publish the part");
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn put_object_part_fences_upload_lock_loss_before_rename() {
+        put_object_part_fences_upload_lock_loss_before_rename_case(false).await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn capped_multipart_part_fences_upload_lock_loss_before_rename() {
+        put_object_part_fences_upload_lock_loss_before_rename_case(true).await;
+    }
+
     #[tokio::test]
     async fn complete_encrypted_multipart_marks_quorum_validated_layout() {
         temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
@@ -11133,7 +11621,7 @@ mod tests {
             *disk = None;
         }
         let (info, _) = set
-            .check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
+            .check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
             .await
             .expect("four primary plus four recovery copies reach metadata read quorum");
         assert_eq!(info.name, path);
@@ -11154,7 +11642,7 @@ mod tests {
             *disk = None;
         }
         assert!(
-            set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
+            set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
                 .await
                 .is_err(),
             "two files on each of four disks must not count as eight voters"
@@ -14471,7 +14959,7 @@ mod tests {
                     object,
                     &u_new,
                     &SetDisks::get_upload_id_dir(bucket, object, &u_new),
-                    true,
+                    MultipartMetadataQuorum::Write,
                     true,
                 )
                 .await
@@ -14516,7 +15004,7 @@ mod tests {
                 object,
                 &upload,
                 &SetDisks::get_upload_id_dir(bucket, object, &upload),
-                true,
+                MultipartMetadataQuorum::Write,
                 true,
             )
             .await
@@ -14599,7 +15087,7 @@ mod tests {
                     assert_completion_files_preserved(&disk_stores, RUSTFS_META_MULTIPART_BUCKET, &source_files).await;
                     set_disks
                         .check_multipart_upload_path_exists(
-                            bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), true, true,
+                            bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), MultipartMetadataQuorum::Write, true,
                         )
                         .await
                         .expect("upload authority must survive the post-commit crash");

crates/ecstore/src/set_disk/ops/multipart_abort_tests.rs

@@ -363,6 +363,202 @@ mod native_abort_closure_tests {
         }
     }
 
+    #[tokio::test]
+    async fn stale_cleanup_reclaims_certified_one_and_two_disk_residuals_after_reopen() {
+        for survivors in [1, 2] {
+            let (dirs, disks, set) = hermetic_set_disks(4).await;
+            let bucket = "native-stale-certified";
+            let object = format!("survivors-{survivors}");
+            make_bucket_on_all(&disks, bucket).await;
+            let opts = ObjectOptions {
+                max_parity: true,
+                ..Default::default()
+            };
+            let (upload, _) = stage_upload_with_create_opts(&set, bucket, &object, &[0x61; 4096], &opts).await;
+            let path = SetDisks::get_upload_id_dir(bucket, &object, &upload);
+            let before = originals(&disks, &path).await;
+            {
+                let physical = disks.iter().cloned().map(Some).collect::<Vec<_>>();
+                let SourceCleanupAuthority::Native { certificate, snapshots } =
+                    prepare_source_cleanup(&physical, bucket, &object, &path, true, &set.multipart_proof_verification_context())
+                        .await
+                        .expect("close every original and persist the certificate")
+                else {
+                    panic!("native upload must retain a native certificate")
+                };
+                for slot in 0..(4 - survivors) {
+                    let response = operation(
+                        &disks[slot],
+                        &path,
+                        MultipartSourceOperation::DeleteAborted {
+                            expected: snapshots[slot].clone().expect("actual certified source"),
+                            certificate: certificate.as_ref().clone(),
+                        },
+                    )
+                    .await;
+                    assert!(matches!(response, MultipartSourceResponse::AbortDeleted(_)));
+                }
+            }
+            let (reopened, cold) = cold_reopen(disks, set).await;
+            for slot in (4 - survivors)..4 {
+                assert_eq!(
+                    retained_native_multipart_original(&reopened[slot], slot, bucket, &object, &path).await,
+                    before[slot]
+                );
+            }
+            cold.lock_stale_multipart_cleanup(&path)
+                .await
+                .expect("complete certificate authorizes the background retry")
+                .delete(&cold)
+                .await
+                .expect("background retry removes certified survivors");
+            for dir in &dirs {
+                assert!(!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&path).exists());
+            }
+            assert!(
+                cold.lock_stale_multipart_cleanup(&path).await.is_err(),
+                "a repeated pass must not recreate the retired upload"
+            );
+        }
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn stale_certified_residual_survives_lock_loss_then_converges() {
+        let target = Arc::new(std::sync::RwLock::new(None));
+        let refresh_calls = Arc::new(AtomicUsize::new(0));
+        let lockers: Vec<Arc<dyn LockClient>> = (0..4)
+            .map(|_| {
+                Arc::new(SelectiveLockLossClient::new(Arc::clone(&target), Arc::clone(&refresh_calls))) as Arc<dyn LockClient>
+            })
+            .collect();
+        let (dirs, disks, set) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+        let bucket = "stale-certified-lock-loss";
+        let object = "object";
+        make_bucket_on_all(&disks, bucket).await;
+        let opts = ObjectOptions {
+            max_parity: true,
+            ..Default::default()
+        };
+        let (upload, _) = stage_upload_with_create_opts(&set, bucket, object, &[0x63; 4096], &opts).await;
+        let path = SetDisks::get_upload_id_dir(bucket, object, &upload);
+        let (fi, _) = set
+            .check_upload_id_exists(bucket, object, &upload, false)
+            .await
+            .expect("source metadata before closure");
+        let part_path = format!("{path}/{}/part.1", fi.data_dir.expect("source data directory"));
+        let original = originals(&disks, &path).await;
+        let surviving_part = disks[3]
+            .read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path)
+            .await
+            .expect("snapshot surviving shard");
+        {
+            let physical = disks.iter().cloned().map(Some).collect::<Vec<_>>();
+            let SourceCleanupAuthority::Native { certificate, snapshots } =
+                prepare_source_cleanup(&physical, bucket, object, &path, true, &set.multipart_proof_verification_context())
+                    .await
+                    .expect("all original participants close and persist the certificate")
+            else {
+                panic!("native certificate required")
+            };
+            let authority = set.multipart_proof_verification_context();
+            let mut ids = certificate.disk_ids.iter().copied().map(Some).collect::<Vec<_>>();
+            ids[0] = None;
+            assert_eq!(
+                crate::set_disk::core::multipart_source::abort_certificate_cohort(&certificate, &ids, &authority),
+                Err(DiskError::ErasureWriteQuorum),
+                "an unavailable participant is a retryable proof gap"
+            );
+            ids[1] = Some(Uuid::new_v4());
+            assert_eq!(
+                crate::set_disk::core::multipart_source::abort_certificate_cohort(&certificate, &ids, &authority),
+                Err(DiskError::FileCorrupt),
+                "a missing participant cannot hide a known identity conflict"
+            );
+            ids[0] = Some(Uuid::new_v4());
+            ids[1] = Some(certificate.disk_ids[1]);
+            assert_eq!(
+                crate::set_disk::core::multipart_source::abort_certificate_cohort(&certificate, &ids, &authority),
+                Err(DiskError::FileCorrupt),
+                "a known different identity remains a certificate conflict"
+            );
+            for slot in 0..3 {
+                let response = operation(
+                    &disks[slot],
+                    &path,
+                    MultipartSourceOperation::DeleteAborted {
+                        expected: snapshots[slot].clone().expect("certified source"),
+                        certificate: certificate.as_ref().clone(),
+                    },
+                )
+                .await;
+                assert!(matches!(response, MultipartSourceResponse::AbortDeleted(_)));
+            }
+        }
+        assert!(dirs[3].path().join(RUSTFS_META_MULTIPART_BUCKET).join(&path).exists());
+        let offline = set.disks.write().await[0]
+            .take()
+            .expect("take an original participant offline");
+        let rejected = set.lock_stale_multipart_cleanup(&path).await;
+        set.disks.write().await[0] = Some(offline);
+        let err = rejected
+            .err()
+            .expect("a certificate must not substitute for an unavailable participant");
+        assert!(
+            matches!(err, StorageError::ErasureWriteQuorum | StorageError::InsufficientWriteQuorum(..)),
+            "the missing participant must be reported as a quorum failure: {err:?}"
+        );
+        assert_eq!(retained_native_multipart_original(&disks[3], 3, bucket, object, &path).await, original[3]);
+        assert_eq!(
+            disks[3]
+                .read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path)
+                .await
+                .expect("surviving shard while a participant was offline"),
+            surviving_part
+        );
+        tokio::time::pause();
+        *target.write().expect("lock-loss target") =
+            Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, path.clone()));
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+        let guard = set
+            .lock_stale_multipart_cleanup(&path)
+            .await
+            .expect("certificate admits the one-disk residual");
+        tokio::time::advance(Duration::from_secs(11)).await;
+        for _ in 0..20 {
+            if guard.is_lock_lost() {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+        assert!(refresh_calls.load(Ordering::Acquire) > 0, "the real lock refresh must fail");
+        assert!(guard.is_lock_lost(), "the cleanup guard must observe lock loss");
+        // Real disk I/O must not run under Tokio's auto-advancing paused clock:
+        // a pending blocking operation could otherwise exhaust its timeout.
+        tokio::time::resume();
+        let err = guard.delete(&set).await.expect_err("lock loss must fence physical deletion");
+        assert!(matches!(err, StorageError::NamespaceLockQuorumUnavailable { .. }), "{err:?}");
+        assert_eq!(retained_native_multipart_original(&disks[3], 3, bucket, object, &path).await, original[3]);
+        assert_eq!(
+            disks[3]
+                .read_all(RUSTFS_META_MULTIPART_BUCKET, &part_path)
+                .await
+                .expect("surviving shard after loss"),
+            surviving_part
+        );
+
+        *target.write().expect("restore lock service") = None;
+        set.lock_stale_multipart_cleanup(&path)
+            .await
+            .expect("reacquire the cleanup lock")
+            .delete(&set)
+            .await
+            .expect("the certified residual converges after lock recovery");
+        for dir in &dirs {
+            assert!(!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&path).exists());
+        }
+    }
+
     #[tokio::test]
     async fn native_abort_partial_delete_surviving_producer_confirms_current_and_historical_exact_target() {
         use crate::set_disk::core::io_primitives::{rename_fault_injection, rollback_fault_injection};
@@ -507,7 +703,7 @@ mod native_abort_closure_tests {
         let path = SetDisks::get_upload_id_dir(bucket, object, &upload);
         pin_historical_unmarked_upload(&disks, bucket, &path).await;
         let before = originals(&disks, &path).await;
-        set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
+        set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
             .await
             .expect("readonly legacy cleanup identity lookup");
         assert_eq!(
@@ -786,7 +982,7 @@ mod native_abort_closure_tests {
             "known orphan native evidence remains unknown: {inspected:?}"
         );
         let readonly = set
-            .check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
+            .check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
             .await;
         assert!(
             matches!(

crates/ecstore/src/set_disk/ops/object.rs

@@ -73,6 +73,7 @@ struct DeleteCommitContext<'a> {
     incarnation: Option<Uuid>,
     options: Option<&'a ObjectOptions>,
     guard: Option<Arc<super::super::ObjectLockDiagGuard>>,
+    transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
 }
 
 impl<'a> DeleteCommitContext<'a> {
@@ -81,17 +82,37 @@ impl<'a> DeleteCommitContext<'a> {
             incarnation: options.expected_bucket_incarnation_id,
             options: Some(options),
             guard: guard.cloned(),
+            transaction_guard: None,
         }
     }
 
+    fn with_transaction_guard(mut self, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) -> Self {
+        self.transaction_guard = guard.cloned();
+        self
+    }
+
+    fn ensure_transaction_lock_held(&self, bucket: &str, object: &str) -> Result<()> {
+        if self.transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            return Err(StorageError::NamespaceLockQuorumUnavailable {
+                mode: "delete_transaction",
+                bucket: bucket.to_owned(),
+                object: object.to_owned(),
+                required: 1,
+                achieved: 0,
+            });
+        }
+        Ok(())
+    }
+
     fn can_commit(&self, bucket: &str, object: &str) -> bool {
-        self.options.is_none_or(|options| {
-            ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok()
-                && options
-                    .scanner_publication_commit_scope
-                    .as_ref()
-                    .is_none_or(|scope| scope.can_commit())
-        })
+        !self.transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
+            && self.options.is_none_or(|options| {
+                ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok()
+                    && options
+                        .scanner_publication_commit_scope
+                        .as_ref()
+                        .is_none_or(|scope| scope.can_commit())
+            })
     }
 }
 
@@ -3852,6 +3873,7 @@ impl SetDisks {
         object: &str,
         opts: &ObjectOptions,
         lock_guard: Option<&Arc<super::super::ObjectLockDiagGuard>>,
+        transaction_guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
     ) -> Result<bool> {
         if is_meta_bucketname(bucket)
             || opts.version_id.is_some()
@@ -3888,6 +3910,7 @@ impl SetDisks {
                 object,
                 DeleteRecoveryContext {
                     explicit_delete: true,
+                    transaction_guard: transaction_guard.cloned(),
                     namespace_owner: Some(Arc::new((
                         self.ctx.begin_namespace_commit(),
                         lock_guard.cloned(),
@@ -8839,7 +8862,7 @@ impl SetDisks {
         // quorum, failing write quorum on update_object_meta (backlog#872).
         let read_opts = opts.as_read_opts();
         let (mut fi, _, disks) = self
-            .get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
+            .get_object_fileinfo_for_metadata_update(bucket, object, &read_opts)
             .await?
             .into_owned();
 
@@ -8877,8 +8900,11 @@ impl SetDisks {
         fi: &FileInfo,
         force_del_marker: bool,
         delete_marker_purge: Option<MrfDeleteMarkerPurge>,
-        commit_context: DeleteCommitContext<'_>,
+        mut commit_context: DeleteCommitContext<'_>,
     ) -> Result<()> {
+        if !is_meta_bucketname(bucket) && commit_context.transaction_guard.is_none() {
+            commit_context.transaction_guard = Some(self.acquire_delete_transaction_lock(bucket, object).await?);
+        }
         let source_bucket_incarnation_id = commit_context.incarnation;
         let transported = delete_file_info_with_replication_transport_metadata(fi);
         let fi = &transported;
@@ -8887,6 +8913,7 @@ impl SetDisks {
             Arc::new((
                 self.ctx.begin_namespace_commit(),
                 commit_context.guard.clone(),
+                commit_context.transaction_guard.clone(),
                 commit_context
                     .options
                     .and_then(|options| options.object_lock_config_snapshot.clone()),
@@ -8926,7 +8953,11 @@ impl SetDisks {
 
         for disk in disks.iter() {
             let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
+            let transaction_guard = commit_context.transaction_guard.as_ref();
             futures.push(async move {
+                if transaction_guard.is_some_and(|guard| guard.is_lock_lost()) {
+                    return Err(DiskError::DiskOngoingReq);
+                }
                 if let Some(disk) = disk {
                     let options = DeleteOptions {
                         old_data_dir: Some(rollback_dir),
@@ -8976,6 +9007,7 @@ impl SetDisks {
             }
         }
 
+        commit_context.ensure_transaction_lock_held(bucket, object)?;
         let mut quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object);
         if !is_meta_bucketname(bucket) && errs.iter().flatten().any(super::delete_recovery::unsupported_recovery) {
             quorum_result = Err(DiskError::UnsupportedDisk.into());
@@ -9008,6 +9040,7 @@ impl SetDisks {
                 write_quorum,
                 DeleteRecoveryContext {
                     recovery_ticket,
+                    transaction_guard: commit_context.transaction_guard.clone(),
                     namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
                     ..DeleteRecoveryContext::new(source_bucket_incarnation_id, &|| commit_context.can_commit(bucket, object))
                 },
@@ -9017,6 +9050,7 @@ impl SetDisks {
         }
         #[cfg(test)]
         pause_delete_before_cleanup(bucket).await;
+        commit_context.ensure_transaction_lock_held(bucket, object)?;
         let mut rollback_futures = Vec::new();
         for (index, err) in errs.iter().enumerate() {
             if structured_cleanup {
@@ -9042,7 +9076,11 @@ impl SetDisks {
             let object = object.to_string();
             let fi = fi.clone();
             let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
+            let transaction_guard = commit_context.transaction_guard.as_ref();
             rollback_futures.push(async move {
+                if transaction_guard.is_some_and(|guard| guard.is_lock_lost()) {
+                    return;
+                }
                 if should_rollback {
                     // The dedicated undo path never forwards the marker-only creation flag.
                     if let Err(err) = disk
@@ -9099,6 +9137,7 @@ impl SetDisks {
         }
 
         join_all(rollback_futures).await;
+        commit_context.ensure_transaction_lock_held(bucket, object)?;
         drop(namespace_owner);
         if quorum_result.is_ok()
             && errs.iter().any(Option::is_some)
@@ -9294,36 +9333,56 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
-            Ok((r, w)) => (
-                usize::try_from(r)
-                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![src_bucket, src_object]))?,
-                usize::try_from(w)
-                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![src_bucket, src_object]))?,
-            ),
+        let src_version_id = src_opts.version_id.as_deref().unwrap_or_default();
+        let quorum = Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count);
+        let selection = match &quorum {
+            Ok((read_quorum, write_quorum)) => {
+                let read_quorum = usize::try_from(*read_quorum)
+                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![src_bucket, src_object]))?;
+                let write_quorum = usize::try_from(*write_quorum)
+                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![src_bucket, src_object]))?;
+                Self::select_fileinfo_for_read(src_bucket, &disks, &metas, &errs, src_version_id, read_quorum, write_quorum)
+                    .map(|(online_disks, fi)| (online_disks, fi, write_quorum))
+            }
+            Err(err) => Err(err.clone()),
+        };
+        let (online_disks, mut fi, write_quorum) = match selection {
+            Ok(selection) => selection,
             Err(mut err) => {
-                if err == DiskError::ErasureReadQuorum
-                    && !src_bucket.starts_with(RUSTFS_META_BUCKET)
-                    && self
-                        .delete_if_dangling(src_bucket, src_object, &metas, &errs, &HashMap::new(), src_opts.clone())
-                        .await
-                        .is_ok()
+                // The merged latest version can be one a failed PUT left below
+                // quorum; resolve it like a latest GET before the dangling
+                // cleanup below can report a key with a committed version absent.
+                if let Some(snapshot) = self
+                    .select_latest_past_dangling_versions(src_bucket, src_object, src_opts, true, &disks, &err)
+                    .await
                 {
-                    if src_opts.version_id.is_some() {
-                        err = DiskError::FileVersionNotFound
-                    } else {
-                        err = DiskError::FileNotFound
+                    // A version-only copy writes the served version's own
+                    // copies, so they and its write quorum replace the latest
+                    // read's.
+                    let (fi, parts_metadata, online_disks) = snapshot.into_owned();
+                    metas = parts_metadata;
+                    let write_quorum = fi.write_quorum(self.default_write_quorum());
+                    (online_disks, fi, write_quorum)
+                } else {
+                    if quorum.is_err()
+                        && err == DiskError::ErasureReadQuorum
+                        && !src_bucket.starts_with(RUSTFS_META_BUCKET)
+                        && self
+                            .delete_if_dangling(src_bucket, src_object, &metas, &errs, &HashMap::new(), src_opts.clone())
+                            .await
+                            .is_ok()
+                    {
+                        if src_opts.version_id.is_some() {
+                            err = DiskError::FileVersionNotFound
+                        } else {
+                            err = DiskError::FileNotFound
+                        }
                     }
+                    return Err(to_object_err(err.into(), vec![src_bucket, src_object]));
                 }
-                return Err(to_object_err(err.into(), vec![src_bucket, src_object]));
             }
         };
 
-        let src_version_id = src_opts.version_id.as_deref().unwrap_or_default();
-        let (online_disks, mut fi) =
-            Self::select_fileinfo_for_read(src_bucket, &disks, &metas, &errs, src_version_id, read_quorum, write_quorum)
-                .map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
-
         if fi.deleted {
             if src_opts.version_id.is_none() {
                 return Err(to_object_err(Error::FileNotFound, vec![src_bucket, src_object]));
@@ -9461,6 +9520,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 }
             }
 
+            // A version-only copy writes and then reverts on a quorum miss;
+            // refuse it before writing when the served version's disks cannot
+            // reach its write quorum even if every write succeeds.
+            let reachable = online_disks
+                .iter()
+                .map(|disk| disk.is_none().then_some(DiskError::DiskNotFound))
+                .collect::<Vec<_>>();
+            if let Some(err) = reduce_write_quorum_errs(&reachable, OBJECT_OP_IGNORED_ERRS, write_quorum) {
+                return Err(to_object_err(err.into(), vec![src_bucket, src_object]));
+            }
             Self::write_unique_file_info(&online_disks, "", src_bucket, src_object, &metas, write_quorum)
                 .await
                 .map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
@@ -9528,9 +9597,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         &self,
         bucket: &str,
         objects: Vec<ObjectToDelete>,
-        opts: ObjectOptions,
+        mut opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
-        let current_generations = opts.conditional_delete_generations.as_deref();
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
         let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
@@ -9625,6 +9693,30 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
+        let outer_namespace_lock_fence = opts.namespace_lock_fence.clone();
+        let mut transaction_guards = std::collections::BTreeMap::new();
+        if !is_meta_bucketname(bucket) {
+            // Every raw batch lock is acquired first. Canonical sorting also
+            // gives aliases of one disk key exactly one transaction owner.
+            let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
+            for key in keys {
+                match self.acquire_delete_transaction_lock(bucket, &key).await {
+                    Ok(guard) => {
+                        opts.add_namespace_lock_guard(&guard);
+                        transaction_guards.insert(key, guard);
+                    }
+                    Err(error) => {
+                        for (index, lock_name) in lock_names.iter().enumerate() {
+                            if lock_name.as_ref() == key {
+                                del_errs[index] = Some(error.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        let transaction_guards = Arc::new(transaction_guards);
+        let current_generations = opts.conditional_delete_generations.as_deref();
         let object_lock_config = if is_meta_bucketname(bucket) {
             Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))
         } else {
@@ -9722,6 +9814,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                         &dobj.object_name,
                         DeleteRecoveryContext {
                             dry_run: !opts.no_lock,
+                            transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
                             namespace_owner: opts.delete_mutation_owner.clone(),
                             ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| {
                                 ensure_delete_commit_locks_held(None, bucket, &dobj.object_name, &opts).is_ok()
@@ -10003,6 +10096,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
             Arc::new((
                 self.ctx.begin_namespace_commit(),
+                Arc::clone(&transaction_guards),
                 opts.delete_mutation_owner.clone(),
                 opts.object_lock_config_snapshot.clone(),
                 opts.scanner_publication_commit_scope.clone(),
@@ -10178,6 +10272,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
 
         let results = join_all(futures).await;
+        if transaction_guards.values().any(|guard| guard.is_lock_lost()) {
+            for (index, object) in objects.iter().enumerate() {
+                if del_errs[index].is_none() {
+                    accounting[index] = None;
+                    del_errs[index] = Some(StorageError::NamespaceLockQuorumUnavailable {
+                        mode: "delete_transaction",
+                        bucket: bucket.to_owned(),
+                        object: object.object_name.clone(),
+                        required: 1,
+                        achieved: 0,
+                    });
+                }
+            }
+            if dist_erasure {
+                self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
+            }
+            return (del_objects, del_errs, accounting);
+        }
 
         let mut del_obj_errs: Vec<Vec<Option<DiskError>>> = vec![vec![None; objects.len()]; disks.len()];
 
@@ -10273,6 +10385,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 let group = &vers[index];
                 let recovery_ticket = recovery_tickets[index].clone();
                 let namespace_owner = namespace_owner.clone();
+                let transaction_guard = transaction_guards
+                    .get(object_name_disk_identity(&group.name).as_ref())
+                    .cloned();
                 let disks = &disks;
                 let opts = &opts;
                 async move {
@@ -10293,6 +10408,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                             DeleteRecoveryContext {
                                 recovery_ticket,
                                 namespace_owner,
+                                transaction_guard,
                                 ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &can_commit)
                             },
                         )
@@ -10339,7 +10455,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 let object = fi_vers.name.clone();
                 let versions = fi_vers.clone();
                 let disk_namespace_owner = namespace_owner.clone();
+                let transaction_guard = transaction_guards.get(object_name_disk_identity(&object).as_ref()).cloned();
                 rollback_futures.push(async move {
+                    if transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+                        return;
+                    }
                     if should_rollback {
                         let errs = disk
                             .delete_versions_with_namespace_owner(
@@ -10415,7 +10535,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             if del_errs[index].is_none() {
                 let object = &objects[index].object_name;
                 let result = match ensure_delete_commit_locks_held(None, bucket, object, &opts) {
-                    Ok(()) => self.reclaim_missing_delete_residue(bucket, object, &opts, None).await,
+                    Ok(()) => {
+                        self.reclaim_missing_delete_residue(
+                            bucket,
+                            object,
+                            &opts,
+                            None,
+                            transaction_guards.get(object_name_disk_identity(object).as_ref()),
+                        )
+                        .await
+                    }
                     Err(err) => Err(err),
                 };
                 if let Err(err) = result {
@@ -10424,6 +10553,13 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
+        // Deferred recovery takes fresh raw and transaction locks. Discard the
+        // completed leases' deadlines while retaining every caller-owned fence.
+        opts.namespace_lock_fence = outer_namespace_lock_fence;
+        // Release inner owners before reacquiring any raw key lock.
+        drop(namespace_owner);
+        drop(transaction_guards);
+
         // TODO(backlog): support partial object deletion for multi-part objects
 
         if dist_erasure {
@@ -10533,6 +10669,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         } else {
             None
         };
+        // Lock order: the caller's raw object lock, then this target set's
+        // transaction key. The inner key remains distinct even when pool
+        // client quorums overlap, and no_lock never bypasses journal ownership.
+        let transaction_guard = if !is_meta_bucketname(bucket) && (!opts.delete_prefix || opts.delete_prefix_object) {
+            let guard = self.acquire_delete_transaction_lock(bucket, object).await?;
+            opts.add_namespace_lock_guard(&guard);
+            Some(guard)
+        } else {
+            None
+        };
         if opts.delete_prefix {
             let destructive_dispatch_phase = opts.lifecycle_delete_all.as_ref().is_none_or(|request| {
                 matches!(
@@ -10633,7 +10779,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 &delete_request,
                                 false,
                                 None,
-                                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
+                                DeleteCommitContext::request(&opts, _lock_guard.as_ref())
+                                    .with_transaction_guard(transaction_guard.as_ref()),
                             )
                             .await?;
                             if let Some((_, deleted_object)) = replication_delete {
@@ -10667,7 +10814,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 &delete_request,
                                 false,
                                 None,
-                                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
+                                DeleteCommitContext::request(&opts, _lock_guard.as_ref())
+                                    .with_transaction_guard(transaction_guard.as_ref()),
                             )
                             .await?;
                         }
@@ -10687,7 +10835,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 &delete_request,
                                 false,
                                 None,
-                                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
+                                DeleteCommitContext::request(&opts, _lock_guard.as_ref())
+                                    .with_transaction_guard(transaction_guard.as_ref()),
                             )
                             .await?;
                         }
@@ -10776,6 +10925,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     bucket,
                     object,
                     DeleteRecoveryContext {
+                        transaction_guard: transaction_guard.clone(),
                         namespace_owner: Some(Arc::new((
                             self.ctx.begin_namespace_commit(),
                             _lock_guard.clone(),
@@ -10816,7 +10966,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 if is_err_object_not_found(err) {
                     ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?;
                     if self
-                        .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref())
+                        .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref(), transaction_guard.as_ref())
                         .await?
                     {
                         if let Some(scope) = scanner_publication_commit_scope.as_ref() {
@@ -10866,7 +11016,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 &dfi,
                 false,
                 None,
-                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
+                DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
             )
             .await
             .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -10968,7 +11118,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 &fi,
                 should_force_delete_marker_for_missing_version(&opts),
                 delete_marker_purge.clone(),
-                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
+                DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
             )
             .await
             .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -11033,7 +11183,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             &dfi,
             opts.delete_marker,
             delete_marker_purge,
-            DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
+            DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
         )
         .await
         .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -11147,45 +11297,63 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
-            Ok((read_quorum, write_quorum)) => (read_quorum, write_quorum),
+        let version_id = opts.version_id.as_deref().unwrap_or_default();
+        let quorum = Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count);
+        let selection = match &quorum {
+            Ok((read_quorum, write_quorum)) => {
+                let read_quorum = usize::try_from(*read_quorum)
+                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
+                let write_quorum = usize::try_from(*write_quorum)
+                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
+                Self::select_fileinfo_for_read(bucket, &disks, &metas, &errs, version_id, read_quorum, write_quorum)
+            }
+            Err(err) => Err(err.clone()),
+        };
+        let (online_disks, mut fi) = match selection {
+            Ok(selection) => selection,
             Err(mut err) => {
-                // A replication terminal CAS is a validation/write-back
-                // attempt, never a heal operation. In particular,
-                // ValidateOnly must remain zero-write, and a worker whose
-                // outer replication lease is already lost must not delete the
-                // surviving xl.meta copies before the namespace fence below
-                // can reject it. Leave dangling cleanup to heal/scanner.
-                if opts.replication_status_writeback.is_some() {
+                // The merged latest version can be one a failed PUT left below
+                // quorum; resolve it like a latest GET before the dangling
+                // cleanup below can report a key with a committed version absent.
+                // A replication terminal CAS stays side-effect free: the walk
+                // queues heal for the versions it skips, so it does not run.
+                let resolved = if opts.replication_status_writeback.is_none() {
+                    self.select_latest_past_dangling_versions(bucket, object, opts, false, &disks, &err)
+                        .await
+                } else {
+                    None
+                };
+                if let Some(snapshot) = resolved {
+                    let (fi, _, online_disks) = snapshot.into_owned();
+                    (online_disks, fi)
+                } else {
+                    // A replication terminal CAS is a validation/write-back
+                    // attempt, never a heal operation. In particular,
+                    // ValidateOnly must remain zero-write, and a worker whose
+                    // outer replication lease is already lost must not delete
+                    // the surviving xl.meta copies before the namespace fence
+                    // below can reject it. Leave dangling cleanup to
+                    // heal/scanner.
+                    if quorum.is_err()
+                        && opts.replication_status_writeback.is_none()
+                        && err == DiskError::ErasureReadQuorum
+                        && !bucket.starts_with(RUSTFS_META_BUCKET)
+                        && self
+                            .delete_if_dangling(bucket, object, &metas, &errs, &HashMap::new(), opts.clone())
+                            .await
+                            .is_ok()
+                    {
+                        if opts.version_id.is_some() {
+                            err = DiskError::FileVersionNotFound
+                        } else {
+                            err = DiskError::FileNotFound
+                        }
+                    }
                     return Err(to_object_err(err.into(), vec![bucket, object]));
                 }
-                if err == DiskError::ErasureReadQuorum
-                    && !bucket.starts_with(RUSTFS_META_BUCKET)
-                    && self
-                        .delete_if_dangling(bucket, object, &metas, &errs, &HashMap::new(), opts.clone())
-                        .await
-                        .is_ok()
-                {
-                    if opts.version_id.is_some() {
-                        err = DiskError::FileVersionNotFound
-                    } else {
-                        err = DiskError::FileNotFound
-                    }
-                }
-                return Err(to_object_err(err.into(), vec![bucket, object]));
             }
         };
 
-        let read_quorum =
-            usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
-        let write_quorum = usize::try_from(write_quorum)
-            .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
-
-        let version_id = opts.version_id.as_deref().unwrap_or_default();
-        let (online_disks, mut fi) =
-            Self::select_fileinfo_for_read(bucket, &disks, &metas, &errs, version_id, read_quorum, write_quorum)
-                .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
-
         if fi.deleted {
             return Err(to_object_err(Error::MethodNotAllowed, vec![bucket, object]));
         }
@@ -11790,13 +11958,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let bucket_lifecycle_guard = if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id
             && opts.bucket_lifecycle_lock_fence.is_none()
         {
-            let guard = metadata_sys::object_store_in(&self.ctx)
-                .await?
-                .acquire_bucket_lifecycle_read_lock(bucket)
-                .await?;
-            self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
-                .await?;
-            Some(guard)
+            Some(
+                metadata_sys::object_store_in(&self.ctx)
+                    .await?
+                    .acquire_bucket_incarnation_fence_with_options(bucket, expected_incarnation_id, Some(opts))
+                    .await?,
+            )
         } else {
             None
         };
@@ -25358,6 +25525,7 @@ mod delete_objects_lock_gating_tests {
     }
 
     #[tokio::test]
+    #[serial_test::serial]
     async fn delete_object_aborts_when_outer_lock_is_lost_at_commit() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
         let bucket = "single-lost-outer-lock";
@@ -25578,6 +25746,13 @@ mod delete_objects_lock_gating_tests {
             .as_fast_lock_manager()
             .expect("isolated fixture should use the local lock manager");
         let lock_metrics = lock_manager.get_metrics().shard_metrics;
+        let raw_key = ObjectKey::new(bucket, "obj-a");
+        let raw_holder = lock_manager.get_lock_info(&raw_key).expect("outer holder must be visible");
+        let transaction_key = ObjectKey::with_version(
+            bucket,
+            "obj-a",
+            format!("rustfs:delete-transaction:{}:{}", set_disks.pool_index, set_disks.set_index),
+        );
         assert_eq!(lock_manager.current_lock_counts(), (1, 0));
         let mut delete = Box::pin(set_disks.delete_objects(
             bucket,
@@ -25594,23 +25769,43 @@ mod delete_objects_lock_gating_tests {
         // than a second on CI without waiting for the namespace lock at all.
         let deletion = futures::future::poll_fn(|cx| {
             let result = std::future::Future::poll(delete.as_mut(), cx);
+            let current_holder = lock_manager.get_lock_info(&raw_key).expect("outer holder must remain held");
+            assert_eq!(
+                (current_holder.mode, &current_holder.owner, current_holder.acquired_at),
+                (raw_holder.mode, &raw_holder.owner, raw_holder.acquired_at),
+                "no_lock must preserve the original raw object holder"
+            );
+            let transaction_holder = lock_manager.get_lock_info(&transaction_key);
+            if let Some(holder) = &transaction_holder {
+                assert_eq!(holder.mode, rustfs_lock::LockMode::Exclusive);
+            }
             assert_eq!(
                 lock_manager.current_lock_counts(),
-                (1, 0),
-                "no_lock batch delete must preserve the outer holder without adding lock waiters"
+                (1 + u64::from(transaction_holder.is_some()), 0),
+                "only the distinct transaction owner may be added, without raw-key waiters"
             );
             let current = lock_manager.get_metrics().shard_metrics;
-            assert_eq!(
-                (current.total_acquisitions(), current.timeouts),
-                (lock_metrics.total_acquisitions(), lock_metrics.timeouts),
-                "no_lock batch delete must not acquire or time out on another namespace lock"
+            assert!(
+                current.total_acquisitions() <= lock_metrics.total_acquisitions() + 1,
+                "the transaction owner must be reused by inline recovery"
             );
+            assert_eq!(current.timeouts, lock_metrics.timeouts, "the raw key must never be reacquired");
             result
         });
         let (deleted, errs) = tokio::time::timeout(Duration::from_secs(30), deletion)
             .await
             .expect("no_lock batch delete disk work should complete");
 
+        assert_eq!(lock_manager.current_lock_counts(), (1, 0));
+        assert!(
+            lock_manager.get_lock_info(&transaction_key).is_none(),
+            "the transaction owner must be released"
+        );
+        assert_eq!(
+            lock_manager.get_metrics().shard_metrics.total_acquisitions(),
+            lock_metrics.total_acquisitions() + 1,
+            "no_lock still requires exactly one distinct transaction owner"
+        );
         assert!(errs[0].is_none(), "no_lock batch delete should not fail with a lock error: {:?}", errs[0]);
         assert!(deleted[0].found, "existing object must still be deleted");
         for disk in &disk_stores {

crates/ecstore/src/set_disk/read.rs

@@ -278,6 +278,10 @@ const MAX_DANGLING_LATEST_VERSIONS: usize = 8;
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum DanglingRead {
     KeepQuorumError,
+    /// Metadata updates of the latest version keep the quorum error but, like
+    /// object reads, look past newer versions proven dangling
+    /// (`read_latest_past_dangling_versions`).
+    SkipDanglingVersions,
     ReportNotFound,
 }
 
@@ -729,9 +733,10 @@ impl SetDisks {
     }
 
     /// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
-    /// quorum fanout. Read-before-write callers (object tagging) must use this:
-    /// the returned online-disk set is the write target, and the early-stop
-    /// subset would fail write quorum (backlog#872 regression fix).
+    /// quorum fanout. Read-before-write callers must use this or
+    /// `get_object_fileinfo_for_metadata_update`: the returned online-disk set
+    /// is the write target, and the early-stop subset would fail write quorum
+    /// (backlog#872 regression fix).
     #[tracing::instrument(level = "debug", skip(self))]
     pub(in crate::set_disk) async fn get_object_fileinfo_gated(
         &self,
@@ -753,6 +758,64 @@ impl SetDisks {
         .await
     }
 
+    /// The full-fanout read behind a metadata update such as object tagging.
+    /// A latest read whose newest versions never committed resolves to the
+    /// version a latest GET serves, while a dangling key keeps the quorum
+    /// error. The returned online disks are the write target.
+    #[tracing::instrument(level = "debug", skip(self))]
+    pub(in crate::set_disk) async fn get_object_fileinfo_for_metadata_update(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+    ) -> Result<GetObjectFileInfo> {
+        self.get_object_fileinfo_gated_inner(
+            bucket,
+            object,
+            opts,
+            false,
+            false,
+            MetadataReadPurpose::FullLayout,
+            DanglingRead::SkipDanglingVersions,
+        )
+        .await
+    }
+
+    /// Recovers a failed latest selection in read-before-write callers that
+    /// read and select metadata themselves (`put_object_metadata` and the
+    /// CopyObject source) the way a latest GET does. `None` keeps the caller's
+    /// error: a failure other than read quorum, an explicit version, the
+    /// system bucket's commit-proof rule, or a walk that finds no readable
+    /// version or cannot read every drive. The selected metadata keeps its
+    /// part checksums because a version-only copy carries them into the new
+    /// version.
+    pub(in crate::set_disk) async fn select_latest_past_dangling_versions(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+        read_data: bool,
+        disks: &[Option<DiskStore>],
+        err: &DiskError,
+    ) -> Option<GetObjectFileInfo> {
+        if *err != DiskError::ErasureReadQuorum || opts.version_id.is_some() || crate::bucket::utils::is_meta_bucketname(bucket) {
+            return None;
+        }
+        let opts = ObjectOptions {
+            include_part_checksums: true,
+            ..opts.clone()
+        };
+        Box::pin(self.read_latest_past_dangling_versions(
+            bucket,
+            object,
+            &opts,
+            read_data,
+            MetadataReadPurpose::FullLayout,
+            disks,
+        ))
+        .await
+    }
+
     #[allow(clippy::too_many_arguments)]
     async fn get_object_fileinfo_gated_inner(
         &self,
@@ -978,7 +1041,7 @@ impl SetDisks {
                 Err(err) => {
                     if err == DiskError::ErasureReadQuorum
                         && vid.is_empty()
-                        && dangling_read == DanglingRead::ReportNotFound
+                        && dangling_read != DanglingRead::KeepQuorumError
                         && metadata_fanout_complete
                         && !crate::bucket::utils::is_meta_bucketname(bucket)
                         && let Some(snapshot) =
@@ -9930,6 +9993,7 @@ mod dangling_read_tests {
     use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
     use crate::storage_api_contracts::object::ObjectOperations;
     use http::HeaderMap;
+    use std::collections::HashMap;
     use std::path::PathBuf;
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
@@ -10451,6 +10515,353 @@ mod dangling_read_tests {
         assert_ne!(latest.version_id, Some(v2));
     }
 
+    /// Version one on every drive under a version two that a failed PUT left
+    /// on drives `0..3`, below the read quorum of four.
+    async fn put_committed_below_dangling(set: &SetDisks, bucket: &str, object: &str) -> (Uuid, Uuid) {
+        let v1 = put(set, bucket, object, b"version one", true)
+            .await
+            .version_id
+            .expect("versioned PUT should assign a version id");
+        let v2 = put(set, bucket, object, b"version two", true)
+            .await
+            .version_id
+            .expect("versioned PUT should assign a version id");
+        drop_version_from(set, bucket, object, v2, 3..DRIVES).await;
+        (v1, v2)
+    }
+
+    /// Each drive's copy of `version_id`, or `None` where the drive lacks it.
+    async fn version_metadata(
+        set: &SetDisks,
+        bucket: &str,
+        object: &str,
+        version_id: Uuid,
+    ) -> Vec<Option<HashMap<String, String>>> {
+        let disks = set.disks.read().await.clone();
+        let mut copies = Vec::with_capacity(disks.len());
+        for disk in &disks {
+            let disk = disk.as_ref().expect("every drive should be online");
+            copies.push(
+                disk.read_version("", bucket, object, &version_id.to_string(), &ReadOptions::default())
+                    .await
+                    .ok()
+                    .map(|fi| fi.metadata),
+            );
+        }
+        copies
+    }
+
+    #[tokio::test]
+    async fn latest_tagging_skips_dangling_versions_above_a_committed_version() {
+        let bucket = "dangling-latest-tagging";
+        let object = "torn/latest";
+        let (_dirs, set) = dangling_test_set(bucket).await;
+        let (v1, v2) = put_committed_below_dangling(&set, bucket, object).await;
+        let tag_of = |metadata: &Option<HashMap<String, String>>| {
+            metadata
+                .as_ref()
+                .and_then(|metadata| metadata.get(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING).cloned())
+        };
+
+        let tagged = set
+            .put_object_tags(bucket, object, "stage=committed", &ObjectOptions::default())
+            .await
+            .expect("PutObjectTagging must update the version a latest GET serves");
+        assert_eq!(tagged.version_id, Some(v1));
+        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
+            assert_eq!(
+                tag_of(metadata).as_deref(),
+                Some("stage=committed"),
+                "drive {disk} must carry the tag on version one"
+            );
+        }
+        for (disk, metadata) in version_metadata(&set, bucket, object, v2).await.iter().enumerate() {
+            assert_eq!(tag_of(metadata), None, "drive {disk} must leave the dangling version untouched");
+        }
+        assert_eq!(*head(&set, bucket, object, None).await.expect("latest HEAD").user_tags, "stage=committed");
+
+        let untagged = set
+            .delete_object_tags(bucket, object, &ObjectOptions::default())
+            .await
+            .expect("DeleteObjectTagging must update the version a latest GET serves");
+        assert_eq!(untagged.version_id, Some(v1));
+        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
+            assert_eq!(tag_of(metadata).as_deref(), Some(""), "drive {disk} must clear the tag on version one");
+        }
+    }
+
+    #[tokio::test]
+    async fn latest_metadata_update_skips_dangling_versions_above_a_committed_version() {
+        let bucket = "dangling-latest-metadata";
+        let object = "torn/latest";
+        let (_dirs, set) = dangling_test_set(bucket).await;
+        let (v1, v2) = put_committed_below_dangling(&set, bucket, object).await;
+
+        let updated = set
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "updated".to_string())])),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("a latest metadata update must target the version a latest GET serves");
+        assert_eq!(updated.version_id, Some(v1));
+        assert!(updated.is_latest, "the updated version is the latest one");
+        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
+            let metadata = metadata.as_ref().expect("version one is on every drive");
+            assert_eq!(metadata.get("x-amz-meta-probe").map(String::as_str), Some("updated"), "drive {disk}");
+        }
+        for (disk, metadata) in version_metadata(&set, bucket, object, v2).await.iter().enumerate() {
+            assert!(
+                metadata
+                    .as_ref()
+                    .is_none_or(|metadata| !metadata.contains_key("x-amz-meta-probe")),
+                "drive {disk} must leave the dangling version untouched"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn latest_copy_source_skips_dangling_versions_above_a_committed_version() {
+        let bucket = "dangling-latest-copy";
+        let (_dirs, set) = dangling_test_set(bucket).await;
+
+        // A metadata-only self-copy rewrites the latest version in place.
+        let object = "torn/in-place";
+        let (v1, _) = put_committed_below_dangling(&set, bucket, object).await;
+        let mut source = head(&set, bucket, object, None).await.expect("latest HEAD");
+        assert_eq!(source.version_id, Some(v1));
+        source.metadata_only = true;
+        source.user_defined = Arc::new(HashMap::from([("x-amz-meta-copied".to_string(), "in-place".to_string())]));
+        let copied = set
+            .copy_object(
+                bucket,
+                object,
+                bucket,
+                object,
+                &mut source,
+                &ObjectOptions::default(),
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("a CopyObject source without a version id must resolve like a latest GET");
+        assert_eq!(copied.version_id, Some(v1));
+        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
+            let metadata = metadata.as_ref().expect("version one is on every drive");
+            assert_eq!(metadata.get("x-amz-meta-copied").map(String::as_str), Some("in-place"), "drive {disk}");
+        }
+
+        // A version-only self-copy commits a new version that shares the
+        // served version's data.
+        let object = "torn/new-version";
+        let (_, v2) = put_committed_below_dangling(&set, bucket, object).await;
+        let v3 = Uuid::new_v4();
+        let mut source = head(&set, bucket, object, None).await.expect("latest HEAD");
+        source.version_only = true;
+        let copied = set
+            .copy_object(
+                bucket,
+                object,
+                bucket,
+                object,
+                &mut source,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+                &ObjectOptions {
+                    version_id: Some(v3.to_string()),
+                    versioned: true,
+                    object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
+                        crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("a version-only CopyObject source without a version id must resolve like a latest GET");
+        assert_eq!(copied.version_id, Some(v3));
+        let latest = head(&set, bucket, object, None).await.expect("latest HEAD");
+        assert_eq!(latest.version_id, Some(v3));
+        assert_eq!(get(&set, bucket, object, None).await.expect("latest GET"), b"version one");
+        assert!(
+            version_metadata(&set, bucket, object, v2).await[..3]
+                .iter()
+                .all(Option::is_some),
+            "the copy must not remove the dangling version; heal owns its cleanup"
+        );
+    }
+
+    #[tokio::test]
+    async fn latest_metadata_update_past_dangling_versions_keeps_write_quorum() {
+        // EC 3+3: read quorum three, write quorum four. Version one reached
+        // three drives, so it is readable but below write quorum; versions two
+        // and three are dangling on two drives each. A latest read serves
+        // version one, and a metadata update of it is refused before writing.
+        let bucket = "dangling-latest-write-quorum";
+        let object = "torn/latest";
+        let (_dirs, set) = dangling_test_set_with(bucket, 6, 3).await;
+        let mut versions = Vec::new();
+        for body in [b"version zero".as_slice(), b"version one", b"version two", b"version three"] {
+            versions.push(
+                put(&set, bucket, object, body, true)
+                    .await
+                    .version_id
+                    .expect("versioned PUT should assign a version id"),
+            );
+        }
+        let [_, v1, v2, v3] = versions[..] else {
+            unreachable!("four versions were written");
+        };
+        drop_version_from(&set, bucket, object, v1, 3..6).await;
+        drop_version_from(&set, bucket, object, v2, 2..6).await;
+        drop_version_from(&set, bucket, object, v3, 0..3).await;
+        drop_version_from(&set, bucket, object, v3, 5..6).await;
+
+        let latest = head(&set, bucket, object, None)
+            .await
+            .expect("version one is readable at read quorum");
+        assert_eq!(latest.version_id, Some(v1));
+
+        let is_write_quorum_error =
+            |err: &StorageError| matches!(err, StorageError::ErasureWriteQuorum | StorageError::InsufficientWriteQuorum(_, _));
+        let Err(err) = set
+            .put_object_tags(bucket, object, "stage=refused", &ObjectOptions::default())
+            .await
+        else {
+            panic!("tagging a version below write quorum must be refused");
+        };
+        assert!(is_write_quorum_error(&err), "tagging keeps the write-quorum error: {err:?}");
+        let Err(err) = set
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "refused".to_string())])),
+                    ..Default::default()
+                },
+            )
+            .await
+        else {
+            panic!("a metadata update of a version below write quorum must be refused");
+        };
+        assert!(is_write_quorum_error(&err), "the metadata update keeps the write-quorum error: {err:?}");
+        let v4 = Uuid::new_v4();
+        let mut source = head(&set, bucket, object, None).await.expect("latest HEAD");
+        source.version_only = true;
+        let Err(err) = set
+            .copy_object(
+                bucket,
+                object,
+                bucket,
+                object,
+                &mut source,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+                &ObjectOptions {
+                    version_id: Some(v4.to_string()),
+                    versioned: true,
+                    object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
+                        crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
+                    ))),
+                    ..Default::default()
+                },
+            )
+            .await
+        else {
+            panic!("a version-only copy of a version below write quorum must be refused");
+        };
+        assert!(is_write_quorum_error(&err), "the version-only copy keeps the write-quorum error: {err:?}");
+        assert!(
+            version_metadata(&set, bucket, object, v4).await.iter().all(Option::is_none),
+            "a refused version-only copy must not write the new version to any drive"
+        );
+        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
+            assert!(
+                metadata.as_ref().is_none_or(|metadata| {
+                    !metadata.contains_key(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING)
+                        && !metadata.contains_key("x-amz-meta-probe")
+                }),
+                "drive {disk} must not be written by a refused update"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn replication_status_writeback_never_walks_past_dangling_versions() {
+        // A replication terminal CAS stays side-effect free: it keeps the
+        // read-quorum error instead of resolving the latest version, so it
+        // neither queues heal for the dangling version nor writes version one.
+        let bucket = "dangling-latest-replication-cas";
+        let object = "torn/latest";
+        let (_dirs, set) = dangling_test_set(bucket).await;
+        let (v1, _) = put_committed_below_dangling(&set, bucket, object).await;
+
+        let Err(err) = set
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "written".to_string())])),
+                    replication_status_writeback: Some(Box::new(crate::object_api::ReplicationStatusWritebackCondition {
+                        mode: crate::object_api::ReplicationStatusWritebackMode::ValidateOnly,
+                        ..Default::default()
+                    })),
+                    ..Default::default()
+                },
+            )
+            .await
+        else {
+            panic!("a replication terminal CAS must not resolve past a dangling version");
+        };
+        assert!(is_read_quorum_error(&err), "the CAS keeps the read-quorum error: {err:?}");
+        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
+            let metadata = metadata.as_ref().expect("version one is on every drive");
+            assert!(!metadata.contains_key("x-amz-meta-probe"), "drive {disk} must not be written");
+        }
+    }
+
+    #[tokio::test]
+    async fn latest_metadata_updates_past_dangling_versions_require_every_drive_to_answer() {
+        let bucket = "dangling-latest-update-offline";
+        let object = "torn/latest";
+        let (_dirs, set) = dangling_test_set(bucket).await;
+        put_committed_below_dangling(&set, bucket, object).await;
+        let offline = set.disks.write().await[5].take().expect("drive five should start online");
+
+        let Err(err) = set
+            .put_object_tags(bucket, object, "stage=uncertain", &ObjectOptions::default())
+            .await
+        else {
+            panic!("tagging must not resolve past a version an offline drive may commit");
+        };
+        assert!(is_read_quorum_error(&err), "tagging keeps the read-quorum error: {err:?}");
+        let Err(err) = set
+            .put_object_metadata(
+                bucket,
+                object,
+                &ObjectOptions {
+                    versioned: true,
+                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "uncertain".to_string())])),
+                    ..Default::default()
+                },
+            )
+            .await
+        else {
+            panic!("a metadata update must not resolve past a version an offline drive may commit");
+        };
+        assert!(is_read_quorum_error(&err), "the metadata update keeps the read-quorum error: {err:?}");
+        set.disks.write().await[5] = Some(offline);
+    }
+
     #[tokio::test]
     async fn dangling_classification_keeps_quorum_error_for_corrupt_metadata() {
         let bucket = "dangling-read-corrupt";

crates/ecstore/src/store/bucket.rs

@@ -448,6 +448,14 @@ impl ECStore {
         await_orphan_bucket_recovery(bucket, &publication_guard, &lifecycle_guard, &metadata_guard, &ns_guard, recover).await
     }
 
+    /// Read quota and bucket identity from one authoritative metadata snapshot.
+    pub async fn bucket_quota_and_incarnation_from_disk(
+        &self,
+        bucket: &str,
+    ) -> Result<(Option<crate::bucket::quota::BucketQuota>, Uuid, OffsetDateTime)> {
+        metadata_sys::get_quota_config_and_incarnation_from_disk_in(&self.ctx, bucket).await
+    }
+
     pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
         let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
         sys.read().await.get(bucket).await
@@ -564,13 +572,83 @@ impl ECStore {
             .await
     }
 
-    pub(super) async fn acquire_bucket_incarnation_fence_with_options(
+    pub(crate) async fn acquire_bucket_incarnation_fence_with_options(
         &self,
         bucket: &str,
         expected: Uuid,
         opts: Option<&ObjectOptions>,
     ) -> Result<super::BucketIncarnationFenceGuard> {
+        Ok(self
+            .acquire_bucket_incarnation_fence_for_options(bucket, Some(expected), opts)
+            .await?
+            .0)
+    }
+
+    pub(crate) async fn acquire_bucket_incarnation_fence_for_options(
+        &self,
+        bucket: &str,
+        expected: Option<Uuid>,
+        opts: Option<&ObjectOptions>,
+    ) -> Result<(super::BucketIncarnationFenceGuard, Uuid)> {
         let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
+        self.validate_bucket_incarnation_fence_guard_with_options(bucket, inner, expected, opts)
+            .await
+    }
+
+    /// Acquire lexicographically ordered bucket lifecycle read locks for a copy
+    /// operation and validate both bucket incarnations through [`super::BucketFenceRegistry`].
+    pub async fn acquire_copy_bucket_incarnation_fences(
+        &self,
+        source_bucket: &str,
+        expected_source: Option<Uuid>,
+        destination_bucket: &str,
+        expected_destination: Option<Uuid>,
+    ) -> Result<(super::BucketIncarnationFenceGuard, Uuid, Option<super::BucketIncarnationFenceGuard>, Uuid)> {
+        let (source_guard, destination_guard) = if source_bucket == destination_bucket {
+            (self.acquire_bucket_lifecycle_read_lock(source_bucket).await?, None)
+        } else if source_bucket < destination_bucket {
+            let source_guard = self.acquire_bucket_lifecycle_read_lock(source_bucket).await?;
+            let destination_guard = self.acquire_bucket_lifecycle_read_lock(destination_bucket).await?;
+            (source_guard, Some(destination_guard))
+        } else {
+            let destination_guard = self.acquire_bucket_lifecycle_read_lock(destination_bucket).await?;
+            let source_guard = self.acquire_bucket_lifecycle_read_lock(source_bucket).await?;
+            (source_guard, Some(destination_guard))
+        };
+        let (source_guard, current_source_incarnation_id) = self
+            .validate_bucket_incarnation_fence_guard_with_options(source_bucket, source_guard, expected_source, None)
+            .await?;
+        let (destination_guard, current_destination_incarnation_id) = if let Some(destination_guard) = destination_guard {
+            let (destination_guard, destination_incarnation_id) = self
+                .validate_bucket_incarnation_fence_guard_with_options(
+                    destination_bucket,
+                    destination_guard,
+                    expected_destination,
+                    None,
+                )
+                .await?;
+            (Some(destination_guard), destination_incarnation_id)
+        } else {
+            if expected_destination.is_some_and(|expected| expected != current_source_incarnation_id) {
+                return Err(StorageError::BucketNotFound(destination_bucket.to_owned()));
+            }
+            (None, current_source_incarnation_id)
+        };
+        Ok((
+            source_guard,
+            current_source_incarnation_id,
+            destination_guard,
+            current_destination_incarnation_id,
+        ))
+    }
+
+    async fn validate_bucket_incarnation_fence_guard_with_options(
+        &self,
+        bucket: &str,
+        inner: rustfs_lock::NamespaceLockGuard,
+        expected: Option<Uuid>,
+        opts: Option<&ObjectOptions>,
+    ) -> Result<(super::BucketIncarnationFenceGuard, Uuid)> {
         let mut guard = super::BucketIncarnationFenceGuard::new(
             Arc::clone(&self.bucket_fence_registry),
             bucket,
@@ -586,10 +664,10 @@ impl ECStore {
                 }
             })
             .await?;
-        if current != expected {
+        if expected.is_some_and(|expected| current != expected) {
             return Err(StorageError::BucketNotFound(bucket.to_owned()));
         }
-        Ok(guard)
+        Ok((guard, current))
     }
 
     /// Hold this guard through recursive-delete authorization and mutation so

crates/ecstore/src/store/bucket_fence.rs

@@ -149,6 +149,18 @@ impl BucketFenceRegistry {
             }
         }
     }
+
+    /// Return the memoized incarnation id for `bucket` when at least one
+    /// registered lifecycle read guard is currently held without lock loss.
+    pub(crate) fn validated_incarnation(&self, bucket: &str) -> Option<Uuid> {
+        let mut entries = self.entries.lock().expect("bucket fence registry poisoned");
+        let entry = entries.get_mut(bucket)?;
+        if entry.guards.iter().any(|guard| guard.loss_probe.is_lost()) {
+            entry.validated = None;
+            return None;
+        }
+        entry.validated
+    }
 }
 
 /// Owns continuous lifecycle coverage and its incarnation validation memo.
@@ -221,7 +233,7 @@ impl BucketIncarnationFenceGuard {
         }
     }
 
-    pub(crate) fn is_lock_lost(&self) -> bool {
+    pub fn is_lock_lost(&self) -> bool {
         self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
     }
 
@@ -310,9 +322,16 @@ mod tests {
         let (first_probe, first_lost) = controllable_probe();
         let first = reg.enter("b", first_probe);
         assert_eq!(first.memoized, None);
+        assert_eq!(reg.validated_incarnation("b"), None);
         reg.memoize("b", uuid(7));
+        assert_eq!(reg.validated_incarnation("b"), Some(uuid(7)));
 
         first_lost.store(true, std::sync::atomic::Ordering::SeqCst);
+        assert_eq!(
+            reg.validated_incarnation("b"),
+            None,
+            "live lost guard must clear and block validated_incarnation"
+        );
 
         let second = reg.enter("b", live_probe());
         assert_eq!(second.memoized, None, "live lost guard must force disk revalidation");
@@ -320,9 +339,11 @@ mod tests {
 
         let third = reg.enter("b", live_probe());
         assert_eq!(third.memoized, None, "memo remains blocked while the lost guard is live");
+        assert_eq!(reg.validated_incarnation("b"), None);
 
         reg.exit("b", first.token, true);
         reg.memoize("b", uuid(8));
+        assert_eq!(reg.validated_incarnation("b"), Some(uuid(8)));
         let fourth = reg.enter("b", live_probe());
         assert_eq!(fourth.memoized, Some(uuid(8)), "memo resumes after lost coverage leaves");
 

crates/ecstore/src/store/init.rs

@@ -21384,6 +21384,141 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn put_object_metadata_and_copy_object_reuse_overlapping_incarnation_fence() {
+        let temp = tempfile::tempdir().expect("create metadata/copy fence store");
+        let (_ctx, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(temp.path(), "metadata-copy-fence-reuse", &[4])).await;
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        let bucket = format!("metadata-copy-fence-{}", Uuid::new_v4());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create bucket");
+        let mut seed_reader = PutObjReader::from_vec(b"seed payload".to_vec());
+        let versioned_opts = ObjectOptions {
+            versioned: true,
+            ..Default::default()
+        };
+        let mut src_info = store
+            .put_object(&bucket, "src.bin", &mut seed_reader, &versioned_opts)
+            .await
+            .expect("seed source object");
+        let src_version_id = src_info
+            .version_id
+            .expect("versioned PUT should return a version ID")
+            .to_string();
+        let expected = store.bucket_incarnation_id(&bucket).await.expect("read incarnation");
+        let coverage = store
+            .acquire_bucket_incarnation_fence(&bucket, expected)
+            .await
+            .expect("validate continuous coverage");
+        let metadata_writer = crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
+            .await
+            .expect("hold metadata writer during covered metadata and copy operations");
+
+        let updated = tokio::time::timeout(
+            Duration::from_secs(10),
+            store.put_object_metadata(
+                &bucket,
+                "src.bin",
+                &ObjectOptions {
+                    version_id: Some(src_version_id.clone()),
+                    versioned: true,
+                    eval_metadata: Some(HashMap::from([("x-amz-meta-fenced".to_string(), "true".to_string())])),
+                    ..Default::default()
+                },
+            ),
+        )
+        .await
+        .expect("covered put_object_metadata must not reacquire the metadata transaction lock")
+        .expect("update object metadata using authoritative memo");
+        assert_eq!(updated.user_defined.get("x-amz-meta-fenced").map(String::as_str), Some("true"));
+
+        Arc::make_mut(&mut src_info.user_defined).insert("x-amz-meta-copied".to_string(), "true".to_string());
+        let src_copy_opts = ObjectOptions {
+            version_id: Some(src_version_id),
+            versioned: true,
+            ..Default::default()
+        };
+        let copied = tokio::time::timeout(
+            Duration::from_secs(10),
+            store.copy_object(&bucket, "src.bin", &bucket, "src.bin", &mut src_info, &src_copy_opts, &versioned_opts),
+        )
+        .await
+        .expect("covered versioned copy_object must not reacquire the metadata transaction lock")
+        .expect("versioned copy using authoritative memo");
+        assert_eq!(copied.user_defined.get("x-amz-meta-copied").map(String::as_str), Some("true"));
+
+        let (src_guard, src_incarnation, dst_guard, dst_incarnation) = tokio::time::timeout(
+            Duration::from_secs(10),
+            store.acquire_copy_bucket_incarnation_fences(&bucket, Some(expected), &bucket, Some(expected)),
+        )
+        .await
+        .expect("covered acquire_copy_bucket_incarnation_fences must not reacquire the metadata transaction lock")
+        .expect("validate same-bucket copy fences using authoritative memo");
+        assert!(dst_guard.is_none());
+        assert_eq!(src_incarnation, expected);
+        assert_eq!(dst_incarnation, expected);
+        let mut borrowed_fence_opts = ObjectOptions {
+            expected_bucket_incarnation_id: Some(expected),
+            ..Default::default()
+        };
+        src_guard.attach_to_object_options(&mut borrowed_fence_opts);
+        let fenced_incarnation = tokio::time::timeout(
+            Duration::from_secs(10),
+            crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&store.ctx, &bucket, &borrowed_fence_opts),
+        )
+        .await
+        .expect("borrowed fence under active coverage must not reacquire the metadata transaction lock")
+        .expect("resolve incarnation under borrowed fence");
+        assert_eq!(fenced_incarnation, expected);
+        assert!(
+            matches!(
+                store
+                    .acquire_copy_bucket_incarnation_fences(&bucket, Some(expected), &bucket, Some(Uuid::new_v4()))
+                    .await,
+                Err(StorageError::BucketNotFound(_))
+            ),
+            "same-bucket copy fence must reject a mismatched destination incarnation"
+        );
+        drop(src_guard);
+
+        let stale_opts = ObjectOptions {
+            expected_bucket_incarnation_id: Some(Uuid::new_v4()),
+            versioned: true,
+            ..Default::default()
+        };
+        assert!(
+            matches!(
+                store.put_object_metadata(&bucket, "src.bin", &stale_opts).await,
+                Err(StorageError::BucketNotFound(_))
+            ),
+            "put_object_metadata must reject a stale expected incarnation even under warm coverage"
+        );
+        assert!(
+            matches!(
+                store
+                    .copy_object(
+                        &bucket,
+                        "src.bin",
+                        &bucket,
+                        "src.bin",
+                        &mut src_info,
+                        &ObjectOptions::default(),
+                        &stale_opts,
+                    )
+                    .await,
+                Err(StorageError::BucketNotFound(_))
+            ),
+            "copy_object must reject a stale expected incarnation even under warm coverage"
+        );
+        drop(metadata_writer);
+        drop(coverage);
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn multipart_snapshot_bypasses_initializer_queued_behind_metadata_writer() {
@@ -21469,10 +21604,25 @@ mod tests {
             .acquire_bucket_incarnation_fence(&bucket, old)
             .await
             .expect("validate with no-op lock");
+        let snapshot = store
+            .object_lock_config_snapshot_under_lifecycle_fence(&bucket, &crate::object_api::NamespaceLockFence::new())
+            .await
+            .expect("capture snapshot under no-op lock");
         let current = Uuid::new_v4();
         crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, current)
             .await
             .expect("replace authority while no-op guard provides no exclusion");
+        let snapshot_opts = ObjectOptions {
+            expected_bucket_incarnation_id: Some(old),
+            object_lock_config_snapshot: Some(snapshot),
+            ..Default::default()
+        };
+        assert_eq!(
+            crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&store.ctx, &bucket, &snapshot_opts)
+                .await
+                .expect("disabled lock manager must re-read incarnation from disk instead of trusting snapshot"),
+            current
+        );
         assert!(
             matches!(
                 store.acquire_bucket_incarnation_fence(&bucket, old).await,

crates/ecstore/src/store/list_objects.rs

@@ -1030,16 +1030,19 @@ impl ECStore {
         // Lock order matches writes: bucket lifecycle, metadata validation,
         // then object namespace. Borrow fences supplied by an outer operation.
         let bucket_guard = if opts.bucket_lifecycle_lock_fence.is_none() {
-            let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-            opts.add_bucket_lifecycle_lock_guard(&guard);
+            let guard = self
+                .acquire_bucket_incarnation_fence_with_options(bucket, bucket_incarnation_id, Some(&opts))
+                .await?;
+            guard.attach_to_object_options(&mut opts);
             Some(guard)
         } else {
+            let incarnation =
+                crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
+            if incarnation != bucket_incarnation_id {
+                return Err(Error::BucketNotFound(bucket.to_owned()));
+            }
             None
         };
-        let incarnation = crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
-        if incarnation != bucket_incarnation_id {
-            return Err(Error::BucketNotFound(bucket.to_owned()));
-        }
         let metadata_sys = crate::bucket::metadata_sys::bucket_metadata_sys_of(&self.ctx)?;
         let metadata_sys = metadata_sys.read().await.clone();
         let (versioning, _) = metadata_sys.get_versioning_config(bucket).await?;

crates/ecstore/src/store/multipart.rs

@@ -412,7 +412,7 @@ impl ECStore {
 
     /// Multipart lock order is bucket lifecycle, generation validation, then
     /// object/upload locks in the selected set.
-    async fn guard_multipart_bucket_incarnation(
+    pub(super) async fn guard_multipart_bucket_incarnation(
         &self,
         bucket: &str,
         opts: &ObjectOptions,

crates/ecstore/src/store/object.rs

@@ -30,8 +30,8 @@ use crate::bucket::lifecycle::{
     },
 };
 use crate::bucket::metadata_sys::{
-    acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
-    get_cached_bucket_incarnation_id_in, get_delete_config_from_disk_in, get_object_lock_config_and_incarnation_from_disk_in,
+    acquire_bucket_metadata_transaction_read_lock_in, get_cached_bucket_incarnation_id_in, get_delete_config_from_disk_in,
+    get_object_lock_config_and_incarnation_from_disk_in,
 };
 use crate::bucket::object_lock::objectlock_sys::{
     check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
@@ -4883,24 +4883,20 @@ impl ECStore {
         if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
             dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
         }
-        let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
-            None
-        } else {
-            Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?)
-        };
-        let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
-            dst_opts.add_bucket_lifecycle_lock_guard(guard);
-            let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
-            if dst_opts
-                .expected_bucket_incarnation_id
-                .is_some_and(|expected| expected != current_incarnation_id)
-            {
-                return Err(StorageError::BucketNotFound(dst_bucket.to_string()));
-            }
-            Some(current_incarnation_id)
-        } else {
-            dst_opts.expected_bucket_incarnation_id
-        };
+        let (_bucket_lifecycle_guard, current_bucket_incarnation_id) =
+            if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
+                (None, dst_opts.expected_bucket_incarnation_id)
+            } else {
+                let (guard, current_incarnation_id) = self
+                    .acquire_bucket_incarnation_fence_for_options(
+                        dst_bucket,
+                        dst_opts.expected_bucket_incarnation_id,
+                        Some(&dst_opts),
+                    )
+                    .await?;
+                guard.attach_to_object_options(&mut dst_opts);
+                (Some(guard), Some(current_incarnation_id))
+            };
         if dst_opts
             .bucket_lifecycle_lock_fence
             .as_ref()
@@ -6504,24 +6500,8 @@ impl ECStore {
         opts: &ObjectOptions,
     ) -> Result<()> {
         let object = encode_dir_object(object);
-        let mut opts = transition_restore_pool_opts(opts);
-        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
-            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
-        }
-        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
-            None
-        } else {
-            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
-        };
-        if let Some(guard) = bucket_lifecycle_guard.as_ref() {
-            opts.add_bucket_lifecycle_lock_guard(guard);
-        }
-        if !is_meta_bucketname(bucket) {
-            let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
-            if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
-                return Err(StorageError::BucketNotFound(bucket.to_string()));
-            }
-        }
+        let opts = transition_restore_pool_opts(opts);
+        let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, &opts).await?;
         if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
             let expected_incarnation_id = opts
                 .expected_bucket_incarnation_id
@@ -6589,16 +6569,11 @@ impl ECStore {
         let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
             None
         } else {
-            let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
-            let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
-            if opts
-                .expected_bucket_incarnation_id
-                .is_some_and(|expected| expected != current_incarnation_id)
-            {
-                return Err(StorageError::BucketNotFound(bucket.to_string()));
-            }
+            let (guard, current_incarnation_id) = self
+                .acquire_bucket_incarnation_fence_for_options(bucket, opts.expected_bucket_incarnation_id, Some(&opts))
+                .await?;
             opts.expected_bucket_incarnation_id = Some(current_incarnation_id);
-            opts.add_bucket_lifecycle_lock_guard(&guard);
+            guard.attach_to_object_options(&mut opts);
             if guard.is_lock_lost() {
                 return Err(Error::other("bucket lifecycle lock was lost before the metadata update"));
             }
@@ -13167,6 +13142,202 @@ mod tests {
         })
         .await;
     }
+
+    async fn assert_delete_excludes_recovery_across_pool_lock_domains(topology: &str, batch: bool) {
+        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
+        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
+        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
+        let mut store = new_prepared_reader_test_store_with_ctx(&[first_set, second_set], Arc::clone(&ctx)).await;
+        let clients: Vec<Arc<dyn rustfs_lock::LockClient>> = (0..4)
+            .map(|_| {
+                Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(
+                    Arc::new(rustfs_lock::FastObjectLockManager::new()),
+                )))) as Arc<dyn rustfs_lock::LockClient>
+            })
+            .collect();
+        let domains = match topology {
+            "disjoint" => [vec![clients[0].clone()], vec![clients[1].clone()]],
+            "identical" => [clients[..3].to_vec(), clients[..3].to_vec()],
+            "overlapping" => [clients[..3].to_vec(), clients[1..].to_vec()],
+            _ => panic!("unknown lock topology"),
+        };
+        for (pool, lockers) in store.pools.iter_mut().zip(domains) {
+            let mut set = (*pool.disk_set[0]).clone();
+            set.lockers = lockers;
+            Arc::make_mut(pool).disk_set[0] = Arc::new(set);
+        }
+        let store = Arc::new(store);
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
+        let bucket = "delete-nonzero-pool-lock";
+        let object = "object.bin";
+        store
+            .handle_make_bucket(bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create the bucket in both pools");
+        if batch {
+            // Latest-marker routing sends this batch only to the nonzero pool,
+            // so the cleanup barrier observes its prepared journal directly.
+            store
+                .update_bucket_metadata_config(
+                    bucket,
+                    crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
+                    b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
+                )
+                .await
+                .expect("enable versioning for the routed batch");
+        }
+        let target = store.pools[1].get_disks_by_key(object);
+        let source = target
+            .put_object(
+                bucket,
+                object,
+                &mut PutObjReader::from_vec(b"delete lock regression".to_vec()),
+                &ObjectOptions {
+                    versioned: batch,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("seed an object only in the nonzero pool");
+        ctx.update_erasure_type(SetupType::DistErasure).await;
+
+        let object_barrier = (!batch).then(|| crate::set_disk::DeleteObjectCommitBarrier::install(bucket, object));
+        let batch_barrier = batch.then(|| crate::set_disk::DeleteCleanupBarrier::install(bucket));
+        let delete_store = Arc::clone(&store);
+        let delete = tokio::spawn(async move {
+            if batch {
+                let (_, errors) = delete_store
+                    .delete_objects(
+                        bucket,
+                        vec![ObjectToDelete {
+                            object_name: object.to_owned(),
+                            ..Default::default()
+                        }],
+                        ObjectOptions::default(),
+                    )
+                    .await;
+                match errors.into_iter().flatten().next() {
+                    Some(error) => Err(error),
+                    None => Ok(()),
+                }
+            } else {
+                delete_store
+                    .delete_object(bucket, object, ObjectOptions::default())
+                    .await
+                    .map(|_| ())
+            }
+        });
+        if let Some(barrier) = &object_barrier {
+            barrier.wait_until_paused().await;
+        }
+        if let Some(barrier) = &batch_barrier {
+            barrier.wait_until_paused().await;
+        }
+        let recovery_store = Arc::clone(&store);
+        let mut recovery = tokio::spawn(async move {
+            recovery_store
+                .recover_delete_transactions_at_incarnation(
+                    bucket,
+                    Some(object),
+                    None,
+                    &HealOpts {
+                        pool: Some(1),
+                        set: Some(0),
+                        ..Default::default()
+                    },
+                )
+                .await
+        });
+        let early_recovery = tokio::time::timeout(Duration::from_millis(100), &mut recovery).await;
+        let recovery_was_blocked = early_recovery.is_err();
+        if let Some(barrier) = &object_barrier {
+            barrier.release();
+        }
+        if let Some(barrier) = &batch_barrier {
+            barrier.release();
+        }
+        tokio::time::timeout(Duration::from_secs(10), delete)
+            .await
+            .expect("the delete should finish after releasing its commit barrier")
+            .expect("the delete task should join")
+            .expect("the delete should succeed");
+        let recovered = match early_recovery {
+            Ok(result) => result,
+            Err(_) => tokio::time::timeout(Duration::from_secs(10), recovery)
+                .await
+                .expect("recovery should finish after the frontend delete"),
+        };
+        recovered
+            .expect("the recovery task should join")
+            .expect("recovery should succeed");
+        assert!(
+            recovery_was_blocked,
+            "recovery must not inspect an in-flight frontend delete with {topology} pool lock clients"
+        );
+        if batch {
+            let marker = target
+                .get_object_info(bucket, object, &ObjectOptions::default())
+                .await
+                .expect("the versioned batch must publish a current delete marker");
+            assert!(marker.delete_marker && marker.is_latest);
+            assert_eq!(marker.num_versions, 2, "the source version must remain beside the new marker");
+            assert_ne!(marker.version_id, source.version_id);
+            let mut reader = target
+                .get_object_reader(
+                    bucket,
+                    object,
+                    None,
+                    HeaderMap::new(),
+                    &ObjectOptions {
+                        version_id: Some(source.version_id.expect("versioned source identity").to_string()),
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("the original source version must remain readable");
+            let mut body = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut body)
+                .await
+                .expect("read the retained source version");
+            assert_eq!(body, b"delete lock regression");
+        } else {
+            let error = target
+                .get_object_info(bucket, object, &ObjectOptions::default())
+                .await
+                .expect_err("the deleted object must be absent");
+            assert!(is_err_object_not_found(&error));
+        }
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    #[serial_test::serial(storage_class_env)]
+    async fn batch_delete_excludes_recovery_on_nonzero_pool() {
+        assert_delete_excludes_recovery_across_pool_lock_domains("disjoint", true).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    #[serial_test::serial(storage_class_env)]
+    async fn delete_excludes_recovery_with_disjoint_pool_lock_clients() {
+        assert_delete_excludes_recovery_across_pool_lock_domains("disjoint", false).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    #[serial_test::serial(storage_class_env)]
+    async fn delete_excludes_recovery_with_identical_pool_lock_clients() {
+        assert_delete_excludes_recovery_across_pool_lock_domains("identical", false).await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    #[serial_test::serial(storage_class_env)]
+    async fn delete_excludes_recovery_with_overlapping_pool_lock_clients() {
+        assert_delete_excludes_recovery_across_pool_lock_domains("overlapping", false).await;
+    }
 }
 
 #[cfg(test)]

crates/heal/src/error.rs

@@ -16,7 +16,7 @@ use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
 use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
 use thiserror::Error;
 
-use super::heal::{DiskError, EcstoreError};
+use super::heal::{DiskError, EcstoreError, storage_api::owner::EcstorePoolMetadataFailure};
 
 const HEAL_DANGLING_DELETE_GRACE_MESSAGE: &str = "dangling object deletion deferred by heal grace window";
 pub(crate) const MRF_MISSING_PROOF_PREFIX: &str = "Missing exact storage proof for durable MRF repair ";
@@ -236,6 +236,14 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
 /// with object absence, the conflation that once let an offline drive be
 /// recorded as absent and skipped (backlog#856 / #799 B7).
 pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
+    if let Some(failure) = error.pool_metadata_failure() {
+        return match failure.kind {
+            EcstorePoolMetadataFailure::ReadUnavailable | EcstorePoolMetadataFailure::TransactionUnknown => {
+                HealErrorClass::Transient
+            }
+            EcstorePoolMetadataFailure::RecoveryRequired | EcstorePoolMetadataFailure::FenceLost => HealErrorClass::Permanent,
+        };
+    }
     if let EcstoreError::Lock(lock_error) = error {
         return if lock_error.is_fatal() {
             HealErrorClass::Permanent
@@ -435,6 +443,34 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
+    #[test]
+    fn pool_metadata_failures_keep_their_typed_recovery_classification() {
+        use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
+
+        for (kind, retryable) in [
+            (EcstorePoolMetadataFailure::ReadUnavailable, true),
+            (EcstorePoolMetadataFailure::TransactionUnknown, true),
+            (EcstorePoolMetadataFailure::RecoveryRequired, false),
+            (EcstorePoolMetadataFailure::FenceLost, false),
+        ] {
+            let error = EcstoreError::other(EcstorePoolMetadataError {
+                kind,
+                operation: "bucket heal".to_owned(),
+                phase: "native_heal",
+                since: std::time::UNIX_EPOCH.into(),
+                source: Some(std::sync::Arc::new(EcstoreError::SlowDown)),
+            });
+            let text = error.to_string();
+            for error in [error.clone(), error] {
+                assert_eq!(Error::Storage(error).is_recoverable_heal(), retryable, "{kind:?}");
+            }
+            assert!(
+                !Error::Storage(EcstoreError::other(text)).is_recoverable_heal(),
+                "display text must not grant recovery authority"
+            );
+        }
+    }
+
     #[test]
     fn namespace_lock_quorum_is_recoverable_across_io_error_conversions() {
         for (required, achieved) in [(2, 1), (1, 0)] {

crates/heal/src/heal/storage_api.rs

@@ -26,7 +26,11 @@ pub(crate) use rustfs_ecstore::api::disk::{
     RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, ReplacementExecutionLease as EcstoreReplacementExecutionLease,
 };
 pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
-pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
+#[cfg(test)]
+pub(crate) use rustfs_ecstore::api::error::PoolMetadataError as EcstorePoolMetadataError;
+pub(crate) use rustfs_ecstore::api::error::{
+    Error as EcstoreErrorType, PoolMetadataFailure as EcstorePoolMetadataFailure, StorageError as EcstoreStorageError,
+};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::layout::{
     EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
@@ -53,6 +57,9 @@ pub(crate) use rustfs_ecstore::api::storage::{
 use rustfs_storage_api as storage_contracts;
 
 pub(crate) mod owner {
+    #[cfg(test)]
+    pub(crate) use super::EcstorePoolMetadataError;
+    pub(crate) use super::EcstorePoolMetadataFailure;
     pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
 
     pub(crate) use super::{

crates/heal/src/heal/task/heal_bucket.rs

@@ -306,9 +306,7 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_stage(3, 3);
                 }
-                Err(Error::TaskExecutionFailed {
-                    message: format!("Failed to heal bucket {bucket}: {e}"),
-                })
+                Err(e)
             }
         }
     }
@@ -520,9 +518,7 @@ impl HealTask {
                     error = %err,
                     "Heal cluster pool metadata failed"
                 );
-                Err(Error::TaskExecutionFailed {
-                    message: format!("Failed to heal cluster pool metadata: {err}"),
-                })
+                Err(err)
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
@@ -539,9 +535,7 @@ impl HealTask {
                     error = %err,
                     "Heal cluster pool metadata failed"
                 );
-                Err(Error::TaskExecutionFailed {
-                    message: format!("Failed to heal cluster pool metadata: {err}"),
-                })
+                Err(err)
             }
         }
     }

crates/heal/src/heal/task/tests.rs

@@ -16,6 +16,7 @@ use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
 use super::*;
 use crate::heal::EcstoreError;
 use crate::heal::storage::HealStorageObjectResult;
+use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
 
 mod absent_verdict;
 mod concurrent_delete;
@@ -784,7 +785,10 @@ mod canonical_outcome {
     async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
         let storage = Arc::new(MockStorage {
             listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
+            bucket_heal_errors: Mutex::new(HashMap::from([(
+                "bucket-a".to_string(),
+                VecDeque::from([Error::other("metadata unavailable")]),
+            )])),
             ..Default::default()
         });
         let task = HealTask::from_request(
@@ -1566,7 +1570,7 @@ struct MockStorage {
     include_object_dir_candidate: Mutex<bool>,
     listed_buckets: Mutex<Option<Vec<String>>>,
     bucket_discovery_incomplete: bool,
-    bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
+    bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
     bucket_heal_calls: Mutex<Vec<String>>,
     delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
     delete_recovery_calls: Mutex<Vec<String>>,
@@ -1990,6 +1994,7 @@ fn replacement_identity(
 
 #[derive(Clone)]
 enum MockHealObjectOutcome {
+    PoolMetadata { kind: EcstorePoolMetadataFailure, outer: bool },
     LegacyAmbiguous { outer: bool },
     MissingObject { outer: bool },
     MissingVersion,
@@ -2009,6 +2014,16 @@ enum MockHealObjectOutcome {
     PermanentOther(&'static str),
 }
 
+fn pool_metadata_error(kind: EcstorePoolMetadataFailure) -> Error {
+    Error::Storage(EcstoreError::other(EcstorePoolMetadataError {
+        kind,
+        operation: "bucket heal".to_owned(),
+        phase: "native_heal",
+        since: std::time::UNIX_EPOCH.into(),
+        source: None,
+    }))
+}
+
 fn unavailable_drive_heal_result(state: DriveState) -> (HealResultItem, Option<Error>) {
     (
         HealResultItem {
@@ -2180,6 +2195,14 @@ impl HealStorageAPI for MockStorage {
             .and_then(VecDeque::pop_front)
         {
             return match outcome {
+                MockHealObjectOutcome::PoolMetadata { kind, outer } => {
+                    let error = pool_metadata_error(kind);
+                    if outer {
+                        Err(error)
+                    } else {
+                        Ok((HealResultItem::default(), Some(error)))
+                    }
+                }
                 MockHealObjectOutcome::LegacyAmbiguous { outer } => {
                     let error = Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous);
                     if outer {
@@ -2256,6 +2279,14 @@ impl HealStorageAPI for MockStorage {
         }
         if let Some(outcome) = self.heal_object_outcome.lock().unwrap().take() {
             return match outcome {
+                MockHealObjectOutcome::PoolMetadata { kind, outer } => {
+                    let error = pool_metadata_error(kind);
+                    if outer {
+                        Err(error)
+                    } else {
+                        Ok((HealResultItem::default(), Some(error)))
+                    }
+                }
                 MockHealObjectOutcome::LegacyAmbiguous { outer } => {
                     let error = Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous);
                     if outer {
@@ -2387,14 +2418,14 @@ impl HealStorageAPI for MockStorage {
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
         self.bucket_heal_opts.lock().unwrap().push(*opts);
-        if let Some(message) = self
+        if let Some(error) = self
             .bucket_heal_errors
             .lock()
             .unwrap()
             .get_mut(bucket)
             .and_then(VecDeque::pop_front)
         {
-            return Err(Error::other(message));
+            return Err(error);
         }
         Ok(HealResultItem::default())
     }
@@ -3429,13 +3460,70 @@ async fn cluster_recreate_fails_when_pool_metadata_heal_fails() {
         .await
         .expect_err("cluster recreate heal must not hide pool metadata failure");
 
-    assert!(matches!(err, Error::TaskExecutionFailed { .. }));
+    assert!(matches!(err, Error::Other(ref message) if message == "pool metadata missing"));
     assert_eq!(
         storage.heal_object_calls.lock().expect("object calls").as_slice(),
         ["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
     );
 }
 
+#[tokio::test]
+async fn cluster_pool_metadata_failures_preserve_recovery_classification_in_both_response_shapes() {
+    for outer in [false, true] {
+        for (kind, retryable) in [
+            (EcstorePoolMetadataFailure::ReadUnavailable, true),
+            (EcstorePoolMetadataFailure::TransactionUnknown, true),
+            (EcstorePoolMetadataFailure::RecoveryRequired, false),
+            (EcstorePoolMetadataFailure::FenceLost, false),
+        ] {
+            let storage = Arc::new(MockStorage {
+                listed_buckets: Mutex::new(Some(Vec::new())),
+                ..Default::default()
+            });
+            storage.heal_object_outcomes.lock().expect("object outcomes").insert(
+                POOL_META_NAME.to_string(),
+                VecDeque::from([MockHealObjectOutcome::PoolMetadata { kind, outer }]),
+            );
+            let mut request = HealRequest::new(
+                HealType::Cluster,
+                HealOptions {
+                    recreate_missing: true,
+                    timeout: None,
+                    ..Default::default()
+                },
+                HealPriority::Normal,
+            );
+            request.source = HealRequestSource::Admin;
+            let task = HealTask::from_request(request, storage.clone());
+            let error = task
+                .execute()
+                .await
+                .expect_err("pool metadata failure must reach the scheduler");
+            assert!(
+                matches!(&error, Error::Storage(error) if error.pool_metadata_failure().is_some_and(|failure| failure.kind == kind)),
+                "the original typed pool metadata state must survive either response shape: {error}"
+            );
+            assert_eq!(error.is_recoverable_heal(), retryable, "{kind:?}, outer={outer}");
+            assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), [POOL_META_NAME]);
+            if retryable {
+                let retry = task
+                    .retry_request_with_remaining_timeout()
+                    .await
+                    .expect("existing retry request");
+                assert_eq!(retry.id, task.id, "recovery retains the accepted task token");
+                HealTask::from_request(retry, storage.clone())
+                    .execute()
+                    .await
+                    .expect("the same task should complete after pool metadata recovery");
+                assert_eq!(
+                    storage.heal_object_calls.lock().expect("object calls").as_slice(),
+                    [POOL_META_NAME, POOL_META_NAME]
+                );
+            }
+        }
+    }
+}
+
 #[tokio::test]
 async fn cluster_dry_run_does_not_heal_pool_metadata() {
     let storage = Arc::new(MockStorage::default());
@@ -3799,7 +3887,10 @@ async fn test_recursive_bucket_heal_reports_typed_exhausted_and_permanent_failur
 async fn test_cluster_heal_continues_after_bucket_failure() {
     let storage = Arc::new(MockStorage {
         listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-        bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
+        bucket_heal_errors: Mutex::new(HashMap::from([(
+            "bucket-a".to_string(),
+            VecDeque::from([Error::other("metadata unavailable")]),
+        )])),
         ..Default::default()
     });
     let request = HealRequest::new(
@@ -3831,7 +3922,10 @@ async fn test_cluster_heal_continues_after_bucket_failure() {
 async fn test_cluster_heal_retries_only_recoverable_bucket() {
     let storage = Arc::new(MockStorage {
         listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-        bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["lock acquisition timeout"]))])),
+        bucket_heal_errors: Mutex::new(HashMap::from([(
+            "bucket-a".to_string(),
+            VecDeque::from([Error::other("lock acquisition timeout")]),
+        )])),
         ..Default::default()
     });
     let request = HealRequest::new(
@@ -3855,6 +3949,47 @@ async fn test_cluster_heal_retries_only_recoverable_bucket() {
     );
 }
 
+#[tokio::test(start_paused = true)]
+async fn cluster_bucket_heal_retries_only_recoverable_pool_metadata_failures() {
+    for (kind, retryable) in [
+        (EcstorePoolMetadataFailure::ReadUnavailable, true),
+        (EcstorePoolMetadataFailure::TransactionUnknown, true),
+        (EcstorePoolMetadataFailure::RecoveryRequired, false),
+        (EcstorePoolMetadataFailure::FenceLost, false),
+    ] {
+        let storage = Arc::new(MockStorage {
+            bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
+            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_owned(), VecDeque::from([pool_metadata_error(kind)]))])),
+            ..Default::default()
+        });
+        let mut request = HealRequest::new(
+            HealType::Cluster,
+            HealOptions {
+                recursive: true,
+                timeout: None,
+                ..Default::default()
+            },
+            HealPriority::Normal,
+        );
+        request.source = HealRequestSource::Admin;
+        let task = HealTask::from_request(request, storage.clone());
+        let result = task.execute().await;
+        if retryable {
+            result.expect("a recoverable pool metadata state must not terminate the root heal");
+            assert_eq!(task.get_status().await, HealTaskStatus::Completed);
+            assert_eq!(
+                storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
+                ["bucket-a", "bucket-a"]
+            );
+        } else {
+            result.expect_err("operator recovery and lost fences must remain terminal");
+            let failure = task.take_batch_failure().await.expect("permanent cluster failure");
+            assert_eq!((failure.retryable, failure.permanent), (0, 1));
+            assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
+        }
+    }
+}
+
 #[tokio::test]
 async fn test_recursive_bucket_heal_does_not_remove_bucket_metadata() {
     let storage = Arc::new(MockStorage::default());
@@ -4617,7 +4752,7 @@ async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let storage = Arc::new(MockStorage {
         bucket_heal_errors: Mutex::new(HashMap::from([(
             "bucket-a".to_string(),
-            VecDeque::from(["injected bucket prepass failure"]),
+            VecDeque::from([Error::other("injected bucket prepass failure")]),
         )])),
         resume_disk: Mutex::new(Some(disk)),
         ..Default::default()

crates/iam/src/delegation.rs

@@ -0,0 +1,313 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! Durable restrictions owned by an authenticated user. A record is not an IAM
+//! grant or credential; issuing credentials still requires current IAM checks.
+use crate::{error::is_err_config_not_found, store::Store};
+use serde::{Deserialize, Serialize};
+
+const PREFIX: &str = "config/iam/plugin-delegations/v1";
+
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct Restrictions {
+    #[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
+    pub version: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
+    pub output_bucket: Option<String>,
+    pub deployment: String,
+    pub bucket: String,
+    pub input_prefix: String,
+    pub output_prefix: String,
+    pub expires_at_unix_seconds: u64,
+}
+
+// An absent v1 field is compatible; an explicitly null v2 field is invalid.
+fn present_value<'de, T: Deserialize<'de>, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Option<T>, D::Error> {
+    T::deserialize(deserializer).map(Some)
+}
+
+impl Restrictions {
+    pub fn output_bucket(&self) -> Result<&str, DelegationError> {
+        match (self.version, self.output_bucket.as_deref()) {
+            (None, None) => Ok(&self.bucket),
+            (Some(2), Some(output)) => Ok(output),
+            _ => Err(DelegationError::Invalid),
+        }
+    }
+
+    pub fn validate(&self, now: u64) -> Result<(), DelegationError> {
+        let output_bucket = self.output_bucket()?;
+        let names: [&str; 5] = [
+            &self.deployment,
+            &self.bucket,
+            &self.input_prefix,
+            &self.output_prefix,
+            output_bucket,
+        ];
+        if names
+            .iter()
+            .any(|s| s.is_empty() || s.len() > 1024 || s.contains("${") || s.chars().any(char::is_control))
+            || self.bucket.contains('/')
+            || self.bucket.contains('*')
+            || self.bucket.contains('?')
+            || output_bucket.contains('/')
+            || output_bucket.contains('*')
+            || output_bucket.contains('?')
+            || self.input_prefix.contains('*')
+            || self.input_prefix.contains('?')
+            || self.output_prefix.contains('*')
+            || self.output_prefix.contains('?')
+            || self.expires_at_unix_seconds <= now
+            || self.expires_at_unix_seconds.saturating_sub(now) > 86400
+        {
+            return Err(DelegationError::Invalid);
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct Delegation {
+    pub version: u32,
+    pub id: String,
+    pub revision: String,
+    pub owner_access_key: String,
+    pub created_at_unix_seconds: u64,
+    pub restrictions: Restrictions,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+struct Tombstone {
+    version: u32,
+    id: String,
+    revision: String,
+    owner_access_key: String,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum DelegationError {
+    #[error("invalid delegation restriction")]
+    Invalid,
+    #[error("delegation not found")]
+    NotFound,
+    #[error("delegation owner rejected")]
+    OwnerRejected,
+    #[error("delegation unavailable")]
+    Unavailable,
+    #[error("delegation revoked")]
+    Revoked,
+    #[error("delegation expired")]
+    Expired,
+}
+
+fn record_path(id: &str) -> Result<String, DelegationError> {
+    if id.len() != 32 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
+        return Err(DelegationError::Invalid);
+    }
+    Ok(format!("{PREFIX}/{id}/record.json"))
+}
+
+async fn load<T: Store>(store: &T, id: &str) -> Result<Delegation, DelegationError> {
+    let record: Delegation = store.load_iam_config_strict(record_path(id)?).await.map_err(|error| {
+        if is_err_config_not_found(&error) {
+            DelegationError::NotFound
+        } else {
+            DelegationError::Unavailable
+        }
+    })?;
+    if !matches!(record.version, 1 | 2)
+        || record.version != record.restrictions.version.unwrap_or(1)
+        || record.id != id
+        || record.revision != id
+        || record.owner_access_key.is_empty()
+    {
+        return Err(DelegationError::Unavailable);
+    }
+    record
+        .restrictions
+        .validate(record.created_at_unix_seconds)
+        .map_err(|_| DelegationError::Unavailable)?;
+    Ok(record)
+}
+
+pub(crate) async fn create<T: Store>(
+    store: &T,
+    owner: &str,
+    restrictions: Restrictions,
+    now: u64,
+) -> Result<Delegation, DelegationError> {
+    restrictions.validate(now)?;
+    if owner.is_empty() {
+        return Err(DelegationError::OwnerRejected);
+    }
+    // IDs are generated exclusively here; no update operation accepts an ID.
+    let id = format!("{:032x}", rand::random::<u128>());
+    let path = record_path(&id)?;
+    match store.load_iam_config_strict::<Delegation>(&path).await {
+        Err(error) if is_err_config_not_found(&error) => {}
+        _ => return Err(DelegationError::Unavailable),
+    }
+    let record = Delegation {
+        version: restrictions.version.unwrap_or(1),
+        revision: id.clone(),
+        id,
+        owner_access_key: owner.to_owned(),
+        created_at_unix_seconds: now,
+        restrictions,
+    };
+    store
+        .save_iam_config(&record, path)
+        .await
+        .map_err(|_| DelegationError::Unavailable)?;
+    if load(store, &record.id).await? != record {
+        return Err(DelegationError::Unavailable);
+    }
+    Ok(record)
+}
+
+pub(crate) async fn revoke<T: Store>(store: &T, owner: &str, id: &str) -> Result<(), DelegationError> {
+    let record = load(store, id).await?;
+    if record.owner_access_key != owner {
+        return Err(DelegationError::OwnerRejected);
+    }
+    let tombstone = Tombstone {
+        version: 1,
+        id: record.id,
+        revision: record.revision,
+        owner_access_key: record.owner_access_key,
+    };
+    let path = format!("{PREFIX}/{id}/revoked.json");
+    // Every retry writes exactly the same revocation; active records never change.
+    store
+        .save_iam_config(&tombstone, &path)
+        .await
+        .map_err(|_| DelegationError::Unavailable)?;
+    let saved: Tombstone = store
+        .load_iam_config_strict(path)
+        .await
+        .map_err(|_| DelegationError::Unavailable)?;
+    if saved != tombstone {
+        return Err(DelegationError::Unavailable);
+    }
+    Ok(())
+}
+
+pub(crate) async fn read_active<T: Store>(store: &T, owner: &str, id: &str, now: u64) -> Result<Delegation, DelegationError> {
+    let record = load(store, id).await?;
+    if record.owner_access_key != owner {
+        return Err(DelegationError::OwnerRejected);
+    }
+    check_active(store, record, now).await
+}
+
+pub(crate) async fn read_owned<T: Store>(store: &T, owner: &str, id: &str) -> Result<Delegation, DelegationError> {
+    let record = load(store, id).await?;
+    if record.owner_access_key != owner {
+        return Err(DelegationError::OwnerRejected);
+    }
+    Ok(record)
+}
+
+pub(crate) async fn read_active_for_broker<T: Store>(store: &T, id: &str, now: u64) -> Result<Delegation, DelegationError> {
+    let record = load(store, id).await?;
+    check_active(store, record, now).await
+}
+async fn check_active<T: Store>(store: &T, record: Delegation, now: u64) -> Result<Delegation, DelegationError> {
+    let id = &record.id;
+    match store
+        .load_iam_config_strict::<Tombstone>(format!("{PREFIX}/{id}/revoked.json"))
+        .await
+    {
+        Ok(_) => return Err(DelegationError::Revoked),
+        Err(error) if is_err_config_not_found(&error) => {}
+        Err(_) => return Err(DelegationError::Unavailable),
+    }
+    if now >= record.restrictions.expires_at_unix_seconds {
+        return Err(DelegationError::Expired);
+    }
+    Ok(record)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn delegation_v1_json_remains_compatible() {
+        let legacy = serde_json::json!({"deployment":"runner-a","bucket":"photos",
+            "input_prefix":"inputs/","output_prefix":"outputs/","expires_at_unix_seconds":200});
+        let restriction: Restrictions = serde_json::from_value(legacy.clone()).expect("legacy restriction");
+        restriction.validate(100).expect("legacy validation");
+        assert_eq!(restriction.output_bucket().expect("v1 output"), "photos");
+        assert_eq!(serde_json::to_value(restriction).expect("legacy serialization"), legacy);
+        let record = serde_json::json!({"version":1,"id":"a".repeat(32),"revision":"a".repeat(32),
+            "owner_access_key":"owner","created_at_unix_seconds":100,"restrictions":legacy});
+        let decoded: Delegation = serde_json::from_value(record.clone()).expect("legacy record");
+        assert_eq!(serde_json::to_value(decoded).expect("legacy record serialization"), record);
+    }
+
+    #[test]
+    fn delegation_v2_requires_explicit_output_bucket() {
+        let valid = serde_json::json!({"version":2,"deployment":"runner-a","bucket":"photos",
+            "input_prefix":"inputs/","output_bucket":"artifacts","output_prefix":"outputs/",
+            "expires_at_unix_seconds":200});
+        let restriction: Restrictions = serde_json::from_value(valid.clone()).expect("v2 restriction");
+        restriction.validate(100).expect("explicit output bucket");
+        assert_eq!(restriction.output_bucket().expect("v2 output"), "artifacts");
+        assert_eq!(serde_json::to_value(restriction).expect("v2 serialization"), valid);
+        for key in ["version", "output_bucket"] {
+            let mut bad = valid.clone();
+            bad.as_object_mut().expect("object").remove(key);
+            assert!(
+                serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
+                "v2 required {key}"
+            );
+        }
+        for (key, value) in [
+            ("version", serde_json::json!(1)),
+            ("version", serde_json::json!(3)),
+            ("version", serde_json::Value::Null),
+            ("output_bucket", serde_json::Value::Null),
+            ("output_bucket", serde_json::json!("")),
+            ("output_bucket", serde_json::json!("other/*")),
+            ("output_bucket", serde_json::json!("${aws:username}")),
+            ("output_bucket", serde_json::json!("other?")),
+            ("output_bucket", serde_json::json!("other\n")),
+            ("unknown", serde_json::json!(true)),
+        ] {
+            let mut bad = valid.clone();
+            bad[key] = value;
+            assert!(
+                serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
+                "invalid v2 {key}"
+            );
+        }
+    }
+
+    #[test]
+    fn restrictions_are_literal_bounded_and_expiring() {
+        let mut restriction = Restrictions {
+            version: None,
+            output_bucket: None,
+            deployment: "runner-a".into(),
+            bucket: "photos".into(),
+            input_prefix: "inputs/".into(),
+            output_prefix: "outputs/".into(),
+            expires_at_unix_seconds: 200,
+        };
+        assert!(restriction.validate(100).is_ok());
+        restriction.output_prefix = "outputs/*".into();
+        assert!(matches!(restriction.validate(100), Err(DelegationError::Invalid)));
+        restriction.output_prefix = "outputs/".into();
+        assert!(matches!(restriction.validate(200), Err(DelegationError::Invalid)));
+        restriction.expires_at_unix_seconds = 200;
+        restriction.input_prefix = "inputs/${aws:username}/".into();
+        assert!(
+            matches!(restriction.validate(100), Err(DelegationError::Invalid)),
+            "C07_LITERAL_SCOPE_BOUND: policy variables are not literal restrictions"
+        );
+        assert!(record_path("../../users").is_err());
+    }
+}

crates/iam/src/lib.rs

@@ -28,6 +28,7 @@ pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
 const EVENT_OIDC_STATE: &str = "oidc_state";
 
 pub mod cache;
+pub mod delegation;
 pub mod error;
 pub mod federation;
 pub mod keyring;

crates/iam/src/storage_api.rs

@@ -91,6 +91,7 @@ pub(crate) mod crate_boundary {
 
 pub(crate) mod object_store {
     pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
+    pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
 }
 
 pub(crate) mod runtime {

crates/iam/src/store.rs

@@ -26,6 +26,11 @@ pub trait Store: Clone + Send + Sync + 'static {
     fn has_watcher(&self) -> bool;
     async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
     async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
+    /// Only confirmed storage absence may return ConfigNotFound. Corrupt or
+    /// undecryptable objects must remain errors, especially for revocations.
+    async fn load_iam_config_strict<Item: DeserializeOwned>(&self, _path: impl AsRef<str> + Send) -> Result<Item> {
+        Err(crate::error::Error::IamSysNotInitialized)
+    }
     async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
 
     async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;

crates/iam/src/store/object.rs

@@ -42,6 +42,7 @@ use tracing::{debug, error, warn};
 
 use crate::storage_api::object_store::{
     HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
+    read_config_preserve_empty_with_metadata_opts,
 };
 
 pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
@@ -611,6 +612,11 @@ impl ObjectStore {
         Ok(serde_json::from_slice(&outcome.plain)?)
     }
 
+    fn decode_iam_config_strict<Item: DeserializeOwned>(data: &[u8]) -> Result<Item> {
+        let outcome = Self::decrypt_data_with_source(data)?;
+        Ok(serde_json::from_slice(&outcome.plain)?)
+    }
+
     /// Parameterized core of [`Store::load_user_identity`].
     async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
         let mut u: UserIdentity = self
@@ -824,6 +830,12 @@ impl Store for ObjectStore {
     async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
         self.load_iam_config_with(path, LoadMode::Locked).await
     }
+    async fn load_iam_config_strict<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
+        let (data, _) =
+            read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path.as_ref(), &LoadMode::Locked.read_opts())
+                .await?;
+        Self::decode_iam_config_strict(&data)
+    }
     /// Saves IAM configuration with a retry mechanism on failure.
     ///
     /// Attempts to save the IAM configuration up to 5 times if the storage layer is not ready,
@@ -1458,6 +1470,40 @@ mod tests {
         assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
     }
 
+    #[test]
+    fn strict_revocation_load_keeps_real_crypto_failure_distinct_from_absence() {
+        let cred = test_cred();
+        let key = format!("{}:{}", cred.access_key, cred.secret_key);
+        let plain = br#"{"version":1,"id":"revoked"}"#;
+        let mut encrypted = rustfs_crypto::encrypt_stream_io(key.as_bytes(), plain).expect("real stored ciphertext");
+        // Exercise the exact decoder used after the actual object read.
+        assert!(ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted).is_ok());
+        encrypted[50] ^= 0xFF;
+        let error = ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted)
+            .expect_err("corrupt tombstone must not become missing");
+        assert!(
+            matches!(error, Error::CryptoError(_)),
+            "C07_REVOCATION_FAIL_CLOSED: preserve actual crypto error"
+        );
+        assert!(
+            !crate::error::is_err_config_not_found(&error),
+            "C07_REVOCATION_FAIL_CLOSED: corrupted revocation is not absence"
+        );
+        // The preserve-empty reader keeps an existing zero-byte object as
+        // Ok(empty); ecstore's read_config_preserve_empty_distinguishes_empty_object
+        // covers that storage boundary. Its payload must fail here, not become
+        // ConfigNotFound and reactivate a revoked delegation.
+        let empty =
+            ObjectStore::decode_iam_config_strict::<serde_json::Value>(&[]).expect_err("an existing empty tombstone is corrupt");
+        assert!(
+            !crate::error::is_err_config_not_found(&empty),
+            "C07_REVOCATION_FAIL_CLOSED: empty existing revocation is not absence"
+        );
+        let invalid_json =
+            ObjectStore::decode_iam_config_strict::<serde_json::Value>(br#"{"version": }"#).expect_err("invalid stored JSON");
+        assert!(!crate::error::is_err_config_not_found(&invalid_json));
+    }
+
     #[test]
     fn test_decrypt_data_short_data_fails() {
         let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON

crates/iam/src/sys.rs

@@ -75,6 +75,7 @@ pub const STATUS_ENABLED: &str = "enabled";
 pub const STATUS_DISABLED: &str = "disabled";
 
 pub const POLICYNAME: &str = "policy";
+pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
 pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
 pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
 const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-policy";
@@ -168,6 +169,12 @@ enum PreparedServicePolicyMode {
 
 #[derive(Clone)]
 enum PreparedIamMode {
+    TerminalDeny,
+    PluginAttemptSts {
+        is_owner: bool,
+        combined_policy: Policy,
+        session_policy: PreparedSessionPolicy,
+    },
     Opa,
     Owner,
     Deny,
@@ -202,7 +209,7 @@ impl PreparedIamAuth {
     pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool {
         match &self.mode {
             PreparedIamMode::Opa => true,
-            PreparedIamMode::Owner | PreparedIamMode::Deny => false,
+            PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => false,
             PreparedIamMode::Regular { combined_policy } => {
                 policy_needs_existing_object_tag_for_args(combined_policy, args).await
             }
@@ -210,6 +217,11 @@ impl PreparedIamAuth {
                 combined_policy,
                 session_policy,
                 ..
+            }
+            | PreparedIamMode::PluginAttemptSts {
+                combined_policy,
+                session_policy,
+                ..
             } => {
                 policy_needs_existing_object_tag_for_args(combined_policy, args).await
                     || prepared_session_policy_needs_existing_object_tag_for_args(session_policy, args).await
@@ -239,14 +251,56 @@ impl PreparedIamAuth {
     pub fn combined_policy_for_view(&self) -> Option<&Policy> {
         match &self.mode {
             PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
-            PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
+            PreparedIamMode::Sts { combined_policy, .. } | PreparedIamMode::PluginAttemptSts { combined_policy, .. } => {
+                Some(combined_policy)
+            }
             PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
-            PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
+            PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => None,
         }
     }
 }
 
 impl<T: Store> IamSys<T> {
+    /// For the authenticated, explicitly configured broker handler only.
+    pub async fn read_active_plugin_delegation_for_broker(
+        &self,
+        id: &str,
+        now: u64,
+    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
+        crate::delegation::read_active_for_broker(&self.store.api, id, now).await
+    }
+    pub async fn create_plugin_delegation(
+        &self,
+        owner: &str,
+        restrictions: crate::delegation::Restrictions,
+        now: u64,
+    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
+        crate::delegation::create(&self.store.api, owner, restrictions, now).await
+    }
+    pub async fn revoke_plugin_delegation(
+        &self,
+        owner: &str,
+        id: &str,
+    ) -> std::result::Result<(), crate::delegation::DelegationError> {
+        crate::delegation::revoke(&self.store.api, owner, id).await
+    }
+    pub async fn read_active_plugin_delegation(
+        &self,
+        owner: &str,
+        id: &str,
+        now: u64,
+    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
+        crate::delegation::read_active(&self.store.api, owner, id, now).await
+    }
+    /// Reads an owned delegation record for historical lookups, including expired or revoked records.
+    pub async fn read_owned_plugin_delegation(
+        &self,
+        owner: &str,
+        id: &str,
+    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
+        crate::delegation::read_owned(&self.store.api, owner, id).await
+    }
+
     /// Create a new IamSys instance with the given IamCache store
     ///
     /// # Arguments
@@ -281,6 +335,16 @@ impl<T: Store> IamSys<T> {
         *guard = PolicyPluginState::Ready(client);
     }
 
+    /// Restricted credentials require the built-in parent/session intersection.
+    /// External authorization bypasses that path, so only Disabled is supported.
+    pub async fn ensure_restricted_credential_vending() -> Result<()> {
+        if matches!(Self::policy_plugin_state().await, PolicyPluginState::Disabled) {
+            Ok(())
+        } else {
+            Err(Error::IAMActionNotAllowed)
+        }
+    }
+
     pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
         let policy_plugin_state = get_policy_plugin_state();
         let guard = policy_plugin_state.read().await;
@@ -1383,17 +1447,50 @@ impl<T: Store> IamSys<T> {
     /// 1) know whether policy evaluation may need `s3:ExistingObjectTag`, and
     /// 2) evaluate with final conditions without re-merging identity policies.
     pub async fn prepare_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
-        if args.is_owner {
+        let marked = args.claims.contains_key(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
+        if args.is_owner && !marked {
             return PreparedIamAuth {
                 needs_existing_object_tag: false,
                 mode: PreparedIamMode::Owner,
             };
         }
-
-        if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
+        // One observed state decides this request's path. Marked credentials
+        // never enter OPA, including after a restart or on another node.
+        let state = Self::policy_plugin_state().await;
+        if marked {
+            if args.is_owner
+                || args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
+                || !matches!(state, PolicyPluginState::Disabled)
+            {
+                return PreparedIamAuth {
+                    needs_existing_object_tag: false,
+                    mode: PreparedIamMode::TerminalDeny,
+                };
+            }
+            let prepared = self.prepare_builtin_auth(args).await;
+            let mode = match prepared.mode {
+                PreparedIamMode::Sts {
+                    is_owner,
+                    combined_policy,
+                    session_policy,
+                } => PreparedIamMode::PluginAttemptSts {
+                    is_owner,
+                    combined_policy,
+                    session_policy,
+                },
+                _ => PreparedIamMode::TerminalDeny,
+            };
+            return PreparedIamAuth {
+                needs_existing_object_tag: prepared.needs_existing_object_tag,
+                mode,
+            };
+        }
+        if let Some(prepared) = state.prepared_iam_auth() {
             return prepared;
         }
-
+        self.prepare_builtin_auth(args).await
+    }
+    async fn prepare_builtin_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
             return PreparedIamAuth {
                 needs_existing_object_tag: false,
@@ -1425,6 +1522,20 @@ impl<T: Store> IamSys<T> {
     /// bucket policies on an IAM denial. An error must not become an implicit deny.
     pub async fn try_eval_prepared(&self, prepared: &PreparedIamAuth, args: &Args<'_>) -> Result<bool> {
         Ok(match &prepared.mode {
+            PreparedIamMode::TerminalDeny => return Err(Error::IAMActionNotAllowed),
+            PreparedIamMode::PluginAttemptSts {
+                is_owner,
+                combined_policy,
+                session_policy,
+            } => {
+                // A false IAM result permits bucket-policy fallback. Restricted
+                // plugin credentials must instead terminate authorization.
+                let session_allowed = evaluate_prepared_session_policy(session_policy, args).await == Some(true);
+                if !session_allowed || !(*is_owner || combined_policy.is_allowed(args).await) {
+                    return Err(Error::IAMActionNotAllowed);
+                }
+                true
+            }
             PreparedIamMode::Opa => {
                 let Some(opa_enable) = Self::get_policy_plugin_client().await else {
                     tracing::warn!("eval_prepared: OPA mode requested but plugin is unavailable");
@@ -2127,6 +2238,145 @@ mod tests {
         assert!(prepared.combined_policy_for_view().is_none());
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn marked_signed_sts_denial_is_terminal_before_public_bucket_fallback() {
+        let iam = IamSys::new(IamCache::new(StsTestMockStore::new(false)).await.unwrap());
+        let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
+        let signing_key = "c07-test-token-signing-key";
+        let mut metadata=HashMap::from([
+            ("parent".into(),Value::String("sts-fallback-test-parent".into())),
+            ("exp".into(),serde_json::json!((OffsetDateTime::now_utc()+time::Duration::hours(1)).unix_timestamp())),
+            (SESSION_POLICY_NAME.into(),Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/inside"]}]}"#))),
+        ]);
+        metadata.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), Value::Bool(true));
+        let mut marked = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
+        marked.parent_user = "sts-fallback-test-parent".into();
+        let marked_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&marked.session_token, signing_key)
+            .expect("signed test credentials must decode")
+            .claims;
+        let marked_user = UserIdentity::from(marked.clone());
+        iam.store
+            .cache
+            .add_or_update_sts_account(&marked.access_key, &marked_user, OffsetDateTime::now_utc());
+        metadata.remove(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
+        let mut ordinary = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
+        ordinary.parent_user = marked.parent_user.clone();
+        let ordinary_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&ordinary.session_token, signing_key)
+            .expect("signed test credentials must decode")
+            .claims;
+        iam.store.cache.add_or_update_sts_account(
+            &ordinary.access_key,
+            &UserIdentity::from(ordinary.clone()),
+            OffsetDateTime::now_utc(),
+        );
+        let groups = None;
+        let conditions = HashMap::new();
+        let args = Args {
+            account: &marked.access_key,
+            groups: &groups,
+            action: Action::S3Action(S3Action::GetObjectAction),
+            bucket: "mybucket",
+            object: "outside",
+            conditions: &conditions,
+            is_owner: false,
+            claims: &marked_claims,
+            deny_only: false,
+        };
+        *get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
+        let denied = iam.prepare_auth(&args).await;
+        let disabled_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
+        let inside = Args {
+            object: "inside",
+            ..args.clone()
+        };
+        let allowed = iam.prepare_auth(&inside).await;
+        let inside_allowed = matches!(iam.try_eval_prepared(&allowed, &inside).await, Ok(true));
+        let unmarked_args = Args {
+            account: &ordinary.access_key,
+            claims: &ordinary_claims,
+            ..args.clone()
+        };
+        let unmarked = iam.prepare_auth(&unmarked_args).await;
+        let ordinary_false = matches!(iam.try_eval_prepared(&unmarked, &unmarked_args).await, Ok(false));
+        let public:rustfs_policy::policy::BucketPolicy=serde_json::from_value(serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]})).unwrap();
+        let public_allows = public
+            .is_allowed(&rustfs_policy::policy::BucketPolicyArgs {
+                account: &marked.access_key,
+                groups: &groups,
+                action: args.action,
+                bucket: args.bucket,
+                object: args.object,
+                conditions: &conditions,
+                is_owner: false,
+            })
+            .await;
+        IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
+            url: "http://127.0.0.1:8181/allow".into(),
+            auth_token: String::new(),
+        }))
+        .await;
+        let opa_marked = iam.prepare_auth(&args).await;
+        let ready_terminal = matches!(iam.try_eval_prepared(&opa_marked, &args).await, Err(Error::IAMActionNotAllowed));
+        let opa_ordinary = iam.prepare_auth(&unmarked_args).await;
+        let ordinary_keeps_opa = matches!(opa_ordinary.mode, PreparedIamMode::Opa);
+        // The captured Disabled preparation stays on built-in intersection even
+        // if an external policy client becomes ready before evaluation.
+        let captured_still_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
+        *get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
+        let pending = iam.prepare_auth(&args).await;
+        let initializing_terminal = matches!(iam.try_eval_prepared(&pending, &args).await, Err(Error::IAMActionNotAllowed));
+        *get_policy_plugin_state().write().await = PolicyPluginState::Failed;
+        let failed = iam.prepare_auth(&args).await;
+        let failed_terminal = matches!(iam.try_eval_prepared(&failed, &args).await, Err(Error::IAMActionNotAllowed));
+        *get_policy_plugin_state().write().await = previous;
+        assert!(public_allows, "C07_PUBLIC_BUCKET_CONTROL: real public policy permits outside object");
+        assert!(
+            disabled_terminal && captured_still_terminal,
+            "C07_SCOPE_TERMINAL_BOUND: bucket policy must not rescue restricted scope denial"
+        );
+        assert!(
+            ready_terminal && initializing_terminal && failed_terminal,
+            "C07_EXTERNAL_POLICY_BOUND: marked persisted STS cannot enter OPA"
+        );
+        assert!(
+            inside_allowed && ordinary_false && ordinary_keeps_opa,
+            "C07_UNMARKED_COMPAT: preserve ordinary STS and valid marked scope"
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn restricted_vending_rejects_actual_external_policy_plugin_state() {
+        let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
+        IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
+            url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".into(),
+            auth_token: String::new(),
+        }))
+        .await;
+        let active_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
+            .await
+            .is_err();
+        *get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
+        let initializing_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
+            .await
+            .is_err();
+        *get_policy_plugin_state().write().await = PolicyPluginState::Failed;
+        let failed_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
+            .await
+            .is_err();
+        *get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
+        let disabled_allowed = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
+            .await
+            .is_ok();
+        *get_policy_plugin_state().write().await = previous;
+        assert!(
+            active_rejected && initializing_rejected && failed_rejected,
+            "C07_EXTERNAL_POLICY_BOUND: external authorization cannot bypass minted session scope"
+        );
+        assert!(disabled_allowed);
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() {

crates/notify/src/config_manager.rs

@@ -42,13 +42,14 @@ enum NotifyConfigStoreError {
     Read(String),
     Save(String),
     Converge { persisted: bool, error: String },
+    Conflict,
 }
 
 fn notification_convergence_error(persisted: bool, error: impl std::fmt::Display) -> NotificationError {
-    let durable_state = if persisted { "persisted" } else { "unchanged" };
-    NotificationError::Configuration(format!(
-        "configuration was {durable_state} but notification runtime convergence failed: {error}"
-    ))
+    NotificationError::ConfigurationConvergence {
+        persisted,
+        detail: error.to_string(),
+    }
 }
 
 async fn supervise_config_update<T>(
@@ -63,12 +64,62 @@ where
     })?
 }
 
+fn configs_equal(first: Option<&KVS>, second: Option<&KVS>) -> bool {
+    match (first, second) {
+        (None, None) => true,
+        (Some(first), Some(second)) => {
+            if first == second {
+                return true;
+            }
+            let mut first: Vec<_> = first.0.iter().collect();
+            let mut second: Vec<_> = second.0.iter().collect();
+            // Lookup uses the first occurrence of a key, so preserve duplicate-key order.
+            first.sort_by(|a, b| a.key.cmp(&b.key));
+            second.sort_by(|a, b| a.key.cmp(&b.key));
+            first == second
+        }
+        _ => false,
+    }
+}
+
+fn replace_target_if_unchanged(
+    config: &mut Config,
+    target_type: &str,
+    target_name: &str,
+    expected: Option<&KVS>,
+    expected_default: Option<&KVS>,
+    kvs: &KVS,
+) -> Result<bool, NotifyConfigStoreError> {
+    let targets = config.0.get(target_type);
+    let mut matching = targets
+        .into_iter()
+        .flat_map(|targets| targets.iter())
+        .filter(|(name, _)| name.eq_ignore_ascii_case(target_name));
+    let current = matching.next();
+    if matching.next().is_some() {
+        return Err(NotifyConfigStoreError::Conflict);
+    }
+    let stored_name = current
+        .map(|(name, _)| name.clone())
+        .unwrap_or_else(|| target_name.to_owned());
+    let current_default = targets.and_then(|targets| targets.get(rustfs_config::DEFAULT_DELIMITER));
+    if !configs_equal(current.map(|(_, kvs)| kvs), expected) || !configs_equal(current_default, expected_default) {
+        return Err(NotifyConfigStoreError::Conflict);
+    }
+    config
+        .0
+        .entry(target_type.to_owned())
+        .or_default()
+        .insert(stored_name, kvs.clone());
+    Ok(true)
+}
+
 async fn update_server_config<F>(
     mut modifier: F,
     lifecycle: NotifyLifecycleCoordinator,
 ) -> Result<Option<(crate::lifecycle::NotificationLifecycleTransition, bool)>, NotifyConfigStoreError>
 where
-    F: FnMut(&mut Config) -> bool + Send + 'static,
+    F: FnMut(&mut Config, Option<String>) -> Result<bool, NotifyConfigStoreError> + Send + 'static,
 {
     let Some(store) = resolve_notify_object_store_handle() else {
         return Err(NotifyConfigStoreError::StorageNotAvailable);
@@ -79,7 +130,8 @@ where
             .await
             .map_err(NotifyConfigStoreError::Read)?;
         let mut config = snapshot.config.clone();
-        if !modifier(&mut config) {
+        let generation = snapshot.generation().map(|value| value.to_string());
+        if !modifier(&mut config, generation)? {
             return Ok(None);
         }
 
@@ -245,6 +297,30 @@ impl NotifyConfigManager {
         .await
     }
 
+    /// Replace a target only when its persisted configuration is unchanged since validation.
+    pub async fn set_target_config_if_unchanged(
+        &self,
+        target_type: &str,
+        target_name: &str,
+        expected: Option<KVS>,
+        expected_default: Option<KVS>,
+        expected_generation: Option<String>,
+        kvs: KVS,
+    ) -> Result<(), NotificationError> {
+        let ttype = target_type.to_lowercase();
+        let tname = target_name.to_lowercase();
+        self.update_config_and_reload_checked(move |config, generation| {
+            if expected_generation
+                .as_ref()
+                .is_some_and(|expected| generation.as_ref() != Some(expected))
+            {
+                return Err(NotifyConfigStoreError::Conflict);
+            }
+            replace_target_if_unchanged(config, &ttype, &tname, expected.as_ref(), expected_default.as_ref(), &kvs)
+        })
+        .await
+    }
+
     pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
         debug!(
             event = EVENT_NOTIFY_CONFIG_UPDATE,
@@ -338,9 +414,17 @@ impl NotifyConfigManager {
         Ok(())
     }
 
-    async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
+    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
     where
         F: FnMut(&mut Config) -> bool + Send + 'static,
+    {
+        self.update_config_and_reload_checked(move |config, _generation| Ok(modifier(config)))
+            .await
+    }
+
+    async fn update_config_and_reload_checked<F>(&self, modifier: F) -> Result<(), NotificationError>
+    where
+        F: FnMut(&mut Config, Option<String>) -> Result<bool, NotifyConfigStoreError> + Send + 'static,
     {
         if self.lifecycle.state() == NotificationRuntimeState::Terminated {
             return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
@@ -356,6 +440,7 @@ impl NotifyConfigManager {
                     NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
                     NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
                     NotifyConfigStoreError::Converge { persisted, error } => notification_convergence_error(persisted, error),
+                    NotifyConfigStoreError::Conflict => NotificationError::ConfigurationConflict,
                 })?
         else {
             debug!(
@@ -394,6 +479,126 @@ mod tests {
     use crate::rules::RulesMap;
     use crate::{NotificationError, NotificationRuntimeState};
 
+    #[test]
+    fn config_comparison_preserves_duplicates_and_hidden_flags() {
+        use rustfs_config::server_config::{KV, KVS};
+        let field = |key: &str, value: &str, hidden_if_empty: bool| KV {
+            key: key.into(),
+            value: value.into(),
+            hidden_if_empty,
+        };
+        let original = KVS(vec![
+            field("password", "secret", false),
+            field("token", "", true),
+            field("password", "secret", false),
+        ]);
+        let reordered = KVS(vec![
+            field("password", "secret", false),
+            field("password", "secret", false),
+            field("token", "", true),
+        ]);
+        let missing_duplicate = KVS(vec![field("password", "secret", false), field("token", "", true)]);
+        let changed_flag = KVS(vec![
+            field("password", "secret", false),
+            field("password", "secret", false),
+            field("token", "", false),
+        ]);
+        assert!(super::configs_equal(Some(&original), Some(&reordered)));
+        assert!(!super::configs_equal(Some(&original), Some(&missing_duplicate)));
+        assert!(!super::configs_equal(Some(&original), Some(&changed_flag)));
+        assert!(!super::configs_equal(Some(&original), None));
+        let first_value = KVS(vec![
+            field("password", "old", false),
+            field("token", "", true),
+            field("password", "new", false),
+        ]);
+        let reordered_keys = KVS(vec![
+            field("token", "", true),
+            field("password", "old", false),
+            field("password", "new", false),
+        ]);
+        let reordered_duplicates = KVS(vec![
+            field("token", "", true),
+            field("password", "new", false),
+            field("password", "old", false),
+        ]);
+        assert!(super::configs_equal(Some(&first_value), Some(&reordered_keys)));
+        assert!(!super::configs_equal(Some(&first_value), Some(&reordered_duplicates)));
+    }
+
+    #[test]
+    fn conditional_target_update_rejects_rotation_and_deletion() {
+        use rustfs_config::server_config::{Config, KVS};
+        let mut old = KVS::new();
+        old.insert("password".into(), "old".into());
+        let mut rotated = KVS::new();
+        rotated.insert("password".into(), "new".into());
+        let mut config = Config::new();
+        config
+            .0
+            .entry("notify_nats".into())
+            .or_default()
+            .insert("primary".into(), rotated.clone());
+        let result = super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", Some(&old), None, &old);
+        assert!(matches!(result, Err(super::NotifyConfigStoreError::Conflict)));
+        assert_eq!(config.0["notify_nats"]["primary"], rotated);
+        config.0.get_mut("notify_nats").expect("subsystem exists").remove("primary");
+        assert!(matches!(
+            super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", Some(&old), None, &old),
+            Err(super::NotifyConfigStoreError::Conflict)
+        ));
+        assert!(
+            super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", None, None, &old)
+                .expect("create unchanged target")
+        );
+        assert!(
+            super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", Some(&old), None, &rotated)
+                .expect("replace unchanged target")
+        );
+        assert_eq!(config.0["notify_nats"]["primary"], rotated);
+
+        let mut first = KVS::new();
+        first.insert("url".into(), "nats://example.com".into());
+        first.insert("token".into(), "secret".into());
+        let mut reordered = KVS::new();
+        reordered.insert("token".into(), "secret".into());
+        reordered.insert("url".into(), "nats://example.com".into());
+        config
+            .0
+            .get_mut("notify_nats")
+            .expect("subsystem")
+            .insert("primary".into(), first);
+        assert!(
+            super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", Some(&reordered), None, &rotated)
+                .expect("key order does not signal a change")
+        );
+        let targets = config.0.get_mut("notify_nats").expect("subsystem");
+        targets.remove("primary");
+        targets.insert("Primary".into(), rotated.clone());
+        assert!(
+            super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", Some(&rotated), None, &old)
+                .expect("case-insensitive match")
+        );
+        let targets = &config.0["notify_nats"];
+        assert!(targets.contains_key("Primary"));
+        assert!(!targets.contains_key("primary"));
+        let mut old_default = KVS::new();
+        old_default.insert("password".into(), "old-default".into());
+        let mut new_default = KVS::new();
+        new_default.insert("password".into(), "new-default".into());
+        config
+            .0
+            .get_mut("notify_nats")
+            .expect("subsystem")
+            .insert("_".into(), new_default.clone());
+        assert!(matches!(
+            super::replace_target_if_unchanged(&mut config, "notify_nats", "primary", Some(&old), Some(&old_default), &rotated),
+            Err(super::NotifyConfigStoreError::Conflict)
+        ));
+        assert_eq!(config.0["notify_nats"]["Primary"], old);
+        assert_eq!(config.0["notify_nats"]["_"], new_default);
+    }
+
     #[tokio::test]
     async fn supervised_config_update_survives_waiter_cancellation() {
         let (started_tx, started_rx) = tokio::sync::oneshot::channel();
@@ -550,13 +755,9 @@ mod tests {
 
     #[test]
     fn convergence_error_reports_durable_write_state() {
-        for (persisted, expected) in [(true, "configuration was persisted"), (false, "configuration was unchanged")] {
+        for persisted in [true, false] {
             let error = notification_convergence_error(persisted, "injected failure");
-            let NotificationError::Configuration(message) = error else {
-                panic!("convergence error should be a configuration error");
-            };
-            assert!(message.contains(expected), "unexpected convergence error: {message}");
-            assert!(message.contains("injected failure"));
+            assert!(matches!(error, NotificationError::ConfigurationConvergence { persisted: value, .. } if value == persisted));
         }
     }
 }

crates/notify/src/error.rs

@@ -38,6 +38,12 @@ pub enum NotificationError {
     #[error("Configuration error: {0}")]
     Configuration(String),
 
+    #[error("Target configuration changed during validation")]
+    ConfigurationConflict,
+
+    #[error("Notification configuration convergence failed (persisted: {persisted}): {detail}")]
+    ConfigurationConvergence { persisted: bool, detail: String },
+
     #[error("ARN not found: {0}")]
     ARNNotFound(String),
 

crates/notify/src/integration.rs

@@ -298,6 +298,22 @@ impl NotificationSystem {
             .await
     }
 
+    /// Replace a target only if its persisted configuration still matches the validated snapshot.
+    pub async fn set_target_config_if_unchanged(
+        &self,
+        target_type: &str,
+        target_name: &str,
+        expected: Option<KVS>,
+        expected_default: Option<KVS>,
+        expected_generation: Option<String>,
+        kvs: KVS,
+    ) -> Result<(), NotificationError> {
+        self.services
+            .config_manager
+            .set_target_config_if_unchanged(target_type, target_name, expected, expected_default, expected_generation, kvs)
+            .await
+    }
+
     /// Removes all notification configurations for a bucket.
     /// If the configuration is successfully removed, the entire notification system will be automatically reloaded.
     ///

crates/obs/src/telemetry/isolation_probe_tests.rs

@@ -1,680 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Test-only diagnostic isolation prototype. Nothing here is a GET hook.
-
-use super::local::build_json_log_layer;
-use opentelemetry::{
-    Context, InstrumentationScope, KeyValue,
-    baggage::BaggageExt,
-    logs::AnyValue,
-    trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState},
-};
-use opentelemetry_appender_tracing::layer::{OpenTelemetryTracingBridge, TracingSpanAttributes};
-use opentelemetry_sdk::{
-    Resource,
-    error::OTelSdkResult,
-    logs::{LogBatch, LogExporter, SdkLogRecord, SdkLoggerProvider},
-};
-use serde_json::Value;
-use std::{
-    collections::BTreeSet,
-    io::{self, Write},
-    sync::{Arc, Mutex},
-};
-use tracing::{Dispatch, dispatcher};
-use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt};
-
-const TARGET: &str = "rustfs_isolation_probe_test";
-const FAKE_URI: &str = "/synthetic/private-object";
-const FAKE_AUTH: &str = "synthetic-private-header";
-
-#[derive(Clone, Debug, Default)]
-struct MemoryWriter(Arc<Mutex<Vec<u8>>>);
-
-impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for MemoryWriter {
-    type Writer = Self;
-
-    fn make_writer(&'a self) -> Self::Writer {
-        self.clone()
-    }
-}
-
-impl Write for MemoryWriter {
-    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
-        self.0.lock().expect("capture JSON output").extend_from_slice(bytes);
-        Ok(bytes.len())
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[derive(Clone, Debug, Default)]
-struct MemoryExporter {
-    records: Arc<Mutex<Vec<(SdkLogRecord, InstrumentationScope)>>>,
-    resource: Arc<Mutex<Option<Resource>>>,
-}
-
-impl LogExporter for MemoryExporter {
-    async fn export(&self, batch: LogBatch<'_>) -> OTelSdkResult {
-        self.records
-            .lock()
-            .expect("capture exporter records")
-            .extend(batch.iter().map(|(record, scope)| (record.clone(), scope.clone())));
-        Ok(())
-    }
-
-    fn set_resource(&mut self, resource: &Resource) {
-        *self.resource.lock().expect("capture exporter resource") = Some(resource.clone());
-    }
-}
-
-struct Sinks {
-    dispatch: Dispatch,
-    json: MemoryWriter,
-    exporter: MemoryExporter,
-    _provider: SdkLoggerProvider,
-}
-
-impl Sinks {
-    fn new(span_list: bool, enrich_spans: bool) -> Self {
-        let json = MemoryWriter::default();
-        let exporter = MemoryExporter::default();
-        let provider = SdkLoggerProvider::builder()
-            .with_resource(Resource::builder_empty().build())
-            .with_simple_exporter(exporter.clone())
-            .build();
-        let bridge = OpenTelemetryTracingBridge::builder(&provider);
-        let bridge = if enrich_spans {
-            bridge.with_tracing_span_attributes(TracingSpanAttributes::all()).build()
-        } else {
-            bridge.build()
-        };
-        let subscriber = tracing_subscriber::registry()
-            .with(build_json_log_layer(json.clone(), false, FmtSpan::NONE, span_list))
-            .with(bridge);
-        Self {
-            dispatch: Dispatch::new(subscriber),
-            json,
-            exporter,
-            _provider: provider,
-        }
-    }
-
-    fn json_records(&self) -> Vec<Value> {
-        let bytes = self.json.0.lock().expect("capture JSON output").clone();
-        let text = String::from_utf8(bytes).expect("JSON output is UTF-8");
-        text.lines()
-            .map(|line| serde_json::from_str(line).expect("each captured record is JSON"))
-            .collect()
-    }
-
-    fn exported_records(&self) -> Vec<(SdkLogRecord, InstrumentationScope)> {
-        self.exporter.records.lock().expect("capture exporter records").clone()
-    }
-}
-
-fn fake_ambient_context() -> Context {
-    Context::new()
-        .with_remote_span_context(SpanContext::new(
-            TraceId::from(17),
-            SpanId::from(19),
-            TraceFlags::SAMPLED,
-            true,
-            TraceState::default(),
-        ))
-        .with_baggage([KeyValue::new("synthetic-secret", FAKE_AUTH)])
-}
-
-fn emit_root_event(phase: u64) {
-    tracing::info!(target: TARGET, parent: None, expected = 1024_u64, emitted = 64_u64, phase, "diagnostic fixture");
-}
-
-// This prototype is synchronous and must use a fresh, span-free dispatcher.
-fn emit_candidate(dispatch: &Dispatch, phase: u64) {
-    let _context_guard = Context::new().attach();
-    dispatcher::with_default(dispatch, || {
-        assert!(
-            Context::current().baggage().get("synthetic-secret").is_none(),
-            "ambient baggage must be hidden"
-        );
-        assert!(!Context::current().has_active_span(), "ambient OTEL span must be hidden");
-        emit_root_event(phase);
-    });
-}
-
-#[test]
-fn parent_none_alone_keeps_json_parent_and_otel_thread_context() {
-    let sinks = Sinks::new(true, true);
-    let _context_guard = fake_ambient_context().attach();
-
-    dispatcher::with_default(&sinks.dispatch, || {
-        let span = tracing::info_span!("http-request", uri = FAKE_URI, authorization = FAKE_AUTH, request_id = "synthetic-id");
-        let _span_guard = span.enter();
-        emit_root_event(0);
-    });
-
-    let json = sinks.json_records();
-    let logs = sinks.exported_records();
-    assert_eq!(json.len(), 1, "the control event must be captured");
-    assert_eq!(logs.len(), 1, "the control event must reach the exporter");
-    assert_eq!(json[0]["span"]["uri"].as_str(), Some(FAKE_URI));
-    assert_eq!(json[0]["span"]["authorization"].as_str(), Some(FAKE_AUTH));
-    let context = logs[0].0.trace_context().expect("ambient OTEL trace context is present");
-    assert_eq!(context.trace_id, TraceId::from(17));
-    assert_eq!(context.span_id, SpanId::from(19));
-}
-
-#[test]
-fn diagnostic_fixture_is_detached_for_both_sinks_and_restores_context() {
-    for span_list in [false, true] {
-        for enrich_spans in [false, true] {
-            let outer = Sinks::new(span_list, enrich_spans);
-            let isolated = Sinks::new(span_list, enrich_spans);
-            let _context_guard = fake_ambient_context().attach();
-
-            dispatcher::with_default(&outer.dispatch, || {
-                let span =
-                    tracing::info_span!("http-request", uri = FAKE_URI, authorization = FAKE_AUTH, request_id = "synthetic-id");
-                let _span_guard = span.enter();
-                for phase in [0, 1] {
-                    emit_candidate(&isolated.dispatch, phase);
-                    assert_eq!(tracing::Span::current().id(), span.id(), "restore the outer tracing span");
-                    assert_eq!(Context::current().span().span_context().trace_id(), TraceId::from(17));
-                    assert!(Context::current().baggage().get("synthetic-secret").is_some());
-                }
-            });
-
-            assert!(outer.json_records().is_empty(), "diagnostic events must not reach the outer JSON sink");
-            assert!(outer.exported_records().is_empty(), "diagnostic events must not reach the outer exporter");
-            let json = isolated.json_records();
-            let logs = isolated.exported_records();
-            assert_eq!(json.len(), 2, "both diagnostic events must reach the isolated JSON sink");
-            assert_eq!(logs.len(), 2, "both diagnostic events must reach the isolated exporter");
-
-            let json_keys: BTreeSet<&str> = [
-                "timestamp",
-                "level",
-                "target",
-                "filename",
-                "line_number",
-                "threadName",
-                "threadId",
-                "message",
-                "expected",
-                "emitted",
-                "phase",
-            ]
-            .into_iter()
-            .collect();
-            let otel_keys: BTreeSet<&str> = [
-                "code.namespace",
-                "code.filepath",
-                "code.filename",
-                "code.lineno",
-                "expected",
-                "emitted",
-                "phase",
-            ]
-            .into_iter()
-            .collect();
-            let resource = isolated.exporter.resource.lock().expect("capture exporter resource");
-            assert!(
-                resource
-                    .as_ref()
-                    .expect("resource was passed to exporter")
-                    .iter()
-                    .next()
-                    .is_none()
-            );
-            for (phase, (entry, (log, scope))) in [0_u64, 1].into_iter().zip(json.iter().zip(&logs)) {
-                let object = entry.as_object().expect("diagnostic JSON must be an object");
-                assert!(object.keys().all(|key| json_keys.contains(key.as_str())), "unexpected JSON field");
-                assert_eq!(entry["target"].as_str(), Some(TARGET));
-                assert_eq!(entry["phase"].as_u64(), Some(phase));
-                assert_eq!(entry["expected"].as_u64(), Some(1024));
-                assert_eq!(entry["emitted"].as_u64(), Some(64));
-                assert!(!entry.to_string().contains(FAKE_URI));
-                assert!(!entry.to_string().contains(FAKE_AUTH));
-
-                assert_eq!(log.target().map(|target| target.as_ref()), Some(TARGET));
-                assert!(log.trace_context().is_none(), "ambient OTEL context must not reach the exporter");
-                assert_eq!(scope.name(), "");
-                assert!(scope.version().is_none());
-                assert!(scope.schema_url().is_none());
-                assert!(scope.attributes().next().is_none());
-                assert!(
-                    log.attributes_iter().all(|(key, _)| otel_keys.contains(key.as_str())),
-                    "unexpected OTEL field"
-                );
-                let phase = i64::try_from(phase).expect("fixture phase fits into i64");
-                assert!(
-                    log.attributes_iter()
-                        .any(|(key, value)| key.as_str() == "phase" && value == &AnyValue::Int(phase))
-                );
-                assert!(
-                    log.attributes_iter()
-                        .any(|(key, value)| key.as_str() == "expected" && value == &AnyValue::Int(1024))
-                );
-                assert!(
-                    log.attributes_iter()
-                        .any(|(key, value)| key.as_str() == "emitted" && value == &AnyValue::Int(64))
-                );
-                assert!(!format!("{log:?}").contains(FAKE_URI));
-                assert!(!format!("{log:?}").contains(FAKE_AUTH));
-            }
-        }
-    }
-}
-
-mod async_isolation {
-    use super::*;
-    use std::{
-        collections::HashSet,
-        future::pending,
-        panic::{AssertUnwindSafe, catch_unwind},
-        sync::atomic::{AtomicBool, AtomicUsize, Ordering},
-        time::Duration,
-    };
-    use tracing::{Instrument, instrument::WithSubscriber};
-    use tracing_subscriber::EnvFilter;
-
-    const PARTICIPANTS: usize = 4;
-    const WAIT: Duration = Duration::from_secs(30);
-
-    fn parent_span(dispatch: &Dispatch) -> tracing::Span {
-        dispatcher::with_default(dispatch, || {
-            tracing::info_span!("http-request", uri = FAKE_URI, authorization = FAKE_AUTH, request_id = "synthetic-id")
-        })
-    }
-
-    fn observe_once(dispatch: &Dispatch, parent: &Option<tracing::span::Id>, phase: u64) {
-        let _context = fake_ambient_context().attach();
-        assert_eq!(&tracing::Span::current().id(), parent);
-        emit_candidate(dispatch, phase);
-        assert_eq!(&tracing::Span::current().id(), parent);
-        assert_eq!(Context::current().span().span_context().trace_id(), TraceId::from(17));
-        assert!(Context::current().baggage().get("synthetic-secret").is_some());
-    }
-
-    fn assert_isolated_events(sinks: &Sinks, expected_phases: &[u64]) {
-        let json = sinks.json_records();
-        let logs = sinks.exported_records();
-        assert_eq!(json.len(), expected_phases.len(), "each event must reach the isolated JSON sink");
-        assert_eq!(logs.len(), expected_phases.len(), "each event must reach the isolated exporter");
-        let json_keys: BTreeSet<&str> = [
-            "timestamp",
-            "level",
-            "target",
-            "filename",
-            "line_number",
-            "threadName",
-            "threadId",
-            "message",
-            "expected",
-            "emitted",
-            "phase",
-        ]
-        .into_iter()
-        .collect();
-        let otel_keys: BTreeSet<&str> = [
-            "code.namespace",
-            "code.filepath",
-            "code.filename",
-            "code.lineno",
-            "expected",
-            "emitted",
-            "phase",
-        ]
-        .into_iter()
-        .collect();
-        let mut json_phases = Vec::new();
-        for entry in &json {
-            let object = entry.as_object().expect("diagnostic JSON must be an object");
-            assert!(object.keys().all(|key| json_keys.contains(key.as_str())), "unexpected JSON field");
-            assert_eq!(entry["target"].as_str(), Some(TARGET));
-            assert_eq!(entry["message"].as_str(), Some("diagnostic fixture"));
-            assert_eq!(entry["expected"].as_u64(), Some(1024));
-            assert_eq!(entry["emitted"].as_u64(), Some(64));
-            json_phases.push(entry["phase"].as_u64().expect("phase is an unsigned number"));
-            assert!(!entry.to_string().contains(FAKE_URI));
-            assert!(!entry.to_string().contains(FAKE_AUTH));
-        }
-        let mut otel_phases = Vec::new();
-        for (log, scope) in &logs {
-            assert_eq!(log.target().map(|target| target.as_ref()), Some(TARGET));
-            assert!(log.trace_context().is_none());
-            assert!(log.attributes_iter().all(|(key, _)| otel_keys.contains(key.as_str())));
-            assert!(matches!(log.body(), Some(AnyValue::String(body)) if body.as_str() == "diagnostic fixture"));
-            assert!(
-                log.attributes_iter()
-                    .any(|(key, value)| key.as_str() == "expected" && value == &AnyValue::Int(1024))
-            );
-            assert!(
-                log.attributes_iter()
-                    .any(|(key, value)| key.as_str() == "emitted" && value == &AnyValue::Int(64))
-            );
-            let phase = log
-                .attributes_iter()
-                .find_map(|(key, value)| match (key.as_str(), value) {
-                    ("phase", AnyValue::Int(phase)) => u64::try_from(*phase).ok(),
-                    _ => None,
-                })
-                .expect("phase is a nonnegative OTEL integer");
-            otel_phases.push(phase);
-            assert_eq!(scope.name(), "");
-            assert!(scope.version().is_none());
-            assert!(scope.schema_url().is_none());
-            assert!(scope.attributes().next().is_none());
-            assert!(!format!("{log:?}").contains(FAKE_URI));
-            assert!(!format!("{log:?}").contains(FAKE_AUTH));
-        }
-        json_phases.sort_unstable();
-        otel_phases.sort_unstable();
-        let mut expected_phases = expected_phases.to_vec();
-        expected_phases.sort_unstable();
-        assert_eq!(json_phases, expected_phases);
-        assert_eq!(otel_phases, expected_phases);
-        let resource = sinks.exporter.resource.lock().expect("capture exporter resource");
-        assert!(resource.as_ref().expect("exporter resource exists").iter().next().is_none());
-    }
-
-    #[test]
-    fn separate_threads_keep_their_contexts_across_async_suspension() {
-        let outer = Sinks::new(true, true);
-        let isolated = Sinks::new(true, true);
-        let barrier = Arc::new(tokio::sync::Barrier::new(PARTICIPANTS));
-        let thread_ids: HashSet<_> = std::thread::scope(|scope| {
-            let mut handles = Vec::new();
-            for _ in 0..PARTICIPANTS {
-                let outer_dispatch = outer.dispatch.clone();
-                let isolated_dispatch = isolated.dispatch.clone();
-                let barrier = Arc::clone(&barrier);
-                handles.push(scope.spawn(move || {
-                    let thread_id = std::thread::current().id();
-                    let span = parent_span(&outer_dispatch);
-                    let span_id = span.id();
-                    let runtime = tokio::runtime::Builder::new_current_thread()
-                        .enable_all()
-                        .build()
-                        .expect("build a test-only runtime");
-                    runtime.block_on(
-                        async move {
-                            for phase in [0_u64, 1] {
-                                tokio::time::timeout(WAIT, barrier.wait())
-                                    .await
-                                    .expect("all threads reach the barrier");
-                                observe_once(&isolated_dispatch, &span_id, phase);
-                                tokio::task::yield_now().await;
-                            }
-                        }
-                        .instrument(span)
-                        .with_subscriber(outer_dispatch),
-                    );
-                    thread_id
-                }));
-            }
-            handles
-                .into_iter()
-                .map(|handle| handle.join().expect("thread completes"))
-                .collect()
-        });
-        assert_eq!(thread_ids.len(), PARTICIPANTS, "the fixture must use distinct OS threads");
-        assert!(outer.json_records().is_empty());
-        assert!(outer.exported_records().is_empty());
-        assert_isolated_events(&isolated, &[0, 1, 0, 1, 0, 1, 0, 1]);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn shared_runtime_reenters_outer_context_after_each_await() {
-        let outer = Sinks::new(true, true);
-        let isolated = Sinks::new(true, true);
-        let barrier = Arc::new(tokio::sync::Barrier::new(PARTICIPANTS));
-        let mut tasks = Vec::new();
-        for _ in 0..PARTICIPANTS {
-            let outer_dispatch = outer.dispatch.clone();
-            let isolated_dispatch = isolated.dispatch.clone();
-            let barrier = Arc::clone(&barrier);
-            let span = parent_span(&outer_dispatch);
-            let span_id = span.id();
-            tasks.push(tokio::spawn(
-                async move {
-                    for phase in [0_u64, 1] {
-                        tokio::time::timeout(WAIT, barrier.wait())
-                            .await
-                            .expect("all tasks reach the barrier");
-                        observe_once(&isolated_dispatch, &span_id, phase);
-                        tokio::task::yield_now().await;
-                    }
-                }
-                .instrument(span)
-                .with_subscriber(outer_dispatch),
-            ));
-        }
-        for task in tasks {
-            tokio::time::timeout(WAIT, task)
-                .await
-                .expect("task finishes")
-                .expect("task succeeds");
-        }
-        assert!(outer.json_records().is_empty());
-        assert!(outer.exported_records().is_empty());
-        assert_isolated_events(&isolated, &[0, 1, 0, 1, 0, 1, 0, 1]);
-    }
-
-    #[test]
-    fn either_isolation_step_alone_leaves_ambient_context() {
-        let outer = Sinks::new(true, true);
-        let separate = Sinks::new(true, true);
-        dispatcher::with_default(&outer.dispatch, || {
-            let span = parent_span(&outer.dispatch);
-            let _span = span.enter();
-            let _context = fake_ambient_context().attach();
-            dispatcher::with_default(&separate.dispatch, || emit_root_event(0));
-            let _empty_context = Context::new().attach();
-            emit_root_event(1);
-        });
-
-        let separate_json = separate.json_records();
-        let separate_logs = separate.exported_records();
-        assert_eq!(separate_json.len(), 1);
-        assert_eq!(separate_logs.len(), 1);
-        assert!(separate_json[0].get("span").is_none());
-        assert_eq!(
-            separate_logs[0]
-                .0
-                .trace_context()
-                .expect("ambient trace still exists")
-                .trace_id,
-            TraceId::from(17)
-        );
-
-        let outer_json = outer.json_records();
-        let outer_logs = outer.exported_records();
-        assert_eq!(outer_json.len(), 1);
-        assert_eq!(outer_logs.len(), 1);
-        assert_eq!(outer_json[0]["span"]["uri"].as_str(), Some(FAKE_URI));
-        assert_eq!(outer_json[0]["span"]["authorization"].as_str(), Some(FAKE_AUTH));
-        assert!(outer_logs[0].0.trace_context().is_none());
-    }
-
-    fn filtered_sinks(filter: &str, sensitive_resource: bool) -> Sinks {
-        let json = MemoryWriter::default();
-        let exporter = MemoryExporter::default();
-        let resource = if sensitive_resource {
-            Resource::builder_empty()
-                .with_attributes([KeyValue::new("synthetic.resource", FAKE_AUTH)])
-                .build()
-        } else {
-            Resource::builder_empty().build()
-        };
-        let provider = SdkLoggerProvider::builder()
-            .with_resource(resource)
-            .with_simple_exporter(exporter.clone())
-            .build();
-        let bridge = OpenTelemetryTracingBridge::builder(&provider)
-            .with_tracing_span_attributes(TracingSpanAttributes::all())
-            .build();
-        let subscriber = tracing_subscriber::registry()
-            .with(EnvFilter::new(filter))
-            .with(build_json_log_layer(json.clone(), false, FmtSpan::NONE, true))
-            .with(bridge);
-        Sinks {
-            dispatch: Dispatch::new(subscriber),
-            json,
-            exporter,
-            _provider: provider,
-        }
-    }
-
-    #[test]
-    fn filters_and_resources_do_not_cross_dispatchers() {
-        let outer = filtered_sinks("info,rustfs_isolation_probe_test=off", true);
-        let isolated = filtered_sinks("off,rustfs_isolation_probe_test=info", false);
-        let disabled = filtered_sinks("off", false);
-        dispatcher::with_default(&outer.dispatch, || {
-            let span = parent_span(&outer.dispatch);
-            let _span = span.enter();
-            let parent_id = span.id();
-            observe_once(&isolated.dispatch, &parent_id, 0);
-            observe_once(&disabled.dispatch, &parent_id, 1);
-            tracing::info!(target: "isolation_outer_control", value = 1_u64, "outer control");
-        });
-        let outer_json = outer.json_records();
-        let outer_logs = outer.exported_records();
-        assert_eq!(outer_json.len(), 1);
-        assert_eq!(outer_logs.len(), 1);
-        assert_eq!(outer_json[0]["target"].as_str(), Some("isolation_outer_control"));
-        assert_eq!(outer_logs[0].0.target().map(|target| target.as_ref()), Some("isolation_outer_control"));
-        assert!(
-            outer
-                .exporter
-                .resource
-                .lock()
-                .expect("capture outer resource")
-                .as_ref()
-                .expect("outer resource exists")
-                .get(&opentelemetry::Key::new("synthetic.resource"))
-                .is_some()
-        );
-        assert_isolated_events(&isolated, &[0]);
-        assert!(disabled.json_records().is_empty());
-        assert!(disabled.exported_records().is_empty());
-    }
-
-    struct CancellationProbe {
-        outer: Dispatch,
-        isolated: Dispatch,
-        drops: Arc<AtomicUsize>,
-        restored: Arc<AtomicBool>,
-    }
-
-    impl Drop for CancellationProbe {
-        fn drop(&mut self) {
-            dispatcher::with_default(&self.outer, || {
-                let span = parent_span(&self.outer);
-                let _span = span.enter();
-                let _context = fake_ambient_context().attach();
-                emit_candidate(&self.isolated, 1);
-                let restored = tracing::Span::current().id() == span.id()
-                    && Context::current().span().span_context().trace_id() == TraceId::from(17)
-                    && Context::current().baggage().get("synthetic-secret").is_some();
-                self.restored.store(restored, Ordering::SeqCst);
-            });
-            self.drops.fetch_add(1, Ordering::SeqCst);
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn cancellation_drop_emits_in_isolation_and_restores_context() {
-        let outer = Sinks::new(true, true);
-        let isolated = Sinks::new(true, true);
-        let drops = Arc::new(AtomicUsize::new(0));
-        let restored = Arc::new(AtomicBool::new(false));
-        let probe = CancellationProbe {
-            outer: outer.dispatch.clone(),
-            isolated: isolated.dispatch.clone(),
-            drops: Arc::clone(&drops),
-            restored: Arc::clone(&restored),
-        };
-        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
-        let task = tokio::spawn(async move {
-            let _probe = probe;
-            started_tx.send(()).expect("signal that the cancellation fixture is active");
-            pending::<()>().await;
-        });
-        tokio::time::timeout(WAIT, started_rx)
-            .await
-            .expect("task starts")
-            .expect("start signal");
-        task.abort();
-        let error = tokio::time::timeout(WAIT, task)
-            .await
-            .expect("cancelled task finishes")
-            .expect_err("the task must be cancelled");
-        assert!(error.is_cancelled());
-        assert_eq!(drops.load(Ordering::SeqCst), 1);
-        assert!(restored.load(Ordering::SeqCst));
-        assert!(outer.json_records().is_empty());
-        assert!(outer.exported_records().is_empty());
-        assert_isolated_events(&isolated, &[1]);
-    }
-
-    #[derive(Clone)]
-    struct PanicWriter;
-
-    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for PanicWriter {
-        type Writer = Self;
-
-        fn make_writer(&'a self) -> Self::Writer {
-            self.clone()
-        }
-    }
-
-    impl Write for PanicWriter {
-        fn write(&mut self, _: &[u8]) -> io::Result<usize> {
-            panic!("synthetic writer panic");
-        }
-
-        fn flush(&mut self) -> io::Result<()> {
-            Ok(())
-        }
-    }
-
-    #[test]
-    fn panic_during_sink_write_restores_the_outer_context() {
-        let outer = Sinks::new(true, true);
-        let panic_dispatch =
-            Dispatch::new(tracing_subscriber::registry().with(build_json_log_layer(PanicWriter, false, FmtSpan::NONE, true)));
-        dispatcher::with_default(&outer.dispatch, || {
-            let span = parent_span(&outer.dispatch);
-            let _span = span.enter();
-            let _context = fake_ambient_context().attach();
-            let panic = catch_unwind(AssertUnwindSafe(|| emit_candidate(&panic_dispatch, 0)));
-            assert!(panic.is_err(), "the fixture must panic while writing the isolated event");
-            assert_eq!(tracing::Span::current().id(), span.id());
-            assert_eq!(Context::current().span().span_context().trace_id(), TraceId::from(17));
-            assert!(Context::current().baggage().get("synthetic-secret").is_some());
-            tracing::info!(target: "isolation_outer_control", value = 1_u64, "outer control");
-        });
-        let json = outer.json_records();
-        let logs = outer.exported_records();
-        assert_eq!(json.len(), 1);
-        assert_eq!(logs.len(), 1);
-        assert_eq!(json[0]["target"].as_str(), Some("isolation_outer_control"));
-        assert_eq!(logs[0].0.target().map(|target| target.as_ref()), Some("isolation_outer_control"));
-    }
-}

crates/obs/src/telemetry/mod.rs

@@ -43,8 +43,6 @@
 pub mod dial9;
 mod filter;
 mod guard;
-#[cfg(test)]
-mod isolation_probe_tests;
 mod local;
 mod otel;
 mod recorder;

crates/policy/src/policy/action.rs

@@ -568,6 +568,24 @@ pub enum AdminAction {
     GetTableLifecycleAction,
     #[strum(serialize = "admin:SetTableLifecycle")]
     SetTableLifecycleAction,
+    #[strum(serialize = "admin:CreatePluginDelegation")]
+    CreatePluginDelegationAction,
+    #[strum(serialize = "admin:IssuePluginAttempt")]
+    IssuePluginAttemptAction,
+    #[strum(serialize = "admin:SubmitPluginJob")]
+    SubmitPluginJobAction,
+    #[strum(serialize = "admin:GetPluginRun")]
+    GetPluginRunAction,
+    #[strum(serialize = "admin:CancelPluginRun")]
+    CancelPluginRunAction,
+    #[strum(serialize = "admin:ListPluginRuns")]
+    ListPluginRunsAction,
+    #[strum(serialize = "admin:ReplayPluginRun")]
+    ReplayPluginRunAction,
+    #[strum(serialize = "admin:GetPluginDelegation")]
+    GetPluginDelegationAction,
+    #[strum(serialize = "admin:RevokePluginDelegation")]
+    RevokePluginDelegationAction,
     #[strum(serialize = "admin:GetTableCredentials")]
     GetTableCredentialsAction,
     #[strum(serialize = "admin:RunTableMaintenance")]
@@ -713,6 +731,15 @@ impl AdminAction {
                 | AdminAction::GetTableLifecycleAction
                 | AdminAction::SetTableLifecycleAction
                 | AdminAction::GetTableCredentialsAction
+                | AdminAction::IssuePluginAttemptAction
+                | AdminAction::SubmitPluginJobAction
+                | AdminAction::GetPluginRunAction
+                | AdminAction::CancelPluginRunAction
+                | AdminAction::ListPluginRunsAction
+                | AdminAction::ReplayPluginRunAction
+                | AdminAction::GetPluginDelegationAction
+                | AdminAction::CreatePluginDelegationAction
+                | AdminAction::RevokePluginDelegationAction
                 | AdminAction::RunTableMaintenanceAction
                 | AdminAction::GetTableMetadataLocationAction
                 | AdminAction::SetTableMetadataLocationAction
@@ -913,6 +940,35 @@ mod tests {
         assert_eq!(arr[0].as_str().unwrap(), "s3:*");
     }
 
+    #[test]
+    fn plugin_delegation_actions_are_explicit_and_unscoped() {
+        for (name, expected) in [
+            ("admin:GetPluginDelegation", AdminAction::GetPluginDelegationAction),
+            ("admin:CreatePluginDelegation", AdminAction::CreatePluginDelegationAction),
+            ("admin:RevokePluginDelegation", AdminAction::RevokePluginDelegationAction),
+            ("admin:SubmitPluginJob", AdminAction::SubmitPluginJobAction),
+            ("admin:GetPluginRun", AdminAction::GetPluginRunAction),
+            ("admin:CancelPluginRun", AdminAction::CancelPluginRunAction),
+            ("admin:ListPluginRuns", AdminAction::ListPluginRunsAction),
+            ("admin:ReplayPluginRun", AdminAction::ReplayPluginRunAction),
+        ] {
+            assert_eq!(AdminAction::try_from(name).unwrap(), expected);
+            assert!(expected.is_valid());
+            assert!(!expected.is_table_resource_scoped());
+        }
+        assert_eq!(
+            AdminAction::try_from("admin:IssuePluginAttempt").unwrap(),
+            AdminAction::IssuePluginAttemptAction
+        );
+        assert!(AdminAction::IssuePluginAttemptAction.is_valid());
+        assert!(AdminAction::SubmitPluginJobAction.is_valid());
+        assert!(AdminAction::GetPluginRunAction.is_valid());
+        assert!(AdminAction::CancelPluginRunAction.is_valid());
+        assert!(AdminAction::ListPluginRunsAction.is_valid());
+        assert!(AdminAction::ReplayPluginRunAction.is_valid());
+        assert!(AdminAction::try_from("admin:IssuePluginCredentials").is_err());
+    }
+
     #[test]
     fn test_get_replication_metrics_admin_action_is_valid() {
         assert!(AdminAction::GetReplicationMetricsAction.is_valid());

crates/scanner/src/error.rs

@@ -53,6 +53,10 @@ pub enum ScannerError {
     #[error("Remote namespace scanner disk is already active")]
     RemoteDiskBusy,
 
+    /// Node resource pressure deferred work before its first storage poll.
+    #[error("Scanner node resource admission deferred")]
+    ResourceDeferred,
+
     /// Partial data usage cache produced before the scanner stopped.
     #[error("Scanner stopped with partial data usage cache")]
     PartialCache(Box<DataUsageCache>),

crates/scanner/src/lib.rs

@@ -50,9 +50,9 @@ use storage_api::owner::{
 };
 #[cfg(test)]
 use storage_api::owner::{
-    EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreInstanceContext,
-    EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
-    ecstore_new_disk,
+    EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreExpiryState,
+    EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
+    ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
 };
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
@@ -123,7 +123,7 @@ pub use storage_api::scan::{
     SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
     SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
 };
-pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
+pub use workload_admission::{set_scanner_node_admission, set_scanner_workload_admission_snapshot_provider};
 
 static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
 static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
@@ -678,8 +678,37 @@ fn reset_tier_name_cache_for_test() {
     TIER_REGISTRY_GENERATION.store(0, Ordering::Relaxed);
 }
 
-pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo) {
-    ecstore_expiry_state_handle().write().await.enqueue_free_version(oi);
+// Only a task running an isolated unit test can override these handles.
+#[cfg(test)]
+struct ExpiryLockTestContext {
+    state: Arc<tokio::sync::RwLock<EcstoreExpiryState>>,
+    store: Option<Arc<ECStore>>,
+    completed_incarnation_reads: Arc<std::sync::atomic::AtomicUsize>,
+}
+
+#[cfg(test)]
+tokio::task_local! {
+    static EXPIRY_LOCK_TEST_CONTEXT: ExpiryLockTestContext;
+}
+
+pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo, cancellation: Option<&CancellationToken>) {
+    #[cfg(test)]
+    let state = EXPIRY_LOCK_TEST_CONTEXT
+        .try_with(|context| context.state.clone())
+        .unwrap_or_else(|_| ecstore_expiry_state_handle());
+    #[cfg(not(test))]
+    let state = ecstore_expiry_state_handle();
+    let state = match cancellation {
+        Some(cancel) => tokio::select! {
+            biased;
+            _ = cancel.cancelled() => return,
+            state = state.write() => state,
+        },
+        None => state.write().await,
+    };
+    if !cancellation.is_some_and(CancellationToken::is_cancelled) {
+        state.enqueue_free_version(oi);
+    }
 }
 
 pub(crate) async fn enqueue_runtime_newer_noncurrent(
@@ -687,20 +716,50 @@ pub(crate) async fn enqueue_runtime_newer_noncurrent(
     to_delete_objs: Vec<ObjectToDelete>,
     event: Event,
     src: &LcEventSrc,
+    cancellation: Option<&CancellationToken>,
 ) -> bool {
-    let Some(store) = ecstore_resolve_object_store_handle() else {
+    if cancellation.is_some_and(CancellationToken::is_cancelled) {
+        return false;
+    }
+    #[cfg(test)]
+    let store = EXPIRY_LOCK_TEST_CONTEXT
+        .try_with(|context| context.store.clone())
+        .unwrap_or_else(|_| ecstore_resolve_object_store_handle());
+    #[cfg(not(test))]
+    let store = ecstore_resolve_object_store_handle();
+    let Some(store) = store else {
         return false;
     };
     let Ok(bucket_incarnation_id) = store.bucket_incarnation_id(bucket).await else {
         return false;
     };
-    ecstore_expiry_state_handle().write().await.enqueue_by_newer_noncurrent(
-        bucket,
-        to_delete_objs,
-        event,
-        src,
-        bucket_incarnation_id,
-    )
+    #[cfg(test)]
+    let _ = EXPIRY_LOCK_TEST_CONTEXT.try_with(|context| {
+        context
+            .completed_incarnation_reads
+            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+    });
+    if cancellation.is_some_and(CancellationToken::is_cancelled) {
+        return false;
+    }
+    #[cfg(test)]
+    let state = EXPIRY_LOCK_TEST_CONTEXT
+        .try_with(|context| context.state.clone())
+        .unwrap_or_else(|_| ecstore_expiry_state_handle());
+    #[cfg(not(test))]
+    let state = ecstore_expiry_state_handle();
+    let mut state = match cancellation {
+        Some(cancel) => tokio::select! {
+            biased;
+            _ = cancel.cancelled() => return false,
+            state = state.write() => state,
+        },
+        None => state.write().await,
+    };
+    if cancellation.is_some_and(CancellationToken::is_cancelled) {
+        return false;
+    }
+    state.enqueue_by_newer_noncurrent(bucket, to_delete_objs, event, src, bucket_incarnation_id)
 }
 
 pub(crate) async fn queue_replication_heal(

crates/scanner/src/remote_scanner/stream.rs

@@ -705,7 +705,7 @@ async fn scan_and_persist_local_bucket(
         RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement")
     })?;
     let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
-    let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
+    let mut guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
         .await
         .map_err(|err| {
             if err.is_contention() {
@@ -791,19 +791,23 @@ async fn scan_and_persist_local_bucket(
 
     let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
     let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
+    let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
     let scan_ctx = ctx.child_token();
-    let scan = ScannerIODisk::nsscanner_disk(
-        disk.clone(),
-        scan_ctx.clone(),
-        budget,
-        set_disks,
-        cache,
-        None,
-        ScannerDiskScanOptions {
-            scan_mode,
-            prefix_scan_scope: None,
-            checkpoint_tx: Some(checkpoint_tx),
-        },
+    let scan = crate::workload_admission::with_scanner_node_requests(
+        node_tx,
+        ScannerIODisk::nsscanner_disk(
+            disk.clone(),
+            scan_ctx.clone(),
+            budget,
+            set_disks,
+            cache,
+            None,
+            ScannerDiskScanOptions {
+                scan_mode,
+                prefix_scan_scope: None,
+                checkpoint_tx: Some(checkpoint_tx),
+            },
+        ),
     );
     tokio::pin!(scan);
     let fence_watch = watch_remote_scanner_request_fence(next_cycle, leader_epoch, store.clone(), NS_SCANNER_FENCE_POLL_INTERVAL);
@@ -813,6 +817,27 @@ async fn scan_and_persist_local_bucket(
     let mut checkpoint_channel_closed = false;
     let outcome = loop {
         tokio::select! {
+            request = async {
+                match node_rx.as_mut() {
+                    Some(receiver) => receiver.recv().await,
+                    None => std::future::pending().await,
+                }
+            } => {
+                let Some(request) = request else {
+                    node_rx = None;
+                    continue;
+                };
+                if let Err(error) = wait_for_remote_node_resumption(
+                    guard.resume_node_operation(set.clone(), &cache_name, source, &revisions, request),
+                    fence_watch.as_mut(),
+                ).await {
+                    scan_ctx.cancel();
+                    let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
+                    return Err(RemoteScannerServerError::retry_bucket(format!(
+                        "remote namespace scanner could not resume node admission: {error}",
+                    )));
+                }
+            }
             result = &mut scan => {
                 match result {
                     Ok(outcome) => break outcome,
@@ -1121,6 +1146,22 @@ pub(crate) async fn scan_remote_bucket(
     result
 }
 
+/// Resource waits must keep checking the persisted leader fence.
+async fn wait_for_remote_node_resumption<F, G>(resume: F, fence_watch: Pin<&mut G>) -> Result<(), StorageError>
+where
+    F: Future<Output = Result<(), StorageError>>,
+    G: Future<Output = Result<(), ScannerError>>,
+{
+    tokio::select! {
+        biased;
+        result = fence_watch => Err(match result {
+            Ok(()) => StorageError::other("remote namespace scanner fence watcher stopped unexpectedly"),
+            Err(error) => StorageError::other(error),
+        }),
+        result = resume => result,
+    }
+}
+
 fn finish_remote_scanner_stream(
     result: RemoteScannerStreamResult<RemoteScannerOutcome>,
     budget: &ScannerCycleBudget,

crates/scanner/src/remote_scanner/stream/tests.rs

@@ -1006,6 +1006,58 @@ async fn fence_watcher_stops_work_after_persisted_leader_changes() {
     assert!(err.to_string().contains("requested_epoch=9"));
 }
 
+#[tokio::test]
+async fn node_admission_wait_does_not_hide_replacement_leader() {
+    use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
+    use std::num::NonZeroU16;
+
+    let store = Arc::new(MutableCycleStateStore {
+        state: RwLock::new(crate::scanner::encode_scanner_cycle_fence_for_test(7, 9)),
+    });
+    let cache = Mutex::new(None);
+    let refresh = AsyncMutex::new(());
+    let watcher = watch_remote_scanner_request_fence_with_cache(
+        7,
+        9,
+        store.clone(),
+        Duration::from_millis(5),
+        &cache,
+        &refresh,
+        Duration::from_millis(5),
+    );
+    tokio::pin!(watcher);
+    let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one shared node unit"));
+    let storage_polled = AtomicBool::new(false);
+    {
+        let resume = async {
+            let _permit = gate.acquire().await;
+            storage_polled.store(true, Ordering::Relaxed);
+            Ok(())
+        };
+        let waiting = wait_for_remote_node_resumption(resume, watcher.as_mut());
+        tokio::pin!(waiting);
+        assert!(futures::poll!(&mut waiting).is_pending(), "no resource sample grants a node unit");
+        *store.state.write().expect("persist replacement leader") = crate::scanner::encode_scanner_cycle_fence_for_test(7, 10);
+        let error = tokio::time::timeout(Duration::from_millis(100), waiting)
+            .await
+            .expect("resource pause must not hide a changed leader")
+            .expect_err("old leader cannot resume storage work");
+        assert!(error.to_string().contains("requested_epoch=9"));
+    }
+    assert!(!storage_polled.load(Ordering::Relaxed));
+    assert_eq!(gate.snapshot().active, 0);
+    gate.update(
+        &mut FeedbackController::new(),
+        Some(FeedbackObservation {
+            generation: 1,
+            max_concurrency: 1,
+            pressure: ResourcePressure::Unknown,
+            completed_units: None,
+        }),
+    );
+    assert!(gate.try_acquire().is_some(), "failed fencing drops the old node queue ticket");
+}
+
 #[tokio::test]
 async fn concurrent_cycle_cache_misses_share_one_backend_read() {
     let request = RemoteScannerRequest(test_request(Uuid::new_v4()));

crates/scanner/src/scanner_folder.rs

@@ -43,6 +43,7 @@ use crate::storage_api::owner::{
 };
 #[cfg(test)]
 use crate::storage_api::owner::{EcstoreExpirationStatus as ExpirationStatus, EcstoreLifecycleRule as LifecycleRule};
+use crate::workload_admission::run_scanner_node_operation;
 use metrics::{counter, describe_counter};
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
 use rustfs_filemeta::{
@@ -795,6 +796,7 @@ pub struct FolderScanner {
     resolved_failed_object_paths: HashSet<String>,
 
     sleeper: DynamicSleeper,
+    node_admission: Option<Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
     // should_heal: Arc<dyn Fn() -> bool + Send + Sync>,
     disks: Vec<Arc<Disk>>,
     disks_quorum: usize,
@@ -828,6 +830,8 @@ pub struct FolderScanner {
     pending_size_reconciliation_truncated: bool,
     #[cfg(test)]
     list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
+    #[cfg(test)]
+    defer_abandoned_entries_until_finished: bool,
 }
 
 struct RawEnumerationProgress {
@@ -998,12 +1002,13 @@ async fn next_scanner_dir_entry(
     ctx: &CancellationToken,
     sleeper: &DynamicSleeper,
     entries: &mut ScannerDirectory,
+    admission: Option<&Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
 ) -> Result<Option<tokio::fs::DirEntry>, ScannerError> {
     if entries.batch_entries == SCANNER_DIRECTORY_BATCH_ENTRIES {
         wait_for_scanner_io(ctx, tokio::task::yield_now()).await?;
         entries.batch_entries = 0;
     }
-    let entry = wait_for_scanner_io(ctx, entries.next_entry()).await??;
+    let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.next_entry())).await???;
     if entry.is_some() {
         // Tokio prefetches bounded chunks. Pacing consumption also paces the
         // next refill, without charging an extra entry for EOF.
@@ -1923,7 +1928,12 @@ impl FolderScanner {
             // directory that vanished during iteration instead of failing the
             // whole parent subtree. The bucket root (no parent) still propagates
             // NotFound so namespace removal is detected by the caller.
-            let mut dir_reader = match wait_for_scanner_io(&ctx, ScannerDirectory::open(&dir_path)).await? {
+            let mut dir_reader = match wait_for_scanner_io(
+                &ctx,
+                run_scanner_node_operation(self.node_admission.as_ref(), &ctx, ScannerDirectory::open(&dir_path)),
+            )
+            .await??
+            {
                 Ok(dir_reader) => Some(dir_reader),
                 Err(e)
                     if folder.parent.is_some()
@@ -1961,7 +1971,7 @@ impl FolderScanner {
                 // Hold at most one metadata entry until a candidate reaches
                 // EOF: a later part must not leave object accounting or ILM work.
                 let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut() {
-                    let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader).await {
+                    let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader, self.node_admission.as_ref()).await {
                         Ok(Some(entry)) => entry,
                         Ok(None) => {
                             raw_enumeration_complete = true;
@@ -2072,7 +2082,12 @@ impl FolderScanner {
 
                 // Ignore entries that disappeared during traversal or hit symlink
                 // loops, but propagate other walk errors.
-                let mut entry_type = match wait_for_scanner_io(&ctx, entry.file_type()).await? {
+                let mut entry_type = match wait_for_scanner_io(
+                    &ctx,
+                    run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()),
+                )
+                .await??
+                {
                     Ok(entry_type) => entry_type,
                     Err(e) if e.kind() == ErrorKind::NotFound => {
                         debug!(
@@ -2123,7 +2138,12 @@ impl FolderScanner {
                 let mut entry_metadata_len = None;
                 if entry_type.is_symlink() {
                     wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
-                    let metadata = match wait_for_scanner_io(&ctx, tokio::fs::metadata(&file_path)).await? {
+                    let metadata = match wait_for_scanner_io(
+                        &ctx,
+                        run_scanner_node_operation(self.node_admission.as_ref(), &ctx, tokio::fs::metadata(&file_path)),
+                    )
+                    .await??
+                    {
                         Ok(metadata) => metadata,
                         Err(e) if e.kind() == ErrorKind::NotFound => {
                             debug!(
@@ -2274,22 +2294,30 @@ impl FolderScanner {
                         len
                     } else {
                         wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
-                        wait_for_scanner_io(&ctx, tokio::fs::metadata(&item.path))
-                            .await?
-                            .map(|metadata| metadata.len())
-                            .unwrap_or(0)
+                        wait_for_scanner_io(
+                            &ctx,
+                            run_scanner_node_operation(self.node_admission.as_ref(), &ctx, tokio::fs::metadata(&item.path)),
+                        )
+                        .await??
+                        .map(|metadata| metadata.len())
+                        .unwrap_or(0)
                     }
                 } else {
                     0
                 };
                 wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, budgeted_bytes)).await?;
 
-                self.object_metadata_visits.increment(1);
-                let sz = match self
-                    .local_disk
-                    .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
-                    .await
-                {
+                let size_result = run_scanner_node_operation(self.node_admission.as_ref(), &ctx, async {
+                    self.object_metadata_visits.increment(1);
+                    self.local_disk
+                        .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
+                        .await
+                })
+                .await?;
+                if self.node_admission.is_some() && ctx.is_cancelled() {
+                    return Err(ScannerError::Other("Operation cancelled".to_string()));
+                }
+                let sz = match size_result {
                     Ok(sz) => sz,
                     Err(e) => {
                         let failure_action = classify_get_size_failure(&item, &e);
@@ -2606,7 +2634,7 @@ impl FolderScanner {
                     {
                         Ok(outcome) => outcome,
                         Err(e) => {
-                            if ctx.is_cancelled() {
+                            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
                                 self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
                                     .await;
                                 return Err(e);
@@ -2847,19 +2875,23 @@ impl FolderScanner {
                 let mut seen_truncated_objects: HashSet<String> = HashSet::new();
 
                 loop {
-                    if agreed_closed && partial_closed && finished_closed {
+                    if child_ctx.is_cancelled() || (agreed_closed && partial_closed && finished_closed) {
                         break;
                     }
+                    #[cfg(test)]
+                    let defer_entries = self.defer_abandoned_entries_until_finished && !finished_closed;
+                    #[cfg(not(test))]
+                    let defer_entries = false;
 
                     select! {
-                        entry_name = agreed_rx.recv(), if !agreed_closed => {
+                        entry_name = agreed_rx.recv(), if !agreed_closed && !defer_entries => {
                             let Some(entry_name) = entry_name else {
                                 agreed_closed = true;
                                 continue;
                             };
                             (self.update_current_path)(&entry_name).await;
                         }
-                        entries = partial_rx.recv(), if !partial_closed => {
+                        entries = partial_rx.recv(), if !partial_closed && !defer_entries => {
                             let Some(entries) = entries else {
                                 partial_closed = true;
                                 continue;
@@ -2985,6 +3017,7 @@ impl FolderScanner {
                                 finished_closed = true;
                                 continue;
                             };
+                            finished_closed = true;
                             if disk_errors_are_only_missing_paths(&errs) {
                                 debug!(
                                     target: "rustfs::scanner::folder",
@@ -3006,7 +3039,9 @@ impl FolderScanner {
                                     "Scanner list_path_raw finished with disk errors"
                                 );
                             }
-                            child_ctx.cancel();
+                            // The producer awaited callback sends, not their
+                            // consumption. Drain the remaining candidate and
+                            // agreed frames before closing this child walk.
                         }
                         _ = child_ctx.cancelled() => {
                             break;
@@ -3024,7 +3059,10 @@ impl FolderScanner {
                     if into.compacted {
                         // In compacted mode child totals are accumulated directly into the parent entry.
                         let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), into));
-                        fut.await.map_err(|e| ScannerError::Other(e.to_string()))?;
+                        fut.await.map_err(|error| match error {
+                            ScannerError::ResourceDeferred => ScannerError::ResourceDeferred,
+                            other => ScannerError::Other(other.to_string()),
+                        })?;
                         self.send_update_for_entry(&this_hash, &folder.parent, into).await;
                         tokio::task::yield_now().await;
                     } else {
@@ -3043,7 +3081,7 @@ impl FolderScanner {
                             )
                             .await
                         {
-                            if ctx.is_cancelled() {
+                            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
                                 self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
                                     .await;
                                 return Err(e);
@@ -3217,6 +3255,7 @@ pub(crate) async fn scan_data_folder_scoped(
         failed_object_paths_seen: HashSet::new(),
         resolved_failed_object_paths: HashSet::new(),
         sleeper,
+        node_admission: crate::workload_admission::scanner_node_admission(),
         disks,
         disks_quorum,
         usage_updates,
@@ -3244,6 +3283,8 @@ pub(crate) async fn scan_data_folder_scoped(
         pending_size_reconciliation_truncated: false,
         #[cfg(test)]
         list_path_raw_options_observer: None,
+        #[cfg(test)]
+        defer_abandoned_entries_until_finished: false,
     };
 
     let now = FolderScanner::now_secs();
@@ -3333,7 +3374,7 @@ pub(crate) async fn scan_data_folder_scoped(
             }
         }
         Err(e) => {
-            if ctx.is_cancelled() {
+            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
                 let root_hash = hash_path(&bucket_name);
                 let root_has_progress = data_usage_root_has_progress(&root);
                 let pending_heals_changed = scanner.pending_heals_changed;
@@ -3408,6 +3449,19 @@ pub(crate) async fn scan_data_folder_scoped(
                 close_disk_guard.close().await;
                 return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
             }
+            if matches!(&e, ScannerError::ResourceDeferred) {
+                // Admission pressure is not disk failure or an empty census.
+                // Keep the previous tree when no new useful progress exists.
+                let mut partial_cache = scanner.old_cache;
+                if partial_cache.cache.is_empty() {
+                    partial_cache
+                        .cache
+                        .insert(hash_path(&bucket_name).0, DataUsageEntry::default());
+                }
+                partial_cache.info.snapshot_complete = false;
+                close_disk_guard.close().await;
+                return Err(ScannerError::PartialCache(Box::new(partial_cache)));
+            }
             close_disk_guard.close().await;
             // No useful information, return original cache
             Err(e)
@@ -3427,3 +3481,15 @@ pub(crate) use tests::checkpoint_fixture::run_checkpoint_fixture;
 
 #[cfg(test)]
 mod tests;
+
+#[cfg(test)]
+tokio::task_local! {
+    pub(crate) static TEST_SIZE_DISPATCH_BOUNDARIES: tokio::sync::mpsc::UnboundedSender<&'static str>;
+}
+
+#[cfg(test)]
+pub(crate) fn observe_test_size_dispatch_boundary(stage: &'static str) -> bool {
+    TEST_SIZE_DISPATCH_BOUNDARIES
+        .try_with(|sender| sender.send(stage).is_ok())
+        .unwrap_or(false)
+}

crates/scanner/src/scanner_folder/item_actions.rs

@@ -628,6 +628,22 @@ impl ScannerItem {
         tier_names: &[String],
         size_summary: &mut SizeSummary,
     ) {
+        self.apply_actions_cancellable(object_infos, lock_retention, versioning_config, tier_names, size_summary, None)
+            .await;
+    }
+
+    pub(crate) async fn apply_actions_cancellable(
+        &mut self,
+        object_infos: Vec<ObjectInfo>,
+        lock_retention: Option<Arc<ObjectLockConfiguration>>,
+        versioning_config: VersioningConfiguration,
+        tier_names: &[String],
+        size_summary: &mut SizeSummary,
+        cancellation: Option<&CancellationToken>,
+    ) {
+        if cancellation.is_some_and(CancellationToken::is_cancelled) {
+            return;
+        }
         let object_path = self.object_path();
         if object_infos.is_empty() {
             debug!(
@@ -671,6 +687,9 @@ impl ScannerItem {
         let Some(lifecycle) = self.lifecycle.clone() else {
             let mut cumulative_size: i64 = 0;
             for (oi, resolved_size) in object_infos.iter().zip(resolved_sizes.iter()) {
+                if cancellation.is_some_and(CancellationToken::is_cancelled) {
+                    return;
+                }
                 let accounting_size = match resolved_size {
                     SizeResolution::Known { logical, .. } => *logical,
                     // A valid compressed legacy sentinel has no logical size,
@@ -678,7 +697,7 @@ impl ScannerItem {
                     // physical size is only an input to those operations; it
                     // is not folded into the logical total below.
                     SizeResolution::Unknown { physical, .. } => {
-                        self.heal_actions(oi, *physical, size_summary).await;
+                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
                         size_summary.actions_accounting_unknown(oi);
                         continue;
                     }
@@ -688,13 +707,16 @@ impl ScannerItem {
                     }
                 };
 
-                let size = self.heal_actions(oi, accounting_size, size_summary).await;
+                let size = self.heal_actions(oi, accounting_size, size_summary, cancellation).await;
 
                 size_summary.actions_accounting(oi, size, accounting_size);
 
                 cumulative_size = cumulative_size.saturating_add(size);
             }
 
+            if cancellation.is_some_and(CancellationToken::is_cancelled) {
+                return;
+            }
             self.alert_excessive_versions(object_infos.len(), cumulative_size);
 
             debug!(
@@ -736,6 +758,10 @@ impl ScannerItem {
             }
         };
 
+        if cancellation.is_some_and(CancellationToken::is_cancelled) {
+            return;
+        }
+
         // Every version handed to the evaluator counts as an ILM-checked
         // version, whether or not a rule matched (NoneAction included). This is
         // reached only for buckets with lifecycle rules; it feeds
@@ -754,6 +780,9 @@ impl ScannerItem {
             .all(|candidate| Self::tier_is_known(candidate, tier_names));
         'eventLoop: {
             for (i, event) in events.iter().enumerate() {
+                if cancellation.is_some_and(CancellationToken::is_cancelled) {
+                    return;
+                }
                 let oi = &object_infos[i];
                 let known_size = resolved_sizes[i].known_size();
                 if has_corrupt_size
@@ -772,7 +801,7 @@ impl ScannerItem {
                     // accounting. Size-filtered or deferred events remain
                     // pending, so retain the version-only physical counters.
                     if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                        self.heal_actions(oi, *physical, size_summary).await;
+                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
                         size_summary.actions_accounting_unknown(oi);
                     }
                     continue;
@@ -829,7 +858,7 @@ impl ScannerItem {
                                     noncurrent_unknown.push(oi);
                                 } else {
                                     if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                                        self.heal_actions(oi, *physical, size_summary).await;
+                                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
                                     }
                                     size_summary.actions_accounting_unknown(oi);
                                 }
@@ -846,7 +875,7 @@ impl ScannerItem {
                             }
                             IlmAction::NoneAction | IlmAction::ActionCount => {
                                 if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                                    self.heal_actions(oi, *physical, size_summary).await;
+                                    self.heal_actions(oi, *physical, size_summary, cancellation).await;
                                 }
                                 size_summary.actions_accounting_unknown(oi);
                             }
@@ -864,7 +893,7 @@ impl ScannerItem {
                 // side-effecting tier actions fail closed until the registry
                 // recognizes the source again.
                 if Self::action_blocked_by_unknown_tier(event.action, oi, all_versions_known, tier_names, &event.storage_class) {
-                    size = self.heal_actions(oi, actual_size, size_summary).await;
+                    size = self.heal_actions(oi, actual_size, size_summary, cancellation).await;
                     size_summary.actions_accounting(oi, size, actual_size);
                     cumulative_size += size;
                     continue;
@@ -973,7 +1002,7 @@ impl ScannerItem {
                     }
 
                     IlmAction::NoneAction | IlmAction::ActionCount => {
-                        size = self.heal_actions(oi, actual_size, size_summary).await;
+                        size = self.heal_actions(oi, actual_size, size_summary, cancellation).await;
                     }
                 }
 
@@ -984,6 +1013,9 @@ impl ScannerItem {
             }
         }
 
+        if cancellation.is_some_and(CancellationToken::is_cancelled) {
+            return;
+        }
         if !to_delete_objs.is_empty()
             && let Some(event) = noncurrent_events.first().cloned()
         {
@@ -991,7 +1023,8 @@ impl ScannerItem {
             let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX);
             let done_ilm = Metrics::time_ilm(action);
             let trace_started_at = trace_start_instant();
-            let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await;
+            let queued =
+                enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner, cancellation).await;
             if let Some(trace_started_at) = trace_started_at {
                 let state = if queued { "queued" } else { "not_queued" };
                 trace_emit(|| {
@@ -1022,15 +1055,28 @@ impl ScannerItem {
                 }
             }
         }
-        self.alert_excessive_versions(remaining_versions, cumulative_size);
+        if !cancellation.is_some_and(CancellationToken::is_cancelled) {
+            self.alert_excessive_versions(remaining_versions, cumulative_size);
+        }
     }
 
-    pub(super) async fn heal_actions(&mut self, oi: &ObjectInfo, actual_size: i64, size_summary: &mut SizeSummary) -> i64 {
+    pub(super) async fn heal_actions(
+        &mut self,
+        oi: &ObjectInfo,
+        actual_size: i64,
+        size_summary: &mut SizeSummary,
+        cancellation: Option<&CancellationToken>,
+    ) -> i64 {
+        if cancellation.is_some_and(CancellationToken::is_cancelled) {
+            return actual_size;
+        }
         if self.heal_enabled {
             self.enqueue_heal(oi).await;
         }
 
-        self.heal_replication(oi, size_summary).await;
+        if !cancellation.is_some_and(CancellationToken::is_cancelled) {
+            self.heal_replication(oi, size_summary).await;
+        }
 
         actual_size
     }

crates/scanner/src/scanner_folder/tests.rs

@@ -30,7 +30,9 @@ use std::sync::Mutex;
 pub(super) mod checkpoint_fixture;
 pub(super) mod enumeration_restart;
 mod incremental_enumeration;
+mod listing_completion;
 mod metadata_visits;
+mod node_admission;
 mod usage_updates;
 
 /// Reset the process-global alert cooldown map; test-only.
@@ -364,6 +366,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         failed_object_paths_seen: HashSet::new(),
         resolved_failed_object_paths: HashSet::new(),
         sleeper: SCANNER_SLEEPER.clone(),
+        node_admission: None,
         disks: Vec::new(),
         disks_quorum: 0,
         usage_updates: None,
@@ -394,6 +397,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         pending_size_reconciliation_scopes: HashSet::new(),
         pending_size_reconciliation_truncated: false,
         list_path_raw_options_observer: None,
+        defer_abandoned_entries_until_finished: false,
     };
 
     (scanner, temp_dir)
@@ -2262,7 +2266,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
     };
     let mut size_summary = SizeSummary::default();
 
-    let size = item.heal_actions(&object_info, 123, &mut size_summary).await;
+    let size = item.heal_actions(&object_info, 123, &mut size_summary, None).await;
     assert_eq!(size, 123);
 }
 

crates/scanner/src/scanner_folder/tests/listing_completion.rs

@@ -0,0 +1,104 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+
+#[tokio::test]
+#[serial]
+async fn abandoned_listing_drains_candidate_buffered_before_finished() {
+    let (mut scanner, root) = build_test_scanner().await;
+    let _guard = TestGuard {
+        temp_dir: Some(root.clone()),
+    };
+    let bucket = "bucket";
+    let object = "abandoned/object";
+    tokio::fs::create_dir_all(root.join(bucket)).await.expect("local bucket");
+    let mut disks = vec![scanner.local_disk.clone()];
+    let mut original_metadata = Vec::new();
+    for (index, version) in [(2, 0x1935), (3, 0x1935), (4, 0x1934)] {
+        let disk_root = root.join(format!("disk-{index}"));
+        let metadata = metadata_for_object_version(bucket, object, Some(Uuid::from_u128(version)));
+        write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
+        original_metadata.push((disk_root.join(bucket).join(object).join(STORAGE_FORMAT_FILE), metadata));
+        let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("listing fixture endpoint");
+        disks.push(
+            new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("listing fixture disk"),
+        );
+    }
+    for cache in [&mut scanner.old_cache, &mut scanner.new_cache] {
+        cache.info.name = bucket.to_owned();
+    }
+    scanner.old_cache.replace(
+        "bucket/abandoned/object",
+        bucket,
+        DataUsageEntry {
+            objects: 1,
+            ..Default::default()
+        },
+    );
+    scanner.heal_object_select = 1;
+    scanner.disks = disks;
+    scanner.disks_quorum = 2;
+    scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
+    // Force the valid select ordering that receives terminal errors before the
+    // final candidate. One candidate frame fits in the bounded partial channel.
+    scanner.defer_abandoned_entries_until_finished = true;
+    let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels().expect("isolated heal channel");
+    let (captured_tx, mut captured_rx) = mpsc::unbounded_channel();
+    let responder = tokio::spawn(async move {
+        while let Some(command) = heal_rx.recv().await {
+            let rustfs_heal_contracts::heal_channel::HealReceiptCommand { request, response_tx } = command;
+            captured_tx
+                .send((request.object_prefix.clone(), request.object_version_id.clone()))
+                .expect("candidate capture remains open");
+            let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt {
+                result: HealAdmissionResult::Accepted,
+                task_id: request.id,
+            }));
+        }
+    });
+    let result = tokio::time::timeout(
+        Duration::from_secs(2),
+        scanner.scan_folder(
+            CancellationToken::new(),
+            CachedFolder {
+                name: bucket.to_owned(),
+                parent: None,
+                object_heal_prob_div: 1,
+            },
+            &mut DataUsageEntry::default(),
+        ),
+    )
+    .await;
+    responder.abort();
+    result
+        .expect("listing completion must close every callback channel")
+        .expect("complete folder scan");
+    let mut candidates = HashSet::new();
+    while let Ok((object, version)) = captured_rx.try_recv() {
+        if let (Some(object), Some(version)) = (object, version) {
+            candidates.insert((object, version));
+        }
+    }
+    let expected = [0x1934, 0x1935]
+        .into_iter()
+        // Raw base-object listings include the trailing separator. Preserve
+        // the producer's candidate identity when draining its callback frame.
+        .map(|version| (format!("{object}/"), Uuid::from_u128(version).to_string()))
+        .collect();
+    assert_eq!(
+        candidates, expected,
+        "terminal disk errors must not discard buffered exact-version candidates"
+    );
+    for (path, metadata) in original_metadata {
+        assert_eq!(tokio::fs::read(path).await.expect("retained metadata"), metadata);
+    }
+}

crates/scanner/src/scanner_io/cache.rs

@@ -172,12 +172,55 @@ fn record_checkpoint_save_failure(cache_name: &str, checkpoint: &DataUsageCache,
 
 #[derive(Debug)]
 pub(crate) struct ScannerCacheLockGuards {
-    scoped: NamespaceLockGuard,
+    scoped: Option<NamespaceLockGuard>,
 }
 
 impl ScannerCacheLockGuards {
     pub(crate) fn is_lock_lost(&self) -> bool {
-        self.scoped.is_lock_lost()
+        self.scoped.as_ref().is_none_or(NamespaceLockGuard::is_lock_lost)
+    }
+
+    /// Suspend the same traversal while yielding its cache lease. Lock order on
+    /// resumption is node admission -> cache lease -> cache revision reads.
+    pub(crate) async fn resume_node_operation(
+        &mut self,
+        store: Arc<SetDisks>,
+        cache_name: &str,
+        source: DataUsageCacheSource,
+        revisions: &DataUsageCacheRevisions,
+        request: crate::workload_admission::ScannerNodeRequest,
+    ) -> Result<()> {
+        if self.is_lock_lost() {
+            return Err(Error::other("scanner cache lock was lost before node admission"));
+        }
+        if request.respond.is_closed() {
+            return Ok(());
+        }
+        let crate::workload_admission::ScannerNodeRequest { owner, respond, cancel } = request;
+        drop(self.scoped.take());
+        let resume = async {
+            let permit = owner.acquire().await;
+            *self = acquire_scanner_cache_locks(store.as_ref(), cache_name, source)
+                .await
+                .map_err(Error::other)?;
+            let current = crate::storage_api::ecstore_with_background_disk_io(
+                cancel.clone(),
+                DataUsageCache::read_revisions(store, cache_name),
+            )
+            .await?;
+            if self.is_lock_lost() || &current != revisions {
+                return Err(Error::other("scanner cache changed while waiting for node admission"));
+            }
+            // If the walk was dropped, send returns ownership and drops the
+            // permit without recording completed work.
+            let _ = respond.send(permit);
+            Ok(())
+        };
+        tokio::select! {
+            biased;
+            _ = cancel.cancelled() => Err(Error::other("scanner node admission cancelled")),
+            result = resume => result,
+        }
     }
 }
 
@@ -215,6 +258,15 @@ impl std::fmt::Display for ScannerCacheLockError {
     }
 }
 
+impl std::error::Error for ScannerCacheLockError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Self::Create { source, .. } => Some(source),
+            Self::Acquire { source, .. } => Some(source),
+        }
+    }
+}
+
 pub(crate) async fn acquire_scanner_cache_locks(
     store: &SetDisks,
     cache_name: &str,
@@ -237,7 +289,7 @@ pub(crate) async fn acquire_scanner_cache_locks(
             source,
         })?;
 
-    Ok(ScannerCacheLockGuards { scoped })
+    Ok(ScannerCacheLockGuards { scoped: Some(scoped) })
 }
 
 pub(super) async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)
@@ -688,6 +740,15 @@ pub(super) fn completed_data_usage_info(
         .map(|candidate| (candidate.data, candidate.last_update))
 }
 
+/// Certify an incarnation only when every contributing set reports the same non-nil identity.
+fn certified_bucket_incarnation(results: &[DataUsageCache], bucket: &str) -> Option<uuid::Uuid> {
+    let mut identities = results
+        .iter()
+        .map(|result| result.info.scan_bucket_incarnations.get(bucket).copied());
+    let first = identities.next().flatten().filter(|id| !id.is_nil())?;
+    identities.all(|id| id == Some(first)).then_some(first)
+}
+
 pub(super) fn completed_usage_candidate(
     results: &[DataUsageCache],
     scope: &ScannerSnapshotScope<'_>,
@@ -740,7 +801,11 @@ pub(super) fn completed_usage_candidate(
         let merged_last_update = validated.last_update;
         let buckets_usage = bucket_entries
             .into_iter()
-            .map(|(bucket, entry)| Ok((bucket, checked_bucket_usage_info(entry).ok_or(CandidateOutcome::UsageCounters)?)))
+            .map(|(bucket, entry)| {
+                let mut usage = checked_bucket_usage_info(entry).ok_or(CandidateOutcome::UsageCounters)?;
+                usage.bucket_incarnation = certified_bucket_incarnation(results, &bucket).map(|id| id.to_string());
+                Ok((bucket, usage))
+            })
             .collect::<std::result::Result<HashMap<_, _>, CandidateOutcome>>()?;
         let bucket_sizes = buckets_usage
             .iter()

crates/scanner/src/scanner_io/dirty_usage.rs

@@ -238,6 +238,71 @@ pub enum ScannerDurableDirtyUsageReplayScope {
     TopLevelEntries { entries: BTreeSet<String> },
 }
 
+impl ScannerDurableDirtyUsageReplayRecord {
+    /// Replays decoded producer hints with the canonical byte limit and semantic checks.
+    ///
+    /// Callers must retain their framing and owner checks. This performs the same
+    /// state update as [`replay_durable_dirty_usage_producer_record`]; it does not
+    /// establish storage-verified source authority.
+    pub fn replay(self) -> std::result::Result<ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayError> {
+        check_durable_dirty_usage_replay_bytes(&self)?;
+        replay_decoded_dirty_usage_record(&self)
+    }
+}
+
+struct ReplayByteLimit {
+    remaining: usize,
+}
+
+impl std::io::Write for ReplayByteLimit {
+    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+        self.remaining = self
+            .remaining
+            .checked_sub(bytes.len())
+            .ok_or_else(|| std::io::Error::other("dirty replay byte limit"))?;
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+fn check_durable_dirty_usage_replay_bytes(
+    record: &ScannerDurableDirtyUsageReplayRecord,
+) -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> {
+    // Bound structure traversal and string scanning before counting escaped JSON.
+    // Each entry or prefix contributes at least one additional serialized byte.
+    let mut remaining = SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES;
+    let mut charge = |size: usize| -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> {
+        remaining = remaining
+            .checked_sub(size)
+            .ok_or(ScannerDurableDirtyUsageReplayError::ByteLimit)?;
+        Ok(())
+    };
+    charge(record.writer_epoch.len())?;
+    charge(record.entries.len())?;
+    for entry in &record.entries {
+        charge(entry.bucket.len())?;
+        if let ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } = &entry.scope {
+            charge(entries.len())?;
+            for entry in entries {
+                charge(entry.len())?;
+            }
+        }
+    }
+    let mut limit = ReplayByteLimit {
+        remaining: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES,
+    };
+    serde_json::to_writer(&mut limit, record).map_err(|error| {
+        if error.is_io() {
+            ScannerDurableDirtyUsageReplayError::ByteLimit
+        } else {
+            ScannerDurableDirtyUsageReplayError::InvalidJson
+        }
+    })
+}
+
 pub fn encode_durable_dirty_usage_producer_replay_record(
     entries: Vec<ScannerDurableDirtyUsageReplayEntry>,
 ) -> std::result::Result<Vec<u8>, ScannerDurableDirtyUsageReplayError> {
@@ -263,7 +328,13 @@ pub fn replay_durable_dirty_usage_producer_record(
     }
     let record: ScannerDurableDirtyUsageReplayRecord =
         serde_json::from_slice(bytes).map_err(|_| ScannerDurableDirtyUsageReplayError::InvalidJson)?;
-    let replay = validate_durable_dirty_usage_replay_record(&record)?;
+    replay_decoded_dirty_usage_record(&record)
+}
+
+fn replay_decoded_dirty_usage_record(
+    record: &ScannerDurableDirtyUsageReplayRecord,
+) -> std::result::Result<ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayError> {
+    let replay = validate_durable_dirty_usage_replay_record(record)?;
     let max_generation = replay.iter().map(|entry| entry.generation).max().unwrap_or(0);
     set_dirty_usage_generation_floor(max_generation)?;
     let pending = {
@@ -772,6 +843,109 @@ mod scoped_dirty_usage_tests {
         );
     }
 
+    fn decoded_replay_fixture() -> ScannerDurableDirtyUsageReplayRecord {
+        ScannerDurableDirtyUsageReplayRecord {
+            schema: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA,
+            cache_key_format: crate::DATA_USAGE_CACHE_KEY_FORMAT,
+            writer_epoch: "writer".to_string(),
+            entries: vec![ScannerDurableDirtyUsageReplayEntry {
+                bucket: "photos".to_string(),
+                generation: 7,
+                scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
+                    entries: BTreeSet::from(["2026".to_string(), "archive".to_string()]),
+                },
+                producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(),
+            }],
+        }
+    }
+
+    #[test]
+    #[serial]
+    fn durable_dirty_usage_decoded_replay_matches_byte_replay_at_size_boundary() {
+        let record = decoded_replay_fixture();
+        let fixed_bytes = serde_json::to_vec(&record).expect("fixture encoding").len() - record.writer_epoch.len();
+        for size in [
+            SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES - 1,
+            SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES,
+            SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1,
+        ] {
+            clear_dirty_usage_buckets_for_tests();
+            let mut record = record.clone();
+            record.writer_epoch = "x".repeat(size - fixed_bytes);
+            let bytes = serde_json::to_vec(&record).expect("bounded fixture encoding");
+            assert_eq!(bytes.len(), size);
+            let expected = replay_durable_dirty_usage_producer_record(&bytes);
+            let buckets = dirty_usage_buckets_for_tests();
+            let scopes = dirty_usage_bucket_scopes_for_tests();
+            let identities = dirty_usage_producer_identities().clone();
+            let coverage = DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire);
+
+            clear_dirty_usage_buckets_for_tests();
+            assert_eq!(record.replay(), expected, "canonical size {size}");
+            assert_eq!(dirty_usage_buckets_for_tests(), buckets);
+            assert_eq!(dirty_usage_bucket_scopes_for_tests(), scopes);
+            assert_eq!(*dirty_usage_producer_identities(), identities);
+            assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), coverage);
+        }
+        clear_dirty_usage_buckets_for_tests();
+    }
+
+    #[test]
+    fn durable_dirty_usage_decoded_size_counts_escaped_utf8() {
+        for repeats in [6000, 8000] {
+            let mut record = decoded_replay_fixture();
+            record.writer_epoch = "\"\n雪\\".repeat(repeats);
+            let bytes = serde_json::to_vec(&record).expect("escaped fixture encoding");
+            let expected = if bytes.len() > SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
+                Err(ScannerDurableDirtyUsageReplayError::ByteLimit)
+            } else {
+                Ok(())
+            };
+            assert_eq!(check_durable_dirty_usage_replay_bytes(&record), expected);
+        }
+    }
+
+    #[test]
+    #[serial]
+    fn durable_dirty_usage_decoded_replay_rejects_large_values_before_state_changes() {
+        for field in ["epoch", "prefix", "entries"] {
+            clear_dirty_usage_buckets_for_tests();
+            let before = scanner_dirty_usage_state();
+            let mut record = decoded_replay_fixture();
+            match field {
+                "epoch" => record.writer_epoch = "x".repeat(1024 * 1024),
+                "prefix" => {
+                    record.entries[0].scope = ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
+                        entries: BTreeSet::from(["x".repeat(1024 * 1024)]),
+                    };
+                }
+                _ => record.entries = vec![record.entries[0].clone(); SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1],
+            }
+            assert_eq!(record.replay(), Err(ScannerDurableDirtyUsageReplayError::ByteLimit), "{field}");
+            assert_eq!(scanner_dirty_usage_state(), before);
+            assert!(dirty_usage_buckets_for_tests().is_empty());
+            assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
+            assert!(dirty_usage_producer_identities().is_empty());
+            assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
+        }
+        clear_dirty_usage_buckets_for_tests();
+    }
+
+    #[test]
+    #[serial]
+    fn durable_dirty_usage_decoded_replay_preserves_generation_exhaustion() {
+        clear_dirty_usage_buckets_for_tests();
+        let generation = DIRTY_USAGE_BUCKET_GENERATION.swap(u64::MAX, Ordering::AcqRel);
+        let result = decoded_replay_fixture().replay();
+        DIRTY_USAGE_BUCKET_GENERATION.store(generation, Ordering::Release);
+        assert_eq!(result, Err(ScannerDurableDirtyUsageReplayError::InvalidGeneration));
+        assert!(dirty_usage_buckets_for_tests().is_empty());
+        assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
+        assert!(dirty_usage_producer_identities().is_empty());
+        assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
+        clear_dirty_usage_buckets_for_tests();
+    }
+
     #[test]
     #[serial]
     fn durable_dirty_usage_replay_rejects_invalid_records_without_partial_state() {
@@ -843,9 +1017,17 @@ mod scoped_dirty_usage_tests {
 
         for (bytes, expected_error) in invalid_cases {
             clear_dirty_usage_buckets_for_tests();
-            assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error));
+            let before = scanner_dirty_usage_state();
+            assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error.clone()));
             assert!(dirty_usage_buckets_for_tests().is_empty());
             assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
+            let record: ScannerDurableDirtyUsageReplayRecord = serde_json::from_slice(&bytes).expect("typed invalid record");
+            assert_eq!(record.replay(), Err(expected_error));
+            assert_eq!(scanner_dirty_usage_state(), before);
+            assert!(dirty_usage_buckets_for_tests().is_empty());
+            assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
+            assert!(dirty_usage_producer_identities().is_empty());
+            assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
         }
 
         clear_dirty_usage_buckets_for_tests();

crates/scanner/src/scanner_io/io_cache.rs

@@ -1090,7 +1090,7 @@ impl ScannerIOCache for SetDisks {
 
                     // Lock order: scanner leader fence -> set-scoped per-bucket cache lock ->
                     // cache object read/write.
-                    let cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
+                    let mut cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
                         Ok(guard) => guard,
                         Err(e) => {
                             if e.is_contention() {
@@ -1320,9 +1320,10 @@ impl ScannerIOCache for SetDisks {
                     );
 
                     let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
+                    let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
 
                     let scan_ctx = ctx_clone.child_token();
-                    let scan = disk.clone().nsscanner_disk(
+                    let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
                         scan_ctx.clone(),
                         budget_clone.clone(),
                         set_disk_inventory_clone.as_ref().clone(),
@@ -1333,7 +1334,7 @@ impl ScannerIOCache for SetDisks {
                             prefix_scan_scope,
                             checkpoint_tx: Some(checkpoint_tx),
                         },
-                    );
+                    ));
                     tokio::pin!(scan);
                     let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
                     lock_watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -1341,6 +1342,24 @@ impl ScannerIOCache for SetDisks {
                     let scan_result = loop {
                         tokio::select! {
                             result = &mut scan => break result,
+                            request = async {
+                                match node_rx.as_mut() {
+                                    Some(receiver) => receiver.recv().await,
+                                    None => std::future::pending().await,
+                                }
+                            } => {
+                                let Some(request) = request else {
+                                    node_rx = None;
+                                    continue;
+                                };
+                                if let Err(error) = cache_guard.resume_node_operation(
+                                    store_clone_clone.clone(), &cache_name, source, &revisions, request,
+                                ).await {
+                                    scan_ctx.cancel();
+                                    await_scanner_disk_shutdown(scan.as_mut()).await;
+                                    break Err(error);
+                                }
+                            }
                             _ = lock_watch.tick() => {
                                 if cache_guard.is_lock_lost() {
                                     scan_ctx.cancel();

crates/scanner/src/scanner_io/io_disk.rs

@@ -38,6 +38,13 @@ pub(super) fn tier_stats_template(tier_names: &[String]) -> HashMap<String, Tier
     tier_stats
 }
 
+fn ensure_scanner_node_operation_active(cancellation: Option<&CancellationToken>) -> Result<()> {
+    if cancellation.is_some_and(CancellationToken::is_cancelled) {
+        return Err(StorageError::other("Scanner operation cancelled"));
+    }
+    Ok(())
+}
+
 #[async_trait::async_trait]
 impl ScannerIODisk for Disk {
     async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary> {
@@ -45,14 +52,18 @@ impl ScannerIODisk for Disk {
     }
 
     async fn get_size_with_tier_names(&self, mut item: ScannerItem, tier_names: &[String]) -> Result<SizeSummary> {
+        let cancellation = crate::workload_admission::scanner_node_operation_cancellation();
         let done_object = Metrics::time(Metric::ScanObject);
 
         if !is_xl_meta_path(&item.path) {
             return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
         }
 
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         let metadata_object_path = item.object_path();
-        let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
+        let metadata_result = self.read_metadata(&item.bucket, &metadata_object_path).await;
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
+        let data = match metadata_result {
             Ok(data) => data,
             Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
                 return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -86,7 +97,10 @@ impl ScannerIODisk for Disk {
         // (which used to query it a second time). On failure keep the
         // historical fallback: default configuration (versioned = false) plus
         // the warn that `apply_actions` used to emit.
-        let versioning_config = match BucketVersioningSys::get(&item.bucket).await {
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
+        let versioning_result = BucketVersioningSys::get(&item.bucket).await;
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
+        let versioning_config = match versioning_result {
             Ok(versioning_config) => versioning_config,
             Err(_) => {
                 warn!(
@@ -122,7 +136,9 @@ impl ScannerIODisk for Disk {
             ..Default::default()
         };
 
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         let lock_config = object_lock_config_for_scanner_item(&item).await;
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
 
         // Count every version this object contributes to the scan, independent
         // of any lifecycle configuration, so scan-coverage metrics stay honest
@@ -130,17 +146,33 @@ impl ScannerIODisk for Disk {
         // `object_infos`.
         global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);
 
-        item.apply_actions(object_infos, lock_config, versioning_config, tier_names, &mut size_summary)
-            .await;
+        #[cfg(test)]
+        let _ = crate::scanner_folder::observe_test_size_dispatch_boundary("apply_actions");
+        item.apply_actions_cancellable(
+            object_infos,
+            lock_config,
+            versioning_config,
+            tier_names,
+            &mut size_summary,
+            cancellation.as_ref(),
+        )
+        .await;
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
 
         if !free_version_infos.is_empty() {
             for oi in free_version_infos {
+                ensure_scanner_node_operation_active(cancellation.as_ref())?;
                 if ScannerItem::tier_is_known(&oi, tier_names) {
-                    enqueue_runtime_free_version(oi).await;
+                    #[cfg(test)]
+                    if crate::scanner_folder::observe_test_size_dispatch_boundary("free_version_dispatch") {
+                        continue;
+                    }
+                    enqueue_runtime_free_version(oi, cancellation.as_ref()).await;
                 }
             }
         }
 
+        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         done_object();
 
         Ok(size_summary)

crates/scanner/src/scanner_io/publish_gate_tests.rs

@@ -996,6 +996,73 @@ fn completed_data_usage_info_adds_same_bucket_across_unique_sets() {
     assert_eq!(bucket.delete_markers_count, 1);
 }
 
+#[test]
+fn bucket_usage_incarnation_is_optional_and_old_readers_ignore_it() {
+    #[derive(serde::Deserialize)]
+    struct LegacyBucketUsage {
+        size: u64,
+    }
+    let legacy = BucketUsageInfo {
+        size: 42,
+        ..Default::default()
+    };
+    let without = serde_json::to_value(&legacy).expect("encode legacy usage");
+    assert!(without.get("bucket_incarnation").is_none());
+    let restored: BucketUsageInfo = serde_json::from_value(without).expect("decode old usage");
+    assert_eq!(restored.bucket_incarnation, None);
+    let certified = BucketUsageInfo {
+        size: 42,
+        bucket_incarnation: Some(uuid::Uuid::from_u128(7).to_string()),
+        ..Default::default()
+    };
+    let encoded = serde_json::to_vec(&certified).expect("encode certified usage");
+    let old: LegacyBucketUsage = serde_json::from_slice(&encoded).expect("old schema ignores additive field");
+    assert_eq!(old.size, 42);
+    let named = rmp_serde::to_vec_named(&certified).expect("encode certified named msgpack");
+    let old: LegacyBucketUsage = rmp_serde::from_slice(&named).expect("old named reader ignores additive field");
+    assert_eq!(old.size, 42);
+    let old_bytes = rmp_serde::to_vec_named(&legacy).expect("encode old shape");
+    let new: BucketUsageInfo = rmp_serde::from_slice(&old_bytes).expect("new reader accepts old shape");
+    assert_eq!(new.bucket_incarnation, None);
+}
+
+#[test]
+fn completed_usage_certifies_only_matching_non_nil_bucket_incarnations() {
+    let buckets = vec!["bucket".to_string()];
+    let id = uuid::Uuid::from_u128(7);
+    let mut a = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0));
+    let mut b = completed_root_cache("bucket", 3, 20, DataUsageCacheSource::new(1, 0));
+    a.info.scan_bucket_incarnations.insert("bucket".to_string(), id);
+    b.info.scan_bucket_incarnations.insert("bucket".to_string(), id);
+    let published = completed_data_usage_info_for_test(&[a.clone(), b.clone()], &buckets, false, false)
+        .unwrap()
+        .0;
+    assert_eq!(
+        published.buckets_usage["bucket"].bucket_incarnation.as_deref(),
+        Some(id.to_string().as_str())
+    );
+    let encoded = serde_json::to_vec(&published).expect("serialize certified usage");
+    let decoded: DataUsageInfo = serde_json::from_slice(&encoded).expect("read certified usage");
+    assert_eq!(decoded.buckets_usage["bucket"].bucket_incarnation, Some(id.to_string()));
+    b.info
+        .scan_bucket_incarnations
+        .insert("bucket".to_string(), uuid::Uuid::from_u128(8));
+    let mismatch = completed_data_usage_info_for_test(&[a.clone(), b.clone()], &buckets, false, false)
+        .unwrap()
+        .0;
+    assert_eq!(mismatch.buckets_usage["bucket"].bucket_incarnation, None);
+    b.info.scan_bucket_incarnations.remove("bucket");
+    let missing = completed_data_usage_info_for_test(&[a.clone(), b.clone()], &buckets, false, false)
+        .unwrap()
+        .0;
+    assert_eq!(missing.buckets_usage["bucket"].bucket_incarnation, None);
+    b.info
+        .scan_bucket_incarnations
+        .insert("bucket".to_string(), uuid::Uuid::nil());
+    let nil = completed_data_usage_info_for_test(&[a, b], &buckets, false, false).unwrap().0;
+    assert_eq!(nil.buckets_usage["bucket"].bucket_incarnation, None);
+}
+
 #[test]
 fn checked_bucket_usage_info_moves_replication_keys_and_preserves_the_snapshot() {
     let first_target = "arn:rustfs:replication:local::target-a".to_string();
@@ -1057,6 +1124,7 @@ fn checked_bucket_usage_info_moves_replication_keys_and_preserves_the_snapshot()
     }
     let expected = BucketUsageInfo {
         size: 32768,
+        bucket_incarnation: None,
         replication_pending_size_v1: 0,
         replication_failed_size_v1: 0,
         replicated_size_v1: 0,

crates/scanner/src/scanner_io/tests.rs

@@ -46,7 +46,9 @@ use uuid::Uuid;
 
 mod checkpoint_coverage_identity;
 mod collector_lifecycle;
+mod expiry_lock;
 mod listing_window_scope;
+mod node_admission;
 mod partial_bucket_carry;
 mod per_bucket_certification;
 mod scoped_entry_fallback;

crates/scanner/src/scanner_io/tests/expiry_lock.rs

@@ -0,0 +1,183 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+use crate::storage_api::owner::EcstoreExpiryState;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::task::Poll;
+use std::time::Duration;
+
+// Return Some(result) if the operation ended too early, None only if it was
+// actually polled pending after the prerequisite read had finished.
+async fn observe_queued<F: Future>(
+    mut operation: Pin<&mut F>,
+    read_complete: impl Fn() -> bool,
+    observation: Duration,
+) -> std::result::Result<Option<F::Output>, tokio::time::error::Elapsed> {
+    tokio::time::timeout(
+        observation,
+        std::future::poll_fn(|cx| match operation.as_mut().poll(cx) {
+            Poll::Ready(value) => Poll::Ready(Some(value)),
+            Poll::Pending if read_complete() => Poll::Ready(None),
+            Poll::Pending => Poll::Pending,
+        }),
+    )
+    .await
+}
+
+#[tokio::test]
+#[serial]
+async fn free_version_cancelled_while_waiting_for_expiry_write_lock_never_queues() {
+    let state = EcstoreExpiryState::new_with_unconsumed_worker_channel(4);
+    let observation = Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS);
+    let context = crate::ExpiryLockTestContext {
+        state: state.clone(),
+        store: None,
+        completed_incarnation_reads: Arc::new(AtomicUsize::new(0)),
+    };
+    crate::EXPIRY_LOCK_TEST_CONTEXT
+        .scope(context, async {
+            let object = crate::ScannerObjectInfo {
+                bucket: "expiry-lock-bucket".to_string(),
+                name: "free-object".to_string(),
+                ..Default::default()
+            };
+            let cancellation = CancellationToken::new();
+            let guard = state.write().await;
+            let mut pending = Box::pin(crate::enqueue_runtime_free_version(object.clone(), Some(&cancellation)));
+            let first = observe_queued(pending.as_mut(), || true, observation).await;
+            let observed_wait = matches!(&first, Ok(None));
+            let already_finished = matches!(&first, Ok(Some(())));
+            cancellation.cancel();
+            let cancelled_while_locked = if already_finished {
+                false
+            } else {
+                tokio::time::timeout(observation, pending.as_mut()).await.is_ok()
+            };
+            let pending_while_locked = guard.pending_tasks();
+            drop(guard);
+            if !already_finished && !cancelled_while_locked {
+                pending.as_mut().await;
+            }
+            drop(pending);
+            let pending_after_cancel = state.read().await.pending_tasks();
+
+            // Positive control queues a real free-version task while using the same
+            // implementation and isolated, unconsumed worker channel.
+            let mut positive = Box::pin(crate::enqueue_runtime_free_version(object, None));
+            let positive_in_time = tokio::time::timeout(observation, positive.as_mut()).await.is_ok();
+            if !positive_in_time {
+                positive.as_mut().await;
+            }
+            drop(positive);
+            let pending_after_control = state.read().await.pending_tasks();
+            assert!(observed_wait, "must poll the real enqueue while the write lock is held");
+            assert!(cancelled_while_locked, "cancellation must finish before the lock is released");
+            assert_eq!(pending_while_locked, 0);
+            assert_eq!(pending_after_cancel, 0);
+            assert!(positive_in_time, "positive control must finish within the observation limit");
+            assert_eq!(pending_after_control, 1, "positive control must queue a task");
+        })
+        .await;
+}
+
+#[tokio::test]
+#[serial]
+async fn newer_noncurrent_cancelled_after_real_read_while_waiting_for_expiry_write_lock_never_queues() {
+    let (_temp_dir, store) = setup_local_scanner_store(1, 4).await;
+    let bucket = "expiry-lock-bucket";
+    store
+        .make_bucket(bucket, &MakeBucketOptions::default())
+        .await
+        .expect("create bucket");
+    let expected_incarnation = store.bucket_incarnation_id(bucket).await.expect("real incarnation metadata");
+    assert!(!expected_incarnation.is_nil());
+    let state = EcstoreExpiryState::new_with_unconsumed_worker_channel(4);
+    let reads = Arc::new(AtomicUsize::new(0));
+    let context = crate::ExpiryLockTestContext {
+        state: state.clone(),
+        store: Some(store.clone()),
+        completed_incarnation_reads: reads.clone(),
+    };
+    let observation = Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS);
+    crate::EXPIRY_LOCK_TEST_CONTEXT
+        .scope(context, async {
+            let versions = vec![crate::ScannerObjectToDelete {
+                object_name: "noncurrent-object".to_string(),
+                version_id: Some(Uuid::new_v4()),
+                ..Default::default()
+            }];
+            let event = crate::Event::default();
+            let src = crate::LcEventSrc::Scanner;
+            let cancellation = CancellationToken::new();
+            let guard = state.write().await;
+            let mut pending = Box::pin(crate::enqueue_runtime_newer_noncurrent(
+                bucket,
+                versions.clone(),
+                event.clone(),
+                &src,
+                Some(&cancellation),
+            ));
+            let first = observe_queued(pending.as_mut(), || reads.load(Ordering::SeqCst) == 1, observation).await;
+            let observed_wait_after_read = matches!(&first, Ok(None));
+            let mut result = match first {
+                Ok(Some(value)) => Some(value),
+                _ => None,
+            };
+            cancellation.cancel();
+            let cancelled_while_locked = if result.is_some() {
+                false
+            } else {
+                match tokio::time::timeout(observation, pending.as_mut()).await {
+                    Ok(value) => {
+                        result = Some(value);
+                        true
+                    }
+                    Err(_) => false,
+                }
+            };
+            let pending_while_locked = guard.pending_tasks();
+            drop(guard);
+            let result = match result {
+                Some(value) => value,
+                None => pending.as_mut().await,
+            };
+            drop(pending);
+            let pending_after_cancel = state.read().await.pending_tasks();
+            let mut positive = Box::pin(crate::enqueue_runtime_newer_noncurrent(bucket, versions, event, &src, None));
+            let bounded_positive = tokio::time::timeout(observation, positive.as_mut()).await;
+            let positive_in_time = bounded_positive.is_ok();
+            let positive_result = match bounded_positive {
+                Ok(value) => value,
+                Err(_) => positive.as_mut().await,
+            };
+            drop(positive);
+            let pending_after_control = state.read().await.pending_tasks();
+            assert!(
+                observed_wait_after_read,
+                "real incarnation read must finish before waiting on the held lock"
+            );
+            assert!(cancelled_while_locked, "cancellation must finish before the lock is released");
+            assert!(!result);
+            assert_eq!(pending_while_locked, 0);
+            assert_eq!(pending_after_cancel, 0);
+            assert!(positive_in_time, "positive control must finish within the observation limit");
+            assert!(positive_result, "positive control must enqueue the nonempty version batch");
+            assert_eq!(pending_after_control, 1);
+            assert_eq!(reads.load(Ordering::SeqCst), 2);
+        })
+        .await;
+}

crates/scanner/src/scanner_io/tests/node_admission.rs

@@ -0,0 +1,136 @@
+// Copyright 2026 RustFS Team
+// Licensed under the Apache License, Version 2.0.
+
+use super::*;
+use crate::workload_admission::ScannerNodeRequest;
+use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
+use std::num::NonZeroU16;
+use tokio::sync::oneshot;
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum ResumeCase {
+    Unchanged,
+    Updated,
+    Cancelled,
+}
+
+async fn check_cache_lease_resume(case: ResumeCase) {
+    let (_directory, store) = setup_local_scanner_store(1, 4).await;
+    let set = store.pools[0].disk_set[0].clone();
+    let source = DataUsageCacheSource::new(0, 0);
+    let cache_name = "node-resume/.usage-cache.bin";
+    let cache = DataUsageCache::default();
+    cache.save(set.clone(), cache_name).await.expect("seed cache");
+    let revisions = DataUsageCache::read_revisions(set.clone(), cache_name)
+        .await
+        .expect("initial revisions");
+    let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one shared unit"));
+    gate.update(
+        &mut FeedbackController::new(),
+        Some(FeedbackObservation {
+            generation: 1,
+            max_concurrency: 1,
+            pressure: ResourcePressure::Unknown,
+            completed_units: None,
+        }),
+    );
+    let mut held = Some(gate.acquire().await);
+    let mut guard = acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
+        .await
+        .expect("initial cache lease");
+    let ctx = CancellationToken::new();
+    let (respond, response) = oneshot::channel();
+    let request = ScannerNodeRequest {
+        owner: gate.clone(),
+        respond,
+        cancel: ctx.clone(),
+    };
+    let mut expected = revisions.clone();
+    let result = {
+        let resume = guard.resume_node_operation(set.clone(), cache_name, source, &revisions, request);
+        tokio::pin!(resume);
+        assert!(futures::poll!(&mut resume).is_pending(), "shared node is occupied");
+        let other = tokio::time::timeout(Duration::from_secs(1), acquire_scanner_cache_locks(set.as_ref(), cache_name, source))
+            .await
+            .expect("cache lock must be released before joining the node queue")
+            .expect("another cache worker can acquire the lease");
+        if case == ResumeCase::Updated {
+            let mut newer = cache;
+            newer.info.next_cycle = 1;
+            newer
+                .save(set.clone(), cache_name)
+                .await
+                .expect("another worker updates the cache");
+            expected = DataUsageCache::read_revisions(set.clone(), cache_name)
+                .await
+                .expect("new cache revisions");
+            assert_ne!(expected, revisions);
+        }
+        drop(other);
+        if case == ResumeCase::Cancelled {
+            ctx.cancel();
+        } else {
+            held.take().expect("held node unit").complete();
+        }
+        tokio::time::timeout(Duration::from_secs(2), resume)
+            .await
+            .expect("node resumption must finish or observe cancellation")
+    };
+    if case == ResumeCase::Unchanged {
+        result.expect("unchanged cache resumes the same walk");
+        let permit = response.await.expect("coordinator returns the node permit");
+        assert_eq!(gate.snapshot().active, 1);
+        assert_eq!(gate.completed(), Some(1), "admitting scanner work must not count it as completed");
+        assert!(!guard.is_lock_lost());
+        let lock = set
+            .new_ns_lock(RUSTFS_META_BUCKET, &scanner_cache_lock_resource(cache_name, source))
+            .await
+            .expect("competing cache lock");
+        assert!(
+            lock.get_write_lock_quiet(Duration::from_millis(20)).await.is_err(),
+            "cache lease must be restored before the operation resumes"
+        );
+        permit.complete();
+    } else {
+        result.expect_err("cancelled or stale work cannot resume");
+        assert!(response.await.is_err(), "failed resumption must not authorize storage work");
+        if case == ResumeCase::Cancelled {
+            assert!(guard.is_lock_lost());
+            assert_eq!(gate.snapshot().active, 1, "the running heal keeps its unit");
+            drop(held.take());
+            assert_eq!(gate.completed(), Some(0));
+        } else {
+            assert_eq!(gate.completed(), Some(1));
+        }
+    }
+    drop(guard);
+    assert_eq!(gate.snapshot().active, 0);
+    assert!(gate.try_acquire().is_some(), "dropped requests must leave no queue or capacity leak");
+    assert_eq!(
+        DataUsageCache::read_revisions(set.clone(), cache_name)
+            .await
+            .expect("final revisions"),
+        expected
+    );
+    acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
+        .await
+        .expect("final cache lease is available");
+}
+
+#[tokio::test]
+#[serial]
+async fn scanner_node_wait_yields_cache_lease_before_fifo_admission() {
+    check_cache_lease_resume(ResumeCase::Unchanged).await;
+}
+
+#[tokio::test]
+#[serial]
+async fn scanner_node_wait_rejects_cache_changed_while_unlocked() {
+    check_cache_lease_resume(ResumeCase::Updated).await;
+}
+
+#[tokio::test]
+#[serial]
+async fn scanner_node_wait_cancellation_releases_queue_without_stealing_running_unit() {
+    check_cache_lease_resume(ResumeCase::Cancelled).await;
+}

crates/scanner/src/storage_api.rs

@@ -22,6 +22,8 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::Notify;
 
 pub(crate) use rustfs_ecstore::with_background_disk_io as ecstore_with_background_disk_io;
+#[cfg(all(test, unix))]
+pub(crate) use rustfs_ecstore::with_background_resource_unit as ecstore_with_background_resource_unit;
 
 pub(crate) use s3s::dto::{
     BucketLifecycleConfiguration as EcstoreBucketLifecycleConfiguration, LifecycleRuleFilter as EcstoreLifecycleRuleFilter,
@@ -39,6 +41,8 @@ pub(crate) use rustfs_ecstore::api::data_usage::{NativeScannerPauseBacklogWriteF
 
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
+#[cfg(test)]
+pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState as EcstoreExpiryState;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
     apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
     lifecycle_version_delete_target as ecstore_lifecycle_version_delete_target,
@@ -99,6 +103,10 @@ pub(crate) use rustfs_ecstore::api::disk::{
 pub(crate) use rustfs_ecstore::api::disk::{
     DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
 };
+#[cfg(all(test, unix))]
+pub(crate) use rustfs_ecstore::api::disk::{
+    LocalPublicationPause as EcstoreLocalPublicationPause, LocalPublicationStage as EcstoreLocalPublicationStage,
+};
 pub(crate) use rustfs_ecstore::api::error::{
     Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
 };
@@ -161,10 +169,17 @@ pub(crate) mod ecstore_diagnostic {
 pub(crate) type EcstoreHealResultItem = <EcstoreStore as storage_contracts::HealOperations>::HealResultItem;
 
 pub(crate) mod owner {
+    #[cfg(test)]
+    pub(crate) use super::EcstoreExpiryState;
     #[cfg(test)]
     pub(crate) use super::ecstore_set_object_store_resolver;
     #[cfg(test)]
     pub(crate) use super::storage_contracts::DiskSetSelector;
+    #[cfg(all(test, unix))]
+    pub(crate) use super::{
+        EcstoreLocalPublicationPause, EcstoreLocalPublicationStage, ecstore_with_background_disk_io,
+        ecstore_with_background_resource_unit,
+    };
     pub(crate) use super::{
         MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
         ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,

crates/scanner/src/workload_admission.rs

@@ -12,9 +12,143 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::{Arc, LazyLock, RwLock};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, LazyLock, OnceLock, RwLock};
 
+use rustfs_concurrency::adaptive::{NodeAdmission, NodePermit};
 use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
+use tokio::sync::{mpsc, oneshot};
+use tokio_util::sync::CancellationToken;
+
+use crate::ScannerError;
+
+static SCANNER_NODE_ADMISSION: OnceLock<Arc<NodeAdmission>> = OnceLock::new();
+
+tokio::task_local! {
+    static SCANNER_NODE_REQUESTS: ScannerNodeRequestSender;
+    static SCANNER_NODE_OPERATION_CANCEL: CancellationToken;
+}
+
+/// Cancellation for the currently admitted scanner operation, when present.
+pub(crate) fn scanner_node_operation_cancellation() -> Option<CancellationToken> {
+    SCANNER_NODE_OPERATION_CANCEL.try_with(Clone::clone).ok()
+}
+
+pub(crate) struct ScannerNodeRequest {
+    pub(crate) owner: Arc<NodeAdmission>,
+    pub(crate) respond: oneshot::Sender<NodePermit>,
+    pub(crate) cancel: CancellationToken,
+}
+
+pub(crate) type ScannerNodeRequestSender = mpsc::Sender<ScannerNodeRequest>;
+pub(crate) type ScannerNodeRequestReceiver = mpsc::Receiver<ScannerNodeRequest>;
+
+/// One suspended operation per walk; disabled admission allocates no channel.
+pub(crate) fn scanner_node_request_channel() -> (Option<ScannerNodeRequestSender>, Option<ScannerNodeRequestReceiver>) {
+    if scanner_node_admission().is_none() {
+        return (None, None);
+    }
+    let (sender, receiver) = mpsc::channel(1);
+    (Some(sender), Some(receiver))
+}
+
+pub(crate) async fn with_scanner_node_requests<F: std::future::Future>(
+    sender: Option<ScannerNodeRequestSender>,
+    operation: F,
+) -> F::Output {
+    match sender {
+        Some(sender) => SCANNER_NODE_REQUESTS.scope(sender, operation).await,
+        None => operation.await,
+    }
+}
+
+#[cfg(test)]
+tokio::task_local! {
+    pub(crate) static TEST_SCANNER_NODE_ADMISSION: Arc<NodeAdmission>;
+    pub(crate) static TEST_SCANNER_AFTER_NODE_ACQUIRE: Arc<dyn Fn() + Send + Sync>;
+}
+
+/// Install the heal manager's node owner before starting scanner workers.
+/// Refuse replacement: existing scans must not outlive an independently renewed owner.
+pub fn set_scanner_node_admission(owner: Arc<NodeAdmission>) -> bool {
+    SCANNER_NODE_ADMISSION.set(owner).is_ok()
+}
+
+pub(crate) fn scanner_node_admission() -> Option<Arc<NodeAdmission>> {
+    #[cfg(test)]
+    if let Ok(owner) = TEST_SCANNER_NODE_ADMISSION.try_with(Arc::clone) {
+        return Some(owner);
+    }
+    SCANNER_NODE_ADMISSION.get().cloned()
+}
+
+/// Try the fast path first. A scoped coordinator releases the cache lease before
+/// joining the node queue and validates the cache before returning a permit.
+/// Uncoordinated callers defer instead of parking under an unknown lock.
+pub(crate) async fn run_scanner_node_operation<T, E>(
+    owner: Option<&Arc<NodeAdmission>>,
+    cancel: &CancellationToken,
+    operation: impl std::future::Future<Output = Result<T, E>>,
+) -> Result<Result<T, E>, ScannerError> {
+    let Some(owner) = owner else {
+        return Ok(operation.await);
+    };
+    let operation = SCANNER_NODE_OPERATION_CANCEL.scope(cancel.clone(), operation);
+    if cancel.is_cancelled() {
+        return Err(ScannerError::Other("Operation cancelled".to_string()));
+    }
+    let permit = match owner.try_acquire() {
+        Some(permit) => permit,
+        None => {
+            let sender = SCANNER_NODE_REQUESTS
+                .try_with(Clone::clone)
+                .map_err(|_| ScannerError::ResourceDeferred)?;
+            let acquire = async {
+                let (respond, response) = oneshot::channel();
+                sender
+                    .send(ScannerNodeRequest {
+                        owner: owner.clone(),
+                        respond,
+                        cancel: cancel.clone(),
+                    })
+                    .await
+                    .map_err(|_| ScannerError::ResourceDeferred)?;
+                response.await.map_err(|_| ScannerError::ResourceDeferred)
+            };
+            tokio::select! {
+                biased;
+                _ = cancel.cancelled() => return Err(ScannerError::Other("Operation cancelled".to_string())),
+                permit = acquire => permit?,
+            }
+        }
+    };
+    #[cfg(test)]
+    let _ = TEST_SCANNER_AFTER_NODE_ACQUIRE.try_with(|hook| hook());
+    let operation_started = AtomicBool::new(false);
+    tokio::pin!(operation);
+    let observed = std::future::poll_fn(|context| {
+        operation_started.store(true, Ordering::Relaxed);
+        std::future::Future::poll(operation.as_mut(), context)
+    });
+    tokio::pin!(observed);
+    let result = tokio::select! {
+        biased;
+        _ = cancel.cancelled() => {
+            // The node slot belongs to the waiter, not the native operation.
+            drop(permit);
+            if !operation_started.load(Ordering::Relaxed) {
+                return Err(ScannerError::Other("Operation cancelled".to_string()));
+            }
+            // Keep polling the started operation under its existing locks.
+            return Ok(observed.await);
+        }
+        result = &mut observed => result,
+    };
+    if result.is_ok() && !cancel.is_cancelled() {
+        permit.complete();
+    }
+    Ok(result)
+}
 
 type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
 

crates/targets/src/runtime/mod.rs

@@ -1471,6 +1471,7 @@ mod tests {
             inner: QueueStore<QueuedPayload>,
             fail_put_failed_raw: Arc<std::sync::atomic::AtomicBool>,
             fail_del: Arc<std::sync::atomic::AtomicBool>,
+            delete_attempts: Arc<Semaphore>,
         }
 
         impl Store<QueuedPayload> for FailingFailedStore {
@@ -1499,10 +1500,13 @@ mod tests {
                 self.inner.get_raw(key)
             }
             fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
-                if self.fail_del.load(Ordering::SeqCst) {
-                    return Err(StoreError::Internal("injected del failure".to_string()));
-                }
-                self.inner.del(key)
+                let result = if self.fail_del.load(Ordering::SeqCst) {
+                    Err(StoreError::Internal("injected del failure".to_string()))
+                } else {
+                    self.inner.del(key)
+                };
+                self.delete_attempts.add_permits(1);
+                result
             }
             fn delete(&self) -> Result<(), Self::Error> {
                 self.inner.delete()
@@ -1986,6 +1990,7 @@ mod tests {
                 inner: inner.clone(),
                 fail_put_failed_raw: fail_flag.clone(),
                 fail_del: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                delete_attempts: Arc::new(Semaphore::new(0)),
             };
             // The target's terminal move writes through the rejecting failed handle, so the injected
             // failure is what the move hits.
@@ -2042,6 +2047,7 @@ mod tests {
                 inner: inner.clone(),
                 fail_put_failed_raw: Arc::new(std::sync::atomic::AtomicBool::new(false)),
                 fail_del: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+                delete_attempts: Arc::new(Semaphore::new(0)),
             };
             let target = Arc::new(ProgrammedTarget::new(
                 Some(TargetError::JetStreamPublish {
@@ -2053,8 +2059,15 @@ mod tests {
             let tally = Arc::new(ReplayTally::default());
             let (cancel_tx, worker) = spawn_failing_store_worker(&store, target, tally.clone());
 
-            // Many scans run while the live delete keeps failing.
-            tokio::time::sleep(Duration::from_secs(300)).await;
+            // Observe repeated durable overwrites without auto-advancing through thousands of
+            // real filesystem writes. Keep the original virtual observation bound on progress.
+            tokio::time::timeout(Duration::from_secs(300), store.delete_attempts.acquire_many(8))
+                .await
+                .expect("replay should retry the failing delete eight times")
+                .expect("delete progress remains open")
+                .forget();
+            cancel_tx.send(()).await.expect("replay worker should remain cancellable");
+            worker.await.expect("replay worker should drain after cancellation");
             assert_eq!(
                 tally.permanent_failure.load(Ordering::SeqCst),
                 0,
@@ -2063,8 +2076,6 @@ mod tests {
             assert_eq!(inner.len(), 1, "the live entry stays for the next scan");
             assert_eq!(inner.failed_len(), 1, "the durable failed copy landed exactly once");
 
-            let _ = cancel_tx.send(()).await;
-            let _ = worker.await;
             let _ = inner.delete();
         }
 
@@ -2084,6 +2095,7 @@ mod tests {
                 inner: store.clone(),
                 fail_put_failed_raw: Arc::new(std::sync::atomic::AtomicBool::new(false)),
                 fail_del: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+                delete_attempts: Arc::new(Semaphore::new(0)),
             };
             let target = Arc::new(ProgrammedTarget::new(
                 Some(TargetError::JetStreamPublish {

docs/architecture/README.md

@@ -50,6 +50,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
 | [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
 | [scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md) | changing what makes a scanner cycle `Complete`, `Superseded`, or converged, adding or re-classifying a dirty-usage producer, deciding whether a background writeback may invalidate a cycle, changing scanner worker concurrency or throttling, or activating scoped dirty acknowledgements |
 | [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
+| [plugin-delegation.md](plugin-delegation.md) | changing original-user plugin delegation, attempt credentials, or Runner task APIs |
 | [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
 | [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
 | [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |

docs/architecture/erasure-coding.md

@@ -313,7 +313,7 @@ Small objects store their payload inline after the container CRC ([filemeta_inli
 ## 8. Read path and read quorum
 
 - **INVARIANT — read quorum = `data_blocks`.** `object_quorum_from_meta` returns `(read_quorum = data_blocks, write_quorum)` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)); `parity_blocks = common_parity(...)` is the parity value held by the most disks that still reaches its own read quorum. When `default_parity_count == 0`, read = write = all shards.
-- Authoritative FileInfo selection — `find_file_info_in_quorum` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)) groups valid metas by a content-identity SHA-256 (`file_info_quorum_hash`) that hashes size/flags/mod_time/transition/version_id/data_dir/parts and, for real objects, data/parity/distribution — **excluding replication-status keys** so replication noise never splits quorum. A meta counts only if its mod_time equals the common mod_time (or etag matches when mod_time is absent). The winning hash must reach quorum, else `ErasureReadQuorum`. S3 reads select at read quorum through `select_valid_fileinfo`, the same rule heal uses and MinIO's `getObjectFileInfo`: the most common mod_time wins and a tie goes to the newer one, so a write that failed after reaching exactly `data_blocks` copies is served as listing shows it, instead of `ErasureReadQuorum` until heal runs. Two identities tied at the selected mod_time (a data = parity set split in half, such as a metadata update that failed after reaching half the drives) resolve to the one on the lowest-index disk, in reads and heal alike, so a retried metadata write can repair the split. A selected version with local shards also needs as many copies as its own `data_blocks`, since delete markers and empty versions can lower the common parity's read quorum. A metadata update whose target disks cannot reach write quorum is refused before any write. A version replaced by a committed write keeps fewer than `data_blocks` copies and can never be selected again; the newer side of a failed write can be served and then lose to the older side if one of its disks fails before heal converges the set. Reads in the system bucket (`is_meta_bucketname`) keep the commit-proof rule in `select_committed_fileinfo`: when enough copies answer to prove write quorum, the latest version must hold it, and a degraded read never serves a newer version below write quorum. Pool metadata's PREPARE/COMMIT recovery and identity epochs rely on an ordinary read implying a committed record, so a split stays unreadable until native recovery resolves it. When every drive answered and an S3 latest read still cannot select a version, `read_latest_past_dangling_versions` walks the version stack newest first through exact-version reads, as MinIO's `pickLatestQuorumFilesInfo` resolves stacks: it serves the readable version with the newest selected mod_time, skips versions whose exact read proves them dangling (the NoSuchVersion an explicit GET reports) and queues heal for them, and keeps the read-quorum error on any other outcome. A failed versioned PUT that left the new version on fewer drives than its data blocks therefore no longer hides the committed version below it; ListObjectVersions can still show the dangling version as latest until heal removes it.
+- Authoritative FileInfo selection — `find_file_info_in_quorum` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)) groups valid metas by a content-identity SHA-256 (`file_info_quorum_hash`) that hashes size/flags/mod_time/transition/version_id/data_dir/parts and, for real objects, data/parity/distribution — **excluding replication-status keys** so replication noise never splits quorum. A meta counts only if its mod_time equals the common mod_time (or etag matches when mod_time is absent). The winning hash must reach quorum, else `ErasureReadQuorum`. S3 reads select at read quorum through `select_valid_fileinfo`, the same rule heal uses and MinIO's `getObjectFileInfo`: the most common mod_time wins and a tie goes to the newer one, so a write that failed after reaching exactly `data_blocks` copies is served as listing shows it, instead of `ErasureReadQuorum` until heal runs. Two identities tied at the selected mod_time (a data = parity set split in half, such as a metadata update that failed after reaching half the drives) resolve to the one on the lowest-index disk, in reads and heal alike, so a retried metadata write can repair the split. A selected version with local shards also needs as many copies as its own `data_blocks`, since delete markers and empty versions can lower the common parity's read quorum. A metadata update whose target disks cannot reach write quorum is refused before any write. A version replaced by a committed write keeps fewer than `data_blocks` copies and can never be selected again; the newer side of a failed write can be served and then lose to the older side if one of its disks fails before heal converges the set. Reads in the system bucket (`is_meta_bucketname`) keep the commit-proof rule in `select_committed_fileinfo`: when enough copies answer to prove write quorum, the latest version must hold it, and a degraded read never serves a newer version below write quorum. Pool metadata's PREPARE/COMMIT recovery and identity epochs rely on an ordinary read implying a committed record, so a split stays unreadable until native recovery resolves it. When every drive answered and an S3 latest read still cannot select a version, `read_latest_past_dangling_versions` walks the version stack newest first through exact-version reads, as MinIO's `pickLatestQuorumFilesInfo` resolves stacks: it serves the readable version with the newest selected mod_time, skips versions whose exact read proves them dangling (the NoSuchVersion an explicit GET reports) and queues heal for them, and keeps the read-quorum error on any other outcome. A failed versioned PUT that left the new version on fewer drives than its data blocks therefore no longer hides the committed version below it; ListObjectVersions can still show the dangling version as latest until heal removes it. Metadata updates of the latest version (object tagging, `put_object_metadata`) and the CopyObject source read resolve a failed selection through the same walk; when the walk finds no readable version, tagging keeps the read-quorum error while `put_object_metadata` and CopyObject keep their existing `delete_if_dangling` cleanup. Replication status write-back never walks. The update targets only the served version's disks and is refused before writing when they cannot reach its write quorum, including a version-only copy.
 - **INVARIANT — decode needs ≥ `data_blocks` shards.** The stripe reader requires `available_shards ≥ data_shards`; below that the read fails closed with a read-quorum error (never silent truncation) ([set_disk/read.rs](../../crates/ecstore/src/set_disk/read.rs), [set_disk/shard_source.rs](../../crates/ecstore/src/set_disk/shard_source.rs)). Codec geometry is validated at construction: `Erasure::try_new` / `try_new_with_options` ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)) return `ErasureConstructionError` for `data_shards == 0`, `block_size == 0`, shard-count overflow, or an unsupported shard configuration, so a read never reaches a `block_size` / `data_shards` division with invalid geometry (§13).
 - If `available ≥ data_blocks` but some shards are missing, the read is served **and** a background read-repair heal is enqueued.
 - **INVARIANT — cross-stripe read verification.** When a data shard is missing and `available > data_blocks`, reconstruction regenerates parity and compares it to the surviving parity; a mismatch is `InvalidData "inconsistent read source shards"` (backlog#832), catching corruption that passed per-shard bitrot but disagrees across the stripe ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)).

docs/architecture/plugin-delegation.md

@@ -0,0 +1,142 @@
+# Plugin delegation
+
+**Use this when:** changing plugin delegation registration, revocation, broker identity, or the scope of attempt credentials.
+
+## Integration boundary
+
+Runner owns scheduling, leases, container execution and result publication. RustFS provides original-user delegation, scoped temporary credentials and authenticated task API forwarding. Inputs and outputs use ordinary versioned S3 buckets; event-driven jobs use the existing bucket notification/Webhook path. Managed artifact sealing, inventory, garbage collection and backup retention require separate storage contracts and are outside this integration.
+
+Set `RUSTFS_PLUGIN_RUNNER_URL` to the Controller HTTPS URL ending in `/v1/ingress/manual/jobs`, and `RUSTFS_PLUGIN_RUNNER_DEPLOYMENT` to the delegation deployment. Forwarding requires the published outbound mTLS client identity and uses the published trust roots when configured. Connections are reused within a TLS generation; publishing new TLS material replaces the pool, and removing the client identity rejects new forwarding requests.
+
+## Authority
+
+A delegation is an immutable restriction record owned by an ordinary IAM user. It is neither a permission grant nor a credential. Creating it requires that user's real signed request and `admin:CreatePluginDelegation`; reading and revoking require the corresponding `admin:GetPluginDelegation` and `admin:RevokePluginDelegation` actions. Root, temporary credentials and service accounts cannot own records.
+
+The record binds a deployment, an input bucket, literal input and output prefixes, and an expiry of at most 24 hours. The v1 profile writes to that same bucket; v2 explicitly names the output bucket. Wildcards, control characters and IAM variable expressions are rejected. Its random ID is also its immutable revision. There is no update operation.
+
+The Controller is responsible for validating accepted task identity, current lease, epoch, node session and control generation before requesting credentials. RustFS authenticates the configured broker and enforces the delegation and current parent IAM. RustFS does not independently verify a remote Controller lease.
+
+```mermaid
+sequenceDiagram
+    participant U as Original IAM user
+    participant C as Controller broker
+    participant R as RustFS
+    participant P as Plugin
+    U->>R: Signed delegation registration
+    R-->>U: Immutable ID and revision
+    C->>C: Validate task and current lease
+    C->>R: Signed issue-attempt request
+    R->>R: Broker mapping + active record + current IAM
+    R->>R: Persist scoped STS and recheck authority
+    R-->>C: Short-lived credentials
+    C-->>P: Attempt credentials
+    P->>R: Exact version GET / attempt-prefix PUT
+    R->>R: Current parent IAM intersected with session policy
+```
+
+## Persistence and revocation
+
+`crates/iam/src/delegation.rs` owns record validation and immutable revocation tombstones. Records use the existing IAM store under `config/iam/plugin-delegations/v1`. Writes require a successful readback. Revocation is idempotent and durable; it does not delete or overwrite the restriction record.
+
+`ObjectStore::load_iam_config_strict` distinguishes a missing object from an empty object, invalid JSON or failed decryption. Only a genuinely absent tombstone permits an active record. Corrupt or unavailable storage rejects access rather than treating revocation as absent. Existing non-delegation readers keep their compatibility behavior.
+
+Revocation prevents future issuance. It does not promise immediate invalidation of credentials already delivered; those remain subject to their expiration and current parent IAM. Registration, IAM changes and issuance are not one atomic transaction. Issuance rechecks after persistence and deletes the generated STS account if authority changed. Failed persistence or invalidation acknowledgment is reported as unknown issuance, never as safe retry permission.
+
+Manual job submission reads the active delegation before calling the Runner. A submission that passed this check is in flight and may be admitted if revocation races with the outbound request; revocation prevents submissions whose authority check occurs afterward, but does not cancel an admitted job. Callers must treat a successful submit receipt as accepted work even if the delegation is revoked later. After an ambiguous timeout or service error, retry the same idempotency key while the delegation remains active; after revocation or expiry, query status instead of retrying submission.
+
+`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/query` accepts `{version:1,run_id}` and requires an ordinary IAM user with `admin:GetPluginRun` who owns the delegation in `{id}`. It reads the immutable owner record without checking expiry or revocation, then queries Runner using the record's delegation ID and revision and the authenticated access key. The result can contain only that caller's run status and committed manifest; it does not perform object reads or reauthorize object access. The Runner response must match the requested run ID and the strict status schema before RustFS returns it.
+
+`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/cancel` accepts the same request and requires `admin:CancelPluginRun` plus ownership of `{id}`. Expiry and revocation do not block cancelling an existing run. Runner scopes the run to that owner; missing and other-owner runs share the same not-found response, while successful and failed runs return conflict. Success returns `{version:1,run_id,state:"CANCELLED",cleanup_pending}` and does not imply that asynchronous cleanup has finished.
+
+`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/page` accepts `{version:1,limit,cursor?}` with a limit from 1 to 100 and requires `admin:ListPluginRuns` plus ownership of `{id}`. Expiry and revocation do not hide historical runs. RustFS supplies signer and delegation identity from authenticated IAM state; the Runner page is scoped to the exact caller and deployment. The strict response contains run summaries and an opaque-to-clients stable cursor; callers pass the returned cursor unchanged until it is null. RustFS bounds request/response sizes and rejects malformed, unordered, over-limit, or identity-invalid summaries.
+
+`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/history` accepts `{version:1,run_id}` and requires `admin:ListPluginRuns` plus ownership of `{id}`. RustFS derives signer and delegation identity from the authenticated user and IAM record, then proxies Runner's bounded history response. The response contains at most the latest 32 committed state events in Runner's history-ID order and a `has_older` flag; timestamps are display data and are not used to reorder events. Missing and foreign runs share the same not-found response. RustFS rejects malformed, empty, over-limit, unexpected-state, or mismatched-run responses and bounds both request and response bodies.
+
+`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/replay` accepts `{version:1,origin_run_id,idempotency_key,deadline_unix_micros}` and requires `admin:ReplayPluginRun`, current IAM authorization, ownership, and an active unexpired delegation. Caller identity fields are not accepted; RustFS derives signer and delegation revision from the authenticated request and IAM record. Runner accepts only an owned failed origin, preserves its fixed plugin/configuration/input version, and requires a new deadline no later than its configured run timeout. RustFS requires a positive Unix-microsecond deadline and verifies the strict receipt echoes it. Missing or foreign origins share not-found behavior; conflicts map to an operation conflict. Requests and receipts are bounded, and responses use `Cache-Control: no-store`.
+
+## Broker profile
+
+`rustfs/src/admin/handlers/plugin_attempt.rs` implements `POST /rustfs/admin/v3/plugin-delegations/{id}/issue-attempt`. The signer must be an ordinary IAM user with `admin:IssuePluginAttempt`, independently listed in `RUSTFS_PLUGIN_BROKERS`:
+
+```json
+{"broker-access-key":"deployment-name"}
+```
+
+This configuration contains public access keys, not secrets. It is immutable per process. Missing or invalid configuration disables broker issuance. The broker is a trusted Controller identity; ordinary users cannot substitute their own request fields for Controller validation.
+
+The request carries `tenant`, `run_id`, `attempt_id`, `revision`, `deployment`, `input_bucket`, `input_key`, `input_version`, `output_bucket`, `output_prefix` and `ttl_seconds`. IDs must be non-nil. Both profiles accept one exact input version and an output prefix equal to:
+
+```text
+<delegation-output-prefix><lowercase-hex-tenant>/<run-uuid>/<attempt-uuid>/
+```
+
+The session policy permits only `s3:GetObjectVersion` for that input with an exact `s3:VersionId`, and `s3:PutObject` beneath that attempt prefix. Credential lifetime is at most 900 seconds and cannot exceed delegation expiry. The Controller must additionally cap it to its validated lease and hard deadline.
+
+The original user's current IAM remains the parent permission boundary on every S3 request. The server's signing key only seals the token; it is not the plugin's authorization identity. Broker identity and original user identity are separate.
+
+External policy plugins must be disabled: their authorization path does not enforce this built-in session-policy boundary. Initializing, failed and ready plugin states all reject issuance. Attempt tokens carry a signed, persisted marker: after restart or on another node, marked credentials are accepted only by built-in STS authorization. A denied marked request returns a terminal error, so public bucket policies cannot bypass its session scope. The first profile also rejects parent policies containing conditions, because a broker request cannot reconstruct the original user's IP, MFA or other request context. It does not silently replace that context with the broker's.
+
+## Response and failures
+
+Successful issuance returns `version`, `delegation_id`, `delegation_revision`, `owner_access_key`, `access_key_id`, `secret_access_key`, `session_token` and `expires_at_unix_seconds`, with `Cache-Control: no-store`. The Controller verifies the returned delegation and owner against the accepted caller before delivery. Credentials must never enter logs, metrics or task result manifests.
+
+Missing records and records owned by a different user have the same read response. Revoked or expired records reject access. Storage failures remain errors. Post-persistence failures carry static `error_code`, `issued` and `issuance_unknown` facts without credentials; callers must reconcile an unknown outcome rather than automatically mint another credential.
+
+The route/action inventory is maintained in `rustfs/src/admin/route_policy.rs`. This contract does not imply event capture, physical container cleanup, restart reconciliation or upstream release acceptance.
+
+
+## Cross-bucket restriction profile v2
+
+The existing registration endpoint accepts this explicit v2 restriction:
+
+```json
+{
+  "version": 2,
+  "deployment": "runner-production",
+  "bucket": "source-images",
+  "input_prefix": "inputs/",
+  "output_bucket": "plugin-artifacts",
+  "output_prefix": "outputs/",
+  "expires_at_unix_seconds": 1893456000
+}
+```
+
+The example expiry must be replaced by a future value within the existing
+24-hour limit. `bucket` still identifies the input bucket. The existing
+`issue-attempt` request and credential response do not change: its
+`input_bucket` must equal `bucket`, and `output_bucket` must equal the explicit
+v2 output bucket. Neither a different bucket nor a sibling attempt prefix can
+be supplied by the broker. The exact input version and deterministic output
+prefix checks remain in force.
+
+A v1 restriction has neither `version` nor `output_bucket`; its JSON fields
+and same-bucket behavior are unchanged. V2 requires both `version: 2` and a
+nonempty, bounded, literal `output_bucket`. Missing pairs, explicit nulls,
+unknown versions and unknown fields are rejected rather than downgraded to v1.
+The immutable record envelope carries version 1 or 2 to match its restriction
+profile. Both remain in the existing IAM record namespace; no data migration
+or mutable record update is introduced. Readers reject mismatched envelope
+and restriction versions. Older readers reject v2, so deploy compatible
+readers before creating v2 records.
+
+V2 does not grant the original user additional permissions. Issuance retains
+its current-parent checks and post-persistence recheck/invalidation, and each
+S3 request still requires both the original user's current IAM policy and the
+attempt session policy to allow it. An explicit parent Deny or removal of
+output permission prevents writes even with an already issued token. No
+administrator credential substitutes for missing parent permission.
+
+Focused tests are `delegation_v1_json_remains_compatible`,
+`delegation_v2_requires_explicit_output_bucket`,
+`attempt_v2_policy_keeps_input_version_and_output_bucket` and
+`delegation_v2_persists_and_vends_with_real_parent_intersection`.
+The last uses the actual issuance handler, persistent IAM and four-disk
+storage, including a real input version and a third bucket allowed by the
+parent but forbidden by the delegation. Removing only the output-bucket
+comparison must fail `A03_CROSS_BUCKET_OUTPUT_BOUND`; parent intersection is
+checked independently with issued temporary credentials. These are internal
+handler/IAM tests, not signed HTTP or complete cross-bucket S3 execution.
+
+This profile adds no bucket-creation API, Runner binding support, credential
+endpoint or GC permission. Coordinating Runner's pinned binding with its
+result storage bucket remains a separate integration requirement.

docs/architecture/s3-compatibility-matrix.md

@@ -100,7 +100,7 @@ Standard S3 areas that must not be described as complete:
 | Bucket access logging | Handlers exist (`get_bucket_logging`, `put_bucket_logging` in `rustfs/src/storage/ecfs.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
 | POST Object form upload checksum handling | Not yet passing | `unimplemented_tests.txt` |
 | Bucket ownership controls | No handler | `unimplemented_tests.txt` |
-| Multipart upload listing and part lookup compatibility edge cases | Not part of default gate | `excluded_tests.txt` |
+| Multipart upload listing and part lookup compatibility edge cases | Not part of default gate; ListParts returns HTTP 503 `SlowDownRead` when no directory was listed and any participant is unavailable, instead of treating one missing directory as an empty upload | `excluded_tests.txt`; `collect_list_parts_results_rejects_unavailable_empty_upload_dirs` |
 | IAM-account or multi-storage-class dependent cases | Not part of default gate | `unimplemented_tests.txt` |
 | Tenanted bucket policy edge cases | Needs investigation | `unimplemented_tests.txt` |
 

docs/architecture/scanner-usage-publication.md

@@ -182,6 +182,18 @@ requires a compatibility window and a dedicated cleanup entry.
 | `.scanner-cycle.lock` | scanner runtime lock | Serializes cycle-level work. A missing lock object is not itself usage evidence. |
 | `.scanner-pause-backlog.json` | scanner pause and catch-up ledger | Tracks dirty usage, discovered lifecycle work, and full-scan catch-up while authoritative publication is fenced by data movement. It never grants publication admission. |
 
+The authoritative snapshot may include an optional `bucket_incarnation` in each
+bucket usage entry. The coordinator writes it only when every contributing set
+reports the same non-nil bucket UUID. Missing or conflicting identities leave
+that entry uncertified; they do not reject the usage publication or change quota
+admission. Veeam SOSAPI capacity reads require this proof to match the bucket
+incarnation read from authoritative storage. An old snapshot, a mixed-version
+writer that omits the field, a pending target bucket, or an unavailable identity
+therefore yields a retryable capacity error until a compatible scanner publishes
+a certified entry. The optional field is omitted when unavailable so older
+snapshot readers continue to see the original shape. This proof identifies the
+bucket generation; it does not make scanner usage a real-time quota ledger.
+
 ## Observed Snapshots
 
 Observed snapshots are a diagnostic and availability layer. They may be served

docs/operations/presigned-size-limits.md

@@ -54,4 +54,5 @@ Properties of a capped upload:
 - Unknown-length or negative-length parts are rejected with `UnexpectedContent` rather than buffered without a bound.
 - Capped parts are admitted under an upload-wide write lock before temporary shards are created, and hold a per-upload staging permit that bounds local in-flight data. The lock is released while the body is read and reacquired for the final check and rename, so `Complete` and `Abort` are not blocked behind a slow upload. The request-body stall timeout releases the staging permit when a client stops sending.
 - An upload created without the parameter stays unlimited.
-- Enforcement runs in the multipart data plane on every node. During a rolling upgrade, route capped uploads only to nodes that carry the V2 implementation; a node without it treats the internal metadata as unknown and cannot enforce the limit.
+- Size admission requires a response from every configured disk, refuses incomplete or sub-quorum part observations, and requires matching part metadata at write quorum. An unknown logical size, or one differing from its stored part size without a recorded transformation, is refused. A missing part directory on a responding disk is an empty observation; an unavailable disk is not. Capped UploadPart may therefore return a retryable storage error during disk outages even when the write quorum is available. Retry after the disks recover.
+- Enforcement runs in the multipart data plane on every node. During a rolling upgrade, route capped uploads only to nodes that carry both V2 and the strict part-size proof; older nodes can treat an unavailable part observation as empty. A node without V2 does not enforce the limit at all. Fleet-wide protection requires upgrading every node that may receive these writes.

docs/operations/scanner-runtime-controls.md

@@ -529,17 +529,21 @@ The OTLP exporter records these node-local histograms for completed manager admi
 
 The two new histograms aggregate all observed admission sources and results without per-request labels. An ordinary uninterrupted admission partitions the phase into wait and hold. Admin forceStart releases these guards while cancelling overlaps, then reacquires them: the cancellation gap is included only in the original total phase, and the second acquisition contributes to wait. Cancellation subroutine work is outside these outer admission guard intervals. Recording happens after all three guards are released. They measure manager admission ownership, not namespace/object-lock waiting during repair. The status field `maxLockPhaseMicros` remains a manager-lifetime maximum, not a p95 sample. Keep same-window histogram counts and bucket resolution when deriving percentiles; missing observations do not mean zero latency. These metrics do not establish performance acceptance by themselves.
 
-## Experimental heal node admission
+## Experimental shared node admission
 
-`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback for the four object-heal entry points owned by the heal manager. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. This does not yet control scanner walks, bucket/format repair or physical devices, and it is not a replacement for existing worker, disk or byte budgets.
+`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback shared by the heal manager’s four object-heal entry points and local Scanner namespace reads and object accounting. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. This does not yet control the abandoned-child streaming list coordinator, bucket/format repair or physical devices, and it is not a replacement for existing worker, disk or byte budgets.
 
 The node samples its effective CPU quota/cpuset, ancestor memory headroom, CPU usage/throttling and CPU/memory/I/O pressure once per second. Sampling uses one blocking worker at a time. Resource reads retain their real start/end times: reads spanning over 20 ms or older than one second after all inputs return cannot renew admission. Counter rates use both read brackets conservatively. The concurrency cap is at most the rounded-up effective CPU count and 256 scheduling units; existing operator limits still apply independently. At most 1% memory headroom blocks new work. This is a pressure safeguard, not a memory reservation or a claim that concurrent repair buffers fit in the remaining bytes.
 
 Each complete sample leases admission for three seconds. Missing resource caps block new work; missing or low-confidence feedback retains at most one maintenance unit. CPU use above 90%, throttling or pressure above 1% of elapsed time, memory headroom at most 10%, foreground errors/cancellations, a foreground queue or a comparable foreground p99 regression reduce concurrency immediately. Recovery requires CPU use at most 70%, throttling/stalls at most 0.1% of elapsed time, at least 20% memory headroom, fresh foreground evidence and actual successful work. The existing controller probes one additional unit only after eight consecutive healthy seconds and cooldown. Limits affect new admission and do not revoke active work.
 
+The Scanner captures the same node owner once per bucket walk. Directory open/refill, entry type and size lookup, and object accounting try admission before their first storage poll; each unit ends when its operation future finishes. On contention, local and remote coordinators suspend the same traversal, release the bucket cache lease, and join the shared FIFO queue. They reacquire the cache lease and verify both cache revisions before resuming. A changed cache rejects the suspended result. Cancellation also drops the queued request; the existing cycle budget still bounds the walk. This retains directory and recursion state while allowing Heal to advance at capacity one, without repeatedly replaying a partially classified directory.
+
+Calls without a coordinating cache lease retain immediate resource deferral and useful partial-cache handling. Existing remote single-session ownership and local-disk admission shares remain in force. Disabled admission creates no request channel. Enabled walks use one bounded channel, with a response allocation and cache revision reads only on contention; the uncontended per-entry path adds no global lookup or task. The streaming abandoned-child coordinator requires a separate ownership boundary: retaining a node unit while it waits for a bounded-channel consumer that needs the same unit would deadlock at capacity one.
+
 Foreground comparison uses the existing complete response-body windows and shared admission snapshots. Each active class needs at least 100 completions and a finite p99 interval; this permits the percentile rank but is not a statistical confidence guarantee. Comparisons require the same active classes and rates within 80–125% of a retained reference. A changed workload resets recovery. The reference is an observed workload window, not an independent background-off baseline. A p99 interval can justify stability only when its upper bound is within 5% of the reference lower bound; a lower bound above 105% of the reference upper bound is pressure. Overlapping/uncertain comparisons cannot justify growth. Missing queues, stale or repeated cuts, uncertain cut boundaries and low counts are not idle evidence.
 
-Admission waits at most 250 ms before returning the existing typed transient deferral, before polling storage or taking namespace locks. No-lock callers never wait. Queued read repair uses the same node gate; direct foreground ECStore calls remain outside this heal-manager adapter. Cancellation releases the non-cloneable node scheduling unit. Native work retains separate physical-I/O units, and unsuccessful/cancelled/dry-run work cannot increment the node's completed-work counter. This scheduling counter is not an authoritative heal receipt and cannot discharge repair responsibility.
+Heal admission waits at most 250 ms before returning the existing typed transient deferral, before polling storage or taking namespace locks. No-lock callers never wait. Queued read repair uses the same node gate; direct foreground ECStore calls remain outside this heal-manager adapter. Cancellation releases the non-cloneable node scheduling unit. Native work retains separate physical-I/O units, and unsuccessful/cancelled/dry-run work cannot increment the node's completed-work counter. This scheduling counter is not an authoritative heal receipt and cannot discharge repair responsibility.
 
 The `rustfs_background_node_admission_limit` and `rustfs_background_node_admission_active` gauges describe the last published sample, not a live reservation proof. A blocked sampler can leave these exports stale even though the admission lease has expired. Full disk-owner feedback, shared scanner scheduling, escaping-buffer byte ownership and the matched-release SSD matrix/hot-shrink/72-hour soak remain required before enabling this by default or claiming optimal performance.
 
@@ -725,7 +729,11 @@ Only two MinIO scanner variables are recognized. `apply_external_env_compat` (`c
 | `scanner alert_excess_folders` (50000) | `scanner.alert_excess_folders` (65538) | Not mapped; see [Scanner Excess Alerts](scanner-excess-alerts.md). |
 | Any other `MINIO_SCANNER_*` | Corresponding `RUSTFS_SCANNER_*` from the tables above | Not mapped; rename explicitly. |
 
-Stale-upload cleanup differs in one crash-recovery detail. RustFS's stale multipart cleanup (`cleanup_stale_multipart_uploads_in_set`) takes a namespace write lock, re-checks the upload under write quorum (`check_multipart_upload_path_exists`), and fans the delete out to every disk, where the local recursive delete renames the directory into `.rustfs.sys/tmp/.trash/<uuid>` (`move_to_trash`). If the process dies mid fan-out after more disks than the parity count have already moved the upload directory, the next cleanup pass's quorum re-check fails (`FileNotFound` is not in `OBJECT_OP_IGNORED_ERRS`) and the candidate is skipped, so the remaining per-disk residue is not reclaimed by that job. The residue is invisible to the S3 API and only consumes disk space; the window is milliseconds wide. MinIO processes each disk independently and converges in the same scenario.
+Stale-upload cleanup takes the upload namespace write lock and rechecks cleanup authority before deletion. An incomplete legacy upload without sufficient identity or deletion quorum remains on disk; age and absence from an S3 listing do not establish that its recovery or publication authority has ended. A native Abort with a complete, validated closure certificate can recover a smaller residual after partial deletion, subject to the original-participant checks. Missing, unavailable or corrupt proof keeps the operation conservative. For an otherwise valid native Abort certificate, an unavailable participant identity is a retryable write-quorum failure; a known conflicting identity remains a corruption error, including when another participant is unavailable. See [the multipart authority contract](../architecture/erasure-coding.md#7-write-path-and-write-quorum).
+
+Size-filtered incomplete-upload rules also require every participant to answer the part listing and matching part metadata at write quorum. Unknown size skips only size-dependent rules. During a mixed-version upgrade, an older node can still evaluate those rules using the older empty-list behavior; fleet-wide protection requires every cleanup worker to run the updated version. Plan the rollout so older workers cannot perform size-filtered aborts while mixed versions are active.
+
+The `rustfs_stale_multipart_cleanup_blocked_total` counter has a bounded `stage` label: `recheck` for lock or authority recheck failures, `lock_lost` when a held cleanup lock is lost, and `delete` for a failed authorized deletion. It counts attempts, not unique uploads or bytes, and includes transient failures; it carries no bucket, key or upload labels. The `multipart_cleanup_lock_or_recheck_failed` debug event identifies a candidate path and the failure; `multipart_cleanup_lock_lost` identifies the path when a previously acquired cleanup lock is lost. Restore unavailable disks and retry through supported S3 or background recovery paths when their prerequisites hold. Preserve the disk state and seek an evidence-based recovery procedure when authority remains uncertain; do not recursively remove a residual solely because it is old or a destination object exists. The default expiry is measured from upload initiation, including internal data-movement uploads, and can affect long-running uploads even without a bucket lifecycle rule. This differs from [Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html), which retains parts until the upload is completed or stopped and offers an explicit lifecycle rule to abort incomplete uploads. The interval is a schedule, not a guaranteed reclamation deadline.
 
 ## Replacement Recovery Completion
 

docs/operations/sftp.md

@@ -158,7 +158,7 @@ When a session ends mid-upload the server aborts the in-flight multipart upload.
 - a burst of simultaneous disconnects exhausts the global abort task pool (sized at twice the CPU parallelism, between `ABORT_PERMITS_FLOOR` 8 and `ABORT_PERMITS_CEILING` 128);
 - the server process is killed outright.
 
-Orphaned uploads are not permanent. The background stale-upload cleanup aborts incomplete multipart uploads older than `DEFAULT_STALE_UPLOADS_EXPIRY` (24 hours), checked every `DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL` (6 hours). An `AbortIncompleteMultipartUpload` bucket lifecycle rule reclaims them as well and gives per-bucket control of the window. The log lines when a session-drop abort is skipped or fails:
+Background stale-upload cleanup considers incomplete multipart uploads after `DEFAULT_STALE_UPLOADS_EXPIRY` (24 hours from initiation), on a `DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL` schedule (6 hours). A bucket `AbortIncompleteMultipartUpload` rule can make an upload eligible earlier. Both paths retain their lock and cleanup-authority checks: low-quorum or otherwise uncertain residue can remain, so neither the schedule nor a lifecycle rule guarantees physical reclamation. See [stale-upload recovery](scanner-runtime-controls.md#migrating-from-minio-scanner-settings). The log lines when a session-drop abort is skipped or fails:
 
 ```text
 skipped abort of orphaned multipart upload on session drop, principal lacks s3:AbortMultipartUpload, bucket lifecycle rules must reclaim parts
@@ -187,6 +187,6 @@ Every line below logs at warn level except `SFTP server listening` and `SFTP aut
 | `SFTP write would exceed the S3 multipart parts limit` | Client hit the per-upload size cap; raise the part size if legitimate. |
 | `SFTP close rejected: trailing part would exceed S3 multipart parts limit` | Same cap hit at file close. |
 | `root READDIR truncated` | A principal can see more than 10000 buckets; listing was cut off. |
-| `skipped abort of orphaned multipart upload` | Orphaned parts on storage until background cleanup or a lifecycle rule reclaims them. |
-| `abort permit pool exhausted on session drop` | Mass-disconnect burst; orphaned parts on storage until reclaimed. |
+| `skipped abort of orphaned multipart upload` | Parts may remain until cleanup has sufficient authority to reclaim them. |
+| `abort permit pool exhausted on session drop` | Mass-disconnect burst; parts may remain when cleanup authority is unavailable. |
 | `RUSTFS_SFTP_` prefix in a warning | A tuning variable was rejected and its default applied. |

docs/testing/distributed-e2e.md

@@ -108,11 +108,17 @@ Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linu
 
 ## Multipart fence fault acceptance
 
+The EC:2 metadata-delay case also holds the three scoped `ListDir` RPCs only after the upload metadata has been read, while the coordinator's part directory is unavailable. It requires the ListParts response to be HTTP 503 `SlowDownRead`, then restores the exact directory, verifies the part ETag and completes the original bytes through every reader. The same case signs a ten-byte multipart limit and races two six-byte parts through different coordinators; exactly one must succeed, and the final object must match that part.
+
+The four-node rolling and direct upgrade cases seed a multipart upload on the verified previous release. Rolling checks compare part number, size and ETag through old and new nodes while multipart mutations are paused. With one old disk offline and the other three part directories temporarily absent, metadata remains readable: the previous coordinator returns an empty list while the current coordinator returns HTTP 503 `SlowDownRead`. After the directories and disk return, both coordinators must list the original ETag and all four original metadata, part and part-metadata files must remain byte-identical. Once all nodes are current, parity-zero probes prove each coordinator can write through every disk before the single completion request; both upgrade cases then check every reader. This read-only fault case does not permit multipart mutations during a mixed rollout.
+
+The nightly stale-multipart cleanup case also retains one and two EC:2 source copies on online disks. It waits for a path-specific background recheck diagnostic, verifies every surviving directory and file byte, and checks that the diagnostic is not attributed to a client UploadPart. This is evidence for conservative preservation and diagnosis, not automatic reclamation without a closure certificate. Node logs are retained when `RUSTFS_E2E_LOG_DIR` is set.
+
 The `distributed::multipart_fence_acceptance_test` cases separate remote storage RPC delays from lock RPC delays through a method-selective proxy. They require the selected RPC to reach its pause before checking quorum, recovery, or cancellation. The proxy forwards opaque bodies and gRPC trailers and is confined to the test process. It does not change server timeouts, locking, retry policy, or the commit path.
 
 After a quorum-minus-one Complete is rejected and faults are retired, ordinary Abort must succeed. `NoSuchUpload` is not accepted as recovery of that failed completion: it can hide staging metadata consumed by partial publication. Already completed or aborted sessions retain their existing missing-upload cleanup semantics. The ECStore multipart regressions assert unchanged per-disk authority bytes across primary/recovery locations, no duplicated disk votes, rejection of completed-upload resends after cleanup failure, orphan-cleanup refusal while authority remains, preservation of the previous object, and write-quorum completion replay. Scoped Abort faults decode the exact multipart volume, root path and recursive option without changing signed bytes. Post-fault read probes can wait within the existing recovery deadline for classified 503 responses, honoring Retry-After; injected writes and successful streaming bodies remain single-attempt.
 
-`native_certified_partial_abort_retry_reclaims_survivors_below_metadata_quorum` first closes all sixteen native participants, then holds five exact-root `RecoverMultipartSource(DeleteAborted)` RPCs in EC12+4. It observes eleven physical deletions and a typed write-quorum failure. The five survivors must retain their complete original shards, Aborting state and identical full-N certificate bound by each actual owner; no ACTIVE head or old capability remains. After peer recovery the same upload retries successfully, authenticating the stored closure and deleting only those five roots, with no remaining source/control record and unchanged committed-object bytes through every reader. Historical unmarked primary/recovery authority still has its separate four-case ECStore regression: without native closure, five standalone originals remain insufficient and must be retained rather than treated as absence. The explicit late-delivery proxy mode retains bounded control packets beyond caller cancellation; dropping that mode discards them, and proxy teardown aborts its retained tasks. Late Complete acceptance delivers the original signed RenameMultipartData packets after successful Abort, staging GC and same-name bucket recreation, and requires an authenticated mutation refusal plus unchanged new-object bytes. This covers empty and non-inline parts; expired-packet discard remains a distinct fault model.
+`native_certified_partial_abort_retry_reclaims_survivors_below_metadata_quorum` first closes all sixteen native participants, then holds five exact-root `RecoverMultipartSource(DeleteAborted)` RPCs in EC12+4. It observes eleven physical deletions and a typed write-quorum failure. The five survivors must retain their complete original shards, Aborting state and identical full-N certificate bound by each actual owner; no ACTIVE head or old capability remains. After peer recovery the test first stops an original participant and requires a `storage_write_quorum` refusal with byte-identical retained sources. It restores that node, then kills and restarts the coordinator and a node owning a surviving root, verifies actual storage routes and byte-identical surviving sources, and retries the same upload once. The retry authenticates the stored closure and deletes only those five roots, with no remaining source/control record and unchanged committed-object bytes through every reader. The ECStore `stale_certified_residual_survives_lock_loss_then_converges` regression separately checks one surviving EC:2 root: an unavailable original participant and a lost cleanup-lock heartbeat both refuse deletion, preserve the source and shard, and permit cleanup after recovery. Historical unmarked primary/recovery authority still has its separate four-case ECStore regression: without native closure, five standalone originals remain insufficient and must be retained rather than treated as absence. The explicit late-delivery proxy mode retains bounded control packets beyond caller cancellation; dropping that mode discards them, and proxy teardown aborts its retained tasks. Late Complete acceptance delivers the original signed RenameMultipartData packets after successful Abort, staging GC and same-name bucket recreation, and requires an authenticated mutation refusal plus unchanged new-object bytes. This covers empty and non-inline parts; expired-packet discard remains a distinct fault model.
 
 The pinned previous-release MPU recovery case bootstraps and seeds all sixteen disks through the release's normal direct transport. It then installs the method observers on the same advertised localhost ports immediately before restarting the current coordinator, whose fresh RPC channels must traverse them. The observers must receive exact, read-only native Inspect rejections from the independently constructed set of four configured disk endpoint URLs on each old peer; different or duplicate routes cannot substitute for those participants, and old direct connections cannot satisfy that evidence. While three peers remain old, the current coordinator must return a single typed write-quorum refusal before dispatching any old or native publication RPC. Every original source directory, metadata file and shard remains byte-for-byte intact, the part ETag stays readable and no target appears. Multipart mutations are not sent through an old coordinator during rollout. After every peer is current and all configured storage routes respond, the same upload must Abort successfully, reclaim all sixteen roots and preserve the unrelated historical object through every reader. This tests the supported mixed-version pause and recovery boundary. The same case runs for every verified source in the [upgrade compatibility workflow](../../.github/workflows/e2e-upgrade.yml), with an exact one-test listing check and per-version node/test logs. This supplements the distributed suite's historical source pin.
 

fuzz/Cargo.toml

@@ -19,6 +19,18 @@ rustfs-security-governance = { path = "../crates/security-governance" }
 rustfs-utils = { path = "../crates/utils", features = ["compress", "path"] }
 serde_json = "1"
 
+[patch.crates-io]
+# Standalone workspaces must repeat the root workspace's fixed SigV4a backend.
+aws-sigv4 = { git = "https://github.com/smithy-lang/smithy-rs", rev = "5c1c4bca2ea7af97496e077c9f1a718eaac64bf8" }
+
+[patch."https://github.com/smithy-lang/smithy-rs"]
+# Keep the signer's path dependencies on the SDK's existing registry types.
+aws-credential-types = { version = "=1.3.1" }
+aws-smithy-eventstream = { version = "=0.61.6" }
+aws-smithy-http = { version = "=0.64.3" }
+aws-smithy-runtime-api = { version = "=1.19.2" }
+aws-smithy-types = { version = "=1.9.1" }
+
 [[bin]]
 name = "bucket_validation"
 path = "fuzz_targets/bucket_validation.rs"

rustfs/Cargo.toml

@@ -297,7 +297,7 @@ tokio-rustls = { workspace = true, default-features = false, features = ["loggin
 aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 aws-smithy-types = { workspace = true }
 google-cloud-auth = { workspace = true, optional = true }
-moka = { workspace = true, features = ["sync"] }
+moka = { workspace = true, features = ["future", "sync"] }
 xxhash-rust = { workspace = true, features = ["xxh3"] }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 tokio-stream.workspace = true
@@ -392,6 +392,7 @@ object = { workspace = true, optional = true, features = ["read", "std", "elf"]
 symbolic-demangle = { version = "=13.9.0", default-features = false, features = ["rust"], optional = true }
 
 [dev-dependencies]
+rustfs-targets = { workspace = true, features = ["test-support"] }
 uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
 rustfs-iam = { workspace = true, features = ["test-util"] }
 serial_test = { workspace = true }

rustfs/src/admin/handlers/event.rs

@@ -50,7 +50,7 @@ use tracing::{error, info, warn};
 
 const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
 
-async fn converge_target_mutation_on_cluster(
+pub(super) async fn converge_target_mutation_on_cluster(
     context: Option<&AppContext>,
     target_type: &str,
     local_result: Result<(), rustfs_notify::NotificationError>,
@@ -60,18 +60,26 @@ async fn converge_target_mutation_on_cluster(
 
     match (local_result, peer_result) {
         (Ok(()), Ok(())) => Ok(()),
-        (Err(local), Ok(())) => {
-            warn!(target_type, error = %local, "Local notification runtime failed to converge");
-            Err(s3_error!(InternalError, "local notification runtime failed to converge"))
-        }
-        (Ok(()), Err(peer)) => Err(peer),
-        (Err(local), Err(peer)) => {
-            warn!(target_type, error = %local, "Local notification runtime failed while peer convergence also failed");
-            Err(s3_error!(
-                InternalError,
-                "local notification runtime and peer convergence failed: {}",
-                peer
-            ))
+        (Ok(()), Err(_)) => Err(s3_error!(
+            InternalError,
+            "notification target update completed locally but peer convergence failed"
+        )),
+        (Err(local), peer) => {
+            let peer_failed = peer.is_err();
+            let message = match local {
+                rustfs_notify::NotificationError::ConfigurationConvergence { persisted: true, .. } => {
+                    "notification target configuration was persisted but local runtime convergence failed"
+                }
+                rustfs_notify::NotificationError::ConfigurationConvergence { persisted: false, .. } => {
+                    "notification target configuration was unchanged but local runtime convergence failed"
+                }
+                _ => "notification target update failed; persistence status is unknown",
+            };
+            if peer_failed {
+                Err(s3_error!(InternalError, "{}; peer convergence also failed", message))
+            } else {
+                Err(s3_error!(InternalError, "{}", message))
+            }
         }
     }
 }
@@ -84,7 +92,7 @@ fn notification_target_subsystem(target_type: &str) -> S3Result<&'static str> {
         .ok_or_else(|| s3_error!(InvalidArgument, "unsupported notification target type: {}", target_type))
 }
 
-async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> {
+pub(super) async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> {
     preflight_dynamic_config_reload_for_context(context, notification_target_subsystem(target_type)?).await
 }
 
@@ -682,7 +690,7 @@ mod tests {
         use crate::admin::handlers::{
             audit::ListAuditTargets,
             extensions::ListExtensionInstancesHandler,
-            plugins_instances::{GetPluginInstanceHandler, ListPluginInstancesHandler},
+            plugins_instances::{GetPluginInstanceHandler, ListPluginInstancesHandler, PutPluginInstanceHandler},
             target_read::load_target_read_snapshot,
         };
         use crate::admin::storage_api::config::{
@@ -758,6 +766,154 @@ mod tests {
                 .await
                 .expect("persist target without loading it into the runtime");
 
+            // A concurrent credential rotation must be observed under the commit transaction.
+            let initial = crate::admin::storage_api::config::read_admin_config_without_migrate(env.ecstore.clone())
+                .await
+                .expect("read initial notify config");
+            let generation_before = {
+                let snapshot = crate::admin::storage_api::config::read_admin_server_config_snapshot(env.ecstore.clone())
+                    .await
+                    .expect("read initial generation");
+                snapshot.generation().expect("test config has an object generation")
+            };
+            let expected = initial.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"].clone();
+            let expected_default = initial.0[NOTIFY_WEBHOOK_SUB_SYS]
+                .get(rustfs_config::DEFAULT_DELIMITER)
+                .cloned();
+            let mut rotated = initial.clone();
+            rotated
+                .0
+                .get_mut(NOTIFY_WEBHOOK_SUB_SYS)
+                .expect("subsystem")
+                .get_mut("unavailable")
+                .expect("target")
+                .insert(rustfs_config::WEBHOOK_AUTH_TOKEN.to_owned(), "rotated-secret".to_owned());
+            save_admin_server_config(env.ecstore.clone(), &rotated)
+                .await
+                .expect("persist concurrent rotation");
+            let generation_after = {
+                let snapshot = crate::admin::storage_api::config::read_admin_server_config_snapshot(env.ecstore.clone())
+                    .await
+                    .expect("read rotated generation");
+                snapshot.generation().expect("rotated config has an object generation")
+            };
+            assert_ne!(generation_before, generation_after, "content change must publish a new generation");
+            let system = rustfs_notify::NotificationSystem::new(Config::default());
+            let result = system
+                .set_target_config_if_unchanged(
+                    NOTIFY_WEBHOOK_SUB_SYS,
+                    "unavailable",
+                    Some(expected.clone()),
+                    expected_default,
+                    None,
+                    expected,
+                )
+                .await;
+            assert!(matches!(result, Err(rustfs_notify::NotificationError::ConfigurationConflict)));
+            let actual = crate::admin::storage_api::config::read_admin_config_without_migrate(env.ecstore.clone())
+                .await
+                .expect("read rotated config");
+            assert_eq!(
+                actual.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"]
+                    .lookup(rustfs_config::WEBHOOK_AUTH_TOKEN)
+                    .as_deref(),
+                Some("rotated-secret")
+            );
+            save_admin_server_config(env.ecstore.clone(), &initial)
+                .await
+                .expect("restore inventory config");
+            let generation = {
+                let snapshot = crate::admin::storage_api::config::read_admin_server_config_snapshot(env.ecstore.clone())
+                    .await
+                    .expect("read revision baseline");
+                snapshot.generation().expect("baseline generation").to_string()
+            };
+            let mut unrelated = initial.clone();
+            let mut other = KVS::new();
+            other.insert("enable".into(), "off".into());
+            other.insert("endpoint".into(), "http://127.0.0.1:9/other".into());
+            unrelated
+                .0
+                .entry(NOTIFY_WEBHOOK_SUB_SYS.to_owned())
+                .or_default()
+                .insert("other".to_owned(), other);
+            save_admin_server_config(env.ecstore.clone(), &unrelated)
+                .await
+                .expect("persist unrelated update");
+            let target = initial.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"].clone();
+            let default = initial.0[NOTIFY_WEBHOOK_SUB_SYS]
+                .get(rustfs_config::DEFAULT_DELIMITER)
+                .cloned();
+            let result = system
+                .set_target_config_if_unchanged(
+                    NOTIFY_WEBHOOK_SUB_SYS,
+                    "unavailable",
+                    Some(target.clone()),
+                    default,
+                    Some(generation),
+                    target,
+                )
+                .await;
+            assert!(
+                matches!(result, Err(rustfs_notify::NotificationError::ConfigurationConflict)),
+                "revision changes must reject even when target remains unchanged"
+            );
+            save_admin_server_config(env.ecstore.clone(), &initial)
+                .await
+                .expect("restore after unrelated update");
+
+            let mut with_default = initial.clone();
+            let mut default = KVS::new();
+            default.insert("queue_limit".into(), "15".into());
+            with_default
+                .0
+                .get_mut(NOTIFY_WEBHOOK_SUB_SYS)
+                .expect("subsystem")
+                .insert("_".into(), default.clone());
+            save_admin_server_config(env.ecstore.clone(), &with_default)
+                .await
+                .expect("persist inherited default");
+            let saved = crate::admin::storage_api::config::read_admin_config_without_migrate(env.ecstore.clone())
+                .await
+                .expect("read inherited config");
+            let original = saved.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"].clone();
+            let old_default = saved.0[NOTIFY_WEBHOOK_SUB_SYS].get("_").cloned();
+            let expected = std::collections::HashMap::from([
+                ("endpoint".to_owned(), "http://127.0.0.1:9/hook".to_owned()),
+                ("queue_limit".to_owned(), "15".to_owned()),
+            ]);
+            let mut validated = original.clone();
+            validated.insert("endpoint".into(), "http://127.0.0.1:9/changed".into());
+            validated.insert("queue_limit".into(), "15".into());
+            let update = crate::admin::handlers::plugins_instances::merge_conditional_target_update(
+                &original,
+                &expected,
+                &["endpoint".into()],
+                &validated,
+            )
+            .expect("merge only edited field");
+            let result = system
+                .set_target_config_if_unchanged(NOTIFY_WEBHOOK_SUB_SYS, "unavailable", Some(original), old_default, None, update)
+                .await;
+            assert!(
+                result.is_ok()
+                    || matches!(
+                        result,
+                        Err(rustfs_notify::NotificationError::ConfigurationConvergence { persisted: true, .. })
+                    )
+            );
+            let saved = crate::admin::storage_api::config::read_admin_config_without_migrate(env.ecstore.clone())
+                .await
+                .expect("read conditional update");
+            assert_eq!(
+                saved.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"].lookup("endpoint").as_deref(),
+                Some("http://127.0.0.1:9/changed")
+            );
+            assert_eq!(saved.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"].lookup("queue_limit"), None);
+            assert_eq!(saved.0[NOTIFY_WEBHOOK_SUB_SYS]["_"].lookup("queue_limit").as_deref(), Some("15"));
+            save_admin_server_config(env.ecstore.clone(), &initial)
+                .await
+                .expect("restore inventory config");
             let (locked_tx, locked_rx) = tokio::sync::oneshot::channel();
             let (release_tx, release_rx) = tokio::sync::oneshot::channel();
             let store = env.ecstore.clone();
@@ -785,6 +941,53 @@ mod tests {
             assert!(observed_config.0[NOTIFY_WEBHOOK_SUB_SYS].contains_key("unavailable"));
             mark_event_notifier_unreconciled();
 
+            // Conditional edits reject stale revisions before target validation or persistence.
+            let mut router = Router::new();
+            router.insert("/{id}", ()).expect("plugin detail route");
+            let matched = router.at("/builtin:webhook:notify:unavailable").expect("plugin params");
+            let transaction_guard = crate::admin::storage_api::config::read_admin_server_config_snapshot(env.ecstore.clone())
+                .await
+                .expect("hold config transaction lock");
+            let generation = transaction_guard.generation().expect("config generation").to_string();
+            let detail = response_json(GetPluginInstanceHandler {}.call(
+                request("/rustfs/admin/v4/plugins/instances/builtin:webhook:notify:unavailable"),
+                matched.params,
+            ))
+            .await
+            .expect("read conditional edit baseline");
+            let revision = detail["config_revision"].as_str().expect("GET includes opaque revision");
+            assert_eq!(
+                revision,
+                crate::admin::handlers::plugins_instances::config_revision("builtin:webhook:notify:unavailable", &generation)
+            );
+            assert_eq!(detail["config"]["auth_token"], "***redacted***");
+            drop(transaction_guard);
+            let mut put = request("/rustfs/admin/v4/plugins/instances/builtin:webhook:notify:unavailable");
+            put.method = Method::PUT;
+            put.input = Body::from(
+                serde_json::to_vec(&serde_json::json!({
+                    "expected_config": detail["config"],
+                    "expected_revision": "stale",
+                    "key_values": [{"key":"auth_token", "value":"replacement-secret"}]
+                }))
+                .expect("encode conditional edit"),
+            );
+            let matched = router.at("/builtin:webhook:notify:unavailable").expect("plugin params");
+            let err = PutPluginInstanceHandler {}
+                .call(put, matched.params)
+                .await
+                .expect_err("stale revision must be rejected");
+            assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest);
+            let after = crate::admin::storage_api::config::read_admin_config_without_migrate(env.ecstore.clone())
+                .await
+                .expect("read unchanged config");
+            assert_eq!(
+                after.0[NOTIFY_WEBHOOK_SUB_SYS]["unavailable"]
+                    .lookup(rustfs_config::WEBHOOK_AUTH_TOKEN)
+                    .as_deref(),
+                Some("inventory-secret-token")
+            );
+
             with_notify_runtime_reconcile_lock(async {
                 for initialize_runtime in [false, true] {
                     if initialize_runtime {

rustfs/src/admin/handlers/mod.rs

@@ -52,6 +52,8 @@ pub mod object_data_cache;
 pub mod object_zip_download;
 pub mod oidc;
 pub mod on_demand_migration;
+pub mod plugin_attempt;
+pub mod plugin_delegation;
 pub mod plugins_catalog;
 pub mod plugins_instances;
 pub mod policies;

rustfs/src/admin/handlers/plugin_attempt.rs

@@ -0,0 +1,568 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+//! Restricted vending for explicitly configured brokers, never AssumeRole chaining.
+use super::plugin_delegation::{owner, response, storage_error};
+use crate::{
+    admin::{
+        router::{AdminOperation, Operation, S3Router},
+        runtime_sources,
+        storage_api::s3::{self, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result},
+    },
+    auth::check_key_valid_with_context,
+    server::ADMIN_PREFIX,
+};
+use http::{Method, StatusCode};
+use matchit::Params;
+use rustfs_iam::{
+    delegation::Delegation,
+    sys::{PLUGIN_ATTEMPT_CREDENTIAL_CLAIM, SESSION_POLICY_NAME},
+};
+use rustfs_policy::{
+    auth::get_new_credentials_with_metadata,
+    policy::{
+        Args, Policy,
+        action::{Action, AdminAction, S3Action},
+    },
+};
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::HashMap,
+    sync::OnceLock,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+use time::OffsetDateTime;
+
+// Operator configuration contains public signer access keys, not credentials.
+// Empty/invalid configuration disables the endpoint. It is immutable per process.
+static BROKERS: OnceLock<Option<HashMap<String, String>>> = OnceLock::new();
+fn configured_deployment(key: &str) -> Option<&'static str> {
+    BROKERS
+        .get_or_init(|| {
+            let raw = std::env::var("RUSTFS_PLUGIN_BROKERS").ok()?;
+            let map: HashMap<String, String> = serde_json::from_str(&raw).ok()?;
+            if map.is_empty()
+                || map
+                    .iter()
+                    .any(|(k, v)| k.is_empty() || v.is_empty() || k.len() > 1024 || v.len() > 1024)
+            {
+                return None;
+            }
+            Some(map)
+        })
+        .as_ref()?
+        .get(key)
+        .map(String::as_str)
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct Attempt {
+    tenant: String,
+    run_id: uuid::Uuid,
+    attempt_id: uuid::Uuid,
+    revision: String,
+    deployment: String,
+    input_bucket: String,
+    input_key: String,
+    input_version: String,
+    output_bucket: String,
+    output_prefix: String,
+    ttl_seconds: u64,
+}
+#[derive(Serialize)]
+struct Issued {
+    version: u32,
+    delegation_id: String,
+    delegation_revision: String,
+    owner_access_key: String,
+    access_key_id: String,
+    secret_access_key: String,
+    session_token: String,
+    expires_at_unix_seconds: i64,
+}
+
+pub fn register_plugin_attempt_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/issue-attempt"),
+        AdminOperation(&Issue),
+    )?;
+    Ok(())
+}
+fn now() -> S3Result<u64> {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))
+}
+fn literal(value: &str) -> bool {
+    !value.is_empty()
+        && value.len() <= 1024
+        && !value.contains("${")
+        && !value.chars().any(|c| c.is_control() || matches!(c, '*' | '?'))
+}
+fn session_policy(record: &Delegation, attempt: &Attempt) -> S3Result<Policy> {
+    let restriction = &record.restrictions;
+    // Stored records from earlier versions must obey the same literal rules.
+    restriction.validate(record.created_at_unix_seconds).map_err(storage_error)?;
+    let exact_output = format!(
+        "{}{}/{}/{}/",
+        restriction.output_prefix,
+        hex_simd::encode_to_string(attempt.tenant.as_bytes(), hex_simd::AsciiCase::Lower),
+        attempt.run_id,
+        attempt.attempt_id
+    );
+    if attempt.tenant.is_empty()
+        || attempt.tenant.len() > 1024
+        || attempt.run_id.is_nil()
+        || attempt.attempt_id.is_nil()
+        || attempt.output_prefix != exact_output
+        || attempt.revision != record.revision
+        || attempt.deployment != restriction.deployment
+        || attempt.input_bucket != restriction.bucket
+        || attempt.output_bucket != restriction.output_bucket().map_err(storage_error)?
+        || !literal(&attempt.input_key)
+        || !attempt.input_key.starts_with(&restriction.input_prefix)
+        || !literal(&attempt.input_version)
+        || attempt.input_version == "null"
+        || !literal(&attempt.output_prefix)
+        || !attempt.output_prefix.starts_with(&restriction.output_prefix)
+        || attempt.output_prefix == restriction.output_prefix
+        || !attempt.output_prefix.ends_with('/')
+        || !(1..=900).contains(&attempt.ttl_seconds)
+    {
+        return Err(s3::error(S3ErrorCode::AccessDenied, "attempt restriction rejected"));
+    }
+    let value = serde_json::json!({"Version":"2012-10-17","Statement":[
+        {"Effect":"Allow","Action":["s3:GetObjectVersion"],"Resource":[format!("arn:aws:s3:::{}/{}",attempt.input_bucket,attempt.input_key)],"Condition":{"StringEquals":{"s3:VersionId":attempt.input_version}}},
+        {"Effect":"Allow","Action":["s3:PutObject"],"Resource":[format!("arn:aws:s3:::{}/{}*",attempt.output_bucket,attempt.output_prefix)]}
+    ]});
+    // Condition operators borrow their keys while the policy is decoded.
+    let bytes = serde_json::to_vec(&value).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))?;
+    serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))
+}
+
+fn issuance_error(code: &'static str, issued: bool, unknown: bool) -> S3Result<S3Response<(StatusCode, Body)>> {
+    let mut answer = response(&serde_json::json!({"error_code":code,"issued":issued,"issuance_unknown":unknown}))?;
+    answer.output.0 = StatusCode::SERVICE_UNAVAILABLE;
+    Ok(answer)
+}
+fn same_parent_authority(before_names: &[String], before_policy: &Policy, after_names: &[String], after_policy: &Policy) -> bool {
+    fn names(names: &[String]) -> Vec<&str> {
+        let mut names: Vec<_> = names.iter().map(String::as_str).collect();
+        names.sort_unstable();
+        names.dedup();
+        names
+    }
+    fn statements(policy: &Policy) -> Option<Vec<Vec<u8>>> {
+        let mut statements = policy
+            .statements
+            .iter()
+            .map(serde_json::to_vec)
+            .collect::<Result<Vec<_>, _>>()
+            .ok()?;
+        statements.sort_unstable();
+        Some(statements)
+    }
+
+    names(before_names) == names(after_names)
+        && matches!(
+            (statements(before_policy), statements(after_policy)),
+            (Some(before), Some(after)) if before == after
+        )
+}
+async fn require_builtin_authorization() -> S3Result<()> {
+    rustfs_iam::sys::IamSys::<rustfs_iam::store::object::ObjectStore>::ensure_restricted_credential_vending()
+        .await
+        .map_err(|_| s3::error(S3ErrorCode::AccessDenied, "external policy plugin unsupported for attempt credentials"))
+}
+
+struct Issue;
+#[async_trait::async_trait]
+impl Operation for Issue {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        require_builtin_authorization().await?;
+        let (iam, broker) = owner(&req, AdminAction::IssuePluginAttemptAction).await?;
+        let deployment =
+            configured_deployment(&broker).ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "broker deployment rejected"))?;
+        let context = runtime_sources::app_context_from_req(&req)
+            .ok_or_else(|| s3::error(S3ErrorCode::InternalError, "application context unavailable"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "attempt body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "attempt body unavailable"))?;
+        let attempt: Attempt =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid attempt request"))?;
+        if attempt.deployment != deployment {
+            return Err(s3::error(S3ErrorCode::AccessDenied, "broker deployment rejected"));
+        }
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let record = iam
+            .read_active_plugin_delegation_for_broker(id, now()?)
+            .await
+            .map_err(storage_error)?;
+        let policy = session_policy(&record, &attempt)?;
+        let (principal, root) = check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
+        if root || principal.is_temp() || principal.is_service_account() {
+            return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
+        }
+        // A broker request cannot reconstruct the original user's IP/MFA/tag
+        // conditions. This first profile rejects conditional parent policies.
+        let policies = iam
+            .policy_db_get(&principal.access_key, &principal.groups)
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InternalError, "parent policy unavailable"))?;
+        let parent_policy = iam.get_combined_policy(&policies).await;
+        if policies.is_empty() || parent_policy.statements.iter().any(|s| !s.conditions.is_empty()) {
+            return Err(s3::error(S3ErrorCode::AccessDenied, "parent condition context unsupported"));
+        }
+        let mut conditions = HashMap::new();
+        conditions.insert("s3:VersionId".to_owned(), vec![attempt.input_version.clone()]);
+        if !iam
+            .is_allowed(&Args {
+                account: &principal.access_key,
+                groups: &principal.groups,
+                action: Action::S3Action(S3Action::GetObjectVersionAction),
+                bucket: &attempt.input_bucket,
+                object: &attempt.input_key,
+                conditions: &conditions,
+                is_owner: false,
+                claims: principal.claims_or_empty(),
+                deny_only: false,
+            })
+            .await
+        {
+            return Err(s3::error(S3ErrorCode::AccessDenied, "parent input rejected"));
+        }
+        let expiration = now()?
+            .checked_add(attempt.ttl_seconds)
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?
+            .min(record.restrictions.expires_at_unix_seconds);
+        let expires_at = OffsetDateTime::from_unix_timestamp(
+            i64::try_from(expiration).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?,
+        )
+        .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?;
+        let mut claims = principal.claims.clone().unwrap_or_default();
+        claims.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), serde_json::Value::Bool(true));
+        claims.insert("exp".into(), serde_json::json!(expires_at.unix_timestamp()));
+        claims.insert("parent".into(), serde_json::json!(principal.access_key));
+        let bytes =
+            serde_json::to_vec(&policy).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))?;
+        claims.insert(
+            SESSION_POLICY_NAME.into(),
+            serde_json::json!(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&bytes)),
+        );
+        let fresh = iam
+            .read_active_plugin_delegation_for_broker(id, now()?)
+            .await
+            .map_err(storage_error)?;
+        if fresh != record {
+            return Err(s3::error(S3ErrorCode::AccessDenied, "attempt authority changed"));
+        }
+        owner(&req, AdminAction::IssuePluginAttemptAction).await?;
+        require_builtin_authorization().await?;
+        let signing = context
+            .action_credentials()
+            .get()
+            .ok_or_else(|| s3::error(S3ErrorCode::InternalError, "token signer unavailable"))?;
+        // The server key seals the token only. Authorization above uses the
+        // original user's current IAM, and S3 continues its parent intersection.
+        let mut credential = get_new_credentials_with_metadata(&claims, &signing.secret_key)
+            .map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt credential unavailable"))?;
+        credential.parent_user = principal.access_key.clone();
+        if iam.set_temp_user(&credential.access_key, &credential, None).await.is_err() {
+            return issuance_error("PLUGIN_ATTEMPT_ISSUANCE_UNKNOWN", false, true);
+        }
+        // No atomic IAM/delegation transaction spans persistence. Recheck and
+        // invalidate the generated credential if any authority changed.
+        let valid = async {
+            require_builtin_authorization().await?;
+            let after = iam
+                .read_active_plugin_delegation_for_broker(id, now()?)
+                .await
+                .map_err(storage_error)?;
+            let (current, current_root) =
+                check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
+            let names = iam
+                .policy_db_get(&current.access_key, &current.groups)
+                .await
+                .map_err(|_| s3::error(S3ErrorCode::InternalError, "parent policy unavailable"))?;
+            let current_policy = iam.get_combined_policy(&names).await;
+            if after != record
+                || current_root
+                || current.is_temp()
+                || current.is_service_account()
+                || current.secret_key != principal.secret_key
+                || !same_parent_authority(&policies, &parent_policy, &names, &current_policy)
+            {
+                return Err(s3::error(S3ErrorCode::AccessDenied, "attempt authority changed"));
+            }
+            // Re-evaluate the broker against its real signed request context.
+            owner(&req, AdminAction::IssuePluginAttemptAction).await?;
+            Ok::<(), S3Error>(())
+        }
+        .await;
+        if valid.is_err() {
+            if iam.delete_temp_account(&credential.access_key, true).await.is_err() {
+                return issuance_error("PLUGIN_ATTEMPT_INVALIDATION_UNKNOWN", true, true);
+            }
+            return issuance_error("PLUGIN_ATTEMPT_POST_ISSUANCE_REJECTED", true, false);
+        }
+        response(&Issued {
+            version: 1,
+            delegation_id: record.id,
+            delegation_revision: record.revision,
+            owner_access_key: record.owner_access_key,
+            access_key_id: credential.access_key,
+            secret_access_key: credential.secret_key,
+            session_token: credential.session_token,
+            expires_at_unix_seconds: expires_at.unix_timestamp(),
+        })
+    }
+}
+
+#[cfg(test)]
+#[path = "plugin_attempt_v2_tests.rs"]
+mod v2_tests;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn parent_authority_recheck_ignores_policy_order_but_detects_changes() {
+        let first: Policy = serde_json::from_value(serde_json::json!({
+            "Version":"2012-10-17",
+            "Statement":[
+                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
+                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"}
+            ]
+        }))
+        .unwrap();
+        let reordered: Policy = serde_json::from_value(serde_json::json!({
+            "Version":"2012-10-17",
+            "Statement":[
+                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"},
+                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"}
+            ]
+        }))
+        .unwrap();
+        let before_names = vec!["owner-policy".to_owned(), "bucket-policy".to_owned()];
+        let after_names = vec!["bucket-policy".to_owned(), "owner-policy".to_owned()];
+        assert!(same_parent_authority(&before_names, &first, &after_names, &reordered));
+
+        let changed: Policy = serde_json::from_value(serde_json::json!({
+            "Version":"2012-10-17",
+            "Statement":[
+                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
+                {"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::photos/output/*"}
+            ]
+        }))
+        .unwrap();
+        assert!(!same_parent_authority(&before_names, &first, &after_names, &changed));
+        let changed_resource: Policy = serde_json::from_value(serde_json::json!({
+            "Version":"2012-10-17",
+            "Statement":[
+                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
+                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/private/*"}
+            ]
+        }))
+        .unwrap();
+        assert!(!same_parent_authority(&before_names, &first, &after_names, &changed_resource));
+        let added_deny: Policy = serde_json::from_value(serde_json::json!({
+            "Version":"2012-10-17",
+            "Statement":[
+                {"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/private/*"},
+                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
+                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"}
+            ]
+        }))
+        .unwrap();
+        assert!(!same_parent_authority(&before_names, &first, &after_names, &added_deny));
+        assert!(!same_parent_authority(&before_names, &first, &["other-policy".to_owned()], &first));
+    }
+    #[tokio::test]
+    async fn attempt_policy_keeps_exact_version_and_narrow_output() {
+        let record = Delegation {
+            version: 1,
+            id: "a".repeat(32),
+            revision: "a".repeat(32),
+            owner_access_key: "user".into(),
+            created_at_unix_seconds: 1,
+            restrictions: rustfs_iam::delegation::Restrictions {
+                version: None,
+                output_bucket: None,
+                deployment: "runner".into(),
+                bucket: "photos".into(),
+                input_prefix: "input/".into(),
+                output_prefix: "output/".into(),
+                expires_at_unix_seconds: 1000,
+            },
+        };
+        let run_id = uuid::Uuid::new_v4();
+        let attempt_id = uuid::Uuid::new_v4();
+        let output_prefix = format!(
+            "output/{}/{}/{}/",
+            hex_simd::encode_to_string(b"tenant", hex_simd::AsciiCase::Lower),
+            run_id,
+            attempt_id
+        );
+        let mut attempt = Attempt {
+            tenant: "tenant".into(),
+            run_id,
+            attempt_id,
+            revision: record.revision.clone(),
+            deployment: "runner".into(),
+            input_bucket: "photos".into(),
+            input_key: "input/photo.png".into(),
+            input_version: "version-1".into(),
+            output_bucket: "photos".into(),
+            output_prefix: output_prefix.clone(),
+            ttl_seconds: 60,
+        };
+        let policy = session_policy(&record, &attempt).unwrap();
+        let scoped_output = format!("{output_prefix}result");
+        let groups = None;
+        let claims = HashMap::new();
+        for (action, object, version, expected) in [
+            (S3Action::GetObjectVersionAction, "input/photo.png", Some("version-1"), true),
+            (S3Action::GetObjectVersionAction, "input/photo.png", Some("version-2"), false),
+            (S3Action::GetObjectVersionAction, "input/photo.png", None, false),
+            (S3Action::GetObjectAction, "input/photo.png", None, false),
+            (S3Action::PutObjectAction, scoped_output.as_str(), None, true),
+            (S3Action::PutObjectAction, "outside/result", None, false),
+            (S3Action::PutObjectAction, "output/result", None, false),
+        ] {
+            let conditions = version
+                .map(|value| HashMap::from([("versionid".into(), vec![value.into()])]))
+                .unwrap_or_default();
+            assert_eq!(
+                policy
+                    .is_allowed(&Args {
+                        account: "user",
+                        groups: &groups,
+                        action: Action::S3Action(action),
+                        bucket: "photos",
+                        object,
+                        conditions: &conditions,
+                        is_owner: false,
+                        claims: &claims,
+                        deny_only: false,
+                    })
+                    .await,
+                expected,
+                "attempt policy action={action:?} object={object} version={version:?}"
+            );
+        }
+        attempt.output_bucket = "artifacts".into();
+        assert!(
+            session_policy(&record, &attempt).is_err(),
+            "v1 still requires the input bucket for output"
+        );
+        attempt.output_bucket = "photos".into();
+        attempt.output_prefix = "output/".into();
+        assert!(session_policy(&record, &attempt).is_err());
+        attempt.output_prefix = output_prefix;
+        attempt.input_key = "outside/photo.png".into();
+        assert!(session_policy(&record, &attempt).is_err());
+    }
+    #[tokio::test]
+    async fn attempt_v2_policy_keeps_input_version_and_output_bucket() {
+        let record: Delegation = serde_json::from_value(serde_json::json!({
+            "version":2,"id":"a".repeat(32),"revision":"a".repeat(32),"owner_access_key":"user",
+            "created_at_unix_seconds":1,"restrictions":{"version":2,"deployment":"runner","bucket":"photos",
+            "input_prefix":"input/","output_bucket":"artifacts","output_prefix":"output/","expires_at_unix_seconds":1000}
+        }))
+        .expect("explicit v2 delegation");
+        let run_id = uuid::Uuid::new_v4();
+        let attempt_id = uuid::Uuid::new_v4();
+        let output = format!("output/74656e616e74/{run_id}/{attempt_id}/");
+        let mut attempt = Attempt {
+            tenant: "tenant".into(),
+            run_id,
+            attempt_id,
+            revision: record.revision.clone(),
+            deployment: "runner".into(),
+            input_bucket: "photos".into(),
+            input_key: "input/photo.png".into(),
+            input_version: "version-1".into(),
+            output_bucket: "artifacts".into(),
+            output_prefix: output.clone(),
+            ttl_seconds: 60,
+        };
+        let policy = session_policy(&record, &attempt).expect("v2 scope");
+        let result = format!("{output}result");
+        for (action, bucket, object, version, expected) in [
+            (S3Action::GetObjectVersionAction, "photos", "input/photo.png", Some("version-1"), true),
+            (S3Action::GetObjectVersionAction, "photos", "input/photo.png", Some("version-2"), false),
+            (S3Action::GetObjectVersionAction, "artifacts", "input/photo.png", Some("version-1"), false),
+            (S3Action::PutObjectAction, "artifacts", result.as_str(), None, true),
+            (S3Action::PutObjectAction, "photos", result.as_str(), None, false),
+            (S3Action::PutObjectAction, "third", result.as_str(), None, false),
+            (S3Action::PutObjectAction, "artifacts", "output/other/result", None, false),
+        ] {
+            let conditions = version
+                .map(|v| HashMap::from([("versionid".into(), vec![v.into()])]))
+                .unwrap_or_default();
+            assert_eq!(
+                policy
+                    .is_allowed(&Args {
+                        account: "user",
+                        groups: &None,
+                        action: Action::S3Action(action),
+                        bucket,
+                        object,
+                        conditions: &conditions,
+                        claims: &HashMap::new(),
+                        is_owner: false,
+                        deny_only: false
+                    })
+                    .await,
+                expected,
+                "A03_CROSS_BUCKET_SCOPE: bucket={bucket} object={object}"
+            );
+        }
+        attempt.output_bucket = "third".into();
+        assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_OUTPUT_BOUND");
+        attempt.output_bucket = "photos".into();
+        assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_OUTPUT_BOUND");
+        attempt.output_bucket = "artifacts".into();
+        attempt.input_bucket = "artifacts".into();
+        assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_INPUT_BOUND");
+    }
+
+    #[tokio::test]
+    async fn actual_policy_variable_resolution_is_rejected_by_literal_scope() {
+        let resource = rustfs_policy::policy::resource::Resource::S3("photos/input/${aws:username}.png".into());
+        let conditions = HashMap::from([("username".into(), vec!["alice".into()])]);
+        let resolver =
+            rustfs_policy::policy::variables::VariableResolver::new(rustfs_policy::policy::variables::VariableContext {
+                username: Some("alice".into()),
+                ..Default::default()
+            });
+        assert!(
+            resource
+                .is_match_with_resolver("photos/input/alice.png", &conditions, Some(&resolver))
+                .await,
+            "C07_LITERAL_SCOPE_BOUND: actual policy engine substitutes resource variables"
+        );
+        assert!(
+            !literal("input/${aws:username}.png"),
+            "C07_LITERAL_SCOPE_BOUND: requested resources must stay literal"
+        );
+        let restriction = rustfs_iam::delegation::Restrictions {
+            version: None,
+            output_bucket: None,
+            deployment: "runner".into(),
+            bucket: "photos".into(),
+            input_prefix: "input/${aws:username}/".into(),
+            output_prefix: "output/".into(),
+            expires_at_unix_seconds: 200,
+        };
+        assert!(
+            restriction.validate(100).is_err(),
+            "C07_LITERAL_SCOPE_BOUND: legacy stored variable restrictions are rejected"
+        );
+    }
+}

rustfs/src/admin/handlers/plugin_attempt_v2_tests.rs

@@ -0,0 +1,365 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+//! Real handler, persisted IAM and four-disk storage; this is not a signed HTTP/S3 end-to-end test.
+use super::*;
+use crate::admin::{
+    runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot},
+    storage_api::{
+        ECStore,
+        contract::bucket::{BucketOperations as _, MakeBucketOptions},
+        contract::object::ObjectIO as _,
+    },
+};
+use crate::storage::storage_api::{StorageObjectOptions, StoragePutObjReader};
+use http_body_util::BodyExt as _;
+use rustfs_iam::{
+    store::{
+        Store as _, UserType,
+        object::{IAM_CONFIG_PREFIX, ObjectStore},
+    },
+    sys::IamSys,
+};
+use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
+use serde_json::{Value, json};
+use std::sync::Arc;
+
+const PARENT: &str = "cross-parent";
+const BROKER: &str = "cross-broker";
+const SECRET: &str = "cross-bucket-test-secret";
+const ROOT: &str = "cross-root";
+const ROOT_SECRET: &str = "cross-bucket-root-test-secret";
+
+struct RequestIam(Arc<IamSys<ObjectStore>>);
+impl IamInterface for RequestIam {
+    fn handle(&self) -> Arc<IamSys<ObjectStore>> {
+        self.0.clone()
+    }
+    fn is_ready(&self) -> bool {
+        true
+    }
+}
+struct RequestKms;
+impl KmsInterface for RequestKms {
+    fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
+        Arc::new(rustfs_kms::KmsServiceManager::new())
+    }
+}
+struct Fixture {
+    _root: tempfile::TempDir,
+    store: Arc<ECStore>,
+    iam: Arc<IamSys<ObjectStore>>,
+    context: Arc<AppContext>,
+    input_version: String,
+}
+impl Fixture {
+    async fn new() -> Self {
+        let root = tempfile::tempdir().expect("cross-bucket storage root");
+        let env = rustfs_test_utils::TestECStoreEnv::builder()
+            .base_dir(root.path())
+            .disk_count(4)
+            .build()
+            .await;
+        assert_eq!(env.disk_paths.len(), 4);
+        let store = env.ecstore;
+        for bucket in ["photos", "artifacts", "third"] {
+            store
+                .make_bucket(
+                    bucket,
+                    &MakeBucketOptions {
+                        versioning_enabled: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("real versioned bucket");
+        }
+        let object = store
+            .put_object(
+                "photos",
+                "input/photo.png",
+                &mut StoragePutObjReader::from_vec(vec![0xA5; 273]),
+                &StorageObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("real input version");
+        let input_version = object.version_id.expect("exact input version").to_string();
+        ObjectStore::new(store.clone())
+            .save_iam_config(json!({"version":1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
+            .await
+            .expect("persistent IAM format");
+        let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("real IAM");
+        let context = Arc::new(AppContext::new(store.clone(), Arc::new(RequestIam(iam.clone())), Arc::new(RequestKms)));
+        assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
+            access_key: ROOT.into(),
+            secret_key: ROOT_SECRET.into(),
+            status: "on".into(),
+            ..Default::default()
+        }));
+        let fixture = Self {
+            _root: root,
+            store,
+            iam,
+            context,
+            input_version,
+        };
+        fixture.user(PARENT, Self::parent_policy(true, false)).await;
+        fixture
+            .user(
+                BROKER,
+                json!({"Version":"2012-10-17","Statement":[{
+            "Effect":"Allow","Action":"admin:IssuePluginAttempt"}]}),
+            )
+            .await;
+        fixture
+    }
+    fn parent_policy(output: bool, deny: bool) -> Value {
+        let mut resources = vec!["arn:aws:s3:::photos/output/*", "arn:aws:s3:::third/output/*"];
+        if output {
+            resources.push("arn:aws:s3:::artifacts/output/*");
+        }
+        let mut statements = vec![
+            json!({"Effect":"Allow","Action":"s3:GetObjectVersion","Resource":"arn:aws:s3:::photos/input/*"}),
+            json!({"Effect":"Allow","Action":"s3:PutObject","Resource":resources}),
+        ];
+        if deny {
+            statements.push(json!({"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::artifacts/*"}));
+        }
+        json!({"Version":"2012-10-17","Statement":statements})
+    }
+    async fn policy(&self, name: &str, value: Value) {
+        let policy = Policy::parse_config(&serde_json::to_vec(&value).expect("policy JSON")).expect("real IAM policy");
+        self.iam.set_policy(name, policy).await.expect("persist policy");
+    }
+    async fn user(&self, name: &str, policy: Value) {
+        self.iam
+            .create_user(
+                name,
+                &AddOrUpdateUserReq {
+                    secret_key: SECRET.into(),
+                    policy: None,
+                    status: AccountStatus::Enabled,
+                },
+            )
+            .await
+            .expect("create real user");
+        self.policy(name, policy).await;
+        self.iam
+            .policy_db_set(name, UserType::Reg, false, name)
+            .await
+            .expect("attach actual IAM policy");
+    }
+    fn request(&self, body: &Value) -> S3Request<Body> {
+        let slot = ServerContextSlot::new();
+        assert!(slot.install(self.context.clone()));
+        let mut req = S3Request {
+            input: Body::from(serde_json::to_vec(body).expect("attempt JSON")),
+            method: Method::POST,
+            uri: http::Uri::default(),
+            headers: http::HeaderMap::new(),
+            extensions: http::Extensions::new(),
+            credentials: None,
+            region: None,
+            service: None,
+            trailing_headers: None,
+        };
+        req.extensions.insert(slot);
+        req.method = Method::POST;
+        req.credentials = Some(s3::auth::Credentials {
+            access_key: BROKER.into(),
+            secret_key: s3::auth::SecretKey::from(SECRET.to_owned()),
+        });
+        req
+    }
+    async fn issue(&self, record: &Delegation, body: &Value) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let mut router = matchit::Router::new();
+        router.insert("/{id}", ()).expect("parameter route");
+        let path = format!("/{}", record.id);
+        Issue
+            .call(self.request(body), router.at(&path).expect("actual delegation ID").params)
+            .await
+    }
+    async fn allowed(
+        &self,
+        credential: &rustfs_credentials::Credentials,
+        action: S3Action,
+        bucket: &str,
+        object: &str,
+        version: Option<&str>,
+    ) -> bool {
+        let conditions = version
+            .map(|v| HashMap::from([("versionid".into(), vec![v.into()])]))
+            .unwrap_or_default();
+        self.iam
+            .is_allowed(&Args {
+                account: &credential.access_key,
+                groups: &credential.groups,
+                action: Action::S3Action(action),
+                bucket,
+                object,
+                conditions: &conditions,
+                claims: credential.claims_or_empty(),
+                is_owner: false,
+                deny_only: false,
+            })
+            .await
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial_test::serial]
+async fn delegation_v2_persists_and_vends_with_real_parent_intersection() {
+    temp_env::async_with_vars(
+        [
+            ("RUSTFS_PLUGIN_BROKERS", Some(r#"{"cross-broker":"runner"}"#)),
+            ("RUSTFS_SKIP_BACKGROUND_TASK", Some("1")),
+        ],
+        async {
+            let fixture = Fixture::new().await;
+            let now = now().expect("clock");
+            let legacy = json!({"deployment":"runner","bucket":"photos","input_prefix":"input/",
+            "output_prefix":"output/","expires_at_unix_seconds":now+3600});
+            let v1 = fixture
+                .iam
+                .create_plugin_delegation(PARENT, serde_json::from_value(legacy.clone()).expect("v1"), now)
+                .await
+                .expect("persist legacy delegation");
+            assert_eq!(v1.version, 1);
+            assert_eq!(serde_json::to_value(&v1.restrictions).expect("v1 wire"), legacy);
+            let mut v2 = legacy;
+            v2["version"] = json!(2);
+            v2["output_bucket"] = json!("artifacts");
+            let record = fixture
+                .iam
+                .create_plugin_delegation(PARENT, serde_json::from_value(v2.clone()).expect("v2"), now)
+                .await
+                .expect("persist v2 delegation");
+            assert_eq!(record.version, 2);
+            let loaded = fixture
+                .iam
+                .read_active_plugin_delegation(PARENT, &record.id, now)
+                .await
+                .expect("strict stored v2");
+            assert!(loaded == record, "same persisted v2 identity");
+            assert_eq!(serde_json::to_value(&loaded.restrictions).expect("v2 wire"), v2);
+            // The record envelope and restriction profile cannot disagree on a persisted read.
+            let path = format!("config/iam/plugin-delegations/v1/{}/record.json", record.id);
+            let mut corrupt = serde_json::to_value(&record).expect("record JSON");
+            corrupt["version"] = json!(1);
+            ObjectStore::new(fixture.store.clone())
+                .save_iam_config(corrupt, &path)
+                .await
+                .expect("inject incompatible stored version");
+            assert!(
+                fixture
+                    .iam
+                    .read_active_plugin_delegation(PARENT, &record.id, now)
+                    .await
+                    .is_err(),
+                "mixed v1/v2 stored profiles fail closed"
+            );
+            ObjectStore::new(fixture.store.clone())
+                .save_iam_config(&record, &path)
+                .await
+                .expect("restore exact stored record");
+            let run = uuid::Uuid::new_v4();
+            let attempt = uuid::Uuid::new_v4();
+            let prefix = format!("output/74656e616e74/{run}/{attempt}/");
+            let body = json!({"tenant":"tenant","run_id":run,"attempt_id":attempt,"revision":record.revision,
+            "deployment":"runner","input_bucket":"photos","input_key":"input/photo.png","input_version":fixture.input_version,
+            "output_bucket":"artifacts","output_prefix":prefix,"ttl_seconds":60});
+            let loaded_v1 = fixture
+                .iam
+                .read_active_plugin_delegation(PARENT, &v1.id, now)
+                .await
+                .expect("read legacy record");
+            assert!(loaded_v1 == v1, "legacy persistent identity");
+            let mut legacy_body = body.clone();
+            legacy_body["revision"] = json!(v1.revision);
+            legacy_body["output_bucket"] = json!("photos");
+            let legacy_response = fixture.issue(&v1, &legacy_body).await.expect("legacy same-bucket issuance");
+            assert_eq!(legacy_response.output.0, StatusCode::OK);
+            let mut wrong = body.clone();
+            wrong["output_bucket"] = json!("third");
+            match fixture.issue(&record, &wrong).await {
+                Err(error) => assert_eq!(error.code(), &S3ErrorCode::AccessDenied),
+                Ok(_) => panic!("A03_CROSS_BUCKET_OUTPUT_BOUND"),
+            }
+            let response = fixture.issue(&record, &body).await.expect("actual v2 issuance");
+            assert_eq!(response.output.0, StatusCode::OK);
+            assert_eq!(response.headers[http::header::CACHE_CONTROL], "no-store");
+            let issued: Value =
+                serde_json::from_slice(&response.output.1.collect().await.expect("issued body").to_bytes()).expect("issued JSON");
+            let (credential, root) = check_key_valid_with_context(
+                issued["session_token"].as_str().expect("token"),
+                issued["access_key_id"].as_str().expect("access key"),
+                Some(fixture.context.as_ref()),
+            )
+            .await
+            .expect("persisted actual temporary identity");
+            assert!(!root && credential.is_temp());
+            let output = format!("{prefix}image.png");
+            for (action, bucket, object, version, expected) in [
+                (
+                    S3Action::GetObjectVersionAction,
+                    "photos",
+                    "input/photo.png",
+                    Some(fixture.input_version.as_str()),
+                    true,
+                ),
+                (
+                    S3Action::GetObjectVersionAction,
+                    "photos",
+                    "input/photo.png",
+                    Some("wrong-version"),
+                    false,
+                ),
+                (S3Action::GetObjectAction, "photos", "input/photo.png", None, false),
+                (S3Action::PutObjectAction, "artifacts", output.as_str(), None, true),
+                (S3Action::PutObjectAction, "photos", output.as_str(), None, false),
+                (S3Action::PutObjectAction, "third", output.as_str(), None, false),
+                (S3Action::PutObjectAction, "artifacts", "output/other/image.png", None, false),
+            ] {
+                assert_eq!(
+                    fixture.allowed(&credential, action, bucket, object, version).await,
+                    expected,
+                    "A03_CROSS_BUCKET_PARENT_INTERSECTION: bucket={bucket} object={object}"
+                );
+            }
+            fixture.policy(PARENT, Fixture::parent_policy(true, true)).await;
+            assert!(
+                !fixture
+                    .allowed(&credential, S3Action::PutObjectAction, "artifacts", &output, None)
+                    .await,
+                "explicit parent Deny remains effective"
+            );
+            fixture.policy(PARENT, Fixture::parent_policy(false, false)).await;
+            assert!(
+                !fixture
+                    .allowed(&credential, S3Action::PutObjectAction, "artifacts", &output, None)
+                    .await,
+                "removing parent output permission remains effective"
+            );
+            assert!(
+                fixture
+                    .allowed(
+                        &credential,
+                        S3Action::GetObjectVersionAction,
+                        "photos",
+                        "input/photo.png",
+                        Some(&fixture.input_version)
+                    )
+                    .await
+            );
+            fixture
+                .iam
+                .revoke_plugin_delegation(PARENT, &record.id)
+                .await
+                .expect("revoke v2");
+            assert!(fixture.issue(&record, &body).await.is_err(), "revoked v2 cannot issue again");
+        },
+    )
+    .await;
+}

rustfs/src/admin/handlers/plugin_delegation.rs

@@ -0,0 +1,624 @@
+// Copyright 2026 RustFS Team
+// SPDX-License-Identifier: Apache-2.0
+
+//! Restriction registration only. These endpoints do not vend credentials.
+use crate::admin::{
+    auth::{AdminResourceScope, validate_admin_action_with_bucket_object_for_iam},
+    plugin_runner::{self, ObjectReference, SubmitRequest},
+    router::{AdminOperation, Operation, S3Router},
+    runtime_sources,
+    storage_api::s3::{self, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result},
+};
+use crate::{
+    auth::{check_key_valid_with_context, get_session_token},
+    server::{ADMIN_PREFIX, RemoteAddr},
+};
+use http::{HeaderMap, HeaderValue, Method, StatusCode};
+use matchit::Params;
+use rustfs_iam::delegation::{DelegationError, Restrictions};
+use rustfs_policy::policy::action::{Action, AdminAction};
+use serde::Deserialize;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+pub fn register_plugin_delegation_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    router.insert(Method::POST, &format!("{ADMIN_PREFIX}/v3/plugin-delegations"), AdminOperation(&Create))?;
+    router.insert(Method::GET, &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}"), AdminOperation(&Get))?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/revoke"),
+        AdminOperation(&Revoke),
+    )?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/submit"),
+        AdminOperation(&Submit),
+    )?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/query"),
+        AdminOperation(&QueryRun),
+    )?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/history"),
+        AdminOperation(&HistoryRun),
+    )?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/cancel"),
+        AdminOperation(&CancelRun),
+    )?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/page"),
+        AdminOperation(&PageRuns),
+    )?;
+    router.insert(
+        Method::POST,
+        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/replay"),
+        AdminOperation(&ReplayRun),
+    )?;
+    Ok(())
+}
+
+pub(super) fn storage_error(error: DelegationError) -> S3Error {
+    match error {
+        DelegationError::Invalid => s3::error(S3ErrorCode::InvalidRequest, "invalid delegation restriction"),
+        DelegationError::NotFound | DelegationError::OwnerRejected => s3::error(S3ErrorCode::NoSuchKey, "delegation not found"),
+        DelegationError::Unavailable => s3::error(S3ErrorCode::InternalError, "delegation unavailable"),
+        _ => s3::error(S3ErrorCode::AccessDenied, "delegation rejected"),
+    }
+}
+
+pub(super) async fn owner(
+    req: &S3Request<Body>,
+    action: AdminAction,
+) -> S3Result<(std::sync::Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>, String)> {
+    let signed = req
+        .credentials
+        .as_ref()
+        .ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?;
+    if get_session_token(&req.uri, &req.headers).is_some() {
+        return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
+    }
+    let context = runtime_sources::app_context_from_req(req)
+        .ok_or_else(|| s3::error(S3ErrorCode::InternalError, "application context unavailable"))?;
+    let (credentials, root) = check_key_valid_with_context("", &signed.access_key, Some(context.as_ref())).await?;
+    if root || credentials.is_temp() || credentials.is_service_account() {
+        return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
+    }
+    if !context.iam().is_ready() {
+        return Err(s3::error(S3ErrorCode::InternalError, "iam unavailable"));
+    }
+    let iam = context.iam().handle();
+    validate_admin_action_with_bucket_object_for_iam(
+        iam.clone(),
+        &req.headers,
+        &credentials,
+        false,
+        Action::AdminAction(action),
+        req.extensions.get::<Option<RemoteAddr>>().and_then(|a| a.map(|a| a.0)),
+        AdminResourceScope::bucket_object("", ""),
+    )
+    .await?;
+    Ok((iam, credentials.access_key))
+}
+
+pub(super) fn response<T: serde::Serialize>(value: &T) -> S3Result<S3Response<(StatusCode, Body)>> {
+    let bytes =
+        serde_json::to_vec(value).map_err(|_| s3::error(S3ErrorCode::InternalError, "delegation response unavailable"))?;
+    let mut headers = HeaderMap::new();
+    headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
+    headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
+    Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers))
+}
+
+struct Create;
+#[async_trait::async_trait]
+impl Operation for Create {
+    async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, owner) = owner(&req, AdminAction::CreatePluginDelegationAction).await?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "delegation body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "delegation body unavailable"))?;
+        let restriction: Restrictions = serde_json::from_slice(&bytes)
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid delegation restriction"))?;
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
+            .as_secs();
+        let record = iam
+            .create_plugin_delegation(&owner, restriction, now)
+            .await
+            .map_err(storage_error)?;
+        response(&record)
+    }
+}
+
+struct Get;
+#[async_trait::async_trait]
+impl Operation for Get {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, owner) = owner(&req, AdminAction::GetPluginDelegationAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
+            .as_secs();
+        let record = iam
+            .read_active_plugin_delegation(&owner, id, now)
+            .await
+            .map_err(storage_error)?;
+        response(&record)
+    }
+}
+
+struct Revoke;
+#[async_trait::async_trait]
+impl Operation for Revoke {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, owner) = owner(&req, AdminAction::RevokePluginDelegationAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        iam.revoke_plugin_delegation(&owner, id).await.map_err(storage_error)?;
+        response(&serde_json::json!({"revoked":true,"id":id}))
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct SubmitInput {
+    version: u32,
+    instance_id: String,
+    idempotency_key: String,
+    object: SubmitObject,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct SubmitObject {
+    bucket: String,
+    key: String,
+    version_id: String,
+}
+
+struct Submit;
+#[async_trait::async_trait]
+impl Operation for Submit {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, signer) = owner(&req, AdminAction::SubmitPluginJobAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "submit body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "submit body unavailable"))?;
+        let input: SubmitInput =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid submit request"))?;
+        if input.version != 1
+            || !plugin_runner::literal(&input.instance_id)
+            || !plugin_runner::valid_idempotency_key(&input.idempotency_key)
+        {
+            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid submit request"));
+        }
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
+            .as_secs();
+        let record = iam
+            .read_active_plugin_delegation(&signer, id, now)
+            .await
+            .map_err(storage_error)?;
+        let deployment = plugin_runner::configured_deployment()?;
+        if !plugin_runner::validate_delegation_scope(
+            &record,
+            &signer,
+            id,
+            &deployment,
+            ObjectReference {
+                bucket: &input.object.bucket,
+                key: &input.object.key,
+                version_id: &input.object.version_id,
+            },
+            now,
+        ) {
+            return Err(s3::error(S3ErrorCode::AccessDenied, "submit restriction rejected"));
+        }
+        let receipt = plugin_runner::submit(&SubmitRequest {
+            version: input.version,
+            instance_id: &input.instance_id,
+            idempotency_key: &input.idempotency_key,
+            object: ObjectReference {
+                bucket: &input.object.bucket,
+                key: &input.object.key,
+                version_id: &input.object.version_id,
+            },
+            delegation_id: &record.id,
+            delegation_revision: &record.revision,
+            actual_signer_access_key: &signer,
+        })
+        .await?;
+        response(&receipt)
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct QueryRunInput {
+    version: u32,
+    run_id: uuid::Uuid,
+}
+
+struct QueryRun;
+#[async_trait::async_trait]
+impl Operation for QueryRun {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, signer) = owner(&req, AdminAction::GetPluginRunAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "query body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "query body unavailable"))?;
+        let input: QueryRunInput =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid query request"))?;
+        if input.version != 1 || input.run_id.is_nil() {
+            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid query request"));
+        }
+        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
+        let result = plugin_runner::query(&plugin_runner::QueryRequest {
+            version: input.version,
+            run_id: input.run_id,
+            delegation_id: &record.id,
+            delegation_revision: &record.revision,
+            actual_signer_access_key: &signer,
+        })
+        .await?;
+        response(&result)
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct HistoryRunInput {
+    version: u32,
+    run_id: uuid::Uuid,
+}
+
+fn valid_history_run_input(input: &HistoryRunInput) -> bool {
+    input.version == 1 && !input.run_id.is_nil()
+}
+
+struct HistoryRun;
+#[async_trait::async_trait]
+impl Operation for HistoryRun {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, signer) = owner(&req, AdminAction::ListPluginRunsAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "history body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "history body unavailable"))?;
+        let input: HistoryRunInput =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid history request"))?;
+        if !valid_history_run_input(&input) {
+            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid history request"));
+        }
+        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
+        let result = plugin_runner::history(&plugin_runner::HistoryRequest {
+            version: input.version,
+            run_id: input.run_id,
+            delegation_id: &record.id,
+            delegation_revision: &record.revision,
+            actual_signer_access_key: &signer,
+        })
+        .await?;
+        response(&result)
+    }
+}
+
+struct CancelRun;
+#[async_trait::async_trait]
+impl Operation for CancelRun {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, signer) = owner(&req, AdminAction::CancelPluginRunAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "cancel body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "cancel body unavailable"))?;
+        let input: QueryRunInput =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid cancel request"))?;
+        if input.version != 1 || input.run_id.is_nil() {
+            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid cancel request"));
+        }
+        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
+        let result = plugin_runner::cancel(&plugin_runner::CancelRequest {
+            version: input.version,
+            run_id: input.run_id,
+            delegation_id: &record.id,
+            delegation_revision: &record.revision,
+            actual_signer_access_key: &signer,
+        })
+        .await?;
+        response(&result)
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct PageRunsInput {
+    version: u32,
+    limit: u32,
+    #[serde(default)]
+    cursor: Option<plugin_runner::RunPageCursor>,
+}
+
+fn valid_page_runs_input(input: &PageRunsInput) -> bool {
+    input.version == 1 && (1..=100).contains(&input.limit) && !input.cursor.is_some_and(|cursor| cursor.run_id.is_nil())
+}
+
+struct PageRuns;
+#[async_trait::async_trait]
+impl Operation for PageRuns {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, signer) = owner(&req, AdminAction::ListPluginRunsAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "page body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "page body unavailable"))?;
+        let input: PageRunsInput =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid page request"))?;
+        if !valid_page_runs_input(&input) {
+            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid page request"));
+        }
+        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
+        let result = plugin_runner::page(&plugin_runner::PageRequest {
+            version: input.version,
+            actual_signer_access_key: &signer,
+            delegation_id: &record.id,
+            delegation_revision: &record.revision,
+            limit: input.limit,
+            cursor: input.cursor,
+        })
+        .await?;
+        response(&result)
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct ReplayRunInput {
+    version: u32,
+    origin_run_id: uuid::Uuid,
+    idempotency_key: String,
+    deadline_unix_micros: i64,
+}
+
+fn valid_replay_run_input(input: &ReplayRunInput) -> bool {
+    input.version == 1
+        && !input.origin_run_id.is_nil()
+        && plugin_runner::valid_idempotency_key(&input.idempotency_key)
+        && input.deadline_unix_micros > 0
+}
+
+struct ReplayRun;
+#[async_trait::async_trait]
+impl Operation for ReplayRun {
+    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let (iam, signer) = owner(&req, AdminAction::ReplayPluginRunAction).await?;
+        let id = params
+            .get("id")
+            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
+        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
+            .await
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "replay body unavailable"))?
+            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "replay body unavailable"))?;
+        let input: ReplayRunInput =
+            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid replay request"))?;
+        if !valid_replay_run_input(&input) {
+            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid replay request"));
+        }
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
+            .as_secs();
+        let record = iam
+            .read_active_plugin_delegation(&signer, id, now)
+            .await
+            .map_err(storage_error)?;
+        let result = plugin_runner::replay(&plugin_runner::ReplayRequest {
+            version: input.version,
+            origin_run_id: input.origin_run_id,
+            idempotency_key: &input.idempotency_key,
+            deadline_unix_micros: input.deadline_unix_micros,
+            actual_signer_access_key: &signer,
+            delegation_id: &record.id,
+            delegation_revision: &record.revision,
+        })
+        .await?;
+        response(&result)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[tokio::test]
+    async fn delegation_mutations_require_authenticated_original_user() {
+        for operation in 0..8 {
+            let req = S3Request {
+                input: Body::from("{}".to_owned()),
+                method: Method::POST,
+                uri: http::Uri::from_static("/rustfs/admin/v3/plugin-delegations"),
+                headers: HeaderMap::new(),
+                extensions: http::Extensions::new(),
+                credentials: None,
+                region: None,
+                service: None,
+                trailing_headers: None,
+            };
+            let result = if operation == 0 {
+                Get.call(req, Params::new()).await
+            } else if operation == 1 {
+                Revoke.call(req, Params::new()).await
+            } else if operation == 2 {
+                Submit.call(req, Params::new()).await
+            } else if operation == 3 {
+                QueryRun.call(req, Params::new()).await
+            } else if operation == 4 {
+                HistoryRun.call(req, Params::new()).await
+            } else if operation == 5 {
+                CancelRun.call(req, Params::new()).await
+            } else if operation == 6 {
+                PageRuns.call(req, Params::new()).await
+            } else if operation == 7 {
+                ReplayRun.call(req, Params::new()).await
+            } else {
+                Create.call(req, Params::new()).await
+            };
+            let error = result.expect_err("unsigned mutations must not reach persistence");
+            assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
+        }
+    }
+
+    #[test]
+    fn submit_dto_rejects_client_authority_fields_and_null_versions() {
+        assert!(serde_json::from_str::<SubmitInput>(
+            r#"{"version":1,"instance_id":"img","idempotency_key":"k","object":{"bucket":"b","key":"k","version_id":"v"},"actual_signer_access_key":"forged"}"#
+        )
+        .is_err());
+        let input: SubmitInput = serde_json::from_str(
+            r#"{"version":1,"instance_id":"img","idempotency_key":"k","object":{"bucket":"b","key":"a/x","version_id":"null"}}"#,
+        )
+        .unwrap();
+        let record = rustfs_iam::delegation::Delegation {
+            version: 1,
+            id: "a".repeat(32),
+            revision: "a".repeat(32),
+            owner_access_key: "owner".into(),
+            created_at_unix_seconds: 1,
+            restrictions: Restrictions {
+                version: None,
+                output_bucket: None,
+                deployment: "prod".into(),
+                bucket: "b".into(),
+                input_prefix: "a/".into(),
+                output_prefix: "out/".into(),
+                expires_at_unix_seconds: 100,
+            },
+        };
+        assert!(!plugin_runner::validate_delegation_scope(
+            &record,
+            "owner",
+            &record.id,
+            "prod",
+            ObjectReference {
+                bucket: &input.object.bucket,
+                key: &input.object.key,
+                version_id: &input.object.version_id,
+            },
+            2,
+        ));
+    }
+
+    #[test]
+    fn query_run_dto_rejects_client_authority_fields() {
+        let run_id = uuid::Uuid::from_u128(1);
+        let input: QueryRunInput =
+            serde_json::from_str(&format!(r#"{{"version":1,"run_id":"{run_id}"}}"#)).expect("valid run query");
+        assert_eq!(input.run_id, run_id);
+        assert!(
+            serde_json::from_str::<QueryRunInput>(&format!(
+                r#"{{"version":1,"run_id":"{run_id}","actual_signer_access_key":"forged"}}"#
+            ))
+            .is_err()
+        );
+    }
+
+    #[test]
+    fn history_run_dto_is_strict_and_rejects_nil_run() {
+        let run_id = uuid::Uuid::from_u128(1);
+        let input: HistoryRunInput =
+            serde_json::from_str(&format!(r#"{{"version":1,"run_id":"{run_id}"}}"#)).expect("valid history request");
+        assert!(valid_history_run_input(&input));
+        for raw in [
+            r#"{"version":1,"run_id":"00000000-0000-0000-0000-000000000000"}"#,
+            r#"{"version":2,"run_id":"00000000-0000-0000-0000-000000000001"}"#,
+            r#"{"version":1,"run_id":"00000000-0000-0000-0000-000000000001","actual_signer_access_key":"forged"}"#,
+        ] {
+            let rejected = match serde_json::from_str::<HistoryRunInput>(raw) {
+                Ok(input) => !valid_history_run_input(&input),
+                Err(_) => true,
+            };
+            assert!(rejected, "accepted invalid history request: {raw}");
+        }
+    }
+
+    #[test]
+    fn page_runs_dto_is_strict_and_bounded() {
+        let cursor = plugin_runner::RunPageCursor {
+            created_at_unix_micros: 10,
+            run_id: uuid::Uuid::from_u128(1),
+        };
+        let input: PageRunsInput = serde_json::from_str(
+            r#"{"version":1,"limit":100,"cursor":{"created_at_unix_micros":10,"run_id":"00000000-0000-0000-0000-000000000001"}}"#,
+        )
+        .expect("valid page request");
+        assert_eq!(input.cursor, Some(cursor));
+        for raw in [
+            r#"{"version":1,"limit":0}"#,
+            r#"{"version":1,"limit":101}"#,
+            r#"{"version":1,"limit":1,"unknown":true}"#,
+            r#"{"version":1,"limit":1,"actual_signer_access_key":"forged"}"#,
+            r#"{"version":1,"limit":1,"cursor":{"created_at_unix_micros":10,"run_id":"00000000-0000-0000-0000-000000000000"}}"#,
+        ] {
+            let rejected = match serde_json::from_str::<PageRunsInput>(raw) {
+                Ok(input) => !valid_page_runs_input(&input),
+                Err(_) => true,
+            };
+            assert!(rejected, "accepted invalid page request: {raw}");
+        }
+    }
+
+    #[test]
+    fn replay_run_dto_is_strict_and_bounded() {
+        let run_id = uuid::Uuid::from_u128(1);
+        let input: ReplayRunInput = serde_json::from_str(&format!(
+            r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"retry-1","deadline_unix_micros":123}}"#
+        ))
+        .expect("valid replay request");
+        assert!(valid_replay_run_input(&input));
+        for raw in [
+            r#"{"version":1,"origin_run_id":"00000000-0000-0000-0000-000000000000","idempotency_key":"k","deadline_unix_micros":123}"#.to_string(),
+            format!(r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"","deadline_unix_micros":123}}"#),
+            format!(r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"k","deadline_unix_micros":0}}"#),
+            format!(
+                r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"k","deadline_unix_micros":123,"actual_signer_access_key":"forged"}}"#
+            ),
+            format!(
+                r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"{}","deadline_unix_micros":123}}"#,
+                "x".repeat(1025)
+            ),
+        ] {
+            let rejected = match serde_json::from_str::<ReplayRunInput>(&raw) {
+                Ok(input) => !valid_replay_run_input(&input),
+                Err(_) => true,
+            };
+            assert!(rejected, "accepted invalid replay request: {raw}");
+        }
+    }
+}