ci: accept interleaved test tracing output

.config/make/tests.mak

@@ -64,8 +64,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py

.config/nextest.toml

@@ -128,7 +128,7 @@ setup = 'ecstore-large-stack'
 # path on a real store; measured with RUST_MIN_STACK on 2026-10-15 they need
 # 2016-2040 KiB of the 2 MiB libtest default and overflowed on Linux CI.
 [[profile.default.scripts]]
-filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
+filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
 setup = 'ecstore-base-stack'
 
 [[profile.default.scripts]]
@@ -419,7 +419,7 @@ setup = 'ecstore-large-stack'
 
 # These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
 [[profile.ci.scripts]]
-filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::plugin_attempt::v2_tests::delegation_v2_persists_and_vends_with_real_parent_intersection|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
+filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair_tests::[a-z_]+|admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
 setup = 'ecstore-base-stack'
 
 [[profile.ci.scripts]]

.github/workflows/README.md

@@ -16,7 +16,7 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
 | `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
 | `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
 | `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
-| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | Docker workloads with a measured resource or environment requirement beyond the hosted runner |
+| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
 | `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
 
 The self-hosted pods and the hosted image are both Ubuntu 24.04, so a binary built on one runs on the other; `ci.yml` builds the debug binary on `sm-standard-4` and runs it on both pools.
@@ -27,15 +27,11 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
 
 1. **Disk.** It restores or produces the workspace `target/`: `cargo build` or `cargo test` with `--all-targets`, `cargo nextest run --all`, or any reader of the `ci-dev` / `ci-feat-*` caches. The workspace test build leaves about 64 GB in `target/`; a hosted runner has about 14 GB.
 2. **Critical-path CPU.** It is a build that several downstream lanes wait on and would blow its timeout at 4 vCPU (`build-rustfs-debug-binary` gates the e2e and s3-tests lanes).
-3. **Hardware or environment.** Native aarch64 or access to the lab machines. Docker alone is not a reason: hosted Ubuntu ships Docker and supports the QEMU and Buildx setup actions.
+3. **Hardware.** Native aarch64, Docker-in-Docker, or the lab machines.
 4. **Cache ownership.** It reads a `rust-cache` key that a self-hosted job writes (see the cache rule below).
 
 Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
 
-The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
-
-Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
-
 A hosted runner is also the only choice for a few things the pods cannot do: `mount -o loop` and `mount -t tmpfs` fail inside the ARC pods, and a container runtime cannot guarantee that `io_uring_setup` is allowed. `uring-integration` and `e2e-distributed` are on `ubuntu-latest` for that reason, not for cost.
 
 ## Cache rule

.github/workflows/docker.yml

@@ -337,7 +337,7 @@ jobs:
     name: Build Docker Images
     needs: build-check
     if: needs.build-check.outputs.should_build == 'true'
-    runs-on: ubuntu-latest
+    runs-on: dind-sm-standard-2
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/runner-hygiene.yml

@@ -44,7 +44,7 @@ jobs:
   check-ephemerality:
     if: github.repository == 'rustfs/rustfs'
     name: Check runner ephemerality
-    runs-on: ubuntu-latest
+    runs-on: sm-standard-2
     timeout-minutes: 15
     steps:
       - name: Checkout repository

.github/workflows/rustfs-functional-chain.yml

@@ -20,91 +20,13 @@ on:
   workflow_dispatch:
     inputs:
       build_run_id:
-        description: Successful nightly or validation package run; empty for package-only build
+        description: Successful nightly build run on main
         type: string
-        required: false
-        default: ''
+        required: true
       build_run_attempt:
         description: Exact successful build attempt
         type: string
-        required: false
-        default: ''
-      validation_package_sha:
-        description: Exact source SHA for an isolated package-only build
-        type: string
-        required: false
-        default: ''
-      validation_case:
-        description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain
-        type: string
-        required: false
-        default: ''
-      i05_diagnostic_branch:
-        description: I05 source-plan runs both H1/H2 controllers with exact native evidence; subsets remain diagnostic
-        type: choice
-        options: [source-plan, timeout, queued-successor]
-        required: false
-        default: source-plan
-      auto_testing_sha:
-        description: Exact auto-testing commit for a validation case
-        type: string
-        required: false
-        default: ''
-      validation_retain_evidence_run_id:
-        description: Upload existing persistent evidence from one interrupted validation run without fleet access
-        type: string
-        required: false
-        default: ''
-      validation_retain_evidence_run_attempt:
-        description: Exact interrupted run attempt to retain
-        type: string
-        required: false
-        default: '1'
-      validation_runner_inspect_only:
-        description: Inspect GitHub runner registration and queued jobs on a hosted runner without fleet access
-        type: boolean
-        required: false
-        default: false
-      validation_tools_only:
-        description: Build candidate-bound validation tools without running a fleet case
-        type: boolean
-        required: false
-        default: false
-      validation_native_tools:
-        description: Include exact-source native regressions in a tools-only build
-        type: boolean
-        required: false
-        default: false
-      validation_shared_tools:
-        description: Include the controlled C10 shared-reference encoder in a tools-only build
-        type: boolean
-        required: false
-        default: false
-      validation_manager_tools:
-        description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build
-        type: boolean
-        required: false
-        default: false
-      validation_cleanup_run_id:
-        description: Failed validation run whose supported retained environment needs evidence-based finalization
-        type: string
-        required: false
-        default: ''
-      validation_cleanup_retained_evidence:
-        description: Recover from verified persistent runner-local evidence when the failed run uploaded no artifact
-        type: boolean
-        required: false
-        default: false
-      validation_tools_run_id:
-        description: Successful tool build run; required by Heal physical metadata checks
-        type: string
-        required: false
-        default: ''
-      validation_tools_run_attempt:
-        description: Exact tool build attempt
-        type: string
-        required: false
-        default: '1'
+        required: true
   workflow_run:
     workflows: ["Nightly GNU Build"]
     types: [completed]
@@ -114,104 +36,13 @@ permissions:
   actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only && 'rustfs-functional-validation-runner-inspection' || github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }}
+  group: rustfs-functional-chain-runs
   cancel-in-progress: false
 
 jobs:
-  validation-runner-inspection:
-    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-      - name: Inspect shared runner registration and queue without fleet access
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}
-        run: >-
-          python3 -B scripts/inspect_validation_runner.py
-          --output "${RUNNER_TEMP}/validation-runner-inspection/report.json"
-      - name: Upload read-only runner inspection including incomplete reports
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-runner-inspection-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-runner-inspection/report.json
-          if-no-files-found: error
-  validation-evidence-retention:
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id != '' }}
-    # The original interrupted evidence exists only on this long-lived runner.
-    runs-on: smoke-testing
-    timeout-minutes: 15
-    concurrency:
-      group: rustfs-shared-functional-tests-v2
-      cancel-in-progress: false
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-      - name: Verify existing interrupted evidence without changing it
-        env:
-          GH_TOKEN: ${{ github.token }}
-          EVIDENCE_RETAIN_INPUTS: ${{ toJSON(inputs) }}
-        run: >-
-          python3 -B scripts/retain_validation_evidence.py
-          --report "${RUNNER_TEMP}/validation-evidence-retention/report.json"
-      - name: Upload admitted original evidence under collection provenance
-        if: ${{ env.VALIDATION_RETAIN_ROOT != '' }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-retained-${{ env.VALIDATION_RETAIN_SOURCE_RUN }}-${{ env.VALIDATION_RETAIN_SOURCE_ATTEMPT }}-by-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.VALIDATION_RETAIN_ROOT }}
-          if-no-files-found: error
-          include-hidden-files: true
-      - name: Upload retention receipt including incomplete reports
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-evidence-retention-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-evidence-retention/report.json
-          if-no-files-found: error
-  validation-package-request:
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Validate exclusive package-only request
-        env:
-          PACKAGE_INPUTS: ${{ toJSON(inputs) }}
-        run: |
-          python3 - <<'PY'
-          import json
-          import os
-          import re
-          value = json.loads(os.environ["PACKAGE_INPUTS"])
-          if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None:
-              raise ValueError("package build requires an exact source SHA")
-          exclusive = ("validation_retain_evidence_run_id", "validation_runner_inspect_only", "build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha",
-                       "validation_tools_only", "validation_native_tools", "validation_shared_tools",
-                       "validation_manager_tools", "validation_cleanup_run_id", "validation_cleanup_retained_evidence", "validation_tools_run_id")
-          if any(value.get(name) for name in exclusive):
-              raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution")
-          PY
-
-  validation-package:
-    needs: validation-package-request
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
-    uses: ./.github/workflows/rustfs-validation-package.yml
-    with:
-      source_sha: ${{ inputs.validation_package_sha }}
-    secrets:
-      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
-      R2_BUCKET: ${{ secrets.R2_BUCKET }}
-
   prepare:
-    if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && !inputs.validation_cleanup_retained_evidence && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }}
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
+    runs-on: smoke-testing
     timeout-minutes: 10
     outputs:
       manifest: ${{ steps.candidate.outputs.manifest }}
@@ -258,411 +89,6 @@ jobs:
             echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
           fi
 
-  validation:
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
-    runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }}
-    timeout-minutes: 90
-    outputs:
-      source_sha: ${{ steps.candidate.outputs.source_sha }}
-    concurrency:
-      group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }}
-      cancel-in-progress: false
-    env:
-      RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }}
-      RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }}
-      RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }}
-      RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }}
-      RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }}
-    steps:
-      - name: Checkout candidate resolver
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize validation evidence
-        env:
-          VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}
-        run: |
-          set -euo pipefail
-          umask 077
-          python3 -B scripts/initialize_validation_evidence.py
-
-      - name: Resolve immutable candidate
-        id: candidate
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          BUILD_RUN_ID: ${{ inputs.build_run_id }}
-          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
-          CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
-          SELECTED_CASE: ${{ inputs.validation_case }}
-          TOOLS_ONLY: ${{ inputs.validation_tools_only }}
-          NATIVE_TOOLS: ${{ inputs.validation_native_tools }}
-          SHARED_TOOLS: ${{ inputs.validation_shared_tools }}
-          MANAGER_TOOLS: ${{ inputs.validation_manager_tools }}
-          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          import re
-          from scripts.resolve_functional_candidate import resolve
-
-          revision = os.environ["AUTO_TESTING_SHA"]
-          cleanup = os.environ["CLEANUP_RUN_ID"]
-          if os.environ["CLEANUP_RETAINED_EVIDENCE"] == "true" and not cleanup:
-              raise ValueError("persistent recovery input requires one exact cleanup run")
-          if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("tools-only execution cannot select a fleet case or cleanup")
-          if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup")
-          if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup")
-          if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
-          if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
-              raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
-          diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "source-plan"
-          if diagnostic_branch not in {"source-plan", "timeout", "queued-successor"}:
-              raise ValueError("I05 diagnostic branch is not registered")
-          if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort":
-              raise ValueError("queued-successor is available only for the I05 diagnostic case")
-          if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
-              raise ValueError("validation requires an exact auto-testing commit SHA")
-          candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True)
-          manifest = candidate["manifest"]
-          root = Path(os.environ["VALIDATION_ROOT"])
-          (root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n")
-          (root / "candidate.json").write_text(json.dumps({
-              "package_url": manifest["package_url"],
-              "package_sha256": manifest["package_sha256"],
-              "source_sha": manifest["source_sha"],
-          }, sort_keys=True) + "\n")
-          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write("source_sha=" + manifest["source_sha"] + "\n")
-          PY
-
-      - name: Checkout exact auto-testing revision
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Prepare validation Python runtime
-        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.12"
-
-      - name: Install validation Python client
-        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
-        run: |
-          set -euo pipefail
-          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          test ! -e "${validation_python}"
-          python3 -m venv "${validation_python}"
-          "${validation_python}/bin/python" -m pip install \
-            -r auto-testing/scripts/validation_ilm/requirements.txt
-          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
-
-      - name: Validate Heal tool build identity
-        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }}
-          TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }}
-        run: |
-          set -euo pipefail
-          [[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
-          [[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
-          gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \
-            > "${VALIDATION_ROOT}/tool-run.json"
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text())
-          if (value.get("conclusion") != "success" or value.get("status") != "completed"
-                  or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"])
-                  or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}):
-              raise ValueError("Heal tool run is not an exact successful validation build attempt")
-          PY
-
-      - name: Download pinned Heal metadata inspector
-        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify Heal metadata inspector
-        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
-        run: |
-          set -euo pipefail
-          TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          python3 -B auto-testing/scripts/validation_heal/inspector.py \
-            --directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            > "${VALIDATION_ROOT}/tool-manifest.json"
-          printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Select exact-source race native evidence
-        id: race_native
-        if: ${{ inputs.validation_case == 'heal.race-unversioned-overwrite' || inputs.validation_case == 'heal.race-versioned-put' || inputs.validation_case == 'heal.race-version-delete' || inputs.validation_case == 'heal.race-markers-and-batch-delete' || inputs.validation_case == 'heal.race-multipart-complete-abort' || inputs.validation_case == 'heal.admin-nolock-safety' || inputs.validation_case == 'heal.race-copy-and-metadata-update' }}
-        env:
-          CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }}
-          VALIDATION_CASE_ID: ${{ inputs.validation_case }}
-        run: |
-          set -euo pipefail
-          required=false
-          if [ -f auto-testing/scripts/validation_heal/race_native.py ]; then
-            status=0
-            python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
-              --commit "${CANDIDATE_SHA}" --case-id "${VALIDATION_CASE_ID}" || status=$?
-            if [ "${status}" -eq 0 ]; then
-              required=true
-            elif [ "${status}" -ne 2 ]; then
-              exit "${status}"
-            fi
-          fi
-          reviewed=false
-          case "${CANDIDATE_SHA}:${VALIDATION_CASE_ID}" in
-            6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca:heal.race-unversioned-overwrite | \
-            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-unversioned-overwrite | \
-            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-versioned-put | \
-            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-version-delete | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-unversioned-overwrite | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-versioned-put | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-version-delete | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-markers-and-batch-delete | \
-            757b751482bb970bc321e92059690a7698554358:heal.admin-nolock-safety | \
-            757b751482bb970bc321e92059690a7698554358:heal.race-multipart-complete-abort | \
-            757b751482bb970bc321e92059690a7698554358:heal.race-copy-and-metadata-update) reviewed=true ;;
-          esac
-          if [ "${reviewed}" = true ] && [ "${required}" != true ]; then
-            printf 'Race candidate requires its reviewed native collector before fleet execution\n' >&2
-            exit 1
-          fi
-          printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
-
-      - name: Download pinned native storage evidence
-        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout') || (inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac') }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify C07 native source and exact test evidence
-        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/authority.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            > "${VALIDATION_ROOT}/native-authority-manifest.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Verify C11 physical storage and exact final-rename barrier
-        if: ${{ inputs.validation_case == 'heal.bucket-incarnation-recreation' }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/incarnation_native.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${{ inputs.auto_testing_sha }}" \
-            > "${VALIDATION_ROOT}/native-incarnation-manifest.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Verify exact-source race native and runtime evidence
-        if: ${{ steps.race_native.outputs.required == 'true' }}
-        env:
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          VALIDATION_CASE_ID: ${{ inputs.validation_case }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${AUTO_TESTING_SHA}" --case-id "${VALIDATION_CASE_ID}" \
-            > "${VALIDATION_ROOT}/native-race-manifest.json"
-          runtime_driver=race.py
-          if [ "${VALIDATION_CASE_ID}" = 'heal.admin-nolock-safety' ]; then
-            runtime_driver=nolock.py
-          fi
-          VALIDATION_HEAL_NATIVE_DIRECTORY="${NATIVE_DIRECTORY}" \
-            python3 -B "auto-testing/scripts/validation_heal/${runtime_driver}" preflight \
-              --candidate "${VALIDATION_ROOT}/candidate.json" --testing-commit "${AUTO_TESTING_SHA}" \
-              --case-id "${VALIDATION_CASE_ID}" \
-              > "${VALIDATION_ROOT}/native-race-runtime-preflight.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Verify I05 source-plan or queued-successor native prerequisites
-        if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout' }}
-        env:
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          if [ "${I05_DIAGNOSTIC_BRANCH}" = 'queued-successor' ]; then
-            python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
-              --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-              --testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
-              > "${VALIDATION_ROOT}/native-i05-prerequisite.json"
-          else
-            python3 -B auto-testing/scripts/validation_heal/lock_source.py \
-              --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-              --testing-commit "${AUTO_TESTING_SHA}" \
-              > "${VALIDATION_ROOT}/native-i05-source-prerequisites.json"
-          fi
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Bind F07 exact-candidate directory fsync evidence
-        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Download pinned manager persistence and incarnation evidence
-        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify exact manager native prerequisites before fleet mutation
-        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
-        env:
-          VALIDATION_CASE: ${{ inputs.validation_case }}
-        run: |
-          set -euo pipefail
-          MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \
-            --directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json"
-          printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Download pinned C10 shared-reference encoder
-        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify C10 exact-source codec and original tests
-        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
-        run: |
-          set -euo pipefail
-          SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \
-            --directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            > "${VALIDATION_ROOT}/shared-fixture-manifest.json"
-          printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Run selected canonical validation case
-        if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
-        env:
-          VALIDATION_CASE: ${{ inputs.validation_case }}
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          ./auto-testing/scripts/run_validation_case.sh \
-            "${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \
-            "${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run"
-
-      - name: Inspect current Heal fleet resources without mutation
-        if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }}
-        env:
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          python3 -B auto-testing/scripts/validation_heal/environment.py \
-            --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment"
-
-      - name: Audit and finalize retained validation environment
-        if: ${{ inputs.validation_cleanup_run_id != '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
-            --json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json"
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text())
-          if (run["status"] != "completed" or run["conclusion"] not in {"failure", "cancelled"}
-              or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]):
-              raise ValueError("cleanup source must be a completed failed or cancelled Functional Chain run")
-          PY
-          if [ "${CLEANUP_RETAINED_EVIDENCE}" = true ]; then
-            python3 -B scripts/prepare_retained_validation_recovery.py \
-              --report "${VALIDATION_ROOT}/retained-recovery-input.json"
-          else
-            gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
-              --pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact"
-          fi
-          python3 -B auto-testing/scripts/validation_heal/recover_environment.py \
-            --failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \
-            --candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery"
-
-      - name: Show validation report
-        if: ${{ always() }}
-        run: |
-          if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then
-            cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          fi
-
-      - name: Upload validation evidence
-        if: ${{ always() && env.VALIDATION_ROOT != '' }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.VALIDATION_ROOT }}
-          if-no-files-found: error
-
-  validation-tools:
-    needs: [validation]
-    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }}
-    uses: ./.github/workflows/rustfs-validation-tools.yml
-    with:
-      candidate_sha: ${{ needs.validation.outputs.source_sha }}
-      auto_testing_sha: ${{ inputs.auto_testing_sha }}
-      build_native_heal: ${{ inputs.validation_native_tools }}
-      build_shared_heal: ${{ inputs.validation_shared_tools }}
-      build_manager_native: ${{ inputs.validation_manager_tools }}
-    secrets: inherit
-
   upgrade:
     needs: [prepare]
     if: ${{ always() && needs.prepare.result == 'success' }}
@@ -762,7 +188,7 @@ jobs:
   complete-chain:
     needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, fault-tolerance, table, performance]
     if: ${{ always() && needs.prepare.result != 'skipped' }}
-    runs-on: ubuntu-latest
+    runs-on: smoke-testing
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/rustfs-performance-test.yml

@@ -90,7 +90,7 @@ env:
 jobs:
   runner-check:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    runs-on: ubuntu-latest
+    runs-on: sm-standard-2
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/rustfs-validation-package.yml

@@ -1,244 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-
-name: RustFS Validation Package
-
-on:
-  workflow_call:
-    inputs:
-      source_sha:
-        description: Exact RustFS commit SHA to build
-        required: true
-        type: string
-    secrets:
-      R2_ACCESS_KEY_ID:
-        required: true
-      R2_SECRET_ACCESS_KEY:
-        required: true
-      R2_ENDPOINT:
-        required: true
-      R2_BUCKET:
-        required: true
-  workflow_dispatch:
-    inputs:
-      source_sha:
-        description: Exact RustFS commit SHA to build
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: rustfs-validation-package-${{ inputs.source_sha }}
-  cancel-in-progress: false
-
-jobs:
-  resolve-source:
-    if: github.repository == 'rustfs/rustfs'
-    name: Resolve exact validation source
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    outputs:
-      source_sha: ${{ steps.source.outputs.sha }}
-      source_ref: ${{ steps.source.outputs.ref }}
-    steps:
-      - name: Validate requested source
-        id: source
-        env:
-          REQUESTED_SHA: ${{ inputs.source_sha }}
-        run: |
-          set -euo pipefail
-          [[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")"
-          trap 'rm -rf "${source_directory}"' EXIT
-          git init -q "${source_directory}"
-          git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}"
-          test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"
-          echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
-          echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
-
-  build:
-    needs: resolve-source
-    name: Build immutable validation DEB
-    runs-on: ubuntu-latest
-    timeout-minutes: 150
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
-    steps:
-      - name: Checkout exact source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-          ref: ${{ needs.resolve-source.outputs.source_sha }}
-
-      - name: Confirm checked out source
-        env:
-          EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }}
-        run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          cache-shared-key: validation-package-hosted-x86_64-unknown-linux-gnu
-          cache-save-if: "false"
-          install-build-packaging-tools: "false"
-          install-test-tools: "false"
-
-      - name: Build RustFS
-        run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
-
-      - name: Build validation DEB
-        shell: bash
-        run: |
-          set -euo pipefail
-          command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot
-          package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}"
-          package_file="${RUNNER_TEMP}/validation-package/rustfs.deb"
-          mkdir -p "${RUNNER_TEMP}/validation-package"
-          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
-          test -x "${binary}"
-          mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \
-            "${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs"
-          cp "${binary}" "${package_dir}/usr/bin/rustfs"
-          chmod 755 "${package_dir}/usr/bin/rustfs"
-          cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/"
-          cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF'
-          # RustFS Environment Configuration
-          # RUSTFS_ACCESS_KEY=<changeme>
-          # RUSTFS_SECRET_KEY=<changeme>
-          # RUSTFS_VOLUMES=""
-          # RUSTFS_ADDRESS=":9000"
-          ENVEOF
-          cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF'
-          Package: rustfs
-          Version: 0.0.0~validation
-          Section: utils
-          Priority: optional
-          Architecture: amd64
-          Depends: libc6 (>= 2.31)
-          Maintainer: RustFS Team <support@rustfs.com>
-          Description: RustFS immutable validation package
-           Candidate package for RustFS validation only.
-          CONTROLEOF
-          printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles"
-          ./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst"
-          ./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm"
-          ./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm"
-          chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm"
-          cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/"
-          fakeroot dpkg-deb --build "${package_dir}" "${package_file}"
-          test -s "${package_file}"
-
-      - name: Upload opaque validation package
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-package/rustfs.deb
-          if-no-files-found: error
-
-  publish:
-    needs: [resolve-source, build]
-    name: Publish immutable validation DEB
-    # Candidate-controlled build scripts never run on this fresh hosted runner.
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    env:
-      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
-      VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}
-    steps:
-      - name: Download opaque validation package
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-package-input
-
-      - name: Validate and stage opaque package bytes
-        shell: bash
-        run: |
-          set -euo pipefail
-          python3 -I - <<'PYTHON'
-          import os
-          from pathlib import Path
-          import shutil
-          import stat
-
-          root = Path(os.environ["RUNNER_TEMP"])
-          incoming = root / "validation-package-input"
-          if incoming.is_symlink() or not incoming.is_dir() or sorted(p.name for p in incoming.iterdir()) != ["rustfs.deb"]:
-              raise SystemExit("Validation artifact must contain exactly one rustfs.deb file")
-          fd = os.open(incoming / "rustfs.deb", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
-          with os.fdopen(fd, "rb") as source:
-              info = os.fstat(source.fileno())
-              if not stat.S_ISREG(info.st_mode) or info.st_size == 0:
-                  raise SystemExit("Validation package must be a nonempty regular file")
-              staged = root / "validation-publication"
-              staged.mkdir(mode=0o700)
-              with (staged / "rustfs.deb").open("xb") as target:
-                  shutil.copyfileobj(source, target)
-          PYTHON
-
-      - name: Prepare trusted publication tools
-        id: tools
-        env:
-          AWS_EC2_METADATA_DISABLED: true
-        shell: bash
-        run: |
-          set -euo pipefail
-          AWS_CLI=aws
-          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y -qq python3-venv
-            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")"
-            python3 -m venv "${AWS_CLI_DIR}"
-            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
-            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
-          fi
-          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
-          echo "aws_cli=${AWS_CLI}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload validation package to immutable R2 path
-        id: publish
-        env:
-          PACKAGE_FILE: ${{ runner.temp }}/validation-publication/rustfs.deb
-          AWS_CLI: ${{ steps.tools.outputs.aws_cli }}
-          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
-          R2_BUCKET: ${{ secrets.R2_BUCKET }}
-          AWS_EC2_METADATA_DISABLED: true
-        shell: bash
-        run: |
-          set -euo pipefail
-          if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then
-            echo "Validation package publication requires complete R2 configuration" >&2
-            exit 1
-          fi
-          [[ "${VALIDATION_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          test "${VALIDATION_SOURCE_REF}" = "${VALIDATION_SOURCE_SHA}"
-          test -s "${PACKAGE_FILE}" && test ! -L "${PACKAGE_FILE}"
-          export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto
-          package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)"
-          key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb"
-          url="https://dl.rustfs.com/${key}"
-          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \
-            --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
-          downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)"
-          test "${downloaded_sha256}" = "${package_sha256}"
-          manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \
-            --arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \
-            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
-            --arg package_url "${url}" --arg package_sha256 "${package_sha256}" \
-            '{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}"
-          echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload validation candidate manifest
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ steps.publish.outputs.candidate_file }}
-          if-no-files-found: error

.github/workflows/rustfs-validation-tools.yml

@@ -1,343 +0,0 @@
-# Build source-bound helpers separately so live validation needs only artifacts.
-name: RustFS Validation Tools
-
-on:
-  workflow_call:
-    inputs:
-      candidate_sha:
-        type: string
-        required: true
-      auto_testing_sha:
-        type: string
-        required: true
-      build_native_heal:
-        type: boolean
-        required: false
-        default: false
-      build_shared_heal:
-        type: boolean
-        required: false
-        default: false
-      build_manager_native:
-        type: boolean
-        required: false
-        default: false
-    secrets:
-      PF_TESTING_GH_TOKEN:
-        required: true
-  workflow_dispatch:
-    inputs:
-      candidate_sha:
-        description: Exact RustFS source SHA from the validation candidate manifest
-        type: string
-        required: true
-      auto_testing_sha:
-        description: Exact auto-testing SHA containing the Heal metadata inspector
-        type: string
-        required: true
-      build_native_heal:
-        description: Also compile and run separate exact-source native Heal regressions
-        type: boolean
-        required: false
-        default: false
-      build_shared_heal:
-        description: Build the controlled C10 shared-reference fixture encoder
-        type: boolean
-        required: false
-        default: false
-      build_manager_native:
-        description: Build and run exact candidate checkpoint/MRF/incarnation regressions
-        type: boolean
-        required: false
-        default: false
-
-permissions:
-  contents: read
-
-jobs:
-  manager-native-regressions:
-    if: ${{ inputs.build_manager_native }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 180
-    env:
-      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
-      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-    steps:
-      - name: Validate immutable manager native inputs
-        run: |
-          set -euo pipefail
-          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
-
-      - name: Checkout exact manager candidate source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          ref: ${{ inputs.candidate_sha }}
-          path: candidate
-          persist-credentials: false
-
-      - name: Checkout exact manager collector source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Install manager native build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
-          df -h .
-
-      - name: Build exact unmodified candidate manager test executable
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          python3 -B auto-testing/scripts/validation_heal/manager_native.py build \
-            --source candidate --commit "${CANDIDATE_SHA}" --out manager-tools
-
-      - name: Run exact manager regressions independently
-        run: |
-          set -euo pipefail
-          python3 -B auto-testing/scripts/validation_heal/manager_native.py run \
-            --directory manager-tools --out manager-evidence
-
-      - name: Upload manager executable identities and original diagnostics
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            manager-tools/heal-manager-native-tests
-            manager-tools/*.json
-            manager-tools/*.jsonl
-            manager-tools/*.stderr
-            manager-tools/*.log
-            manager-evidence/
-          if-no-files-found: warn
-
-  native-heal-regressions:
-    if: ${{ inputs.build_native_heal }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 180
-    env:
-      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
-      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-    steps:
-      - name: Validate immutable native source inputs
-        run: |
-          set -euo pipefail
-          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
-
-      - name: Checkout exact native candidate source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          ref: ${{ inputs.candidate_sha }}
-          path: candidate
-          persist-credentials: false
-
-      - name: Checkout exact native collector source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Install native build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
-          df -h .
-
-      - name: Build and enumerate the unmodified candidate test executable
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          use_race_native=false
-          if [[ -f auto-testing/scripts/validation_heal/race_native.py ]]; then
-            support_rc=0
-            python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
-              --commit "${CANDIDATE_SHA}" || support_rc=$?
-            case "${support_rc}" in
-              0) use_race_native=true ;;
-              2) ;; # The original exact-source collector handles legacy candidates.
-              *) exit "${support_rc}" ;;
-            esac
-          fi
-          if [[ "${use_race_native}" == true ]]; then
-            python3 -B auto-testing/scripts/validation_heal/race_native.py build \
-              --source candidate --commit "${CANDIDATE_SHA}" --testing-commit "${AUTO_TESTING_SHA}" --out native-tools
-          else
-            python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \
-              --source candidate --commit "${CANDIDATE_SHA}" --out native-tools
-          fi
-
-      - name: Run exact native regressions separately from fleet validation
-        run: |
-          set -euo pipefail
-          if [[ -f native-tools/race-native.json ]]; then
-            test ! -e native-tools/native-regressions.json
-            python3 -B auto-testing/scripts/validation_heal/race_native.py run \
-              --directory native-tools --out native-evidence
-          else
-            python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \
-              --directory native-tools --out native-evidence
-          fi
-
-      - name: Run existing F07 directory fsync failure regressions
-        if: ${{ inputs.candidate_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
-        run: |
-          set -euo pipefail
-          # Older immutable harness pins retain their original native suite;
-          # absence of this additive collector must never fabricate its proof.
-          if [[ ! -f auto-testing/scripts/validation_heal/persistence_native.py ]]; then
-            mkdir native-persistence-evidence
-            echo '{"complete":false,"reason":"pinned harness predates F07 directory-fsync collector","requiredNativeCoverageComplete":false,"canonicalCasePassClaimed":false}' \
-              > native-persistence-evidence/not-available.json
-            exit 0
-          fi
-          python3 -B auto-testing/scripts/validation_heal/persistence_native.py run \
-            --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
-            --testing-commit "${AUTO_TESTING_SHA}"
-          python3 -B auto-testing/scripts/validation_heal/persistence_native.py verify \
-            --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
-            --testing-commit "${AUTO_TESTING_SHA}"
-
-      - name: Upload native executable identities and original diagnostics
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            native-tools/ecstore-native-tests
-            native-tools/ecstore-race-native-tests
-            native-tools/*.json
-            native-tools/*.jsonl
-            native-tools/*.stderr
-            native-tools/*.log
-            native-evidence/
-            native-persistence-evidence/
-          if-no-files-found: warn
-
-  heal-metadata-inspector:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
-      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-    steps:
-      - name: Validate immutable source inputs
-        run: |
-          set -euo pipefail
-          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
-
-      - name: Checkout exact candidate source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          ref: ${{ inputs.candidate_sha }}
-          path: candidate
-          persist-credentials: false
-
-      - name: Checkout exact inspector source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Build the read-only candidate metadata decoder
-        run: |
-          set -euo pipefail
-          test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}"
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          df -h .
-          python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \
-            --source candidate --commit "${CANDIDATE_SHA}" --out inspector-build
-          # Seed the helper's separate graph from the candidate's frozen graph.
-          # Cargo adds the helper package; both lock identities are retained.
-          cp candidate/Cargo.lock inspector-build/Cargo.lock
-          cargo build --release --manifest-path inspector-build/Cargo.toml
-          python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \
-            --binary inspector-build/target/release/heal-meta-inspect --source candidate
-          mkdir validation-tools
-          cp inspector-build/target/release/heal-meta-inspect validation-tools/
-          cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock
-          python3 - <<'PY'
-          import hashlib
-          import json
-          import os
-          from pathlib import Path
-          import subprocess
-
-          source = json.loads(Path("inspector-build/source.json").read_text())
-          binary = Path("validation-tools/heal-meta-inspect")
-          source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"],
-                        binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
-                        inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(),
-                        toolchain=subprocess.check_output(["rustc", "-vV"], text=True),
-                        target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"],
-                        runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
-          Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n")
-          PY
-          git -C candidate diff --exit-code
-
-      - name: Build and test the controlled C10 metadata encoder
-        if: ${{ inputs.build_shared_heal }}
-        run: |
-          set -euo pipefail
-          python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \
-            --source candidate --commit "${CANDIDATE_SHA}" --out shared-build
-          cp candidate/Cargo.lock shared-build/Cargo.lock
-          export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target"
-          cargo build --release --manifest-path shared-build/Cargo.toml
-          mkdir shared-tools
-          if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \
-            > shared-tools/shared-fixture-tests.log 2>&1; then
-            cat shared-tools/shared-fixture-tests.log
-            exit 1
-          fi
-          cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/
-          cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock
-          python3 - <<'PY'
-          import hashlib
-          import json
-          import os
-          from pathlib import Path
-          import subprocess
-
-          source = json.loads(Path("shared-build/source.json").read_text())
-          source.update(schemaVersion=1, built=True, sourceUnmodified=True,
-              candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(),
-              testingCommit=os.environ["AUTO_TESTING_SHA"],
-              binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(),
-              sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(),
-              toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu",
-              runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
-          Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n")
-          PY
-          git -C candidate diff --exit-code
-
-      - name: Upload controlled C10 fixture encoder evidence
-        if: ${{ always() && inputs.build_shared_heal }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
-          path: shared-tools/
-          if-no-files-found: warn
-
-      - name: Upload immutable validation tools
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
-          path: validation-tools/
-          if-no-files-found: error

.github/workflows/windows-filesystem.yml

@@ -1,224 +1,119 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Windows Filesystem Tests
+name: Scanner Read Repair Queue Preflight
 
 on:
-  push:
-    branches: [ main ]
-    paths:
-      - "crates/ecstore/src/disk/**"
-      - "crates/ecstore/src/layout/**"
-      - "crates/ecstore/src/store/init_format.rs"
-      - "crates/ecstore/src/set_disk/ops/locking.rs"
-      - "crates/ecstore/src/object_api/write_commit_context.rs"
-      - "crates/ecstore/src/bucket/utils.rs"
-      - "crates/ecstore/Cargo.toml"
-      - "crates/utils/src/dunce.rs"
-      - "crates/utils/src/os/**"
-      - "crates/utils/Cargo.toml"
-      - "crates/utils/src/path.rs"
-      - "rustfs/build.rs"
-      - "rustfs/Cargo.toml"
-      - "rustfs/src/main.rs"
-      - "rustfs/src/server/runtime.rs"
-      - "rustfs/src/startup_*.rs"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - ".github/actions/setup/**"
-      - ".github/workflows/windows-filesystem.yml"
-  pull_request:
-    branches: [ main ]
-    paths:
-      - "crates/ecstore/src/disk/**"
-      - "crates/ecstore/src/layout/**"
-      - "crates/ecstore/src/store/init_format.rs"
-      - "crates/ecstore/src/set_disk/ops/locking.rs"
-      - "crates/ecstore/src/object_api/write_commit_context.rs"
-      - "crates/ecstore/src/bucket/utils.rs"
-      - "crates/ecstore/Cargo.toml"
-      - "crates/utils/src/dunce.rs"
-      - "crates/utils/src/os/**"
-      - "crates/utils/Cargo.toml"
-      - "crates/utils/src/path.rs"
-      - "rustfs/build.rs"
-      - "rustfs/Cargo.toml"
-      - "rustfs/src/main.rs"
-      - "rustfs/src/server/runtime.rs"
-      - "rustfs/src/startup_*.rs"
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - ".github/actions/setup/**"
-      - ".github/workflows/windows-filesystem.yml"
   workflow_dispatch:
+    inputs:
+      expected_head:
+        description: Full SHA of this workflow commit
+        required: true
+        type: string
 
 permissions:
   contents: read
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-env:
-  CARGO_TERM_COLOR: always
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-  RUST_BACKTRACE: 1
-
 jobs:
-  rename-safety:
-    name: Rename Safety
-    runs-on: windows-latest
-    timeout-minutes: 60
+  focused-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      CARGO_TERM_COLOR: never
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'
+      NO_PROXY: 127.0.0.1,localhost
     steps:
-      - name: Checkout repository
+      - name: Checkout candidate
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
+          fetch-depth: 6
           persist-credentials: false
 
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: build-x86_64-pc-windows-msvc
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
-          install-test-tools: 'false'
-
-      - name: Check production Windows dependencies
-        shell: pwsh
-        run: cargo check -p rustfs-ecstore --lib
-
-      - name: Test guarded rename publication
-        shell: pwsh
-        run: cargo test -p rustfs-ecstore --lib rename_all_ -- --nocapture
-
-      - name: Test multipart parent durability barriers
-        shell: pwsh
-        run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
-
-      - name: Test Windows path conversion
-        shell: pwsh
-        run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
-
-      - name: Test Windows volume and disk-space probes
-        shell: pwsh
-        run: |
-          # Map a drive letter to the temp drive's administrative share so the
-          # remote-volume probe also covers mapped network drives.
-          $drive = (Split-Path -Qualifier ([System.IO.Path]::GetTempPath())).TrimEnd(':')
-          net use R: "\\localhost\$drive`$" /persistent:no
-          if ($LASTEXITCODE -eq 0) {
-            $env:RUSTFS_TEST_WINDOWS_MAPPED_DRIVE = 'R:'
-          } else {
-            Write-Output "::warning::could not map R: to \\localhost\$drive`$; mapped-drive test will skip"
-          }
-          cargo test -p rustfs-utils --features os --lib os:: -- --nocapture
-          $status = $LASTEXITCODE
-          if ($env:RUSTFS_TEST_WINDOWS_MAPPED_DRIVE) { net use R: /delete /y | Out-Null }
-          exit $status
-
-      - name: Test Windows handle guards
-        shell: pwsh
-        run: cargo test -p rustfs-ecstore --lib windows_ -- --nocapture
-
-      - name: Test endpoint path parsing
-        shell: pwsh
-        run: "cargo test -p rustfs-ecstore --lib layout::endpoint:: -- --nocapture"
-
-      - name: Test startup temporary-directory cleanup
-        shell: pwsh
-        run: cargo test -p rustfs-ecstore --lib cleanup_tmp_on_startup_ -- --nocapture
-
-      - name: Test object key Windows admission
-        shell: pwsh
-        run: "cargo test -p rustfs-ecstore --lib bucket::utils:: -- --nocapture"
-
-      - name: Test Windows file-name rules
-        shell: pwsh
-        run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
-
-      - name: Test fresh format publication
-        shell: pwsh
-        run: cargo test -p rustfs-ecstore --lib fresh_format_load_initializes_all_disks -- --nocapture
-
-  # The native Windows main thread only gets the 1 MiB PE stack reserve, while
-  # Linux and macOS hand out 8 MiB. Startup runs the whole root future on the
-  # thread that calls `block_on`, so a debug server once overflowed before it
-  # listened on Windows only. Start a real server to keep that covered.
-  startup-smoke:
-    name: Startup Smoke
-    runs-on: windows-latest
-    timeout-minutes: 90
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          rust-version: stable
-          cache-shared-key: build-x86_64-pc-windows-msvc
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
-          install-test-tools: 'false'
-
-      - name: Build debug server binary
-        shell: pwsh
-        run: cargo build --locked -p rustfs --bin rustfs
-
-      - name: Print version
-        shell: pwsh
-        run: |
-          & .\target\debug\rustfs.exe --version
-          if ($LASTEXITCODE -ne 0) { throw "rustfs --version exited with $LASTEXITCODE" }
-
-      - name: Start server and wait for readiness
-        shell: pwsh
+      - name: Verify candidate and workflow head
+        shell: bash
         env:
-          RUSTFS_ACCESS_KEY: rustfsadmin
-          RUSTFS_SECRET_KEY: rustfsadmin
+          EXPECTED_HEAD: ${{ inputs.expected_head }}
         run: |
-          $volume = Join-Path $env:RUNNER_TEMP 'rustfs-startup-smoke'
-          $stdout = Join-Path $env:RUNNER_TEMP 'rustfs-startup-smoke.out.log'
-          $stderr = Join-Path $env:RUNNER_TEMP 'rustfs-startup-smoke.err.log'
-          New-Item -ItemType Directory -Force -Path $volume | Out-Null
-          $server = Start-Process -FilePath .\target\debug\rustfs.exe `
-            -ArgumentList @('--address', '127.0.0.1:9000', "`"$volume`"") `
-            -RedirectStandardOutput $stdout -RedirectStandardError $stderr `
-            -NoNewWindow -PassThru
-          # Cache the process handle now; otherwise ExitCode reads back empty.
-          $null = $server.Handle
-          $ready = $false
-          try {
-            for ($i = 0; $i -lt 180; $i++) {
-              if ($server.HasExited) { break }
-              try {
-                $response = Invoke-WebRequest -Uri 'http://127.0.0.1:9000/health/ready' -UseBasicParsing -NoProxy -TimeoutSec 2
-                if ($response.StatusCode -eq 200) { $ready = $true; break }
-              } catch {}
-              Start-Sleep -Seconds 1
-            }
-          } finally {
-            if (-not $server.HasExited) { Stop-Process -Id $server.Id -Force }
-            $server.WaitForExit()
-            Write-Host '--- server stdout (tail) ---'
-            Get-Content $stdout -Tail 100 -ErrorAction SilentlyContinue
-            Write-Host '--- server stderr (tail) ---'
-            Get-Content $stderr -Tail 100 -ErrorAction SilentlyContinue
-          }
-          if (-not $ready) {
-            throw ("rustfs did not report ready on /health/ready (exit code: {0})" -f $server.ExitCode)
+          set -euo pipefail
+          actual_head=$(git rev-parse HEAD)
+          source=$(git rev-parse 'e8382ebefc8f6feac681fbbd67d30fefdd3bf46b^{commit}')
+          test_base=$(git rev-parse "${source}^")
+          main_base=$(git rev-parse "${source}~3")
+          base_tree=$(git rev-parse "${test_base}^{tree}")
+          candidate_tree=$(git rev-parse "${source}^{tree}")
+          workflow_tree=$(git rev-parse "${actual_head}^{tree}")
+          test_diff=$(git diff --full-index --binary "$test_base" "$source" | sha256sum | cut -d' ' -f1)
+          combined_diff=$(git diff --full-index --binary "$main_base" "$source" | sha256sum | cut -d' ' -f1)
+          source_files=$(git diff --name-only "$test_base" "$source")
+          workflow_files=$(git diff --name-only "$source" "$actual_head")
+          printf 'head=%s source=%s test_base=%s main_base=%s base_tree=%s candidate_tree=%s workflow_tree=%s test_diff=%s combined_diff=%s\n' \
+            "$actual_head" "$source" "$test_base" "$main_base" "$base_tree" "$candidate_tree" "$workflow_tree" "$test_diff" "$combined_diff"
+          printf 'source_files=%s workflow_files=%s\n' "$source_files" "$workflow_files"
+          [[ "$EXPECTED_HEAD" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$actual_head" == "$EXPECTED_HEAD" && "$actual_head" == "$GITHUB_SHA" ]]
+          git merge-base --is-ancestor "$source" "$actual_head"
+          [[ "$source" == e8382ebefc8f6feac681fbbd67d30fefdd3bf46b ]]
+          [[ "$test_base" == 41d06b4d7a1eee1cec4db8df65bc741b834831eb ]]
+          [[ "$main_base" == 7a3bf8cdbd781116a0970585b995ade668c0e662 ]]
+          [[ "$base_tree" == 01b101acf0119bec8243cab1556cc0218170b6bf ]]
+          [[ "$candidate_tree" == 7cb63e65ca31372a85fd06657608c24cc3b8d063 ]]
+          [[ "$test_diff" == d7dd41fa96e9ee83e7a3c64a0b27efbe2265771e2b026e29e12e94393fab9dd2 ]]
+          [[ "$combined_diff" == 8b898f5821031f5e567c0a0ab51c4039c39467c42e32c54d916aca20e59eba2d ]]
+          [[ "$source_files" == crates/scanner/tests/read_repair_scanner_queue.rs ]]
+          [[ "$workflow_files" == .github/workflows/windows-filesystem.yml ]]
+          [[ -z "$(git status --porcelain)" ]]
+
+      - name: Setup Rust
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e-distributed-hosted
+          cache-save-if: 'false'
+          install-build-packaging-tools: 'false'
+          install-test-tools: 'false'
+
+      # The single integration test uses local disks and an in-process channel.
+      - name: Run the named Scanner and ReadRepair test
+        id: scoped_test
+        shell: bash
+        env:
+          CARGO_BUILD_JOBS: '2'
+          CARGO_INCREMENTAL: '0'
+          CARGO_PROFILE_TEST_DEBUG: '0'
+          RUST_BACKTRACE: '1'
+          TEST_LOG: ${{ runner.temp }}/read-repair-scanner-test.log
+          TEST_NAME: dangling_metadata_read_repair_and_scanner_have_distinct_queued_owners
+        run: |
+          set -euo pipefail
+          validate_test_log() {
+            local test_status="$1" log_file="$2" test_name="$3"
+            if (( test_status != 0 )); then
+              printf 'cargo test exited %s\n' "$test_status" >&2
+              return "$test_status"
+            fi
+            awk -v name="$test_name" '
+              $0 == "running 1 test" { runs++ }
+              index($0, "test " name " ...") == 1 {
+                suffix = substr($0, length("test " name " ...") + 1)
+                if (suffix == "" || suffix ~ /^[[:space:]]/) names++
+              }
+              /^test result: ok[.] 1 passed; 0 failed; 0 ignored; [0-9]+ measured; [0-9]+ filtered out; finished in / { passed++ }
+              END {
+                printf "Harness markers: running=%d named=%d passed=%d\n", runs, names, passed
+                if (runs != 1 || names != 1 || passed != 1) exit 1
+              }
+            ' "$log_file"
           }
+          [[ ! -e "$TEST_LOG" ]]
+          set +e
+          cargo test --locked -p rustfs-scanner --test read_repair_scanner_queue -- "$TEST_NAME" --exact --test-threads=1 >"$TEST_LOG" 2>&1
+          status=$?
+          set -e
+          cat "$TEST_LOG"
+          validate_test_log "$status" "$TEST_LOG" "$TEST_NAME"
+
+      - name: Upload test log
+        if: ${{ always() && (steps.scoped_test.outcome == 'success' || steps.scoped_test.outcome == 'failure') }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: scanner-read-repair-queue-${{ github.run_id }}
+          path: ${{ runner.temp }}/read-repair-scanner-test.log
+          retention-days: 7
+          if-no-files-found: error

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ rustfs-io-core = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 
 # Async runtime
-tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "time"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
 
 # Logging
 tracing = { workspace = true }

crates/ecstore/src/api/mod.rs

@@ -371,15 +371,6 @@ pub mod compression {
 
 pub mod config {
     pub mod com {
-        /// Preserve empty existing objects so callers can distinguish corruption from absence.
-        pub async fn read_config_preserve_empty_with_metadata_opts(
-            api: std::sync::Arc<crate::store::ECStore>,
-            file: &str,
-            opts: &crate::object_api::ObjectOptions,
-        ) -> crate::error::Result<(Vec<u8>, crate::object_api::ObjectInfo)> {
-            crate::config::com::read_config_preserve_empty_with_metadata_opts(api, file, opts).await
-        }
-
         pub use crate::config::com::{
             COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
             ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -1086,9 +1086,8 @@ impl ExpiryState {
         }))
     }
 
-    #[cfg(any(test, feature = "test-util"))]
-    #[doc(hidden)]
-    pub fn new_with_unconsumed_worker_channel(capacity: usize) -> Arc<RwLock<Self>> {
+    #[cfg(test)]
+    fn new_with_unconsumed_worker_channel(capacity: usize) -> Arc<RwLock<Self>> {
         let (tx, rx) = mpsc::channel(capacity);
         Arc::new(RwLock::new(Self {
             tasks_tx: vec![tx],
@@ -3517,21 +3516,13 @@ async fn stale_upload_current_size_with_opts(
         .await
         .ok()?;
 
-    stale_upload_size_from_parts(&parts)
-}
-
-fn stale_upload_size_from_parts(parts: &crate::storage_api_contracts::multipart::ListPartsInfo) -> Option<usize> {
-    if parts.is_truncated {
-        return None;
-    }
-    parts.parts.iter().try_fold(0usize, |total, part| {
-        // ListParts represents a quorum-confirmed missing part with an empty ETag.
-        if part.etag.as_deref().is_none_or(str::is_empty) {
-            return None;
-        }
-        let actual_size = usize::try_from(part.actual_size.max(0)).ok()?;
-        total.checked_add(part.size.max(actual_size))
-    })
+    Some(
+        parts
+            .parts
+            .iter()
+            .map(|part| part.actual_size.max(part.size as i64).max(0) as usize)
+            .sum(),
+    )
 }
 
 async fn stale_upload_lifecycle_due(
@@ -3548,43 +3539,18 @@ async fn stale_upload_lifecycle_due(
     let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
     let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
 
-    let mut lifecycle = match metadata_boundary::get_lifecycle_config(bucket).await {
+    let lifecycle = match metadata_boundary::get_lifecycle_config(bucket).await {
         Ok((lifecycle, _)) => lifecycle,
         Err(_) => return None,
     };
 
-    let has_size_filter = |filter: &s3s::dto::LifecycleRuleFilter| {
-        filter.object_size_greater_than.is_some()
-            || filter.object_size_less_than.is_some()
-            || filter
-                .and
-                .as_ref()
-                .is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
-    };
-    let needs_size = lifecycle
-        .rules
-        .iter()
-        .any(|rule| rule.abort_incomplete_multipart_upload.is_some() && rule.filter.as_ref().is_some_and(has_size_filter));
-    let size = if needs_size {
-        match stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock).await {
-            Some(size) => size,
-            None => {
-                // An unknown size cannot authorize a size-filtered abort.
-                // Unrelated rules can still be evaluated independently.
-                lifecycle
-                    .rules
-                    .retain(|rule| !rule.filter.as_ref().is_some_and(has_size_filter));
-                0
-            }
-        }
-    } else {
-        0
-    };
     let object_opts = ObjectOpts {
         name: object.clone(),
         user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
         mod_time: Some(initiated),
-        size,
+        size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock)
+            .await
+            .unwrap_or_default(),
         is_latest: true,
         delete_marker: false,
         user_defined: metadata.clone(),
@@ -3837,8 +3803,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
     for candidate in candidates.into_values() {
         let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
         let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
-        if now < due
-            && let Some(metadata) = candidate.metadata.as_ref()
+        if let Some(metadata) = candidate.metadata.as_ref()
             && let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await
             && lifecycle_due < due
         {
@@ -3867,9 +3832,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
         let current_metadata = cleanup_guard.file_info().metadata.clone();
         let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time);
         let mut current_due = stale_upload_default_due(current_initiated, default_expiry);
-        if now < current_due
-            && let Some(lifecycle_due) =
-                stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
+        if let Some(lifecycle_due) =
+            stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
             && lifecycle_due < current_due
         {
             current_due = lifecycle_due;
@@ -15904,261 +15868,6 @@ mod tests {
         assert!(ecstore.is_ok(), "fresh local ECStore boot should succeed, got {ecstore:?}");
     }
 
-    #[tokio::test(flavor = "current_thread")]
-    #[serial]
-    async fn stale_multipart_cleanup_preserves_partial_metadata_without_upload_part_warning() {
-        use tracing::instrument::WithSubscriber as _;
-        #[derive(Clone, Default)]
-        struct CapturedLogs(Arc<StdMutex<Vec<u8>>>);
-        struct LogWriter(Arc<StdMutex<Vec<u8>>>);
-        impl std::io::Write for LogWriter {
-            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
-                self.0.lock().unwrap().extend_from_slice(bytes);
-                Ok(bytes.len())
-            }
-            fn flush(&mut self) -> std::io::Result<()> {
-                Ok(())
-            }
-        }
-        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
-            type Writer = LogWriter;
-            fn make_writer(&'a self) -> Self::Writer {
-                LogWriter(self.0.clone())
-            }
-        }
-
-        let (paths, ecstore) = setup_test_env().await;
-        let bucket = format!("stale-partial-{}", Uuid::new_v4().simple());
-        create_test_bucket(&ecstore, &bucket).await;
-        for copies in [1, 2] {
-            let object = format!("partial-{copies}.bin");
-            let initiated = OffsetDateTime::now_utc() - time::Duration::hours(30);
-            let upload = ecstore
-                .new_multipart_upload(
-                    &bucket,
-                    &object,
-                    &ObjectOptions {
-                        mod_time: Some(initiated),
-                        max_parity: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("multipart upload should be created");
-            let decoded = base64_simd::URL_SAFE_NO_PAD
-                .decode_to_vec(upload.upload_id.as_bytes())
-                .unwrap();
-            let decoded = String::from_utf8(decoded).unwrap();
-            let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload id");
-            let upload_path = format!("{}/{upload_dir}", multipart_sha_dir(&bucket, &object));
-            let physical_paths: Vec<_> = paths
-                .iter()
-                .map(|path| path.join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path))
-                .collect();
-            let original: Vec<_> = physical_paths
-                .iter()
-                .take(copies)
-                .map(|path| std::fs::read(path.join(STORAGE_FORMAT_FILE)).unwrap())
-                .collect();
-            let info = super::get_file_info(
-                &original[0],
-                RUSTFS_META_MULTIPART_BUCKET,
-                &upload_path,
-                "",
-                super::FileInfoOpts {
-                    data: false,
-                    include_free_versions: false,
-                    include_part_checksums: false,
-                },
-            )
-            .unwrap();
-            assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (2, 2));
-            for path in physical_paths.iter().skip(copies) {
-                fs::remove_dir_all(path).await.expect("remove selected metadata copies");
-            }
-            if copies == 2 {
-                ecstore
-                    .get_multipart_info(&bucket, &object, &upload.upload_id, &ObjectOptions::default())
-                    .await
-                    .expect("two metadata copies must meet read quorum");
-            }
-
-            let logs = CapturedLogs::default();
-            let subscriber = tracing_subscriber::fmt()
-                .with_max_level(tracing::Level::DEBUG)
-                .with_writer(logs.clone())
-                .with_ansi(false)
-                .without_time()
-                .finish();
-            cleanup_stale_multipart_uploads_once_at(
-                ecstore.clone(),
-                OffsetDateTime::now_utc(),
-                StdDuration::from_secs(24 * 60 * 60),
-            )
-            .with_subscriber(subscriber)
-            .await;
-
-            for (path, expected) in physical_paths.iter().zip(&original) {
-                assert_eq!(fs::read(path.join(STORAGE_FORMAT_FILE)).await.unwrap(), *expected);
-            }
-            let log_output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
-            assert!(
-                log_output.contains(&upload_path),
-                "missing background diagnostic for {upload_path}: {log_output}"
-            );
-            assert!(log_output.contains("multipart_cleanup_lock_or_recheck_failed"), "{log_output}");
-            assert!(!log_output.contains("op=\"upload_part\""), "unexpected upload-part warning: {log_output}");
-            for path in physical_paths.iter().take(copies) {
-                fs::remove_dir_all(path).await.unwrap();
-            }
-        }
-    }
-
-    #[tokio::test(flavor = "current_thread")]
-    #[serial]
-    async fn stale_multipart_cleanup_lists_parts_only_when_size_is_needed() {
-        use std::sync::atomic::{AtomicUsize, Ordering};
-        use tracing::instrument::WithSubscriber as _;
-        use tracing_subscriber::prelude::*;
-
-        struct PartListCounter(Arc<AtomicUsize>);
-        impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for PartListCounter {
-            fn on_new_span(
-                &self,
-                attrs: &tracing::span::Attributes<'_>,
-                _id: &tracing::span::Id,
-                _ctx: tracing_subscriber::layer::Context<'_, S>,
-            ) {
-                if attrs.metadata().name() == "list_object_parts" {
-                    self.0.fetch_add(1, Ordering::Relaxed);
-                }
-            }
-        }
-
-        let (_, ecstore) = setup_test_env().await;
-        let bucket = format!("stale-list-parts-{}", Uuid::new_v4().simple());
-        create_test_bucket(&ecstore, &bucket).await;
-        set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await;
-        let now = OffsetDateTime::now_utc();
-        for (object, age, size_filter, should_list) in [
-            ("logs/default-expired", time::Duration::hours(30), false, false),
-            ("logs/lifecycle-only", time::Duration::minutes(5), false, false),
-            ("logs/size-filter", time::Duration::minutes(5), true, true),
-        ] {
-            if size_filter {
-                set_abort_incomplete_lifecycle_with_size(&bucket, "logs/", 0, 0).await;
-            }
-            let upload = ecstore
-                .new_multipart_upload(
-                    &bucket,
-                    object,
-                    &ObjectOptions {
-                        mod_time: Some(now - age),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("create multipart upload");
-            if size_filter {
-                let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]);
-                ecstore
-                    .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
-                    .await
-                    .expect("upload a part for size filtering");
-            }
-            let calls = Arc::new(AtomicUsize::new(0));
-            let subscriber = tracing_subscriber::registry().with(PartListCounter(Arc::clone(&calls)));
-            cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60))
-                .with_subscriber(subscriber)
-                .await;
-            let listed = calls.load(Ordering::Relaxed);
-            assert_eq!(listed > 0, should_list, "unexpected ListParts calls for {object}: {listed}");
-            let result = ecstore
-                .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
-                .await;
-            assert!(
-                matches!(result, Err(ref err) if is_err_invalid_upload_id(err)),
-                "upload must be removed: {result:?}"
-            );
-        }
-    }
-
-    #[test]
-    fn stale_multipart_cleanup_rejects_incomplete_or_overflowing_part_sizes() {
-        use crate::storage_api_contracts::multipart::{ListPartsInfo, PartInfo};
-
-        let part = |size, actual_size, etag: &str| PartInfo {
-            size,
-            actual_size,
-            etag: Some(etag.to_owned()),
-            ..Default::default()
-        };
-        let mut parts = ListPartsInfo {
-            parts: vec![part(4, 6, "valid")],
-            ..Default::default()
-        };
-        assert_eq!(super::stale_upload_size_from_parts(&parts), Some(6));
-        parts.is_truncated = true;
-        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
-        parts.is_truncated = false;
-        parts.parts = vec![part(0, 0, "")];
-        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
-        parts.parts = vec![part(usize::MAX, 0, "valid"), part(1, 1, "valid")];
-        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn stale_multipart_cleanup_handles_unknown_size_without_false_abort() {
-        let (_, ecstore) = setup_test_env().await;
-        let bucket = format!("stale-unknown-size-{}", Uuid::new_v4().simple());
-        let object = "logs/object";
-        create_test_bucket(&ecstore, &bucket).await;
-        let size_rule = r#"<Rule><ID>size</ID><Status>Enabled</Status><Filter><ObjectSizeLessThan>5</ObjectSizeLessThan></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule>"#;
-        let independent_rule = r#"<Rule><ID>prefix</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule>"#;
-        let lifecycle = |rules: &str| format!("<LifecycleConfiguration>{rules}</LifecycleConfiguration>").into_bytes();
-        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle(size_rule))
-            .await
-            .expect("install upper-size lifecycle rule");
-        let metadata = HashMap::from([
-            (super::RUSTFS_MULTIPART_BUCKET_KEY.to_owned(), bucket.clone()),
-            (super::RUSTFS_MULTIPART_OBJECT_KEY.to_owned(), object.to_owned()),
-        ]);
-        let initiated = OffsetDateTime::now_utc() - time::Duration::hours(1);
-        let set = &ecstore.pools[0].disk_set[0];
-        let upload = ecstore
-            .new_multipart_upload(&bucket, object, &ObjectOptions::default())
-            .await
-            .expect("create empty multipart upload");
-        let decoded = base64_simd::URL_SAFE_NO_PAD
-            .decode_to_vec(upload.upload_id.as_bytes())
-            .expect("decode upload ID");
-        let decoded = String::from_utf8(decoded).expect("upload ID must be UTF-8");
-        let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload ID");
-        assert!(
-            super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
-                .await
-                .is_some(),
-            "a known empty upload must match the upper-size rule"
-        );
-        let missing_upload = format!("{}x{}", Uuid::new_v4(), initiated.unix_timestamp_nanos());
-        assert!(
-            super::stale_upload_lifecycle_due(set, &metadata, initiated, &missing_upload, false)
-                .await
-                .is_none(),
-            "an unreadable size must not match the upper-size rule"
-        );
-        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle(&format!("{size_rule}{independent_rule}")))
-            .await
-            .expect("install size-independent lifecycle rule");
-        assert!(
-            super::stale_upload_lifecycle_due(set, &metadata, initiated, &missing_upload, false)
-                .await
-                .is_some(),
-            "a size-independent rule remains eligible when size is unknown"
-        );
-    }
-
     #[tokio::test]
     #[serial]
     async fn stale_multipart_cleanup_uses_default_expiry_without_lifecycle() {

crates/ecstore/src/bucket/metadata_sys.rs

@@ -3186,17 +3186,9 @@ pub mod test_support {
     use crate::store::init_local_disks_with_instance_ctx;
 
     pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
-        isolated_store_over_temp_disks_with_count(4).await
-    }
-
-    pub async fn isolated_store_over_temp_disks_with_count(disk_count: usize) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
-        assert!(
-            matches!(disk_count, 4 | 16),
-            "isolated store fixture requires a registered logical set geometry"
-        );
-        let mut dirs = Vec::with_capacity(disk_count);
-        let mut endpoints = Vec::with_capacity(disk_count);
-        for disk_idx in 0..disk_count {
+        let mut dirs = Vec::with_capacity(4);
+        let mut endpoints = Vec::with_capacity(4);
+        for disk_idx in 0..4 {
             let dir = tempfile::tempdir().expect("tempdir should be created");
             let mut endpoint =
                 Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
@@ -3209,7 +3201,7 @@ pub mod test_support {
         let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
             legacy: false,
             set_count: 1,
-            drives_per_set: disk_count,
+            drives_per_set: 4,
             endpoints: Endpoints::from(endpoints),
             cmd_line: "metadata-sys-cache-test".to_string(),
             platform: "test".to_string(),

crates/ecstore/src/config/com.rs

@@ -488,18 +488,6 @@ where
     Ok(data)
 }
 
-/// Read config bytes and metadata while preserving an existing empty payload.
-pub(crate) async fn read_config_preserve_empty_with_metadata_opts<S>(
-    api: Arc<S>,
-    file: &str,
-    opts: &ObjectOptions,
-) -> Result<(Vec<u8>, ObjectInfo)>
-where
-    S: EcstoreObjectIO,
-{
-    read_config_with_metadata_inner(api, file, opts, true, None).await
-}
-
 pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
 where
     S: ObjectIO<

crates/ecstore/src/disk/local.rs

@@ -8450,23 +8450,6 @@ impl LocalDisk {
                     return Err(std::io::Error::other("metadata read worker cancelled"));
                 }
                 let _io_permit = io_permit;
-                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
-                {
-                    let opened = root.open_readonly(&path);
-                    if opened.is_ok() {
-                        os::prepared_publication_test_hooks::run(
-                            os::prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
-                            &path,
-                        );
-                    }
-                    let result = read_fn(opened);
-                    os::prepared_publication_test_hooks::run(
-                        os::prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
-                        &path,
-                    );
-                    Ok(result)
-                }
-                #[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
                 Ok(read_fn(root.open_readonly(&path)))
             })
             .await

crates/ecstore/src/disk/os.rs

@@ -333,8 +333,6 @@ pub(crate) mod prepared_publication_test_hooks {
         PreparedRename,
         Rename,
         Remove,
-        MetadataReadAfterOpen,
-        MetadataReadAfterRead,
         #[cfg(test)]
         Rollback,
         #[cfg(test)]
@@ -459,8 +457,6 @@ pub enum LocalPublicationStage {
     PreparedRename,
     Rename,
     Remove,
-    MetadataReadAfterOpen,
-    MetadataReadAfterRead,
 }
 
 #[cfg(all(feature = "test-util", not(windows)))]
@@ -473,8 +469,6 @@ impl LocalPublicationPause {
             LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
             LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
             LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
-            LocalPublicationStage::MetadataReadAfterOpen => prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
-            LocalPublicationStage::MetadataReadAfterRead => prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
         };
         let (entered_tx, entered) = oneshot::channel();
         let (release, release_rx) = std::sync::mpsc::channel::<()>();

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -603,10 +603,6 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
     request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string);
     // A degraded read's surviving minority is not a recreate instruction; the
     // locked storage read decides whether the target is absent (backlog#2900).
-    // Reader failures include checksum damage that metadata-only healing cannot detect.
-    if matches!(reason, "missing_shards" | "decode_error") {
-        request.scan_mode = Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
-    }
 
     let request_id = request.id.clone();
     let bucket = bucket.to_string();
@@ -13831,61 +13827,6 @@ mod tests {
         release_read_repair_heal_reservation(&key).await;
     }
 
-    #[tokio::test]
-    async fn read_repair_reader_failure_requests_deep_scan_for_the_exact_version() {
-        fn captured() -> &'static std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>> {
-            static REQUEST: OnceLock<std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>>> =
-                OnceLock::new();
-            REQUEST.get_or_init(|| std::sync::Mutex::new(None))
-        }
-        fn capture(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
-            *captured().lock().expect("read-repair request capture must be available") = Some(request);
-            Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
-        }
-
-        for (reason, expected_scan) in [
-            ("missing_shards", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
-            ("decode_error", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
-            ("metadata_read_error", None),
-        ] {
-            let object = format!("reader-failure-{}", Uuid::new_v4());
-            let version = Uuid::new_v4().to_string();
-            submit_read_repair_heal_with_submitter(
-                ReadRepairHealSubmission {
-                    bucket: "reader-failure",
-                    object: &object,
-                    version_id: Some(&version),
-                    pool_index: 1,
-                    set_index: 2,
-                    part_number: Some(1),
-                    reason,
-                    mrf_intent: None,
-                },
-                capture,
-            )
-            .await;
-            let request = tokio::time::timeout(Duration::from_secs(5), async {
-                loop {
-                    if let Some(request) = captured().lock().expect("captured request must be readable").take() {
-                        break request;
-                    }
-                    tokio::task::yield_now().await;
-                }
-            })
-            .await
-            .expect("the actual asynchronous admission must be captured");
-            assert_eq!(request.scan_mode, expected_scan, "reader damage needs payload verification");
-            assert_eq!(request.source, HealRequestSource::ReadRepair);
-            assert_eq!(request.bucket, "reader-failure");
-            assert_eq!(request.object_prefix.as_deref(), Some(object.as_str()));
-            assert_eq!(request.object_version_id.as_deref(), Some(version.as_str()));
-            assert_eq!((request.pool_index, request.set_index), (Some(1), Some(2)));
-            assert_eq!(request.recreate_missing, None, "storage evidence decides absence");
-            let key = ReadRepairHealCacheKey::new("reader-failure", &object, Some(&version), 1, 2);
-            release_read_repair_heal_reservation(&key).await;
-        }
-    }
-
     // ========================================================================
     // backlog#898 — groups A/C: pure old-data-dir cleanup classification.
     // ========================================================================

crates/ecstore/src/set_disk/mod.rs

@@ -889,8 +889,6 @@ pub(crate) use ops::heal::AbsenceProofRequest;
 #[cfg(test)]
 pub(crate) use ops::heal::DanglingDeleteFailure;
 pub(crate) use ops::heal::HealedObjectAbsence;
-#[cfg(test)]
-pub(crate) use ops::heal::ReadRepairPauseScope;
 
 #[cfg(test)]
 pub(crate) use ops::hermetic_set_disks_isolated;
@@ -4247,23 +4245,14 @@ pub(super) struct MultipartWriteQuorumContext<'a> {
     part_number: Option<usize>,
 }
 
-fn record_multipart_write_quorum_failure(
-    stage: &'static str,
-    errs: &[Option<DiskError>],
-    write_quorum: usize,
-) -> crate::disk::error_reduce::WriteQuorumFailureSummary {
-    let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
-    runtime_sources::record_erasure_write_quorum_failure(stage, summary.dominant_error_label);
-    summary
-}
-
 fn log_multipart_write_quorum_failure(
     context: MultipartWriteQuorumContext<'_>,
     errs: &[Option<DiskError>],
     write_quorum: usize,
     returned_error: &DiskError,
 ) {
-    let summary = record_multipart_write_quorum_failure(context.stage, errs, write_quorum);
+    let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
+    runtime_sources::record_erasure_write_quorum_failure(context.stage, summary.dominant_error_label);
     warn!(
         target: "rustfs_ecstore::set_disk",
         event = EVENT_SET_DISK_MULTIPART,

crates/ecstore/src/set_disk/ops/delete_recovery.rs

@@ -40,35 +40,9 @@ pub(crate) mod test_hooks {
     use std::sync::{Arc, Mutex};
 
     type Hook = Arc<dyn Fn() -> BoxFuture<'static, ()> + Send + Sync>;
-    type TransactionLockHook = Arc<dyn Fn(&rustfs_lock::NamespaceLockGuard) + Send + Sync>;
 
     static BEFORE_GC_FINALIZE: Mutex<Vec<(String, String, Hook)>> = Mutex::new(Vec::new());
     static CLEANUP_ATTEMPTS: Mutex<Vec<(String, String)>> = Mutex::new(Vec::new());
-    static AFTER_TRANSACTION_LOCK: Mutex<Vec<(String, String, TransactionLockHook)>> = Mutex::new(Vec::new());
-
-    pub(crate) fn after_transaction_lock_acquired(bucket: &str, object: &str, hook: TransactionLockHook) {
-        AFTER_TRANSACTION_LOCK
-            .lock()
-            .expect("transaction lock hooks")
-            .push((bucket.to_owned(), object.to_owned(), hook));
-    }
-
-    pub(in crate::set_disk) fn run_after_transaction_lock_acquired(
-        bucket: &str,
-        object: &str,
-        guard: &rustfs_lock::NamespaceLockGuard,
-    ) {
-        let hook = {
-            let mut hooks = AFTER_TRANSACTION_LOCK.lock().expect("transaction lock hooks");
-            hooks
-                .iter()
-                .position(|(hook_bucket, hook_object, _)| hook_bucket == bucket && hook_object == object)
-                .map(|index| hooks.remove(index).2)
-        };
-        if let Some(hook) = hook {
-            hook(guard);
-        }
-    }
 
     pub(super) fn record_cleanup_attempt(bucket: &str, object: &str) {
         CLEANUP_ATTEMPTS
@@ -138,7 +112,6 @@ pub(crate) struct DeleteRecoveryContext<'a> {
     /// Only the locked heal path may repair a live marker-creation source.
     pub repair_marker_source: bool,
     pub namespace_owner: Option<Arc<dyn Send + Sync>>,
-    pub transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
     pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
     pub published_hint: Option<(Uuid, &'a [Result<DeleteRecoveryInspection>])>,
     pub can_commit: &'a (dyn Fn() -> bool + Sync),
@@ -152,7 +125,6 @@ impl<'a> DeleteRecoveryContext<'a> {
             explicit_delete: false,
             repair_marker_source: false,
             namespace_owner: None,
-            transaction_guard: None,
             recovery_ticket: None,
             published_hint: None,
             can_commit,
@@ -401,22 +373,9 @@ impl SetDisks {
         write_quorum: usize,
         mut context: DeleteRecoveryContext<'_>,
     ) -> Result<()> {
-        let transaction_guard = match context.transaction_guard.take() {
-            Some(guard) => guard,
-            None => self
-                .acquire_delete_transaction_lock(bucket, object)
-                .await
-                .map_err(DiskError::other)?,
-        };
-        let outer_can_commit = context.can_commit;
-        let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
-        context.can_commit = &can_commit;
-        context.transaction_guard = Some(Arc::clone(&transaction_guard));
-        let owner = context
-            .namespace_owner
-            .take()
-            .unwrap_or_else(|| self.ctx.begin_namespace_commit());
-        context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
+        if context.namespace_owner.is_none() {
+            context.namespace_owner = Some(self.ctx.begin_namespace_commit());
+        }
         let can_commit = context.can_commit;
         let states = inspect(disks, bucket, object, transaction).await;
         let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
@@ -588,25 +547,8 @@ impl SetDisks {
         &self,
         bucket: &str,
         object: &str,
-        mut context: DeleteRecoveryContext<'_>,
+        context: DeleteRecoveryContext<'_>,
     ) -> Result<DeleteRecoverySummary> {
-        let transaction_guard = match context.transaction_guard.take() {
-            Some(guard) => guard,
-            None => self
-                .acquire_delete_transaction_lock(bucket, object)
-                .await
-                .map_err(DiskError::other)?,
-        };
-        let outer_can_commit = context.can_commit;
-        let can_commit = || outer_can_commit() && !transaction_guard.is_lock_lost();
-        context.can_commit = &can_commit;
-        if !context.dry_run {
-            let owner = context
-                .namespace_owner
-                .take()
-                .unwrap_or_else(|| self.ctx.begin_namespace_commit());
-            context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
-        }
         let DeleteRecoveryContext {
             incarnation,
             dry_run,
@@ -616,7 +558,6 @@ impl SetDisks {
             published_hint,
             can_commit,
             recovery_ticket: _,
-            transaction_guard: _,
         } = context;
         let disks = self.disk_inventory().await;
         let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
@@ -1316,135 +1257,6 @@ mod tests {
         assert_eq!(body(&store, bucket, None).await, original);
     }
 
-    #[tokio::test]
-    async fn delete_recovery_deferred_batch_discards_released_alias_fence_and_preserves_caller_fence() {
-        use crate::bucket::metadata_sys::ObjectLockConfigState;
-        use crate::object_api::{NamespaceLockFence, NamespaceLockSignalTestFence, ObjectLockConfigSnapshot};
-        use crate::storage_api_contracts::object::ObjectToDelete;
-        use std::sync::Mutex;
-        use std::sync::atomic::{AtomicBool, Ordering};
-
-        for lose_caller in [false, true] {
-            let bucket = if lose_caller {
-                "deferred-batch-lost-caller"
-            } else {
-                "deferred-batch-released-alias"
-            };
-            let (_dirs, store, disks) = fixture(bucket).await;
-            put(&store, bucket, &[27; 4096], false).await;
-            let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
-            let before = join_all(
-                disks
-                    .iter()
-                    .map(|disk| disk.inspect_delete_transaction(bucket, "object", transaction)),
-            )
-            .await
-            .into_iter()
-            .collect::<Result<Vec<_>>>()
-            .expect("capture pending recovery evidence");
-            let mut set = (*store.pools[0].disk_set[0]).clone();
-            set.lockers = vec![Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(
-                rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::FastObjectLockManager::new())),
-            )))];
-            set.ctx
-                .update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
-                .await;
-
-            let (caller_fence, caller_lost) = NamespaceLockFence::loss_handle_for_test();
-            let released_alias_lost = Arc::new(AtomicBool::new(false));
-            let signal_fence = Arc::new(Mutex::new(None));
-            let captured_signal_fence = Arc::clone(&signal_fence);
-            let captured_alias_lost = Arc::clone(&released_alias_lost);
-            test_hooks::after_transaction_lock_acquired(
-                bucket,
-                "object",
-                Arc::new(move |guard| {
-                    let signal = guard.lock_lost_signal().expect("distributed alias signal");
-                    let fence = NamespaceLockSignalTestFence::install_with_loss_handle(&signal, Arc::clone(&captured_alias_lost));
-                    // Retain the signal so a later allocation cannot reuse its
-                    // registry identity after the original guard is dropped.
-                    *captured_signal_fence.lock().expect("captured alias fence") = Some((fence, signal));
-                }),
-            );
-            let replay_started = Arc::new(AtomicBool::new(false));
-            let captured_replay_started = Arc::clone(&replay_started);
-            let expired_alias = Arc::clone(&released_alias_lost);
-            test_hooks::after_transaction_lock_acquired(
-                bucket,
-                "object",
-                Arc::new(move |_| {
-                    // Acquiring the same transaction key again proves the first
-                    // lease was released. Only its copied fence becomes stale.
-                    expired_alias.store(true, Ordering::Release);
-                    caller_lost.store(lose_caller, Ordering::Release);
-                    captured_replay_started.store(true, Ordering::Release);
-                }),
-            );
-
-            let (deleted, errors) = set
-                .delete_objects(
-                    bucket,
-                    vec![ObjectToDelete {
-                        object_name: "object".into(),
-                        ..Default::default()
-                    }],
-                    ObjectOptions {
-                        namespace_lock_fence: Some(caller_fence),
-                        object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
-                            ObjectLockConfigState::ConfirmedAbsent,
-                        ))),
-                        ..Default::default()
-                    },
-                )
-                .await;
-            assert!(
-                replay_started.load(Ordering::Acquire),
-                "missing metadata must take the deferred replay path"
-            );
-            assert!(
-                released_alias_lost.load(Ordering::Acquire),
-                "the completed batch alias must be marked lost"
-            );
-            if lose_caller {
-                assert!(errors[0].is_some(), "losing the original caller fence must reject deferred replay");
-                for (disk, original) in disks.iter().zip(before) {
-                    let after = disk
-                        .inspect_delete_transaction(bucket, "object", transaction)
-                        .await
-                        .expect("retained proof");
-                    assert_eq!(
-                        after.fingerprint, original.fingerprint,
-                        "lost caller authority must preserve every prepared journal"
-                    );
-                }
-            } else {
-                assert!(
-                    errors.iter().all(Option::is_none),
-                    "released alias loss must not reject fresh recovery: {errors:?}"
-                );
-                assert!(
-                    !deleted[0].found,
-                    "fresh unversioned recovery commits absence before the idempotent batch retry"
-                );
-                for disk in &disks {
-                    assert!(
-                        discover(disk, bucket, Some("object"))
-                            .await
-                            .expect("final recovery evidence")
-                            .is_empty()
-                    );
-                    assert_eq!(
-                        disk.read_version("", bucket, "object", "", &ReadOptions::default())
-                            .await
-                            .expect_err("replayed delete must remove the source"),
-                        DiskError::FileNotFound
-                    );
-                }
-            }
-            drop(signal_fence);
-        }
-    }
-
     #[tokio::test]
     async fn delete_recovery_corrupt_bucket_does_not_prevent_other_bucket_rollback() {
         let bad_bucket = "recover-corrupt-bucket";

crates/ecstore/src/set_disk/ops/locking.rs

@@ -44,21 +44,6 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
 
     #[tracing::instrument(level = "trace", skip(self))]
     async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
-        // Lock the file the disk resolves, not the spelling: on Windows `a\b`
-        // and `a/b` share one xl.meta, so they must share one lock. The key is
-        // folded by the requesting node only; lock servers compare it verbatim.
-        let resource = ObjectKey {
-            bucket: Arc::from(bucket),
-            object: Arc::from(object_name_disk_identity(object).as_ref()),
-            version: None,
-        };
-
-        Ok(self.namespace_lock_for_key(resource).await)
-    }
-}
-
-impl SetDisks {
-    async fn namespace_lock_for_key(&self, resource: ObjectKey) -> NamespaceLockWrapper {
         // Resolved from this set's own instance context (backlog#1052), not the
         // ambient facade: the facade tracks whichever context is currently
         // published process-wide, so a second instance (or, in tests, another
@@ -84,33 +69,20 @@ impl SetDisks {
             NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
         };
 
-        NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone())
-    }
+        // Lock the file the disk resolves, not the spelling: on Windows `a\b`
+        // and `a/b` share one xl.meta, so they must share one lock. The key is
+        // folded by the requesting node only; lock servers compare it verbatim.
+        let resource = ObjectKey {
+            bucket: Arc::from(bucket),
+            object: Arc::from(object_name_disk_identity(object).as_ref()),
+            version: None,
+        };
 
-    /// Raw object locks precede this set-local transaction lock. Its versioned
-    /// resource cannot collide with normal namespace keys, which use None.
-    /// Keeping the target set's clients avoids nesting overlapping raw quorums.
-    pub(in crate::set_disk) async fn acquire_delete_transaction_lock(
-        &self,
-        bucket: &str,
-        object: &str,
-    ) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
-        let resource = ObjectKey::with_version(
-            bucket,
-            object_name_disk_identity(object).as_ref(),
-            format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
-        );
-        let guard = self
-            .namespace_lock_for_key(resource)
-            .await
-            .get_write_lock(super::super::get_lock_acquire_timeout())
-            .await
-            .map_err(|error| self.map_namespace_lock_error(bucket, object, "delete_transaction", error))?;
-        #[cfg(test)]
-        super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
-        Ok(Arc::new(guard))
+        Ok(NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone()))
     }
+}
 
+impl SetDisks {
     #[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
     pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
         match err {

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -44,9 +44,9 @@ use super::super::{
     create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
     get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
     log_multipart_write_quorum_failure, parts_after_marker, path_join_buf, prepare_complete_multipart_parts,
-    record_compression_total_memory, record_multipart_write_quorum_failure, reduce_read_quorum_errs, reduce_write_quorum_errs,
-    remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size,
-    strip_internal_multipart_metadata, to_object_err, warn,
+    record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
+    restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
+    to_object_err, warn,
 };
 use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
 #[cfg(test)]
@@ -233,19 +233,6 @@ fn admitted_multipart_size(current: u64, candidate: u64, limit: u64) -> Result<u
     Ok(total)
 }
 
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum MultipartMetadataQuorum {
-    Read,
-    Write,
-    Cleanup,
-}
-
-impl From<bool> for MultipartMetadataQuorum {
-    fn from(write: bool) -> Self {
-        if write { Self::Write } else { Self::Read }
-    }
-}
-
 pub(crate) struct StaleMultipartCleanupGuard {
     file_info: FileInfo,
     upload_path: String,
@@ -1152,7 +1139,7 @@ impl SetDisks {
             }
             let upload_id = runtime_sources::deployment_upload_id(raw_upload_id);
             let file_info = match self
-                .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, MultipartMetadataQuorum::Read, true)
+                .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, false, true)
                 .await
             {
                 Ok((file_info, _)) => file_info,
@@ -1239,7 +1226,7 @@ impl SetDisks {
             .acquire_write_lock_diag("abort_data_movement_multipart", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
             .await?;
         let file_info = match self
-            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, MultipartMetadataQuorum::Write, true)
+            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, true, true)
             .await
         {
             Ok((file_info, _)) => file_info,
@@ -1346,7 +1333,7 @@ impl SetDisks {
         data_movement: bool,
     ) -> Result<(FileInfo, Vec<FileInfo>)> {
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, data_movement)?;
-        self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write.into(), false)
+        self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write, false)
             .await
     }
 
@@ -1386,7 +1373,7 @@ impl SetDisks {
         object: &str,
         upload_id: &str,
         upload_id_path: &str,
-        quorum_mode: MultipartMetadataQuorum,
+        write: bool,
         include_recovery: bool,
     ) -> Result<(FileInfo, Vec<FileInfo>)> {
         // Slow metadata I/O must not hold the topology registry's read lock.
@@ -1502,26 +1489,22 @@ impl SetDisks {
         }
 
         let mut quorum = read_quorum as usize;
-        if quorum_mode != MultipartMetadataQuorum::Read {
+        if write {
             quorum = write_quorum as usize;
 
             if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
-                if quorum_mode == MultipartMetadataQuorum::Write {
-                    log_multipart_write_quorum_failure(
-                        MultipartWriteQuorumContext {
-                            stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
-                            bucket,
-                            object,
-                            upload_id,
-                            part_number: None,
-                        },
-                        &errs,
-                        quorum,
-                        &err,
-                    );
-                } else {
-                    record_multipart_write_quorum_failure(MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, &errs, quorum);
-                }
+                log_multipart_write_quorum_failure(
+                    MultipartWriteQuorumContext {
+                        stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
+                        bucket,
+                        object,
+                        upload_id,
+                        part_number: None,
+                    },
+                    &errs,
+                    quorum,
+                    &err,
+                );
                 return Err(map_error(err));
             }
         } else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
@@ -1540,7 +1523,7 @@ impl SetDisks {
             .acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path)
             .await?;
         let (file_info, _) = self
-            .check_multipart_upload_path_exists("", "", "", upload_path, MultipartMetadataQuorum::Cleanup, true)
+            .check_multipart_upload_path_exists("", "", "", upload_path, true, true)
             .await?;
         let write_quorum = file_info.write_quorum(self.default_write_quorum());
         Ok(StaleMultipartCleanupGuard {
@@ -3049,7 +3032,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         // its full write quorum and the upload's incarnation/lifecycle fences.
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
         let (fi, _) = self
-            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, MultipartMetadataQuorum::Read, true)
+            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
             .await?;
         ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
         ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
@@ -3209,14 +3192,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     }
                     SourceRecovery::Missing => {
                         return match self
-                            .check_multipart_upload_path_exists(
-                                bucket,
-                                object,
-                                upload_id,
-                                &upload_id_path,
-                                MultipartMetadataQuorum::Read,
-                                true,
-                            )
+                            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
                             .await
                         {
                             Ok(_) => Err(Error::ErasureWriteQuorum),
@@ -11157,7 +11133,7 @@ mod tests {
             *disk = None;
         }
         let (info, _) = set
-            .check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+            .check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
             .await
             .expect("four primary plus four recovery copies reach metadata read quorum");
         assert_eq!(info.name, path);
@@ -11178,7 +11154,7 @@ mod tests {
             *disk = None;
         }
         assert!(
-            set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+            set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
                 .await
                 .is_err(),
             "two files on each of four disks must not count as eight voters"
@@ -14495,7 +14471,7 @@ mod tests {
                     object,
                     &u_new,
                     &SetDisks::get_upload_id_dir(bucket, object, &u_new),
-                    MultipartMetadataQuorum::Write,
+                    true,
                     true,
                 )
                 .await
@@ -14540,7 +14516,7 @@ mod tests {
                 object,
                 &upload,
                 &SetDisks::get_upload_id_dir(bucket, object, &upload),
-                MultipartMetadataQuorum::Write,
+                true,
                 true,
             )
             .await
@@ -14623,7 +14599,7 @@ mod tests {
                     assert_completion_files_preserved(&disk_stores, RUSTFS_META_MULTIPART_BUCKET, &source_files).await;
                     set_disks
                         .check_multipart_upload_path_exists(
-                            bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), MultipartMetadataQuorum::Write, true,
+                            bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), true, true,
                         )
                         .await
                         .expect("upload authority must survive the post-commit crash");

crates/ecstore/src/set_disk/ops/multipart_abort_tests.rs

@@ -507,7 +507,7 @@ mod native_abort_closure_tests {
         let path = SetDisks::get_upload_id_dir(bucket, object, &upload);
         pin_historical_unmarked_upload(&disks, bucket, &path).await;
         let before = originals(&disks, &path).await;
-        set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+        set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
             .await
             .expect("readonly legacy cleanup identity lookup");
         assert_eq!(
@@ -786,7 +786,7 @@ mod native_abort_closure_tests {
             "known orphan native evidence remains unknown: {inspected:?}"
         );
         let readonly = set
-            .check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+            .check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
             .await;
         assert!(
             matches!(

crates/ecstore/src/set_disk/ops/object.rs

@@ -73,7 +73,6 @@ struct DeleteCommitContext<'a> {
     incarnation: Option<Uuid>,
     options: Option<&'a ObjectOptions>,
     guard: Option<Arc<super::super::ObjectLockDiagGuard>>,
-    transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
 }
 
 impl<'a> DeleteCommitContext<'a> {
@@ -82,37 +81,17 @@ impl<'a> DeleteCommitContext<'a> {
             incarnation: options.expected_bucket_incarnation_id,
             options: Some(options),
             guard: guard.cloned(),
-            transaction_guard: None,
         }
     }
 
-    fn with_transaction_guard(mut self, guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>) -> Self {
-        self.transaction_guard = guard.cloned();
-        self
-    }
-
-    fn ensure_transaction_lock_held(&self, bucket: &str, object: &str) -> Result<()> {
-        if self.transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
-            return Err(StorageError::NamespaceLockQuorumUnavailable {
-                mode: "delete_transaction",
-                bucket: bucket.to_owned(),
-                object: object.to_owned(),
-                required: 1,
-                achieved: 0,
-            });
-        }
-        Ok(())
-    }
-
     fn can_commit(&self, bucket: &str, object: &str) -> bool {
-        !self.transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
-            && self.options.is_none_or(|options| {
-                ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok()
-                    && options
-                        .scanner_publication_commit_scope
-                        .as_ref()
-                        .is_none_or(|scope| scope.can_commit())
-            })
+        self.options.is_none_or(|options| {
+            ensure_delete_commit_locks_held(self.guard.as_deref(), bucket, object, options).is_ok()
+                && options
+                    .scanner_publication_commit_scope
+                    .as_ref()
+                    .is_none_or(|scope| scope.can_commit())
+        })
     }
 }
 
@@ -3873,7 +3852,6 @@ impl SetDisks {
         object: &str,
         opts: &ObjectOptions,
         lock_guard: Option<&Arc<super::super::ObjectLockDiagGuard>>,
-        transaction_guard: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
     ) -> Result<bool> {
         if is_meta_bucketname(bucket)
             || opts.version_id.is_some()
@@ -3910,7 +3888,6 @@ impl SetDisks {
                 object,
                 DeleteRecoveryContext {
                     explicit_delete: true,
-                    transaction_guard: transaction_guard.cloned(),
                     namespace_owner: Some(Arc::new((
                         self.ctx.begin_namespace_commit(),
                         lock_guard.cloned(),
@@ -8200,23 +8177,13 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
     }
 }
 
-#[cfg(test)]
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum DeleteObjectCommitPause {
-    BeforeNamespace,
-    BeforeCommit,
-    AfterPublish,
-}
-
 #[cfg(test)]
 struct DeleteObjectCommitBarrierState {
     bucket: String,
     object: String,
-    pause: DeleteObjectCommitPause,
+    pause_after_publish: bool,
     arrived: tokio::sync::Notify,
     release: tokio::sync::Notify,
-    namespace_pending: tokio::sync::Notify,
-    namespace_acquired: std::sync::atomic::AtomicBool,
 }
 
 #[cfg(test)]
@@ -8231,27 +8198,21 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
 #[cfg(test)]
 impl DeleteObjectCommitBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeCommit)
-    }
-
-    pub(crate) fn install_before_namespace(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeNamespace)
+        Self::install_with_mode(bucket, object, false)
     }
 
     #[cfg(feature = "test-util")]
     pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::AfterPublish)
+        Self::install_with_mode(bucket, object, true)
     }
 
-    fn install_with_mode(bucket: &str, object: &str, pause: DeleteObjectCommitPause) -> Self {
+    fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self {
         let state = Arc::new(DeleteObjectCommitBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
-            pause,
+            pause_after_publish,
             arrived: tokio::sync::Notify::new(),
             release: tokio::sync::Notify::new(),
-            namespace_pending: tokio::sync::Notify::new(),
-            namespace_acquired: std::sync::atomic::AtomicBool::new(false),
         });
         let mut slot = DELETE_OBJECT_COMMIT_BARRIER
             .get_or_init(|| std::sync::Mutex::new(None))
@@ -8271,19 +8232,6 @@ impl DeleteObjectCommitBarrier {
     pub(crate) fn release(&self) {
         self.state.release.notify_one();
     }
-
-    pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
-        assert_eq!(self.state.pause, DeleteObjectCommitPause::BeforeNamespace);
-        let pending = self.state.namespace_pending.notified();
-        self.release();
-        tokio::time::timeout(Duration::from_secs(5), pending)
-            .await
-            .expect("delete object must poll its actual namespace acquisition to Pending");
-    }
-
-    pub(crate) fn namespace_acquired(&self) -> bool {
-        self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
-    }
 }
 
 #[cfg(test)]
@@ -8300,43 +8248,6 @@ impl Drop for DeleteObjectCommitBarrier {
     }
 }
 
-#[cfg(test)]
-async fn pause_delete_object_before_namespace(bucket: &str, object: &str) {
-    let barrier = DELETE_OBJECT_COMMIT_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("delete object commit barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
-        })
-        .cloned();
-    if let Some(barrier) = barrier {
-        barrier.arrived.notify_one();
-        barrier.release.notified().await;
-    }
-}
-
-#[cfg(test)]
-fn notify_delete_object_namespace(bucket: &str, object: &str, acquired: bool) {
-    let barrier = DELETE_OBJECT_COMMIT_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("delete object commit barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
-        })
-        .cloned();
-    if let Some(barrier) = barrier {
-        if acquired {
-            barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
-        } else {
-            barrier.namespace_pending.notify_one();
-        }
-    }
-}
-
 #[cfg(test)]
 async fn pause_delete_object_commit(bucket: &str, object: &str) {
     let barrier = DELETE_OBJECT_COMMIT_BARRIER
@@ -8344,9 +8255,7 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) {
         .lock()
         .expect("delete object commit barrier mutex should not poison")
         .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeCommit
-        })
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish)
         .cloned();
     if let Some(barrier) = barrier {
         barrier.arrived.notify_one();
@@ -8361,9 +8270,7 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
         .lock()
         .expect("delete object commit barrier mutex should not poison")
         .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::AfterPublish
-        })
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish)
         .cloned();
     if let Some(barrier) = barrier {
         barrier.arrived.notify_one();
@@ -8970,11 +8877,8 @@ impl SetDisks {
         fi: &FileInfo,
         force_del_marker: bool,
         delete_marker_purge: Option<MrfDeleteMarkerPurge>,
-        mut commit_context: DeleteCommitContext<'_>,
+        commit_context: DeleteCommitContext<'_>,
     ) -> Result<()> {
-        if !is_meta_bucketname(bucket) && commit_context.transaction_guard.is_none() {
-            commit_context.transaction_guard = Some(self.acquire_delete_transaction_lock(bucket, object).await?);
-        }
         let source_bucket_incarnation_id = commit_context.incarnation;
         let transported = delete_file_info_with_replication_transport_metadata(fi);
         let fi = &transported;
@@ -8983,7 +8887,6 @@ impl SetDisks {
             Arc::new((
                 self.ctx.begin_namespace_commit(),
                 commit_context.guard.clone(),
-                commit_context.transaction_guard.clone(),
                 commit_context
                     .options
                     .and_then(|options| options.object_lock_config_snapshot.clone()),
@@ -9023,11 +8926,7 @@ impl SetDisks {
 
         for disk in disks.iter() {
             let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
-            let transaction_guard = commit_context.transaction_guard.as_ref();
             futures.push(async move {
-                if transaction_guard.is_some_and(|guard| guard.is_lock_lost()) {
-                    return Err(DiskError::DiskOngoingReq);
-                }
                 if let Some(disk) = disk {
                     let options = DeleteOptions {
                         old_data_dir: Some(rollback_dir),
@@ -9077,7 +8976,6 @@ impl SetDisks {
             }
         }
 
-        commit_context.ensure_transaction_lock_held(bucket, object)?;
         let mut quorum_result = resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object);
         if !is_meta_bucketname(bucket) && errs.iter().flatten().any(super::delete_recovery::unsupported_recovery) {
             quorum_result = Err(DiskError::UnsupportedDisk.into());
@@ -9110,7 +9008,6 @@ impl SetDisks {
                 write_quorum,
                 DeleteRecoveryContext {
                     recovery_ticket,
-                    transaction_guard: commit_context.transaction_guard.clone(),
                     namespace_owner: namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>),
                     ..DeleteRecoveryContext::new(source_bucket_incarnation_id, &|| commit_context.can_commit(bucket, object))
                 },
@@ -9120,7 +9017,6 @@ impl SetDisks {
         }
         #[cfg(test)]
         pause_delete_before_cleanup(bucket).await;
-        commit_context.ensure_transaction_lock_held(bucket, object)?;
         let mut rollback_futures = Vec::new();
         for (index, err) in errs.iter().enumerate() {
             if structured_cleanup {
@@ -9146,11 +9042,7 @@ impl SetDisks {
             let object = object.to_string();
             let fi = fi.clone();
             let disk_namespace_owner = namespace_owner.clone().map(|owner| owner as Arc<dyn Send + Sync>);
-            let transaction_guard = commit_context.transaction_guard.as_ref();
             rollback_futures.push(async move {
-                if transaction_guard.is_some_and(|guard| guard.is_lock_lost()) {
-                    return;
-                }
                 if should_rollback {
                     // The dedicated undo path never forwards the marker-only creation flag.
                     if let Err(err) = disk
@@ -9207,7 +9099,6 @@ impl SetDisks {
         }
 
         join_all(rollback_futures).await;
-        commit_context.ensure_transaction_lock_held(bucket, object)?;
         drop(namespace_owner);
         if quorum_result.is_ok()
             && errs.iter().any(Option::is_some)
@@ -9667,8 +9558,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         &self,
         bucket: &str,
         objects: Vec<ObjectToDelete>,
-        mut opts: ObjectOptions,
+        opts: ObjectOptions,
     ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
+        let current_generations = opts.conditional_delete_generations.as_deref();
         let mut del_objects = vec![DeletedObject::default(); objects.len()];
         let mut accounting = vec![None; objects.len()];
         let delete_config_snapshot = opts
@@ -9763,30 +9655,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        let outer_namespace_lock_fence = opts.namespace_lock_fence.clone();
-        let mut transaction_guards = std::collections::BTreeMap::new();
-        if !is_meta_bucketname(bucket) {
-            // Every raw batch lock is acquired first. Canonical sorting also
-            // gives aliases of one disk key exactly one transaction owner.
-            let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
-            for key in keys {
-                match self.acquire_delete_transaction_lock(bucket, &key).await {
-                    Ok(guard) => {
-                        opts.add_namespace_lock_guard(&guard);
-                        transaction_guards.insert(key, guard);
-                    }
-                    Err(error) => {
-                        for (index, lock_name) in lock_names.iter().enumerate() {
-                            if lock_name.as_ref() == key {
-                                del_errs[index] = Some(error.clone());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        let transaction_guards = Arc::new(transaction_guards);
-        let current_generations = opts.conditional_delete_generations.as_deref();
         let object_lock_config = if is_meta_bucketname(bucket) {
             Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))
         } else {
@@ -9884,7 +9752,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                         &dobj.object_name,
                         DeleteRecoveryContext {
                             dry_run: !opts.no_lock,
-                            transaction_guard: transaction_guards.get(lock_names[i].as_ref()).cloned(),
                             namespace_owner: opts.delete_mutation_owner.clone(),
                             ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &|| {
                                 ensure_delete_commit_locks_held(None, bucket, &dobj.object_name, &opts).is_ok()
@@ -10166,7 +10033,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
             Arc::new((
                 self.ctx.begin_namespace_commit(),
-                Arc::clone(&transaction_guards),
                 opts.delete_mutation_owner.clone(),
                 opts.object_lock_config_snapshot.clone(),
                 opts.scanner_publication_commit_scope.clone(),
@@ -10342,24 +10208,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         }
 
         let results = join_all(futures).await;
-        if transaction_guards.values().any(|guard| guard.is_lock_lost()) {
-            for (index, object) in objects.iter().enumerate() {
-                if del_errs[index].is_none() {
-                    accounting[index] = None;
-                    del_errs[index] = Some(StorageError::NamespaceLockQuorumUnavailable {
-                        mode: "delete_transaction",
-                        bucket: bucket.to_owned(),
-                        object: object.object_name.clone(),
-                        required: 1,
-                        achieved: 0,
-                    });
-                }
-            }
-            if dist_erasure {
-                self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
-            }
-            return (del_objects, del_errs, accounting);
-        }
 
         let mut del_obj_errs: Vec<Vec<Option<DiskError>>> = vec![vec![None; objects.len()]; disks.len()];
 
@@ -10455,9 +10303,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 let group = &vers[index];
                 let recovery_ticket = recovery_tickets[index].clone();
                 let namespace_owner = namespace_owner.clone();
-                let transaction_guard = transaction_guards
-                    .get(object_name_disk_identity(&group.name).as_ref())
-                    .cloned();
                 let disks = &disks;
                 let opts = &opts;
                 async move {
@@ -10478,7 +10323,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                             DeleteRecoveryContext {
                                 recovery_ticket,
                                 namespace_owner,
-                                transaction_guard,
                                 ..DeleteRecoveryContext::new(opts.expected_bucket_incarnation_id, &can_commit)
                             },
                         )
@@ -10525,11 +10369,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 let object = fi_vers.name.clone();
                 let versions = fi_vers.clone();
                 let disk_namespace_owner = namespace_owner.clone();
-                let transaction_guard = transaction_guards.get(object_name_disk_identity(&object).as_ref()).cloned();
                 rollback_futures.push(async move {
-                    if transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
-                        return;
-                    }
                     if should_rollback {
                         let errs = disk
                             .delete_versions_with_namespace_owner(
@@ -10605,16 +10445,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             if del_errs[index].is_none() {
                 let object = &objects[index].object_name;
                 let result = match ensure_delete_commit_locks_held(None, bucket, object, &opts) {
-                    Ok(()) => {
-                        self.reclaim_missing_delete_residue(
-                            bucket,
-                            object,
-                            &opts,
-                            None,
-                            transaction_guards.get(object_name_disk_identity(object).as_ref()),
-                        )
-                        .await
-                    }
+                    Ok(()) => self.reclaim_missing_delete_residue(bucket, object, &opts, None).await,
                     Err(err) => Err(err),
                 };
                 if let Err(err) = result {
@@ -10623,13 +10454,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        // Deferred recovery takes fresh raw and transaction locks. Discard the
-        // completed leases' deadlines while retaining every caller-owned fence.
-        opts.namespace_lock_fence = outer_namespace_lock_fence;
-        // Release inner owners before reacquiring any raw key lock.
-        drop(namespace_owner);
-        drop(transaction_guards);
-
         // TODO(backlog): support partial object deletion for multi-part objects
 
         if dist_erasure {
@@ -10735,30 +10559,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         // Guard lock for single object delete
         let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock {
-            #[cfg(test)]
-            let guard = {
-                pause_delete_object_before_namespace(bucket, object).await;
-                let guard = self
-                    .acquire_write_lock_diag_with_pending_hook("delete_object", bucket, object, || {
-                        notify_delete_object_namespace(bucket, object, false);
-                    })
-                    .await?;
-                notify_delete_object_namespace(bucket, object, true);
-                guard
-            };
-            #[cfg(not(test))]
-            let guard = self.acquire_write_lock_diag("delete_object", bucket, object).await?;
-            Some(Arc::new(guard))
-        } else {
-            None
-        };
-        // Lock order: the caller's raw object lock, then this target set's
-        // transaction key. The inner key remains distinct even when pool
-        // client quorums overlap, and no_lock never bypasses journal ownership.
-        let transaction_guard = if !is_meta_bucketname(bucket) && (!opts.delete_prefix || opts.delete_prefix_object) {
-            let guard = self.acquire_delete_transaction_lock(bucket, object).await?;
-            opts.add_namespace_lock_guard(&guard);
-            Some(guard)
+            Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?))
         } else {
             None
         };
@@ -10862,8 +10663,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 &delete_request,
                                 false,
                                 None,
-                                DeleteCommitContext::request(&opts, _lock_guard.as_ref())
-                                    .with_transaction_guard(transaction_guard.as_ref()),
+                                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
                             )
                             .await?;
                             if let Some((_, deleted_object)) = replication_delete {
@@ -10897,8 +10697,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 &delete_request,
                                 false,
                                 None,
-                                DeleteCommitContext::request(&opts, _lock_guard.as_ref())
-                                    .with_transaction_guard(transaction_guard.as_ref()),
+                                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
                             )
                             .await?;
                         }
@@ -10918,8 +10717,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                                 &delete_request,
                                 false,
                                 None,
-                                DeleteCommitContext::request(&opts, _lock_guard.as_ref())
-                                    .with_transaction_guard(transaction_guard.as_ref()),
+                                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
                             )
                             .await?;
                         }
@@ -11008,7 +10806,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                     bucket,
                     object,
                     DeleteRecoveryContext {
-                        transaction_guard: transaction_guard.clone(),
                         namespace_owner: Some(Arc::new((
                             self.ctx.begin_namespace_commit(),
                             _lock_guard.clone(),
@@ -11049,7 +10846,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 if is_err_object_not_found(err) {
                     ensure_delete_commit_locks_held(_lock_guard.as_deref(), bucket, object, &opts)?;
                     if self
-                        .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref(), transaction_guard.as_ref())
+                        .reclaim_missing_delete_residue(bucket, object, &opts, _lock_guard.as_ref())
                         .await?
                     {
                         if let Some(scope) = scanner_publication_commit_scope.as_ref() {
@@ -11099,7 +10896,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 &dfi,
                 false,
                 None,
-                DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
+                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
             )
             .await
             .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -11201,7 +10998,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 &fi,
                 should_force_delete_marker_for_missing_version(&opts),
                 delete_marker_purge.clone(),
-                DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
+                DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
             )
             .await
             .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -11266,7 +11063,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             &dfi,
             opts.delete_marker,
             delete_marker_purge,
-            DeleteCommitContext::request(&opts, _lock_guard.as_ref()).with_transaction_guard(transaction_guard.as_ref()),
+            DeleteCommitContext::request(&opts, _lock_guard.as_ref()),
         )
         .await
         .map_err(|e| to_object_err(e, vec![bucket, object]))?;
@@ -21687,206 +21484,6 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
-    #[tokio::test]
-    async fn deep_heal_preserves_committed_generation_during_old_data_cleanup() {
-        use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
-
-        for (disk_count, parity, read_repair) in [(4, 2, false), (4, 2, true), (16, 4, false), (16, 4, true)] {
-            let (dirs, disks, set) = super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated(
-                disk_count, 0, parity,
-            )
-            .await;
-            let bucket = "native-heal-commit-cleanup";
-            let object = format!("generation-{disk_count}-{parity}-{read_repair}.bin");
-            for disk in &disks {
-                disk.make_volume(bucket).await.expect("create actual fixture volume");
-            }
-            let options = ObjectOptions {
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
-                ..Default::default()
-            };
-            set.put_object(bucket, &object, &mut PutObjReader::from_vec(vec![0x17; TEST_OBJECT_SIZE]), &options)
-                .await
-                .expect("acknowledge original generation on every disk");
-            let old = disks[0]
-                .read_version("", bucket, &object, "", &ReadOptions::default())
-                .await
-                .expect("read actual original generation");
-            let old_dir = old.data_dir.expect("original external data directory");
-            let old_part = dirs[0]
-                .path()
-                .join(bucket)
-                .join(&object)
-                .join(old_dir.to_string())
-                .join("part.1");
-            assert!(tokio::fs::try_exists(&old_part).await.expect("original part exists"));
-
-            // Cleanup slots follow the writer's shuffled erasure distribution,
-            // not the physical fixture order. Pause every real cleanup slot so
-            // no physical old shard can disappear before the proof below.
-            let barriers: Vec<_> = (0..disk_count)
-                .map(|slot| rename_fanout_barrier::arm(&object, slot, rename_fanout_barrier::PHASE_CLEANUP))
-                .collect();
-            let writer_set = Arc::clone(&set);
-            let writer_object = object.clone();
-            let writer_options = options.clone();
-            let payload = vec![0xa3; TEST_OBJECT_SIZE];
-            let writer_payload = payload.clone();
-            let writer = tokio::spawn(async move {
-                writer_set
-                    .put_object(bucket, &writer_object, &mut PutObjReader::from_vec(writer_payload), &writer_options)
-                    .await
-            });
-            tokio::time::timeout(Duration::from_secs(30), async {
-                for barrier in &barriers {
-                    barrier.wait_until_paused().await;
-                }
-            })
-            .await
-            .expect("overwrite commits before its real old-data cleanup pause");
-            assert!(!writer.is_finished(), "tail-drained ACK must remain pending at cleanup");
-            assert!(
-                tokio::fs::try_exists(&old_part)
-                    .await
-                    .expect("old source still present at pause")
-            );
-
-            let mut committed = Vec::new();
-            let mut healthy_metadata = Vec::new();
-            for disk in &disks {
-                let info = disk
-                    .read_version("", bucket, &object, "", &ReadOptions::default())
-                    .await
-                    .expect("read independently committed physical generation without GET");
-                assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (disk_count - parity, parity));
-                assert!(info.data.is_none(), "use actual external parts");
-                assert_ne!(info.data_dir, Some(old_dir));
-                committed.push(info);
-                healthy_metadata.push(
-                    disk.read_all(bucket, &format!("{object}/xl.meta"))
-                        .await
-                        .expect("committed raw metadata"),
-                );
-            }
-            let new_dir = committed[0].data_dir.expect("new committed data directory");
-            assert!(committed.iter().all(|info| info.data_dir == Some(new_dir)));
-            let new_part = dirs[0]
-                .path()
-                .join(bucket)
-                .join(&object)
-                .join(new_dir.to_string())
-                .join("part.1");
-            let expected_part = tokio::fs::read(&new_part).await.expect("exact new source bytes");
-            let mut committed_parts = Vec::new();
-            for dir in &dirs {
-                let original = dir.path().join(bucket).join(&object).join(old_dir.to_string()).join("part.1");
-                assert!(
-                    tokio::fs::try_exists(original)
-                        .await
-                        .expect("every old cleanup slot is paused")
-                );
-                committed_parts.push(
-                    tokio::fs::read(dir.path().join(bucket).join(&object).join(new_dir.to_string()).join("part.1"))
-                        .await
-                        .expect("every new encoded shard before fault"),
-                );
-            }
-            tokio::fs::remove_file(&new_part)
-                .await
-                .expect("remove only the newly committed target shard");
-            println!(
-                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} old_cleanup_paused new_generation_committed target_part_missing"
-            );
-
-            let (result, error) = tokio::time::timeout(
-                Duration::from_secs(30),
-                set.heal_object(
-                    bucket,
-                    &object,
-                    "",
-                    &HealOpts {
-                        scan_mode: HealScanMode::Deep,
-                        read_repair,
-                        ..Default::default()
-                    },
-                ),
-            )
-            .await
-            .expect("real locked Heal must progress during old cleanup")
-            .expect("complete actual deep Heal");
-            assert!(error.is_none(), "new generation must remain repairable: {error:?}");
-            assert_eq!(result.drives_healed(), Some(1));
-            // Shared data-phase repair deliberately does not certify full integrity.
-            assert_eq!(result.integrity_verified, !read_repair);
-            assert_eq!(tokio::fs::read(&new_part).await.expect("rebuilt part before any GET"), expected_part);
-            let repaired = disks[0]
-                .read_version("", bucket, &object, "", &ReadOptions::default())
-                .await
-                .expect("read actual repaired generation");
-            assert_eq!(repaired.data_dir, Some(new_dir));
-            assert_eq!(repaired.mod_time, committed[0].mod_time);
-            for (disk, expected) in disks.iter().zip(&healthy_metadata).skip(1) {
-                assert_eq!(
-                    disk.read_all(bucket, &format!("{object}/xl.meta"))
-                        .await
-                        .expect("healthy raw metadata after Heal"),
-                    *expected
-                );
-            }
-            println!(
-                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} locked_heal_completed new_generation_part_rebuilt_before_get"
-            );
-
-            for barrier in &barriers {
-                barrier.release();
-            }
-            let acknowledged = tokio::time::timeout(Duration::from_secs(30), writer)
-                .await
-                .expect("cleanup drains")
-                .expect("writer joins")
-                .expect("overwrite ACK remains successful");
-            assert_eq!(acknowledged.size, i64::try_from(payload.len()).expect("fixture size fits i64"));
-            drop(barriers);
-            for (slot, dir) in dirs.iter().enumerate() {
-                let root = dir.path().join(bucket).join(&object);
-                assert!(
-                    !tokio::fs::try_exists(root.join(old_dir.to_string()))
-                        .await
-                        .expect("old cleanup completed")
-                );
-                assert_eq!(
-                    tokio::fs::read(root.join(new_dir.to_string()).join("part.1"))
-                        .await
-                        .expect("new shard survives cleanup before GET"),
-                    committed_parts[slot],
-                    "old cleanup must preserve every exact committed encoded shard"
-                );
-            }
-            let mut reader = set
-                .get_object_reader(bucket, &object, None, HeaderMap::new(), &options)
-                .await
-                .expect("read final acknowledged generation");
-            let mut body = Vec::new();
-            reader
-                .stream
-                .read_to_end(&mut body)
-                .await
-                .expect("read complete final payload");
-            assert_eq!(body, payload);
-            for disk in &disks {
-                let final_info = disk
-                    .read_version("", bucket, &object, "", &ReadOptions::default())
-                    .await
-                    .expect("final physical generation");
-                assert_eq!(final_info.data_dir, Some(new_dir), "old cleanup must not revert the healed generation");
-            }
-            println!(
-                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} overwrite_ack_preserved final_physical_generation_and_full_body_verified"
-            );
-        }
-    }
-
     #[tokio::test]
     async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -25809,7 +25406,6 @@ mod delete_objects_lock_gating_tests {
     }
 
     #[tokio::test]
-    #[serial_test::serial]
     async fn delete_object_aborts_when_outer_lock_is_lost_at_commit() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
         let bucket = "single-lost-outer-lock";
@@ -26030,13 +25626,6 @@ mod delete_objects_lock_gating_tests {
             .as_fast_lock_manager()
             .expect("isolated fixture should use the local lock manager");
         let lock_metrics = lock_manager.get_metrics().shard_metrics;
-        let raw_key = ObjectKey::new(bucket, "obj-a");
-        let raw_holder = lock_manager.get_lock_info(&raw_key).expect("outer holder must be visible");
-        let transaction_key = ObjectKey::with_version(
-            bucket,
-            "obj-a",
-            format!("rustfs:delete-transaction:{}:{}", set_disks.pool_index, set_disks.set_index),
-        );
         assert_eq!(lock_manager.current_lock_counts(), (1, 0));
         let mut delete = Box::pin(set_disks.delete_objects(
             bucket,
@@ -26053,43 +25642,23 @@ mod delete_objects_lock_gating_tests {
         // than a second on CI without waiting for the namespace lock at all.
         let deletion = futures::future::poll_fn(|cx| {
             let result = std::future::Future::poll(delete.as_mut(), cx);
-            let current_holder = lock_manager.get_lock_info(&raw_key).expect("outer holder must remain held");
-            assert_eq!(
-                (current_holder.mode, &current_holder.owner, current_holder.acquired_at),
-                (raw_holder.mode, &raw_holder.owner, raw_holder.acquired_at),
-                "no_lock must preserve the original raw object holder"
-            );
-            let transaction_holder = lock_manager.get_lock_info(&transaction_key);
-            if let Some(holder) = &transaction_holder {
-                assert_eq!(holder.mode, rustfs_lock::LockMode::Exclusive);
-            }
             assert_eq!(
                 lock_manager.current_lock_counts(),
-                (1 + u64::from(transaction_holder.is_some()), 0),
-                "only the distinct transaction owner may be added, without raw-key waiters"
+                (1, 0),
+                "no_lock batch delete must preserve the outer holder without adding lock waiters"
             );
             let current = lock_manager.get_metrics().shard_metrics;
-            assert!(
-                current.total_acquisitions() <= lock_metrics.total_acquisitions() + 1,
-                "the transaction owner must be reused by inline recovery"
+            assert_eq!(
+                (current.total_acquisitions(), current.timeouts),
+                (lock_metrics.total_acquisitions(), lock_metrics.timeouts),
+                "no_lock batch delete must not acquire or time out on another namespace lock"
             );
-            assert_eq!(current.timeouts, lock_metrics.timeouts, "the raw key must never be reacquired");
             result
         });
         let (deleted, errs) = tokio::time::timeout(Duration::from_secs(30), deletion)
             .await
             .expect("no_lock batch delete disk work should complete");
 
-        assert_eq!(lock_manager.current_lock_counts(), (1, 0));
-        assert!(
-            lock_manager.get_lock_info(&transaction_key).is_none(),
-            "the transaction owner must be released"
-        );
-        assert_eq!(
-            lock_manager.get_metrics().shard_metrics.total_acquisitions(),
-            lock_metrics.total_acquisitions() + 1,
-            "no_lock still requires exactly one distinct transaction owner"
-        );
         assert!(errs[0].is_none(), "no_lock batch delete should not fail with a lock error: {:?}", errs[0]);
         assert!(deleted[0].found, "existing object must still be deleted");
         for disk in &disk_stores {

crates/ecstore/src/set_disk/read.rs

@@ -1818,10 +1818,7 @@ impl SetDisks {
                         "Create bitrot reader failed read quorum"
                     );
                     record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
-                    // The metadata selected this object, but the part cannot
-                    // be reconstructed. A dominant disk I/O/not-found error
-                    // must not erase the object's read-quorum failure.
-                    return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+                    return Err(to_object_err(read_err.into(), vec![bucket, object]));
                 }
                 let reason = GetObjectFailureReason::ReadQuorum;
                 error!(
@@ -1846,7 +1843,7 @@ impl SetDisks {
                     "Create bitrot reader did not have enough disks"
                 );
                 record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
-                return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+                return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
             }
 
             // Check if we have missing shards even though we can read successfully
@@ -2381,10 +2378,10 @@ impl SetDisks {
             if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
                 let reason = classify_disk_error(&read_err);
                 record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
-                return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+                return Err(to_object_err(read_err.into(), vec![bucket, object]));
             }
             record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
-            return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+            return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
         }
 
         let missing_shards = reader_setup.completed_failed_shards();
@@ -8195,105 +8192,6 @@ mod tests {
         (data_dir, files)
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn external_missing_shards_keep_typed_read_quorum() {
-        use tokio::io::AsyncReadExt;
-
-        let payload = vec![0x37; 256 * 1024];
-        for readable_shards in [1, 2, 4] {
-            let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
-            let mut fi = codec_streaming_test_fileinfo(i64::try_from(payload.len()).expect("fixture size fits i64"), 1);
-            fi.erasure.data_blocks = erasure.data_shards;
-            fi.erasure.parity_blocks = erasure.parity_shards;
-            fi.erasure.block_size = erasure.block_size;
-            fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
-            let (dirs, disks) = local_test_disks(erasure.total_shard_count(), CODEC_STREAMING_TEST_BUCKET).await;
-            let (data_dir, files) = codec_streaming_external_files(&erasure, &payload, &disks).await;
-            fi.data_dir = Some(data_dir);
-            for directory in dirs.iter().skip(readable_shards) {
-                tokio::fs::remove_file(
-                    directory
-                        .path()
-                        .join(CODEC_STREAMING_TEST_BUCKET)
-                        .join(CODEC_STREAMING_TEST_OBJECT)
-                        .join(data_dir.to_string())
-                        .join("part.1"),
-                )
-                .await
-                .expect("remove real part while retaining its selected metadata");
-            }
-
-            let mut output = Vec::new();
-            let legacy = SetDisks::get_object_with_fileinfo(
-                CODEC_STREAMING_TEST_BUCKET,
-                CODEC_STREAMING_TEST_OBJECT,
-                Arc::new(ErasureCache::new()),
-                0,
-                fi.size,
-                &mut output,
-                fi.clone(),
-                files.clone(),
-                &disks,
-                0,
-                0,
-                false,
-                true,
-                false,
-                false,
-                GET_OBJECT_PATH_LEGACY_DUPLEX,
-                "plain_single_part",
-                "le_1mib",
-            )
-            .await;
-            let streaming = SetDisks::get_object_mid_size_reader_with_fileinfo(
-                CODEC_STREAMING_TEST_BUCKET,
-                CODEC_STREAMING_TEST_OBJECT,
-                Arc::new(ErasureCache::new()),
-                &fi,
-                &files,
-                &disks,
-                0,
-                0,
-                false,
-                true,
-                "plain_single_part",
-                "le_1mib",
-                false,
-            )
-            .await;
-            if readable_shards < erasure.data_shards {
-                for error in [
-                    legacy.expect_err("legacy read must reject quorum minus one"),
-                    streaming.err().expect("streaming setup must reject quorum minus one"),
-                ] {
-                    assert_eq!(
-                        error,
-                        Error::InsufficientReadQuorum(
-                            CODEC_STREAMING_TEST_BUCKET.to_owned(),
-                            CODEC_STREAMING_TEST_OBJECT.to_owned(),
-                        )
-                    );
-                }
-                assert!(output.is_empty(), "quorum rejection must precede all response bytes");
-            } else {
-                legacy.expect("legacy reader retains reconstructable quorum");
-                assert_eq!(output, payload, "legacy read returns every original byte at quorum");
-                let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) =
-                    streaming.expect("streaming quorum remains readable")
-                else {
-                    panic!("mid-size quorum must yield an actual reader");
-                };
-                let mut decoded = Vec::new();
-                reader
-                    .read_to_end(&mut decoded)
-                    .await
-                    .expect("streaming reader returns the complete body");
-                assert_eq!(decoded, payload, "streaming read returns every original byte at quorum");
-            }
-        }
-    }
-
     #[test]
     #[serial]
     fn payload_read_repair_is_suppressed_only_for_read_only_requests() {

crates/ecstore/src/store/heal.rs

@@ -2167,210 +2167,6 @@ mod tests {
         shutdown.cancel();
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit() {
-        use crate::set_disk::ReadRepairPauseScope;
-
-        let (_root, store, shutdown) = multi_pool_heal_store().await;
-        let bucket = format!("heal-commit-owner-{}", Uuid::new_v4().simple());
-        let object = "history.bin";
-        let version = Uuid::new_v4().to_string();
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("original bucket");
-        let old = store
-            .bucket_incarnation_id_from_disk(&bucket)
-            .await
-            .expect("original identity");
-        let set = store.pools[0].get_disks(0);
-        let mut reader = PutObjReader::from_vec(vec![0x17; 1024 * 1024]);
-        set.put_object(
-            &bucket,
-            object,
-            &mut reader,
-            &ObjectOptions {
-                versioned: true,
-                version_id: Some(version.clone()),
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("original acknowledged version");
-        let missing = set.disks.read().await[0].clone().expect("repair target");
-        let healthy = set.disks.read().await[1].clone().expect("healthy target");
-        let baseline = healthy
-            .read_all(&bucket, &format!("{object}/xl.meta"))
-            .await
-            .expect("healthy metadata baseline");
-        let original = healthy
-            .read_version("", &bucket, object, &version, &Default::default())
-            .await
-            .expect("original version metadata");
-        assert!(original.data.is_none(), "the commit barrier fixture must use actual external data shards");
-        missing
-            .delete(
-                &bucket,
-                object,
-                DeleteOptions {
-                    recursive: true,
-                    immediate: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("remove one physical shard");
-
-        // The scope pauses the real repair after reconstruction, immediately
-        // before its generation check and final shard rename.
-        let (_pause, arrived, release) = ReadRepairPauseScope::install_incarnation_commit(&bucket, object);
-        let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
-        let worker_store = store.clone();
-        let worker_bucket = bucket.clone();
-        let worker_version = version.clone();
-        let opts = HealOpts {
-            pool: Some(0),
-            set: Some(0),
-            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
-            ..Default::default()
-        };
-        let mut waiter = tokio::spawn(async move {
-            worker_store
-                .run_bucket_heal_at_incarnation(&worker_bucket, old, &opts, move |store, bucket, opts| async move {
-                    let result = store.heal_object_with_proof(&bucket, object, &worker_version, &opts).await?;
-                    drained_tx
-                        .send(result)
-                        .expect("retain the actual storage result after caller cancellation");
-                    Ok(())
-                })
-                .await
-        });
-        tokio::select! {
-            paused = tokio::time::timeout(std::time::Duration::from_secs(10), arrived.notified()) => {
-                paused.expect("real missing-shard repair reaches the final commit pause");
-            }
-            completed = &mut waiter => panic!("repair ended before the actual final commit barrier: {completed:?}"),
-        }
-        assert!(
-            missing.read_xl(&bucket, object, false).await.is_err(),
-            "final shard is not published at the pause"
-        );
-        println!("incarnation_commit_barrier: final_rename_paused");
-        waiter.abort();
-        assert!(waiter.await.expect_err("caller aborted").is_cancelled());
-        let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
-        assert!(
-            futures::poll!(&mut writer).is_pending(),
-            "bucket deletion/recreation cannot acquire its generation writer while the cancelled repair still owns commit"
-        );
-        println!("incarnation_commit_barrier: caller_aborted_writer_pending");
-        release.notify_one();
-        let result = tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
-            .await
-            .expect("actual repair drains after release")
-            .expect("retained repair result");
-        assert!(
-            result.error.is_none(),
-            "physical repair must succeed before lifecycle handoff: {:?}",
-            result.error
-        );
-        let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
-            .await
-            .expect("generation writer acquires only after physical repair drains")
-            .expect("generation writer");
-        assert!(
-            missing.read_xl(&bucket, object, false).await.is_ok(),
-            "the physical shard is published before writer admission"
-        );
-        let repaired = missing
-            .read_version("", &bucket, object, &version, &Default::default())
-            .await
-            .expect("repaired exact version metadata");
-        assert_eq!(repaired.version_id, original.version_id);
-        assert_eq!(repaired.data_dir, original.data_dir);
-        let checked = missing
-            .check_parts(&bucket, object, &repaired)
-            .await
-            .expect("verify the published physical shard");
-        assert_eq!(
-            checked.results,
-            vec![crate::disk::CHECK_PART_SUCCESS],
-            "the actual published part passes bitrot verification"
-        );
-        assert_eq!(
-            healthy
-                .read_all(&bucket, &format!("{object}/xl.meta"))
-                .await
-                .expect("healthy after repair"),
-            baseline,
-            "repair does not overwrite the surviving version"
-        );
-        println!("incarnation_commit_barrier: repair_drained_writer_acquired");
-        drop(guard);
-
-        set.delete_object(
-            &bucket,
-            object,
-            ObjectOptions {
-                versioned: true,
-                version_id: Some(version.clone()),
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("normal explicit deletion of the old version");
-        store
-            .delete_bucket(&bucket, &DeleteBucketOptions::default())
-            .await
-            .expect("normal empty bucket deletion");
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("same-name successor bucket");
-        let new = store
-            .bucket_incarnation_id_from_disk(&bucket)
-            .await
-            .expect("successor identity");
-        assert_ne!(old, new);
-        let successor_version = Uuid::new_v4().to_string();
-        let mut successor_reader = PutObjReader::from_vec(vec![0x29; 512 * 1024]);
-        set.put_object(
-            &bucket,
-            object,
-            &mut successor_reader,
-            &ObjectOptions {
-                versioned: true,
-                version_id: Some(successor_version),
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("acknowledged successor version");
-        let successor = healthy
-            .read_all(&bucket, &format!("{object}/xl.meta"))
-            .await
-            .expect("successor metadata");
-        assert!(
-            store
-                .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
-                .await
-                .is_err()
-        );
-        assert_eq!(
-            healthy
-                .read_all(&bucket, &format!("{object}/xl.meta"))
-                .await
-                .expect("successor after obsolete admission"),
-            successor,
-            "obsolete admission cannot change the same-name successor"
-        );
-        println!("incarnation_commit_barrier: successor_rejects_obsolete_admission");
-        shutdown.cancel();
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {

crates/ecstore/src/store/object.rs

@@ -13167,202 +13167,6 @@ mod tests {
         })
         .await;
     }
-
-    async fn assert_delete_excludes_recovery_across_pool_lock_domains(topology: &str, batch: bool) {
-        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
-        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
-        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
-        let mut store = new_prepared_reader_test_store_with_ctx(&[first_set, second_set], Arc::clone(&ctx)).await;
-        let clients: Vec<Arc<dyn rustfs_lock::LockClient>> = (0..4)
-            .map(|_| {
-                Arc::new(rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(
-                    Arc::new(rustfs_lock::FastObjectLockManager::new()),
-                )))) as Arc<dyn rustfs_lock::LockClient>
-            })
-            .collect();
-        let domains = match topology {
-            "disjoint" => [vec![clients[0].clone()], vec![clients[1].clone()]],
-            "identical" => [clients[..3].to_vec(), clients[..3].to_vec()],
-            "overlapping" => [clients[..3].to_vec(), clients[1..].to_vec()],
-            _ => panic!("unknown lock topology"),
-        };
-        for (pool, lockers) in store.pools.iter_mut().zip(domains) {
-            let mut set = (*pool.disk_set[0]).clone();
-            set.lockers = lockers;
-            Arc::make_mut(pool).disk_set[0] = Arc::new(set);
-        }
-        let store = Arc::new(store);
-        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
-        let bucket = "delete-nonzero-pool-lock";
-        let object = "object.bin";
-        store
-            .handle_make_bucket(bucket, &MakeBucketOptions::default())
-            .await
-            .expect("create the bucket in both pools");
-        if batch {
-            // Latest-marker routing sends this batch only to the nonzero pool,
-            // so the cleanup barrier observes its prepared journal directly.
-            store
-                .update_bucket_metadata_config(
-                    bucket,
-                    crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
-                    b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
-                )
-                .await
-                .expect("enable versioning for the routed batch");
-        }
-        let target = store.pools[1].get_disks_by_key(object);
-        let source = target
-            .put_object(
-                bucket,
-                object,
-                &mut PutObjReader::from_vec(b"delete lock regression".to_vec()),
-                &ObjectOptions {
-                    versioned: batch,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("seed an object only in the nonzero pool");
-        ctx.update_erasure_type(SetupType::DistErasure).await;
-
-        let object_barrier = (!batch).then(|| crate::set_disk::DeleteObjectCommitBarrier::install(bucket, object));
-        let batch_barrier = batch.then(|| crate::set_disk::DeleteCleanupBarrier::install(bucket));
-        let delete_store = Arc::clone(&store);
-        let delete = tokio::spawn(async move {
-            if batch {
-                let (_, errors) = delete_store
-                    .delete_objects(
-                        bucket,
-                        vec![ObjectToDelete {
-                            object_name: object.to_owned(),
-                            ..Default::default()
-                        }],
-                        ObjectOptions::default(),
-                    )
-                    .await;
-                match errors.into_iter().flatten().next() {
-                    Some(error) => Err(error),
-                    None => Ok(()),
-                }
-            } else {
-                delete_store
-                    .delete_object(bucket, object, ObjectOptions::default())
-                    .await
-                    .map(|_| ())
-            }
-        });
-        if let Some(barrier) = &object_barrier {
-            barrier.wait_until_paused().await;
-        }
-        if let Some(barrier) = &batch_barrier {
-            barrier.wait_until_paused().await;
-        }
-        let recovery_store = Arc::clone(&store);
-        let mut recovery = tokio::spawn(async move {
-            recovery_store
-                .recover_delete_transactions_at_incarnation(
-                    bucket,
-                    Some(object),
-                    None,
-                    &HealOpts {
-                        pool: Some(1),
-                        set: Some(0),
-                        ..Default::default()
-                    },
-                )
-                .await
-        });
-        let early_recovery = tokio::time::timeout(Duration::from_millis(100), &mut recovery).await;
-        let recovery_was_blocked = early_recovery.is_err();
-        if let Some(barrier) = &object_barrier {
-            barrier.release();
-        }
-        if let Some(barrier) = &batch_barrier {
-            barrier.release();
-        }
-        tokio::time::timeout(Duration::from_secs(10), delete)
-            .await
-            .expect("the delete should finish after releasing its commit barrier")
-            .expect("the delete task should join")
-            .expect("the delete should succeed");
-        let recovered = match early_recovery {
-            Ok(result) => result,
-            Err(_) => tokio::time::timeout(Duration::from_secs(10), recovery)
-                .await
-                .expect("recovery should finish after the frontend delete"),
-        };
-        recovered
-            .expect("the recovery task should join")
-            .expect("recovery should succeed");
-        assert!(
-            recovery_was_blocked,
-            "recovery must not inspect an in-flight frontend delete with {topology} pool lock clients"
-        );
-        if batch {
-            let marker = target
-                .get_object_info(bucket, object, &ObjectOptions::default())
-                .await
-                .expect("the versioned batch must publish a current delete marker");
-            assert!(marker.delete_marker && marker.is_latest);
-            assert_eq!(marker.num_versions, 2, "the source version must remain beside the new marker");
-            assert_ne!(marker.version_id, source.version_id);
-            let mut reader = target
-                .get_object_reader(
-                    bucket,
-                    object,
-                    None,
-                    HeaderMap::new(),
-                    &ObjectOptions {
-                        version_id: Some(source.version_id.expect("versioned source identity").to_string()),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("the original source version must remain readable");
-            let mut body = Vec::new();
-            reader
-                .stream
-                .read_to_end(&mut body)
-                .await
-                .expect("read the retained source version");
-            assert_eq!(body, b"delete lock regression");
-        } else {
-            let error = target
-                .get_object_info(bucket, object, &ObjectOptions::default())
-                .await
-                .expect_err("the deleted object must be absent");
-            assert!(is_err_object_not_found(&error));
-        }
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    #[serial_test::serial(storage_class_env)]
-    async fn batch_delete_excludes_recovery_on_nonzero_pool() {
-        assert_delete_excludes_recovery_across_pool_lock_domains("disjoint", true).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    #[serial_test::serial(storage_class_env)]
-    async fn delete_excludes_recovery_with_disjoint_pool_lock_clients() {
-        assert_delete_excludes_recovery_across_pool_lock_domains("disjoint", false).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    #[serial_test::serial(storage_class_env)]
-    async fn delete_excludes_recovery_with_identical_pool_lock_clients() {
-        assert_delete_excludes_recovery_across_pool_lock_domains("identical", false).await;
-    }
-
-    #[tokio::test]
-    #[serial_test::serial]
-    #[serial_test::serial(storage_class_env)]
-    async fn delete_excludes_recovery_with_overlapping_pool_lock_clients() {
-        assert_delete_excludes_recovery_across_pool_lock_domains("overlapping", false).await;
-    }
 }
 
 #[cfg(test)]

crates/filemeta/src/filemeta.rs

@@ -2429,52 +2429,6 @@ mod test {
         }
     }
 
-    #[test]
-    fn truncated_inline_xlmeta_is_file_corrupt_only_for_supported_formats() {
-        let mut meta = FileMeta::new();
-        meta.data
-            .replace("fixture", vec![0x5a; 4095])
-            .expect("seed real inline framing");
-        let inline_len = meta.data.as_slice().len();
-        let original = meta.marshal_msg().expect("serialize valid inline metadata");
-        let inline_start = original.len() - inline_len;
-        let parsed = FileMeta::load(&original).expect("complete inline metadata must decode");
-        assert_eq!(parsed.data, meta.data);
-        for cut in inline_start + 1..original.len() {
-            assert_eq!(
-                FileMeta::load(&original[..cut]).expect_err("incomplete inline framing must fail"),
-                Error::FileCorrupt,
-                "inline truncation at byte {cut} must remain repairable"
-            );
-        }
-
-        for inline_version in [0, 2] {
-            let mut unsupported = original[..original.len() / 2].to_vec();
-            unsupported[inline_start] = inline_version;
-            assert_ne!(
-                FileMeta::load(&unsupported).expect_err("malformed unsupported inline format must fail"),
-                Error::FileCorrupt,
-                "unknown inline version must not authorize repair"
-            );
-        }
-        for (header_ver, meta_ver) in [
-            (XL_HEADER_VERSION + 1, XL_META_VERSION),
-            (XL_HEADER_VERSION, XL_META_VERSION + 1),
-        ] {
-            let mut headers = Vec::new();
-            rmp::encode::write_uint(&mut headers, u64::from(header_ver)).expect("write future header");
-            rmp::encode::write_uint(&mut headers, u64::from(meta_ver)).expect("write future metadata version");
-            rmp::encode::write_uint(&mut headers, 0).expect("write empty version count");
-            let mut unsupported = build_xl_buffer(&headers);
-            unsupported.extend_from_slice(&meta.data.as_slice()[..inline_len / 2]);
-            assert_ne!(
-                FileMeta::load(&unsupported).expect_err("unsupported metadata with truncated inline data must fail"),
-                Error::FileCorrupt,
-                "future metadata schema must not authorize repair"
-            );
-        }
-    }
-
     #[test]
     fn truncated_xlmeta_index_and_format_reads_are_file_corrupt() {
         let buf = FileMeta::default()

crates/filemeta/src/filemeta/codec.rs

@@ -188,13 +188,7 @@ impl FileMeta {
         if !buf.is_empty() {
             InlineData::validate_slice(buf).map_err(|e| {
                 error!("data validation failed: {}", e);
-                // Validation only examines bytes already read. Classify damage
-                // as repairable corruption only for formats we understand.
-                if InlineData::version_ok_slice(buf) && Self::decode_xl_headers(meta).is_ok() {
-                    Error::FileCorrupt
-                } else {
-                    e
-                }
+                e
             })?;
         }
 

crates/filemeta/src/filemeta_inline.rs

@@ -168,11 +168,11 @@ impl InlineData {
         self.0.as_slice()
     }
     pub fn version_ok(&self) -> bool {
-        Self::version_ok_slice(&self.0)
-    }
+        if self.0.is_empty() {
+            return true;
+        }
 
-    pub(crate) fn version_ok_slice(raw: &[u8]) -> bool {
-        raw.first().is_none_or(|version| *version > 0 && *version <= INLINE_DATA_VER)
+        self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
     }
 
     pub fn after_version(&self) -> &[u8] {

crates/heal/src/error.rs

@@ -16,7 +16,7 @@ use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
 use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
 use thiserror::Error;
 
-use super::heal::{DiskError, EcstoreError, storage_api::owner::EcstorePoolMetadataFailure};
+use super::heal::{DiskError, EcstoreError};
 
 const HEAL_DANGLING_DELETE_GRACE_MESSAGE: &str = "dangling object deletion deferred by heal grace window";
 pub(crate) const MRF_MISSING_PROOF_PREFIX: &str = "Missing exact storage proof for durable MRF repair ";
@@ -236,14 +236,6 @@ pub(crate) fn classify_heal_error(error: &Error) -> HealErrorClass {
 /// with object absence, the conflation that once let an offline drive be
 /// recorded as absent and skipped (backlog#856 / #799 B7).
 pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
-    if let Some(failure) = error.pool_metadata_failure() {
-        return match failure.kind {
-            EcstorePoolMetadataFailure::ReadUnavailable | EcstorePoolMetadataFailure::TransactionUnknown => {
-                HealErrorClass::Transient
-            }
-            EcstorePoolMetadataFailure::RecoveryRequired | EcstorePoolMetadataFailure::FenceLost => HealErrorClass::Permanent,
-        };
-    }
     if let EcstoreError::Lock(lock_error) = error {
         return if lock_error.is_fatal() {
             HealErrorClass::Permanent
@@ -443,34 +435,6 @@ mod tests {
     use super::Error;
     use crate::heal::{DiskError, EcstoreError};
 
-    #[test]
-    fn pool_metadata_failures_keep_their_typed_recovery_classification() {
-        use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
-
-        for (kind, retryable) in [
-            (EcstorePoolMetadataFailure::ReadUnavailable, true),
-            (EcstorePoolMetadataFailure::TransactionUnknown, true),
-            (EcstorePoolMetadataFailure::RecoveryRequired, false),
-            (EcstorePoolMetadataFailure::FenceLost, false),
-        ] {
-            let error = EcstoreError::other(EcstorePoolMetadataError {
-                kind,
-                operation: "bucket heal".to_owned(),
-                phase: "native_heal",
-                since: std::time::UNIX_EPOCH.into(),
-                source: Some(std::sync::Arc::new(EcstoreError::SlowDown)),
-            });
-            let text = error.to_string();
-            for error in [error.clone(), error] {
-                assert_eq!(Error::Storage(error).is_recoverable_heal(), retryable, "{kind:?}");
-            }
-            assert!(
-                !Error::Storage(EcstoreError::other(text)).is_recoverable_heal(),
-                "display text must not grant recovery authority"
-            );
-        }
-    }
-
     #[test]
     fn namespace_lock_quorum_is_recoverable_across_io_error_conversions() {
         for (required, achieved) in [(2, 1), (1, 0)] {

crates/heal/src/heal/channel.rs

@@ -196,20 +196,6 @@ impl HealChannelProcessor {
             .map_err(Error::other)
     }
 
-    /// Query a local automatic task; other tokens require coordinator routing.
-    pub async fn execute_local_automatic_query(
-        &self,
-        heal_path: &str,
-        client_token: &str,
-        since_seq: Option<u64>,
-    ) -> Result<Option<HealChannelResponse>> {
-        self.heal_manager
-            .get_local_automatic_task_report(heal_path, client_token, since_seq)
-            .await?
-            .map(|report| Self::query_report_response(client_token.to_owned(), Ok(report)))
-            .transpose()
-    }
-
     /// Execute cancellation directly against the manager.
     pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
         let (response_tx, response_rx) = oneshot::channel();
@@ -477,13 +463,6 @@ impl HealChannelProcessor {
                 .await
         };
 
-        let response = Self::query_report_response(client_token, report)?;
-        let _ = response_tx.send(Ok(response.clone()));
-        self.publish_response(response);
-        Ok(())
-    }
-
-    fn query_report_response(client_token: String, report: Result<HealTaskReport>) -> Result<HealChannelResponse> {
         let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
         let settings = report
             .as_ref()
@@ -602,21 +581,27 @@ impl HealChannelProcessor {
                 0,
             ),
             Err(crate::Error::InvalidClientToken) => {
-                return Ok(HealChannelResponse {
+                let response = HealChannelResponse {
                     request_id: client_token,
                     success: false,
                     data: None,
                     error: Some("invalid heal client token".to_string()),
-                });
+                };
+                let _ = response_tx.send(Ok(response.clone()));
+                self.publish_response(response);
+                return Ok(());
             }
             Err(err) => {
                 let error_text = err.to_string();
-                return Ok(HealChannelResponse {
+                let response = HealChannelResponse {
                     request_id: client_token,
                     success: false,
                     data: None,
                     error: Some(error_text),
-                });
+                };
+                let _ = response_tx.send(Ok(response.clone()));
+                self.publish_response(response);
+                return Ok(());
             }
         };
 
@@ -633,12 +618,17 @@ impl HealChannelProcessor {
             },
         )?;
 
-        Ok(HealChannelResponse {
+        let response = HealChannelResponse {
             request_id: client_token,
             success: true,
             data: Some(data),
             error: detail,
-        })
+        };
+
+        let _ = response_tx.send(Ok(response.clone()));
+        self.publish_response(response);
+
+        Ok(())
     }
 
     /// Process cancel request

crates/heal/src/heal/manager.rs

@@ -234,8 +234,6 @@ fn record_displaced_terminal(
     request: &HealRequest,
 ) -> Arc<CompletedHealStatus> {
     let terminal = Arc::new(CompletedHealStatus {
-        task_id: request.id.clone(),
-        source: request.source,
         outcome: None,
         progress: None,
         retained_bytes: std::sync::OnceLock::new(),
@@ -627,6 +625,37 @@ fn request_matches_request(request: &HealRequest, existing: &HealRequest) -> boo
         && request.heal_endpoints == existing.heal_endpoints
 }
 
+fn lookup_request_id_admission(
+    request: &HealRequest,
+    active_heals: &HashMap<String, Arc<HealTask>>,
+    queue: &PriorityHealQueue,
+    retrying_heals: &HashMap<String, RetryingHeal>,
+    durable_owners: &[HealRequest],
+) -> Option<(bool, &'static str)> {
+    active_heals
+        .get(&request.id)
+        .map(|task| (request_matches_task(request, task), "active"))
+        .or_else(|| {
+            queue
+                .requests()
+                .find(|queued| queued.id == request.id)
+                .map(|queued| (request_matches_request(request, queued), "queued"))
+        })
+        .or_else(|| {
+            retrying_heals
+                .get(&request.id)
+                .map(|retrying| (request_matches_request(request, &retrying.request), "retrying"))
+        })
+        // A durable-only owner must be resumed by recovery, not overwritten
+        // with a new execution budget by a replay before recovery completes.
+        .or_else(|| {
+            durable_owners
+                .iter()
+                .find(|owner| owner.id == request.id)
+                .map(|_| (false, "durable"))
+        })
+}
+
 fn retrying_heal_for_dedup_key(retrying_heals: &HashMap<String, RetryingHeal>, key: &str) -> Option<(String, HealType)> {
     retrying_heals
         .iter()
@@ -671,10 +700,7 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
     if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
         return None;
     }
-    // Admin traversals already exhaust their per-object retries. Restarting
-    // the whole traversal under the same token discards those receipts and
-    // reports previously repaired objects as unchanged on the next scan.
-    if task.has_batch_failure() && (task.source == HealRequestSource::Admin || !retryable_batch_failure) {
+    if task.has_batch_failure() && !retryable_batch_failure {
         return None;
     }
 
@@ -1076,9 +1102,9 @@ pub struct HealManager {
 /// cascade without re-locking.
 enum TaskStateLookup {
     Active(Arc<HealTask>),
-    Retrying(HealTaskStatus, HealOptions, HealRequestSource),
+    Retrying(HealTaskStatus, HealOptions),
     Completed(Arc<CompletedHealStatus>),
-    Queued(HealOptions, HealRequestSource),
+    Queued(HealOptions),
     /// A durable root heal record that cannot be admitted or quarantined;
     /// it answers status queries with the reason until an operator repairs it.
     Unrunnable(String),
@@ -2124,28 +2150,59 @@ impl HealManager {
         let mut queue = self.heal_queue.lock().await;
         let mut retrying_heals = self.retrying_heals.lock().await;
 
-        let request_id_admission = active_heals
-            .get(&request.id)
-            .map(|task| (request_matches_task(&request, task), "active"))
-            .or_else(|| {
-                queue
-                    .requests()
-                    .find(|queued| queued.id == request.id)
-                    .map(|queued| (request_matches_request(&request, queued), "queued"))
-            })
-            .or_else(|| {
-                retrying_heals
-                    .get(&request.id)
-                    .map(|retrying| (request_matches_request(&request, &retrying.request), "retrying"))
-            })
-            // A durable-only owner must be resumed by recovery, not overwritten
-            // with a new execution budget by a replay before recovery completes.
-            .or_else(|| {
-                durable_owners
-                    .iter()
-                    .find(|owner| owner.id == request.id)
-                    .map(|_| (false, "durable"))
-            });
+        let mut request_id_admission =
+            lookup_request_id_admission(&request, &active_heals, &queue, &retrying_heals, &durable_owners);
+
+        if request_id_admission.is_none() && source == HealRequestSource::Admin && force_start {
+            let overlapping = active_heals
+                .values()
+                .map(|task| (&task.id, &task.heal_type, &task.options, task.source))
+                .chain(
+                    queue
+                        .requests()
+                        .chain(retrying_heals.values().map(|retrying| &retrying.request))
+                        .chain(durable_owners.iter())
+                        .map(|owner| (&owner.id, &owner.heal_type, &owner.options, owner.source)),
+                )
+                .filter(|(id, heal_type, options, source)| {
+                    **id != request.id
+                        && *source == HealRequestSource::Admin
+                        && heal_scopes_overlap(&request.heal_type, &request.options, heal_type, options)
+                            != OverlapVerdict::Disjoint
+                })
+                .map(|(id, _, _, _)| id.clone())
+                .collect::<HashSet<_>>();
+            drop(retrying_heals);
+            drop(queue);
+            drop(active_heals);
+            active_lock_hold += active_lock_acquired.elapsed();
+            for task_id in &overlapping {
+                match self.cancel_task(task_id).await {
+                    Ok(_) => info!(
+                        target: "rustfs::heal::manager",
+                        event = EVENT_HEAL_QUEUE_ADMISSION,
+                        component = LOG_COMPONENT_HEAL,
+                        subsystem = LOG_SUBSYSTEM_MANAGER,
+                        request_id = %request.id,
+                        cancelled_task_id = %task_id,
+                        result = "force_start_cancelled_overlap",
+                        "Admin forceStart cancelled an overlapping heal task"
+                    ),
+                    Err(Error::TaskNotFound { .. }) => {} // The owner completed before cancellation acquired its state.
+                    Err(err) => return Err(err),
+                }
+            }
+            durable_owners.retain(|owner| !overlapping.contains(&owner.id));
+            let reacquire_start = Instant::now();
+            active_heals = self.active_heals.lock().await;
+            active_lock_acquired = Instant::now();
+            active_lock_wait += active_lock_acquired.duration_since(reacquire_start);
+            queue = self.heal_queue.lock().await;
+            retrying_heals = self.retrying_heals.lock().await;
+            // Non-admin admission can claim this id while cancellation drops the state locks.
+            request_id_admission = lookup_request_id_admission(&request, &active_heals, &queue, &retrying_heals, &durable_owners);
+        }
+
         if let Some((matches_existing, duplicate_state)) = request_id_admission {
             let admission = if matches_existing {
                 if accept_same_request_id_replay {
@@ -2194,54 +2251,6 @@ impl HealManager {
             });
         }
 
-        if source == HealRequestSource::Admin && force_start {
-            let overlapping = active_heals
-                .values()
-                .map(|task| (&task.id, &task.heal_type, &task.options, task.source))
-                .chain(
-                    queue
-                        .requests()
-                        .chain(retrying_heals.values().map(|retrying| &retrying.request))
-                        .chain(durable_owners.iter())
-                        .map(|owner| (&owner.id, &owner.heal_type, &owner.options, owner.source)),
-                )
-                .filter(|(id, heal_type, options, source)| {
-                    **id != request.id
-                        && *source == HealRequestSource::Admin
-                        && heal_scopes_overlap(&request.heal_type, &request.options, heal_type, options)
-                            != OverlapVerdict::Disjoint
-                })
-                .map(|(id, _, _, _)| id.clone())
-                .collect::<HashSet<_>>();
-            drop(retrying_heals);
-            drop(queue);
-            drop(active_heals);
-            active_lock_hold += active_lock_acquired.elapsed();
-            for task_id in &overlapping {
-                match self.cancel_task(task_id).await {
-                    Ok(_) => info!(
-                        target: "rustfs::heal::manager",
-                        event = EVENT_HEAL_QUEUE_ADMISSION,
-                        component = LOG_COMPONENT_HEAL,
-                        subsystem = LOG_SUBSYSTEM_MANAGER,
-                        request_id = %request.id,
-                        cancelled_task_id = %task_id,
-                        result = "force_start_cancelled_overlap",
-                        "Admin forceStart cancelled an overlapping heal task"
-                    ),
-                    Err(Error::TaskNotFound { .. }) => {} // The owner completed before cancellation acquired its state.
-                    Err(err) => return Err(err),
-                }
-            }
-            durable_owners.retain(|owner| !overlapping.contains(&owner.id));
-            let reacquire_start = Instant::now();
-            active_heals = self.active_heals.lock().await;
-            active_lock_acquired = Instant::now();
-            active_lock_wait += active_lock_acquired.duration_since(reacquire_start);
-            queue = self.heal_queue.lock().await;
-            retrying_heals = self.retrying_heals.lock().await;
-        }
-
         let rejection = if source == HealRequestSource::Admin {
             let live_owners = active_heals
                 .values()
@@ -2287,13 +2296,22 @@ impl HealManager {
             None
         };
 
+        let scanner_queued_owner_required = request.source == HealRequestSource::Scanner
+            && mrf_notice_target.is_none()
+            && request.expected_mrf_bucket_incarnation_id.is_none();
         let duplicate = (!request.force_start && rejection.is_none()).then(|| {
             active_heal_for_dedup_key(&active_heals, &dedup_key)
                 .map(|(task_id, _)| (task_id, "active"))
                 .or_else(|| {
-                    queue
-                        .queued_request_id_for_dedup_key(&dedup_key)
-                        .map(|queued_id| (queued_id.to_string(), "queued"))
+                    let queued_id = if scanner_queued_owner_required {
+                        queue.scanner_queued_request_id_for_dedup_key(&dedup_key)
+                    } else {
+                        queue.queued_request_id_for_dedup_key(&dedup_key)
+                    };
+                    queued_id.map(|queued_id| (queued_id.to_string(), "queued")).or_else(|| {
+                        (scanner_queued_owner_required && queue.dedup_keys.contains_key(&dedup_key))
+                            .then(|| (request.id.clone(), "queued_owner_unknown"))
+                    })
                 })
                 .or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying")))
         });
@@ -2301,12 +2319,12 @@ impl HealManager {
             // HS-06: under the minio_error overlap policy an exact duplicate
             // admin start reports the typed AlreadyRunning rejection instead
             // of the silent merge (MinIO's ErrHealAlreadyRunning).
-            let mut admission = if (request.source == HealRequestSource::Admin
-                && config.overlap_policy == HealOverlapPolicy::MinioError)
+            let mut admission = if duplicate_state == "queued_owner_unknown"
+                || (request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError)
                 || (duplicate_state != "queued" && mrf_notice_target.as_ref().is_some_and(|target| target.kind.is_durable()))
             {
-                // A running/retrying task may have observed the object before
-                // this write committed. Its receipt cannot prove the new lease.
+                // An unknown queued owner cannot certify a receipt. A running or
+                // retrying task may have observed the object before this write.
                 HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
             } else {
                 Self::duplicate_admission_for_request(&request, &config)
@@ -2519,10 +2537,17 @@ impl HealManager {
                 .remove(&request_id, &request_heal_type, request_source)
                 .await?;
         }
-        if admission == HealAdmissionResult::Merged
-            && let Some(queued_id) = queue.queued_request_id_for_dedup_key(&dedup_key)
-        {
-            task_id = queued_id.to_owned();
+        if admission == HealAdmissionResult::Merged {
+            let queued_id = if scanner_queued_owner_required {
+                queue.scanner_queued_request_id_for_dedup_key(&dedup_key)
+            } else {
+                queue.queued_request_id_for_dedup_key(&dedup_key)
+            };
+            if let Some(queued_id) = queued_id {
+                task_id = queued_id.to_owned();
+            } else if scanner_queued_owner_required {
+                return Err(Error::Other("Scanner queued owner is unavailable".to_owned()));
+            }
         }
         let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
         let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned);
@@ -2591,23 +2616,6 @@ impl HealManager {
     /// completed entry. `heal_path` additionally constrains the map matches
     /// the way the `*_for_path` variants always have.
     async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
-        let state = self.lookup_memory_task_state(canonical_task_id, heal_path).await;
-        if !matches!(state, TaskStateLookup::NotFound) {
-            return Ok(state);
-        }
-        if let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
-            && heal_path.is_none_or(|path| heal_type_matches_path(&completed.heal_type, path))
-        {
-            return Ok(TaskStateLookup::Completed(Arc::new(completed)));
-        }
-        // A durable record that cannot run still answers with its reason.
-        if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
-            return Ok(TaskStateLookup::Unrunnable(reason));
-        }
-        Ok(TaskStateLookup::NotFound)
-    }
-
-    async fn lookup_memory_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
         let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
 
         {
@@ -2616,7 +2624,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|task| matches_path(&task.heal_type))
             {
-                return TaskStateLookup::Active(Arc::clone(task));
+                return Ok(TaskStateLookup::Active(Arc::clone(task)));
             }
         }
 
@@ -2626,7 +2634,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|retrying| matches_path(&retrying.request.heal_type))
             {
-                return TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone(), retrying.request.source);
+                return Ok(TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone()));
             }
         }
 
@@ -2639,7 +2647,7 @@ impl HealManager {
             prune_completed_heal_statuses(&mut completed_heals);
             if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
                 if completed_status_is_retrying(&completed.status) {
-                    return TaskStateLookup::Completed(Arc::clone(completed));
+                    return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
                 }
                 terminal_completed = Some(Arc::clone(completed));
             }
@@ -2648,7 +2656,7 @@ impl HealManager {
         {
             let queue = self.heal_queue.lock().await;
             if let Some(request) = queue.request_matching_id_and_path(canonical_task_id, heal_path) {
-                return TaskStateLookup::Queued(request.options.clone(), request.source);
+                return Ok(TaskStateLookup::Queued(request.options.clone()));
             }
         }
 
@@ -2661,10 +2669,22 @@ impl HealManager {
                 .cloned();
         }
 
-        if let Some(completed) = terminal_completed {
-            return TaskStateLookup::Completed(completed);
+        if terminal_completed.is_none()
+            && let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
+            && matches_path(&completed.heal_type)
+        {
+            terminal_completed = Some(Arc::new(completed));
         }
-        TaskStateLookup::NotFound
+
+        if let Some(completed) = terminal_completed {
+            return Ok(TaskStateLookup::Completed(completed));
+        }
+        // A durable root record that can neither run nor be quarantined still
+        // answers status queries with its reason.
+        if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
+            return Ok(TaskStateLookup::Unrunnable(reason));
+        }
+        Ok(TaskStateLookup::NotFound)
     }
 
     async fn publish_admin_terminal(
@@ -2688,8 +2708,6 @@ impl HealManager {
     ) -> Result<bool> {
         let previous = self.completed_heals.lock().await.get(task_id).cloned();
         let mut completed = CompletedHealStatus {
-            task_id: task_id.to_owned(),
-            source,
             outcome: None,
             progress: None,
             retained_bytes: std::sync::OnceLock::new(),
@@ -2725,9 +2743,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
-            TaskStateLookup::Retrying(status, _, _) => Ok(status),
+            TaskStateLookup::Retrying(status, _) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
-            TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
+            TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
             TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
             TaskStateLookup::NotFound => Err(Error::TaskNotFound {
                 task_id: task_id.to_string(),
@@ -2746,9 +2764,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
-            TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
+            TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
-            TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
+            TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
             TaskStateLookup::Unrunnable(error) => {
                 Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
             }
@@ -2762,42 +2780,6 @@ impl HealManager {
         self.get_task_report_for_path_since(heal_path, task_id, None).await
     }
 
-    /// Resolve a node-owned automatic task without granting local control over
-    /// Admin tasks. The source and report come from the same state snapshot.
-    pub async fn get_local_automatic_task_report(
-        &self,
-        heal_path: &str,
-        task_id: &str,
-        since: Option<u64>,
-    ) -> Result<Option<HealTaskReport>> {
-        let canonical_task_id = self.canonical_task_id(task_id).await;
-        if canonical_task_id != task_id {
-            return Ok(None);
-        }
-        let state = self.lookup_memory_task_state(&canonical_task_id, Some(heal_path)).await;
-        let source = match &state {
-            TaskStateLookup::Active(task) => task.source,
-            TaskStateLookup::Retrying(_, _, source) | TaskStateLookup::Queued(_, source) => *source,
-            TaskStateLookup::Completed(completed) if completed.task_id == task_id => completed.source,
-            TaskStateLookup::Completed(_) => return Ok(None),
-            TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
-        };
-        if !matches!(
-            source,
-            HealRequestSource::Scanner | HealRequestSource::Mrf | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
-        ) {
-            return Ok(None);
-        }
-        let report = match state {
-            TaskStateLookup::Active(task) => active_task_report(&task, since).await,
-            TaskStateLookup::Retrying(status, options, _) => empty_task_report_with_options(status, options),
-            TaskStateLookup::Completed(completed) => completed_task_report(&completed, since),
-            TaskStateLookup::Queued(options, _) => empty_task_report_with_options(HealTaskStatus::Pending, options),
-            TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
-        };
-        Ok(Some(report))
-    }
-
     /// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
     pub async fn get_task_report_for_path_since(
         &self,
@@ -2808,9 +2790,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
-            TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
+            TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
-            TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
+            TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
             TaskStateLookup::Unrunnable(error) => {
                 Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
             }
@@ -2834,9 +2816,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
-            TaskStateLookup::Retrying(status, _, _) => Ok(status),
+            TaskStateLookup::Retrying(status, _) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
-            TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
+            TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
             TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
             TaskStateLookup::NotFound => {
                 if self.path_has_task(heal_path).await {

crates/heal/src/heal/manager/queue.rs

@@ -13,15 +13,23 @@
 // limitations under the License.
 /// The priority heal queue and its per-key dedup index.
 use super::*;
+use std::collections::BTreeMap;
 
-/// Per-key bookkeeping for the queued-request dedup index: how many queued
-/// requests hold the key, and the id of the first request that opened it —
-/// the O(1) stand-in for the former heap scan when a merge receipt needs to
-/// name a queued representative.
+/// Per-key bookkeeping for the queued-request dedup index. The legacy
+/// representative is retained for existing non-scanner callers; scanner
+/// receipts use the separately tracked queued holders.
 #[derive(Debug)]
 pub(super) struct DedupKeyEntry {
     pub(super) refcount: usize,
     pub(super) representative_request_id: String,
+    pub(super) scanner_queued_owners: ScannerQueuedOwners,
+}
+
+#[derive(Debug)]
+pub(super) enum ScannerQueuedOwners {
+    Single(u64),
+    // Once a key has forced duplicates, keep every live holder until it is released.
+    Indexed(BTreeMap<u64, String>),
 }
 
 /// Priority queue wrapper for heal requests
@@ -109,9 +117,7 @@ pub(super) enum QueuePushOutcome {
 
 #[derive(Debug, Clone)]
 pub(super) struct CompletedHealStatus {
-    pub(super) task_id: String,
     pub(super) heal_type: HealType,
-    pub(super) source: HealRequestSource,
     /// Options used to execute the task, retained for token-scoped status.
     pub(super) options: HealOptions,
     pub(super) status: HealTaskStatus,
@@ -139,7 +145,6 @@ impl CompletedHealStatus {
     fn measure_retained_bytes(&self) -> usize {
         let mut bytes = size_of::<Self>();
         let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
-        add(self.task_id.capacity());
         add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
         match &self.heal_type {
             HealType::Cluster => {}
@@ -261,9 +266,7 @@ impl CompletedHealStatus {
             outcome.finish(Some(crate::heal::outcome::HealAbortReason::Cancelled));
         }
         let mut snapshot = Self {
-            task_id: task.id.clone(),
             heal_type: task.heal_type.clone(),
-            source: task.source,
             options: task.options.clone(),
             status,
             progress: Some(task.get_progress().await),
@@ -311,11 +314,7 @@ impl PriorityHealQueue {
 
     #[cfg(test)]
     pub(super) fn pop_next(&mut self) -> Option<HealRequest> {
-        self.heap.pop().map(|item| {
-            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
-            self.remove_best_effort_count(item.request.source);
-            item.request
-        })
+        self.heap.pop().map(|item| self.remove_queued_item(item))
     }
 
     pub(super) fn is_empty(&self) -> bool {
@@ -337,14 +336,32 @@ impl PriorityHealQueue {
         // also reserves the dedup key for later non-forced duplicates. The first
         // request that opens the key becomes the named representative for merge
         // receipts (taken before `request` moves into the heap).
-        self.dedup_keys
-            .entry(key.clone())
-            .or_insert_with(|| DedupKeyEntry {
-                refcount: 0,
-                representative_request_id: request.id.clone(),
-            })
-            .refcount += 1;
         self.sequence += 1;
+        if let Some(entry) = self.dedup_keys.get_mut(&key) {
+            match &mut entry.scanner_queued_owners {
+                ScannerQueuedOwners::Single(first_sequence) => {
+                    let mut owners = BTreeMap::new();
+                    if entry.refcount == 1 {
+                        owners.insert(*first_sequence, entry.representative_request_id.clone());
+                    }
+                    owners.insert(self.sequence, request.id.clone());
+                    entry.scanner_queued_owners = ScannerQueuedOwners::Indexed(owners);
+                }
+                ScannerQueuedOwners::Indexed(owners) => {
+                    owners.insert(self.sequence, request.id.clone());
+                }
+            }
+            entry.refcount += 1;
+        } else {
+            self.dedup_keys.insert(
+                key.clone(),
+                DedupKeyEntry {
+                    refcount: 1,
+                    representative_request_id: request.id.clone(),
+                    scanner_queued_owners: ScannerQueuedOwners::Single(self.sequence),
+                },
+            );
+        }
         let source = request.source;
         self.heap.push(PriorityQueueItem {
             priority: request.priority,
@@ -407,7 +424,7 @@ impl PriorityHealQueue {
         self.heap = retained;
 
         let displaced = displaced.map(|item| {
-            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
+            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item);
             self.remove_best_effort_count(item.request.source);
             self.refresh_dedup_representative(&item.dedup_key);
             item.request
@@ -451,11 +468,7 @@ impl PriorityHealQueue {
 
     #[cfg(test)]
     pub(super) fn pop(&mut self) -> Option<HealRequest> {
-        self.heap.pop().map(|item| {
-            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
-            self.remove_best_effort_count(item.request.source);
-            item.request
-        })
+        self.heap.pop().map(|item| self.remove_queued_item(item))
     }
 
     pub(super) fn pop_runnable_with_skips<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
@@ -480,14 +493,7 @@ impl PriorityHealQueue {
 
         self.restore_deferred_items(deferred);
 
-        (
-            selected.map(|item| {
-                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
-                self.remove_best_effort_count(item.request.source);
-                item.request
-            }),
-            skipped,
-        )
+        (selected.map(|item| self.remove_queued_item(item)), skipped)
     }
 
     pub(super) fn pop_runnable_with_fairness<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
@@ -559,15 +565,19 @@ impl PriorityHealQueue {
         self.heap = BinaryHeap::from(items);
 
         let chosen = chosen.map(|item| {
-            let request = item.request;
-            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
-            self.remove_best_effort_count(request.source);
+            let request = self.remove_queued_item(item);
             self.record_dispatch(&request);
             request
         });
         (chosen, skipped)
     }
 
+    fn remove_queued_item(&mut self, item: PriorityQueueItem) -> HealRequest {
+        Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item);
+        self.remove_best_effort_count(item.request.source);
+        item.request
+    }
+
     fn record_dispatch(&mut self, request: &HealRequest) {
         if is_best_effort_source(request.source) {
             self.mrf_normal_dispatches_since_best_effort = 0;
@@ -672,10 +682,19 @@ impl PriorityHealQueue {
         }
     }
 
-    pub(super) fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap<String, DedupKeyEntry>, key: &str) {
-        if let Some(entry) = dedup_keys.get_mut(key) {
+    fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap<String, DedupKeyEntry>, item: &PriorityQueueItem) {
+        if let Some(entry) = dedup_keys.get_mut(&item.dedup_key) {
+            match &mut entry.scanner_queued_owners {
+                ScannerQueuedOwners::Single(sequence) if entry.refcount != 1 || *sequence != item.sequence => {
+                    entry.scanner_queued_owners = ScannerQueuedOwners::Indexed(BTreeMap::new());
+                }
+                ScannerQueuedOwners::Single(_) => {}
+                ScannerQueuedOwners::Indexed(owners) => {
+                    owners.remove(&item.sequence);
+                }
+            }
             if entry.refcount <= 1 {
-                dedup_keys.remove(key);
+                dedup_keys.remove(&item.dedup_key);
             } else {
                 entry.refcount -= 1;
             }
@@ -752,15 +771,19 @@ impl PriorityHealQueue {
         self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str())
     }
 
-    /// Re-elect the representative for `key` from the queue entries holding
-    /// it. Needed after a holder leaves the queue *without* becoming active
-    /// (canceled by id, or displaced): the former opener may be the request
-    /// that just left, and a merge receipt must never name an id that
-    /// resolves nowhere. The scheduler pop path does not need this — the
-    /// popped request surfaces in `active_heals` under the same id and the
-    /// duplicate pre-check consults active heals before the queue. No-op for
-    /// released keys; the survivor scan only runs when a key still has
-    /// holders, which under forced duplicates is the rare admin path.
+    pub(super) fn scanner_queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> {
+        let entry = self.dedup_keys.get(key)?;
+        match &entry.scanner_queued_owners {
+            ScannerQueuedOwners::Single(_) if entry.refcount == 1 => Some(entry.representative_request_id.as_str()),
+            ScannerQueuedOwners::Indexed(owners) if owners.len() == entry.refcount => {
+                owners.first_key_value().map(|(_, id)| id.as_str())
+            }
+            _ => None,
+        }
+    }
+
+    /// Re-elect the legacy representative from the heap after cancellation or
+    /// displacement. Dequeue leaves it unchanged for existing callers.
     pub(super) fn refresh_dedup_representative(&mut self, key: &str) {
         if !self.dedup_keys.contains_key(key) {
             return;
@@ -791,7 +814,7 @@ impl PriorityHealQueue {
         while let Some(item) = self.heap.pop() {
             if removed.is_none() && item.request.id == request_id {
                 let key = item.dedup_key.clone();
-                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
+                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item);
                 self.remove_best_effort_count(item.request.source);
                 affected_key = Some(key);
                 removed = Some(item.request);
@@ -817,7 +840,7 @@ impl PriorityHealQueue {
 
         while let Some(item) = self.heap.pop() {
             if should_remove(&item.request) {
-                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
+                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item);
                 self.remove_best_effort_count(item.request.source);
                 affected_keys.push(item.dedup_key);
                 removed.push(item.request);

crates/heal/src/heal/manager/root_recovery.rs

@@ -460,8 +460,6 @@ impl RootHealTerminal {
 
     fn into_completed(self) -> CompletedHealStatus {
         CompletedHealStatus {
-            task_id: self.task_id,
-            source: HealRequestSource::Admin,
             outcome: None,
             progress: self.progress,
             retained_bytes: std::sync::OnceLock::new(),

crates/heal/src/heal/manager/tests.rs

@@ -34,6 +34,7 @@ mod auto_scan;
 mod completion;
 mod disposition;
 mod legacy_ambiguity_logging;
+mod queued_representative;
 mod running_mainline;
 
 use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
@@ -118,8 +119,6 @@ struct MockStorage;
 
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
-        task_id: "fixture-token".to_string(),
-        source: HealRequestSource::Internal,
         outcome: None,
         heal_type: HealType::Cluster,
         options: HealOptions::default(),
@@ -3073,181 +3072,6 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
     );
 }
 
-#[tokio::test]
-async fn local_automatic_query_preserves_source_path_token_and_state() {
-    use crate::heal::channel::HealChannelProcessor;
-    use crate::heal::outcome::{
-        HealExecutionOutcome, HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome,
-        HealTraversalCoverage,
-    };
-
-    for source in [
-        HealRequestSource::Scanner,
-        HealRequestSource::Mrf,
-        HealRequestSource::AutoHeal,
-        HealRequestSource::ReadRepair,
-        HealRequestSource::Admin,
-        HealRequestSource::Internal,
-    ] {
-        for state in ["queued", "active", "retrying", "completed", "displaced"] {
-            let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-            let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
-            let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
-            request.source = source;
-            let token = request.id.clone();
-            let task = Arc::new(HealTask::from_request(request.clone(), storage));
-            let mut snapshot = CompletedHealStatus::snapshot(&task, HealTaskStatus::Completed).await;
-            assert_eq!(snapshot.source, source, "completion must retain actual task source");
-            let mut outcome = HealTaskOutcome::default();
-            outcome.execution = HealExecutionOutcome::Completed;
-            outcome.coverage = HealTraversalCoverage::Complete;
-            outcome.record(HealObjectOutcome {
-                identity: HealObjectIdentity {
-                    kind: HealObjectKind::Object,
-                    bucket: "bucket".to_string(),
-                    object: "object".to_string(),
-                    version_id: None,
-                    bucket_incarnation_id: Some(uuid::Uuid::nil()),
-                    pool_index: Some(0),
-                    set_index: Some(0),
-                },
-                disposition: HealObjectDisposition::VerifiedHealthy,
-                detail: None,
-            });
-            snapshot.outcome = Some(Arc::new(outcome));
-            snapshot.seqed_items = vec![(
-                3,
-                HealResultItem {
-                    bucket: "bucket".to_string(),
-                    object: "object".to_string(),
-                    ..Default::default()
-                },
-            )];
-            snapshot.min_seq = 3;
-            snapshot.next_seq = 4;
-            let expected_status = match state {
-                "queued" => {
-                    manager.heal_queue.lock().await.push(request.clone());
-                    HealTaskStatus::Pending
-                }
-                "active" => {
-                    manager.active_heals.lock().await.insert(token.clone(), task);
-                    HealTaskStatus::Pending
-                }
-                "retrying" => {
-                    manager.retrying_heals.lock().await.insert(
-                        token.clone(),
-                        RetryingHeal {
-                            request: request.clone(),
-                            error: "transient".to_string(),
-                            cancel_token: CancellationToken::new(),
-                        },
-                    );
-                    HealTaskStatus::Retrying {
-                        error: "transient".to_string(),
-                        retry_attempt: request.retry_attempts,
-                    }
-                }
-                "completed" => {
-                    manager.completed_heals.lock().await.insert(token.clone(), Arc::new(snapshot));
-                    HealTaskStatus::Completed
-                }
-                "displaced" => {
-                    let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
-                    terminal.status.clone()
-                }
-                _ => unreachable!("fixed test states"),
-            };
-            let automatic = !matches!(source, HealRequestSource::Admin | HealRequestSource::Internal);
-            let report = manager
-                .get_local_automatic_task_report("bucket/object", &token, Some(2))
-                .await
-                .expect("local source lookup");
-            assert_eq!(report.is_some(), automatic, "source={source:?} state={state}");
-            if let Some(report) = report {
-                assert_eq!(report.status, expected_status);
-                if state == "completed" {
-                    let outcome = report.outcome.as_ref().expect("retained structured outcome");
-                    assert_eq!(outcome.objects.len(), 1);
-                    assert_eq!(outcome.objects[0].identity.object, "object");
-                    assert_eq!(outcome.objects[0].identity.pool_index, Some(0));
-                    assert_eq!(outcome.objects[0].identity.set_index, Some(0));
-                    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::VerifiedHealthy);
-                    assert_eq!(report.min_seq, 3);
-                    assert_eq!(report.next_seq, 4);
-                    assert_eq!(report.result_items.len(), 1);
-                    assert_eq!(report.result_items[0].object, "object");
-                }
-                let processor = HealChannelProcessor::new(manager.clone());
-                let local = processor
-                    .execute_local_automatic_query("bucket/object", &token, Some(2))
-                    .await
-                    .expect("local response")
-                    .expect("automatic task response");
-                let direct = processor
-                    .execute_query_request_since("bucket/object".to_string(), token.clone(), Some(2))
-                    .await
-                    .expect("original query response");
-                assert!(local.success);
-                assert_eq!(local.request_id, direct.request_id);
-                assert_eq!(local.data, direct.data, "same report encoding and cursor contract");
-                assert_eq!(local.error, direct.error);
-                let coordinator = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
-                let missing = HealChannelProcessor::new(coordinator)
-                    .execute_query_request("bucket/object".to_string(), token.clone())
-                    .await
-                    .expect("coordinator query");
-                let body: serde_json::Value =
-                    serde_json::from_slice(missing.data.as_deref().expect("coordinator response")).expect("coordinator JSON");
-                assert_eq!(body["summary"], "notFound", "a remote manager does not own the automatic task");
-                if state == "completed" {
-                    let completed = manager
-                        .completed_heals
-                        .lock()
-                        .await
-                        .get(&token)
-                        .cloned()
-                        .expect("original completion");
-                    manager
-                        .completed_heals
-                        .lock()
-                        .await
-                        .insert("retired-alias".to_string(), completed);
-                    assert!(
-                        manager
-                            .get_local_automatic_task_report("bucket/object", "retired-alias", None)
-                            .await
-                            .expect("retired alias lookup")
-                            .is_none()
-                    );
-                }
-            }
-            assert!(
-                manager
-                    .get_local_automatic_task_report("bucket/other", &token, None)
-                    .await
-                    .expect("wrong path lookup")
-                    .is_none()
-            );
-            assert!(
-                manager
-                    .get_local_automatic_task_report("bucket/object", "unknown", None)
-                    .await
-                    .expect("unknown token lookup")
-                    .is_none()
-            );
-            manager.insert_task_alias("admin-alias", &token).await;
-            assert!(
-                manager
-                    .get_local_automatic_task_report("bucket/object", "admin-alias", None)
-                    .await
-                    .expect("alias lookup")
-                    .is_none()
-            );
-        }
-    }
-}
-
 #[tokio::test]
 async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -4063,38 +3887,6 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab
     }
 }
 
-#[tokio::test]
-async fn admin_exhausted_batches_do_not_restart_the_traversal() {
-    for heal_type in [
-        HealType::Bucket {
-            bucket: "bucket".to_owned(),
-        },
-        HealType::Prefix {
-            bucket: "bucket".to_owned(),
-            prefix: "prefix/".to_owned(),
-        },
-        HealType::Cluster,
-    ] {
-        let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::Normal);
-        request.source = HealRequestSource::Admin;
-        let task = HealTask::from_request(request, Arc::new(MockStorage));
-        let result = Err(task
-            .record_batch_failure(BatchHealFailure {
-                scope: "completed-admin-traversal".to_owned(),
-                failed: 1,
-                retryable: 1,
-                permanent: 0,
-                first_object: "exhausted-object".to_owned(),
-                first_error: Error::Storage(EcstoreError::SlowDown).to_string(),
-            })
-            .await);
-        assert!(
-            retry_request_for_result_with_budget(&task, &result).await.is_none(),
-            "the exhausted admin batch must retain its original object receipts"
-        );
-    }
-}
-
 #[tokio::test]
 async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
     for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
@@ -4177,8 +3969,6 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: request.source,
-            task_id: request.id.clone(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type,
             options: request.options.clone(),
@@ -5027,8 +4817,6 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: request.source,
-            task_id: request.id.clone(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type.clone(),
             options: request.options.clone(),
@@ -5068,8 +4856,6 @@ async fn test_get_task_status_reads_recent_completed_status() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: HealRequestSource::Internal,
-            task_id: "completed-token".to_string(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Bucket {
                 bucket: "bucket".to_string(),
@@ -5103,8 +4889,6 @@ async fn test_get_task_report_for_path_reads_completed_items() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: HealRequestSource::Internal,
-            task_id: "completed-token".to_string(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Object {
                 bucket: "bucket".to_string(),

crates/heal/src/heal/manager/tests/admin_overlap.rs

@@ -450,6 +450,176 @@ async fn admin_overlap_normal_start_cannot_enter_force_start_cancellation_window
     assert_eq!(manager.heal_queue.lock().await.len(), 1);
 }
 
+#[tokio::test]
+async fn admin_force_start_rechecks_request_id_after_cancelling_overlaps() {
+    for source in [HealRequestSource::Scanner, HealRequestSource::Internal] {
+        let manager = manager_with_policy(HealOverlapPolicy::Merge);
+        let previous_id = install_owner(&manager, admin_prefix_request("bucket", "scope/child/"), OwnerState::Active).await;
+        let aliases = manager.task_aliases.lock().await;
+        let mut replacement = admin_prefix_request("bucket", "scope/");
+        replacement.force_start = true;
+        let request_id = replacement.id.clone();
+        let mut forced = Box::pin(manager.submit_heal_request_with_receipt(replacement));
+
+        tokio::time::timeout(
+            Duration::from_secs(5),
+            std::future::poll_fn(|cx| {
+                assert!(
+                    std::pin::pin!(tokio::task::unconstrained(forced.as_mut()))
+                        .poll(cx)
+                        .is_pending()
+                );
+                if manager.active_heals.try_lock().is_ok() {
+                    Poll::Ready(())
+                } else {
+                    cx.waker().wake_by_ref();
+                    Poll::Pending
+                }
+            }),
+        )
+        .await
+        .expect("forceStart reaches cancellation with registry locks released");
+
+        let mut competing = HealRequest::object("other-bucket".to_string(), "object".to_string(), None);
+        competing.id = request_id.clone();
+        competing.source = source;
+        competing.options.recreate_missing = false;
+        let receipt = manager
+            .submit_heal_request_with_receipt(competing)
+            .await
+            .expect("non-admin admission during forceStart cancellation");
+        assert_eq!(receipt.result, HealAdmissionResult::Accepted);
+        assert_eq!(receipt.task_id, request_id);
+
+        drop(aliases);
+        let receipt = forced.await.expect("forceStart returns a conflict");
+        assert_eq!(receipt.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning));
+        assert_eq!(receipt.task_id, request_id);
+        let queue = manager.heal_queue.lock().await;
+        let same_id: Vec<_> = queue.requests().filter(|request| request.id == request_id).collect();
+        assert_eq!(same_id.len(), 1, "conflicting requests must not share an id");
+        assert_eq!(same_id[0].source, source);
+        drop(queue);
+        assert!(
+            matches!(manager.get_task_status(&previous_id).await, Ok(HealTaskStatus::Cancelled)),
+            "overlapping admin cancellation precedes the conflict recheck"
+        );
+    }
+}
+
+#[tokio::test]
+async fn forced_id_conflict_preserves_scanner_queued_owner() {
+    for competing_source in [HealRequestSource::Scanner, HealRequestSource::Internal] {
+        let manager = manager_with_policy(HealOverlapPolicy::Merge);
+        let previous_id = install_owner(&manager, admin_object_request("object"), OwnerState::Active).await;
+        let mut queued_ids = Vec::new();
+        for source in [
+            HealRequestSource::Scanner,
+            HealRequestSource::Internal,
+            HealRequestSource::Scanner,
+        ] {
+            let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+            request.source = source;
+            request.force_start = true;
+            let id = request.id.clone();
+            let receipt = manager
+                .submit_heal_request_with_receipt(request)
+                .await
+                .expect("queue forced work");
+            assert_eq!(receipt.result, HealAdmissionResult::Accepted);
+            assert_eq!(receipt.task_id, id);
+            queued_ids.push(id);
+        }
+
+        // The first holder leaves the queue, leaving two forced tasks queued.
+        let selected = {
+            let mut queue = manager.heal_queue.lock().await;
+            let (selected, skipped) = queue.pop_runnable_with_skips(|_| true, |_| None);
+            assert!(skipped.is_empty());
+            let selected = selected.expect("first queued task should run");
+            assert_eq!(selected.id, queued_ids[0]);
+            assert_eq!(queue.len(), 2);
+            selected
+        };
+        let completed_task = HealTask::from_request(selected, manager.storage.clone());
+        let mut completed = CompletedHealStatus::snapshot(&completed_task, HealTaskStatus::Completed).await;
+        Arc::make_mut(completed.outcome.as_mut().expect("completed task outcome")).finish(None);
+        manager
+            .completed_heals
+            .lock()
+            .await
+            .insert(queued_ids[0].clone(), Arc::new(completed));
+
+        let aliases = manager.task_aliases.lock().await;
+        let mut replacement = admin_object_request("object");
+        replacement.force_start = true;
+        let competing_id = replacement.id.clone();
+        let mut forced = Box::pin(manager.submit_heal_request_with_receipt(replacement));
+        tokio::time::timeout(
+            Duration::from_secs(5),
+            std::future::poll_fn(|cx| {
+                assert!(
+                    std::pin::pin!(tokio::task::unconstrained(forced.as_mut()))
+                        .poll(cx)
+                        .is_pending()
+                );
+                if manager.active_heals.try_lock().is_ok() {
+                    Poll::Ready(())
+                } else {
+                    cx.waker().wake_by_ref();
+                    Poll::Pending
+                }
+            }),
+        )
+        .await
+        .expect("forceStart reaches cancellation with registry locks released");
+
+        let mut competing = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+        competing.id = competing_id.clone();
+        competing.source = competing_source;
+        competing.force_start = true;
+        let receipt = manager
+            .submit_heal_request_with_receipt(competing)
+            .await
+            .expect("competing same-key forced work");
+        assert_eq!(receipt.result, HealAdmissionResult::Accepted);
+        assert_eq!(receipt.task_id, competing_id);
+
+        drop(aliases);
+        let receipt = forced.await.expect("forceStart returns an ID conflict");
+        assert_eq!(receipt.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning));
+        assert_eq!(receipt.task_id, competing_id);
+        assert!(!manager.active_heals.lock().await.contains_key(&competing_id));
+        assert!(!manager.retrying_heals.lock().await.contains_key(&competing_id));
+        assert!(matches!(manager.get_task_status(&previous_id).await, Ok(HealTaskStatus::Cancelled)));
+        {
+            let queue = manager.heal_queue.lock().await;
+            let same_id: Vec<_> = queue.requests().filter(|request| request.id == competing_id).collect();
+            assert_eq!(same_id.len(), 1, "the competing ID must have one live owner");
+            assert_eq!(same_id[0].source, competing_source);
+            assert_eq!(queue.len(), 3);
+        }
+
+        let mut scanner = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+        scanner.source = HealRequestSource::Scanner;
+        let receipt = manager
+            .submit_heal_request_with_receipt(scanner)
+            .await
+            .expect("scanner should merge into queued work");
+        assert_eq!(receipt.result, HealAdmissionResult::Merged);
+        assert_ne!(receipt.task_id, queued_ids[0], "the completed holder cannot own a queued receipt");
+        assert!(
+            manager
+                .heal_queue
+                .lock()
+                .await
+                .requests()
+                .any(|request| request.id == receipt.task_id),
+            "the canonical receipt must name a live queued task"
+        );
+    }
+}
+
 #[tokio::test]
 async fn admin_overlap_cancel_cannot_miss_a_queued_to_active_transition() {
     let manager = manager_with_policy(HealOverlapPolicy::Merge);

crates/heal/src/heal/manager/tests/queued_representative.rs

@@ -0,0 +1,524 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::*;
+
+fn object_request(source: HealRequestSource, priority: HealPriority, force_start: bool) -> HealRequest {
+    let mut request = HealRequest::new(
+        HealType::Object {
+            bucket: "canonical-owner".to_owned(),
+            object: "object".to_owned(),
+            version_id: None,
+        },
+        HealOptions::default(),
+        priority,
+    );
+    request.source = source;
+    request.force_start = force_start;
+    request
+}
+
+fn scoped_object_request(
+    source: HealRequestSource,
+    version_id: &str,
+    pool_index: Option<usize>,
+    set_index: Option<usize>,
+    force_start: bool,
+) -> HealRequest {
+    let mut request = object_request(source, HealPriority::Low, force_start);
+    let HealType::Object { version_id: version, .. } = &mut request.heal_type else {
+        unreachable!("object_request always constructs an object heal")
+    };
+    *version = Some(version_id.to_owned());
+    request.options.pool_index = pool_index;
+    request.options.set_index = set_index;
+    request.options.recreate_missing = source != HealRequestSource::Scanner;
+    request
+}
+
+#[tokio::test]
+async fn scanner_receipt_tracks_scoped_read_repair_forced_holders() {
+    let manager = HealManager::new_without_root_recovery_for_test(
+        Arc::new(MockStorage),
+        Some(HealConfig {
+            queue_size: 16,
+            low_priority_merge_enable: true,
+            ..HealConfig::default()
+        }),
+    );
+    let version = "00000000-0000-4000-8000-000000000001";
+    let original = scoped_object_request(HealRequestSource::ReadRepair, version, Some(0), Some(1), false);
+    let original_id = original.id.clone();
+    let key = PriorityHealQueue::make_dedup_key(&original);
+    let admitted = manager
+        .submit_heal_request_with_receipt(original)
+        .await
+        .expect("admit read-repair owner");
+    assert_eq!(admitted.result, HealAdmissionResult::Accepted);
+    assert_eq!(admitted.task_id, original_id);
+
+    let receipt = manager
+        .submit_heal_request_with_receipt(scoped_object_request(HealRequestSource::Scanner, version, Some(0), Some(1), false))
+        .await
+        .expect("scanner merges into the scoped read-repair owner");
+    assert_eq!(receipt.result, HealAdmissionResult::Merged);
+    assert_eq!(receipt.task_id, original_id);
+
+    let first_forced = scoped_object_request(HealRequestSource::Internal, version, Some(0), Some(1), true);
+    let first_forced_id = first_forced.id.clone();
+    let second_forced = scoped_object_request(HealRequestSource::Internal, version, Some(0), Some(1), true);
+    let second_forced_id = second_forced.id.clone();
+    {
+        let mut queue = manager.heal_queue.lock().await;
+        assert_eq!(queue.push(first_forced), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(second_forced), QueuePushOutcome::Accepted);
+        let (selected, skipped) = queue.pop_runnable_with_skips(|_| true, |_| None);
+        assert!(skipped.is_empty());
+        assert_eq!(selected.expect("read-repair owner should leave first").id, original_id);
+        assert_eq!(queue.queued_request_id_for_dedup_key(&key), Some(original_id.as_str()));
+    }
+
+    let receipt = manager
+        .submit_heal_request_with_receipt(scoped_object_request(HealRequestSource::Scanner, version, Some(0), Some(1), false))
+        .await
+        .expect("scanner merges into the remaining owner");
+    assert_eq!(receipt.result, HealAdmissionResult::Merged);
+    assert_eq!(receipt.task_id, first_forced_id);
+
+    {
+        let mut queue = manager.heal_queue.lock().await;
+        let (selected, _) = queue.pop_runnable_with_skips(|request| request.id == first_forced_id, |_| None);
+        assert_eq!(selected.expect("first forced holder should leave").id, first_forced_id);
+        assert!(queue.contains_request_id(&second_forced_id));
+    }
+    let receipt = manager
+        .submit_heal_request_with_receipt(scoped_object_request(HealRequestSource::Scanner, version, Some(0), Some(1), false))
+        .await
+        .expect("scanner merges into the last queued holder");
+    assert_eq!(receipt.result, HealAdmissionResult::Merged);
+    assert_eq!(receipt.task_id, second_forced_id);
+}
+
+#[tokio::test]
+async fn scanner_receipt_does_not_merge_read_repair_with_a_different_scope_or_version() {
+    let version = "00000000-0000-4000-8000-000000000001";
+    let other_version = "00000000-0000-4000-8000-000000000002";
+    for (candidate_version, pool, set) in [
+        (version, None, None),
+        (version, Some(1), Some(1)),
+        (version, Some(0), Some(2)),
+        (other_version, Some(0), Some(1)),
+    ] {
+        let manager = HealManager::new_without_root_recovery_for_test(
+            Arc::new(MockStorage),
+            Some(HealConfig {
+                queue_size: 16,
+                low_priority_merge_enable: true,
+                ..HealConfig::default()
+            }),
+        );
+        let original = scoped_object_request(HealRequestSource::ReadRepair, version, Some(0), Some(1), false);
+        let original_id = original.id.clone();
+        let initial = manager
+            .submit_heal_request_with_receipt(original)
+            .await
+            .expect("admit read-repair owner");
+        assert_eq!(initial.result, HealAdmissionResult::Accepted);
+        assert_eq!(initial.task_id, original_id);
+
+        let candidate = scoped_object_request(HealRequestSource::Scanner, candidate_version, pool, set, false);
+        let candidate_id = candidate.id.clone();
+        let receipt = manager
+            .submit_heal_request_with_receipt(candidate)
+            .await
+            .expect("admit scanner with a distinct scope or version");
+        assert_eq!(
+            receipt.result,
+            HealAdmissionResult::Accepted,
+            "version={candidate_version}, pool={pool:?}, set={set:?}"
+        );
+        assert_eq!(receipt.task_id, candidate_id);
+        let queue = manager.heal_queue.lock().await;
+        assert!(queue.contains_request_id(&original_id));
+        assert!(queue.contains_request_id(&candidate_id));
+    }
+}
+
+async fn scanner_receipt_tracks_queued_owner(use_fairness: bool) {
+    let manager = HealManager::new_without_root_recovery_for_test(
+        Arc::new(MockStorage),
+        Some(HealConfig {
+            queue_size: 16,
+            low_priority_merge_enable: true,
+            overlap_policy: HealOverlapPolicy::Merge,
+            ..HealConfig::default()
+        }),
+    );
+
+    if use_fairness {
+        let mut queue = manager.heal_queue.lock().await;
+        for index in 0..4 {
+            let request = bucket_request(&format!("internal-{index}"), HealPriority::Normal, HealRequestSource::Internal);
+            assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+        }
+        for _ in 0..4 {
+            let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
+            assert_eq!(request.expect("internal work should run").source, HealRequestSource::Internal);
+        }
+    }
+
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let first_id = first.id.clone();
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+    let second_id = second.id.clone();
+    for request in [first, second] {
+        assert_eq!(
+            manager
+                .submit_heal_request_with_receipt(request)
+                .await
+                .expect("admit task")
+                .result,
+            HealAdmissionResult::Accepted
+        );
+    }
+
+    let first = {
+        let mut queue = manager.heal_queue.lock().await;
+        let (selected, skipped) = if use_fairness {
+            queue.pop_runnable_with_fairness(|_| true, |_| None)
+        } else {
+            queue.pop_runnable_with_skips(|_| true, |_| None)
+        };
+        assert!(skipped.is_empty());
+        let selected = selected.expect("the first same-priority task should run");
+        assert_eq!(selected.id, first_id);
+        assert!(queue.contains_request_id(&second_id));
+        assert_eq!(
+            queue.queued_request_id_for_dedup_key(&key),
+            Some(first_id.as_str()),
+            "the existing representative for other sources must remain unchanged"
+        );
+        selected
+    };
+    manager
+        .active_heals
+        .lock()
+        .await
+        .insert(first_id.clone(), Arc::new(HealTask::from_request(first, Arc::new(MockStorage))));
+
+    let active = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::Low, false))
+        .await
+        .expect("scanner admission while the first task is active");
+    assert_eq!(active.result, HealAdmissionResult::Merged);
+    assert_eq!(active.task_id, first_id, "active ownership keeps precedence");
+
+    let completed_task = manager
+        .active_heals
+        .lock()
+        .await
+        .remove(&first_id)
+        .expect("first task should leave active ownership");
+    let mut retained = CompletedHealStatus::snapshot(&completed_task, HealTaskStatus::Completed).await;
+    Arc::make_mut(retained.outcome.as_mut().expect("retained task outcome")).finish(None);
+    manager
+        .completed_heals
+        .lock()
+        .await
+        .insert(first_id.clone(), Arc::new(retained));
+    let queued = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::Low, false))
+        .await
+        .expect("scanner admission after the first task leaves active state");
+    assert_eq!(queued.result, HealAdmissionResult::Merged);
+    assert_eq!(queued.task_id, second_id, "the receipt must name the remaining queued task");
+}
+
+#[tokio::test]
+async fn scanner_receipt_tracks_queued_owner_after_normal_dequeue() {
+    scanner_receipt_tracks_queued_owner(false).await;
+}
+
+#[tokio::test]
+async fn scanner_receipt_tracks_queued_owner_after_fair_dequeue() {
+    scanner_receipt_tracks_queued_owner(true).await;
+}
+
+#[tokio::test]
+async fn scanner_receipt_keeps_queued_owner_when_a_different_forced_task_runs() {
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let first = object_request(HealRequestSource::Scanner, HealPriority::Low, false);
+    let first_id = first.id.clone();
+    let second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+    let second_id = second.id.clone();
+    {
+        let mut queue = manager.heal_queue.lock().await;
+        assert_eq!(queue.push(first), QueuePushOutcome::Accepted);
+        assert_eq!(queue.push(second), QueuePushOutcome::Accepted);
+        let (selected, _) = queue.pop_runnable_with_skips(|_| true, |_| None);
+        assert_eq!(selected.expect("higher-priority forced task should run").id, second_id);
+    }
+    let receipt = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::High, false))
+        .await
+        .expect("scanner admission after a non-representative task leaves the queue");
+    assert_eq!(receipt.result, HealAdmissionResult::Merged);
+    assert_eq!(receipt.task_id, first_id);
+}
+
+#[tokio::test]
+async fn scanner_receipt_reselects_after_the_queued_owner_is_cancelled() {
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let first_id = first.id.clone();
+    let second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+    let second_id = second.id.clone();
+    let third = object_request(HealRequestSource::Internal, HealPriority::High, true);
+    let third_id = third.id.clone();
+    {
+        let mut queue = manager.heal_queue.lock().await;
+        for request in [first, second, third] {
+            assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+        }
+        let (selected, _) = queue.pop_runnable_with_skips(|_| true, |_| None);
+        assert_eq!(selected.expect("first task should run").id, first_id);
+    }
+    let first_receipt = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::High, false))
+        .await
+        .expect("first scanner admission");
+    assert_eq!(first_receipt.result, HealAdmissionResult::Merged);
+    assert!(first_receipt.task_id == second_id || first_receipt.task_id == third_id);
+    let remaining = if first_receipt.task_id == second_id {
+        third_id
+    } else {
+        second_id
+    };
+    manager.heal_queue.lock().await.remove_request_id(&first_receipt.task_id);
+
+    let next_receipt = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::High, false))
+        .await
+        .expect("scanner admission after the queued owner is removed");
+    assert_eq!(next_receipt.result, HealAdmissionResult::Merged);
+    assert_eq!(next_receipt.task_id, remaining);
+}
+
+#[tokio::test]
+async fn scanner_receipt_reselects_after_multiple_forced_dequeues() {
+    let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let first_id = first.id.clone();
+    let second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+    let second_id = second.id.clone();
+    let third = object_request(HealRequestSource::Internal, HealPriority::High, true);
+    let third_id = third.id.clone();
+    {
+        let mut queue = manager.heal_queue.lock().await;
+        for request in [first, second, third] {
+            assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+        }
+        let (selected, _) = queue.pop_runnable_with_skips(|_| true, |_| None);
+        assert_eq!(selected.expect("first task should run").id, first_id);
+    }
+    let first_receipt = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::High, false))
+        .await
+        .expect("first scanner admission");
+    assert_eq!(first_receipt.result, HealAdmissionResult::Merged);
+    assert!(first_receipt.task_id == second_id || first_receipt.task_id == third_id);
+    let remaining = if first_receipt.task_id == second_id {
+        third_id
+    } else {
+        second_id
+    };
+    {
+        let mut queue = manager.heal_queue.lock().await;
+        let (selected, _) = queue.pop_runnable_with_skips(|request| request.id == first_receipt.task_id, |_| None);
+        assert_eq!(selected.expect("selected queued owner should run").id, first_receipt.task_id);
+    }
+    let next_receipt = manager
+        .submit_heal_request_with_receipt(object_request(HealRequestSource::Scanner, HealPriority::High, false))
+        .await
+        .expect("scanner admission after another queued owner starts");
+    assert_eq!(next_receipt.result, HealAdmissionResult::Merged);
+    assert_eq!(next_receipt.task_id, remaining);
+}
+
+#[test]
+fn scanner_owner_index_survives_a_single_remaining_holder_and_new_forced_work() {
+    let mut queue = PriorityHealQueue::new();
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let first_id = first.id.clone();
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+    let second_id = second.id.clone();
+    let third = object_request(HealRequestSource::Internal, HealPriority::High, true);
+    let third_id = third.id.clone();
+
+    assert_eq!(queue.push(first), QueuePushOutcome::Accepted);
+    assert!(matches!(&queue.dedup_keys[&key].scanner_queued_owners, ScannerQueuedOwners::Single(_)));
+    assert_eq!(queue.push(second), QueuePushOutcome::Accepted);
+    assert_eq!(queue.pop().expect("first queued task").id, first_id);
+    assert_eq!(queue.queued_request_id_for_dedup_key(&key), Some(first_id.as_str()));
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(second_id.as_str()));
+
+    assert_eq!(queue.push(third), QueuePushOutcome::Accepted);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(second_id.as_str()));
+    assert_eq!(queue.pop().expect("second queued task").id, second_id);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(third_id.as_str()));
+}
+
+#[test]
+fn scanner_owner_index_distinguishes_holders_with_the_same_request_id() {
+    let mut queue = PriorityHealQueue::new();
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let shared_id = first.id.clone();
+    let mut second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+    second.id.clone_from(&shared_id);
+    let third = object_request(HealRequestSource::Internal, HealPriority::High, true);
+    let third_id = third.id.clone();
+    for request in [first, second, third] {
+        assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+    }
+
+    assert_eq!(queue.pop().expect("first holder").id, shared_id);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(shared_id.as_str()));
+    assert_eq!(queue.remove_request_id(&shared_id).expect("second holder").id, shared_id);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(third_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&key), Some(third_id.as_str()));
+}
+
+#[test]
+fn scanner_owner_index_tracks_bulk_removal_and_key_release() {
+    let mut queue = PriorityHealQueue::new();
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let first_id = first.id.clone();
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let second = object_request(HealRequestSource::Admin, HealPriority::Normal, true);
+    let second_id = second.id.clone();
+    let third = object_request(HealRequestSource::Internal, HealPriority::Low, true);
+    let third_id = third.id.clone();
+    for request in [first, second, third] {
+        assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+    }
+
+    let removed = queue.remove_matching(|request| request.id == first_id || request.id == second_id);
+    assert_eq!(removed.len(), 2);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(third_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&key), Some(third_id.as_str()));
+    assert_eq!(queue.remove_matching(|request| request.id == third_id).len(), 1);
+    assert!(!queue.dedup_keys.contains_key(&key));
+
+    let next = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let next_id = next.id.clone();
+    assert_eq!(queue.push(next), QueuePushOutcome::Accepted);
+    assert!(matches!(&queue.dedup_keys[&key].scanner_queued_owners, ScannerQueuedOwners::Single(_)));
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(next_id.as_str()));
+}
+
+#[test]
+fn scanner_owner_index_tracks_displacement_without_changing_legacy_reselection() {
+    let mut queue = PriorityHealQueue::new();
+    let first = object_request(HealRequestSource::Scanner, HealPriority::Low, false);
+    let first_id = first.id.clone();
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let second = object_request(HealRequestSource::Admin, HealPriority::Normal, true);
+    let second_id = second.id.clone();
+    assert_eq!(queue.push(first), QueuePushOutcome::Accepted);
+    assert_eq!(queue.push(second), QueuePushOutcome::Accepted);
+
+    let incoming = bucket_request("displacement", HealPriority::Urgent, HealRequestSource::Internal);
+    let displaced = queue
+        .push_displacing_lower_priority(incoming)
+        .expect("lower priority task should be displaced");
+    assert_eq!(displaced.id, first_id);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(second_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&key), Some(second_id.as_str()));
+}
+
+#[tokio::test]
+async fn scanner_receipt_rejects_a_missing_or_empty_holder_index() {
+    for missing in [true, false] {
+        let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
+        let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+        let key = PriorityHealQueue::make_dedup_key(&first);
+        let second = object_request(HealRequestSource::Admin, HealPriority::High, true);
+        {
+            let mut queue = manager.heal_queue.lock().await;
+            assert_eq!(queue.push(first), QueuePushOutcome::Accepted);
+            assert_eq!(queue.push(second), QueuePushOutcome::Accepted);
+            let entry = queue.dedup_keys.get_mut(&key).expect("dedup key should exist");
+            entry.scanner_queued_owners = if missing {
+                ScannerQueuedOwners::Single(0)
+            } else {
+                ScannerQueuedOwners::Indexed(std::collections::BTreeMap::new())
+            };
+        }
+
+        for leave_one in [false, true] {
+            if leave_one {
+                manager.heal_queue.lock().await.pop().expect("one holder should leave");
+            }
+            let request = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+            let request_id = request.id.clone();
+            let receipt = manager
+                .submit_heal_request_with_receipt(request)
+                .await
+                .expect("admission should fail closed");
+            assert_eq!(receipt.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning));
+            assert_eq!(receipt.task_id, request_id);
+        }
+    }
+}
+
+#[test]
+fn scanner_and_legacy_owners_keep_their_distinct_reselection_orders() {
+    let mut queue = PriorityHealQueue::new();
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let first_id = first.id.clone();
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let second = object_request(HealRequestSource::Admin, HealPriority::Low, true);
+    let second_id = second.id.clone();
+    let third = object_request(HealRequestSource::Internal, HealPriority::Normal, true);
+    let third_id = third.id.clone();
+    for request in [first, second, third] {
+        assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+    }
+
+    assert_eq!(queue.remove_request_id(&first_id).expect("first holder should be removed").id, first_id);
+    assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(second_id.as_str()));
+    assert_eq!(queue.queued_request_id_for_dedup_key(&key), Some(third_id.as_str()));
+}
+
+#[test]
+fn scanner_owner_index_tracks_a_series_of_forced_holders() {
+    let mut queue = PriorityHealQueue::new();
+    let first = object_request(HealRequestSource::Scanner, HealPriority::High, false);
+    let key = PriorityHealQueue::make_dedup_key(&first);
+    let mut ids = vec![first.id.clone()];
+    assert_eq!(queue.push(first), QueuePushOutcome::Accepted);
+    for _ in 0..128 {
+        let request = object_request(HealRequestSource::Internal, HealPriority::High, true);
+        ids.push(request.id.clone());
+        assert_eq!(queue.push(request), QueuePushOutcome::Accepted);
+    }
+    for id in &ids {
+        assert_eq!(queue.scanner_queued_request_id_for_dedup_key(&key), Some(id.as_str()));
+        assert_eq!(&queue.pop().expect("indexed holder should remain queued").id, id);
+    }
+    assert!(!queue.dedup_keys.contains_key(&key));
+}

crates/heal/src/heal/manager/tests/root_recovery.rs

@@ -117,8 +117,6 @@ fn admin_request(heal_type: HealType) -> HealRequest {
 
 fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
-        task_id: "admin-fixture".to_string(),
-        source: HealRequestSource::Admin,
         outcome: None,
         heal_type: heal_type.clone(),
         options: HealOptions::default(),
@@ -1241,8 +1239,6 @@ async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() {
         .expect("durable bucket responsibility");
     let completed = CompletedHealStatus {
         outcome: None,
-        source: request.source,
-        task_id: request.id.clone(),
         heal_type: request.heal_type.clone(),
         options: request.options.clone(),
         status: HealTaskStatus::Completed,

crates/heal/src/heal/storage_api.rs

@@ -26,11 +26,7 @@ pub(crate) use rustfs_ecstore::api::disk::{
     RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, ReplacementExecutionLease as EcstoreReplacementExecutionLease,
 };
 pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
-#[cfg(test)]
-pub(crate) use rustfs_ecstore::api::error::PoolMetadataError as EcstorePoolMetadataError;
-pub(crate) use rustfs_ecstore::api::error::{
-    Error as EcstoreErrorType, PoolMetadataFailure as EcstorePoolMetadataFailure, StorageError as EcstoreStorageError,
-};
+pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
 #[cfg(test)]
 pub(crate) use rustfs_ecstore::api::layout::{
     EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
@@ -57,9 +53,6 @@ pub(crate) use rustfs_ecstore::api::storage::{
 use rustfs_storage_api as storage_contracts;
 
 pub(crate) mod owner {
-    #[cfg(test)]
-    pub(crate) use super::EcstorePoolMetadataError;
-    pub(crate) use super::EcstorePoolMetadataFailure;
     pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
 
     pub(crate) use super::{

crates/heal/src/heal/task.rs

@@ -37,7 +37,7 @@ use std::{
     collections::VecDeque,
     future::Future,
     sync::{
-        Arc, OnceLock,
+        Arc,
         atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
     },
     time::{Duration, Instant, SystemTime},
@@ -59,19 +59,6 @@ pub(crate) struct OutcomeFinishTestHook {
 pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
     std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
 
-#[cfg(test)]
-static STORAGE_RECEIPT_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
-    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
-
-#[cfg(test)]
-async fn pause_storage_receipt(task_id: &str) {
-    let hook = STORAGE_RECEIPT_TEST_HOOK.lock().await.clone();
-    if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
-        hook.reached.notify_one();
-        hook.release.notified().await;
-    }
-}
-
 #[cfg(test)]
 async fn pause_outcome_finish(task_id: &str) {
     let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
@@ -449,9 +436,6 @@ pub struct HealTask {
     replacement_start_retry_count: Arc<AtomicU32>,
     /// Set when the locked storage read classified the single target as absent.
     object_absent: Arc<AtomicBool>,
-    /// The first resolved incarnation belongs to this task even if storage
-    /// fails or the bucket is recreated before its terminal outcome is recorded.
-    single_object_incarnation: Arc<OnceLock<Option<Uuid>>>,
     /// Task status
     pub status: Arc<RwLock<HealTaskStatus>>,
     /// Progress tracking
@@ -524,7 +508,6 @@ impl HealTask {
             replacement_running: Arc::new(AtomicBool::new(false)),
             replacement_start_retry_count: Arc::new(AtomicU32::new(0)),
             object_absent: Arc::new(AtomicBool::new(false)),
-            single_object_incarnation: Arc::new(OnceLock::new()),
             status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
             progress: Arc::new(RwLock::new(HealProgress::new())),
             result_items: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_RETAINED_HEAL_RESULT_ITEMS))),
@@ -686,9 +669,7 @@ impl HealTask {
             } => (bucket, object, Some(version_id.as_str())),
             _ => return None,
         };
-        let mut identity = self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index);
-        identity.bucket_incarnation_id = self.single_object_incarnation.get().copied().flatten();
-        Some(identity)
+        Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
     }
 
     pub(super) async fn record_verified_storage_receipt(

crates/heal/src/heal/task/heal_bucket.rs

@@ -306,9 +306,9 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_stage(3, 3);
                 }
-                // Keep storage/quorum identity for the cluster retry classifier.
-                // The structured log above already carries the bucket context.
-                Err(e)
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal bucket {bucket}: {e}"),
+                })
             }
         }
     }
@@ -520,7 +520,9 @@ impl HealTask {
                     error = %err,
                     "Heal cluster pool metadata failed"
                 );
-                Err(err)
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal cluster pool metadata: {err}"),
+                })
             }
             Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
             Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
@@ -537,7 +539,9 @@ impl HealTask {
                     error = %err,
                     "Heal cluster pool metadata failed"
                 );
-                Err(err)
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal cluster pool metadata: {err}"),
+                })
             }
         }
     }
@@ -792,27 +796,21 @@ impl HealTask {
                                 match unavailable_recreate_error(&storage_result.item, &heal_opts) {
                                     Some(error) => Some(error),
                                     None => {
-                                        #[cfg(test)]
-                                        super::pause_storage_receipt(&self.id).await;
+                                        telemetry_unknown |= !increment_counter(&mut healed);
+                                        telemetry_unknown |= !add_bytes(
+                                            &mut bytes,
+                                            u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
+                                        );
                                         recorded_authoritative_outcome =
                                             self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
-                                        if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
-                                            Some(Error::TaskCancelled)
-                                        } else {
-                                            telemetry_unknown |= !increment_counter(&mut healed);
-                                            telemetry_unknown |= !add_bytes(
-                                                &mut bytes,
-                                                u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
-                                            );
-                                            if !recorded_authoritative_outcome
-                                                && !heal_opts.dry_run
-                                                && !storage_result.item.detail.is_empty()
-                                            {
-                                                detail = Some(storage_result.item.detail.clone());
-                                            }
-                                            self.record_result_item(storage_result.item).await;
-                                            None
+                                        if !recorded_authoritative_outcome
+                                            && !heal_opts.dry_run
+                                            && !storage_result.item.detail.is_empty()
+                                        {
+                                            detail = Some(storage_result.item.detail.clone());
                                         }
+                                        self.record_result_item(storage_result.item).await;
+                                        None
                                     }
                                 }
                             }

crates/heal/src/heal/task/heal_object.rs

@@ -220,11 +220,6 @@ impl HealTask {
         let mut expected_identity =
             self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
         expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
-        if matches!(self.heal_type, HealType::Object { .. }) {
-            expected_identity.bucket_incarnation_id = *self
-                .single_object_incarnation
-                .get_or_init(|| expected_identity.bucket_incarnation_id);
-        }
 
         let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
         let heal_result = if self.source == HealRequestSource::ReadRepair {

crates/heal/src/heal/task/tests.rs

@@ -16,7 +16,6 @@ use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
 use super::*;
 use crate::heal::EcstoreError;
 use crate::heal::storage::HealStorageObjectResult;
-use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
 
 mod absent_verdict;
 mod concurrent_delete;
@@ -306,53 +305,6 @@ mod canonical_outcome {
         assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
     }
 
-    #[tokio::test]
-    async fn bucket_cancellation_after_storage_success_keeps_typed_cancelled_responsibility() {
-        let incarnation = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            heal_object_receipts: Mutex::new(HashMap::from([(
-                "object-a".to_string(),
-                VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
-            )])),
-            bucket_incarnation_id: Mutex::new(Some(incarnation)),
-            ..Default::default()
-        });
-        let task = Arc::new(bucket_task(storage.clone()));
-        let hook = Arc::new(OutcomeFinishTestHook {
-            task_id: task.id.clone(),
-            reached: tokio::sync::Notify::new(),
-            release: tokio::sync::Notify::new(),
-        });
-        *STORAGE_RECEIPT_TEST_HOOK.lock().await = Some(hook.clone());
-        let runner = {
-            let task = task.clone();
-            tokio::spawn(async move { task.execute().await })
-        };
-        tokio::time::timeout(Duration::from_secs(10), hook.reached.notified())
-            .await
-            .expect("storage success must reach the receipt boundary");
-        assert_eq!(storage.healed_objects.lock().expect("storage mutations").as_slice(), ["object-a"]);
-        task.cancel().await.expect("cancel before receipt recording");
-        hook.release.notify_one();
-        let result = tokio::time::timeout(Duration::from_secs(10), runner)
-            .await
-            .expect("cancelled worker must stop")
-            .expect("worker must not panic");
-        STORAGE_RECEIPT_TEST_HOOK.lock().await.take();
-        assert!(matches!(result, Err(Error::TaskCancelled)));
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Cancelled));
-        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
-        assert_eq!(outcome.counters.processed, 1);
-        assert_eq!(outcome.counters.healed, 0);
-        assert_eq!(outcome.counters.unchanged, 0);
-        assert_eq!(outcome.counters.unknown, 0);
-        assert_eq!(outcome.counters.skipped, 1);
-        assert_eq!(outcome.objects.len(), 1);
-        assert_eq!(outcome.objects[0].identity.object, "object-a");
-        assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
-    }
-
     #[tokio::test]
     async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
         let expected_incarnation = Uuid::new_v4();
@@ -832,10 +784,7 @@ mod canonical_outcome {
     async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() {
         let storage = Arc::new(MockStorage {
             listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-            bucket_heal_errors: Mutex::new(HashMap::from([(
-                "bucket-a".to_string(),
-                VecDeque::from([Error::other("metadata unavailable")]),
-            )])),
+            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
             ..Default::default()
         });
         let task = HealTask::from_request(
@@ -860,85 +809,6 @@ mod canonical_outcome {
         );
     }
 
-    #[tokio::test(start_paused = true)]
-    async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() {
-        for (failures, succeeds) in [(1, true), (4, false)] {
-            let storage = Arc::new(MockStorage {
-                listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-                typed_bucket_heal_errors: Mutex::new(HashMap::from([(
-                    "bucket-a".to_string(),
-                    (0..failures)
-                        .map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum))
-                        .collect(),
-                )])),
-                ..Default::default()
-            });
-            let task = HealTask::from_request(
-                HealRequest::new(
-                    HealType::Cluster,
-                    HealOptions {
-                        recursive: true,
-                        timeout: None,
-                        ..Default::default()
-                    },
-                    HealPriority::Normal,
-                ),
-                storage.clone(),
-            );
-            let result = task.execute().await;
-            assert_eq!(
-                result.is_ok(),
-                succeeds,
-                "original cluster task must honor its bounded bucket retry budget"
-            );
-            let outcome = task.get_outcome().await;
-            if succeeds {
-                assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
-                assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
-            } else {
-                assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
-                assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
-                let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure");
-                assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
-            }
-            let calls = storage.bucket_heal_calls.lock().expect("original bucket calls");
-            assert_eq!(
-                calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(),
-                if succeeds { 2 } else { 4 }
-            );
-            assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited");
-        }
-    }
-
-    #[tokio::test]
-    async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() {
-        for error in [
-            Error::Storage(EcstoreError::ErasureWriteQuorum),
-            Error::Disk(DiskError::FileAccessDenied),
-        ] {
-            let retryable = error.is_recoverable_heal();
-            let storage = Arc::new(MockStorage {
-                typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])),
-                ..Default::default()
-            });
-            let task = bucket_task(storage.clone());
-            let observed = task
-                .heal_bucket("bucket-a")
-                .await
-                .expect_err("bucket prepass must retain the storage error");
-            assert!(matches!(
-                observed,
-                Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied)
-            ));
-            assert_eq!(observed.is_recoverable_heal(), retryable);
-            assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
-            assert!(
-                storage.heal_object_calls.lock().expect("object calls").is_empty(),
-                "failed prepass must not heal objects"
-            );
-        }
-    }
-
     #[tokio::test(start_paused = true)]
     async fn cancellation_and_deadline_leave_partial_coverage() {
         for cancel in [false, true] {
@@ -1696,8 +1566,7 @@ struct MockStorage {
     include_object_dir_candidate: Mutex<bool>,
     listed_buckets: Mutex<Option<Vec<String>>>,
     bucket_discovery_incomplete: bool,
-    bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
-    typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
+    bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
     bucket_heal_calls: Mutex<Vec<String>>,
     delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
     delete_recovery_calls: Mutex<Vec<String>>,
@@ -1946,127 +1815,6 @@ async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
                 | HealObjectDisposition::AuthoritativelyAbsent
         )
     }));
-    assert_eq!(outcome.objects.len(), 1);
-    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
-}
-
-#[tokio::test]
-async fn object_failure_retains_incarnation_for_inner_and_outer_storage_errors() {
-    for failure in [
-        MockHealObjectOutcome::PermissionDenied,
-        MockHealObjectOutcome::RetryableReadQuorum,
-        MockHealObjectOutcome::RetryableLockTimeout,
-        MockHealObjectOutcome::ErrOther("backend failed"),
-    ] {
-        let incarnation = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            bucket_incarnation_id: Mutex::new(Some(incarnation)),
-            heal_object_outcome: Mutex::new(Some(failure)),
-            ..Default::default()
-        });
-        let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), Some("version-a".to_owned()));
-        request.options.pool_index = Some(2);
-        request.options.set_index = Some(3);
-        let task = HealTask::from_request(request, storage);
-        task.execute()
-            .await
-            .expect_err("the failed storage operation must remain failed");
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.objects.len(), 1);
-        assert_eq!(
-            (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
-            (1, 1, 1)
-        );
-        assert_eq!((outcome.counters.healed, outcome.counters.unchanged, outcome.counters.unknown), (0, 0, 0));
-        let object = &outcome.objects[0];
-        assert_eq!(object.identity.bucket_incarnation_id, Some(incarnation));
-        assert_eq!(object.identity.bucket, "bucket-a");
-        assert_eq!(object.identity.object, "object-a");
-        assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
-        assert_eq!((object.identity.pool_index, object.identity.set_index), (Some(2), Some(3)));
-        assert!(matches!(object.disposition, HealObjectDisposition::Failed(_)));
-    }
-}
-
-#[tokio::test(start_paused = true)]
-async fn object_abort_keeps_pre_storage_incarnation_after_bucket_recreation() {
-    for cancel in [false, true] {
-        let original = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            bucket_incarnation_id: Mutex::new(Some(original)),
-            block_heal_object: Mutex::new(true),
-            ..Default::default()
-        });
-        let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
-        request.options.timeout = Some(Duration::from_secs(10));
-        let task = HealTask::from_request(request, storage.clone());
-        let execution = task.execute();
-        tokio::pin!(execution);
-        assert!(futures::poll!(&mut execution).is_pending());
-        assert_eq!(storage.heal_object_calls.lock().expect("storage boundary").as_slice(), ["object-a"]);
-        *storage.bucket_incarnation_id.lock().expect("recreated bucket") = Some(Uuid::new_v4());
-        if cancel {
-            task.cancel().await.expect("cancel blocked storage");
-        } else {
-            tokio::time::advance(Duration::from_secs(11)).await;
-        }
-        let result = execution.await;
-        assert!(matches!(
-            (&result, cancel),
-            (Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
-        ));
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.objects.len(), 1);
-        assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
-        assert_eq!((outcome.counters.processed, outcome.counters.skipped), (1, 1));
-        assert_eq!((outcome.counters.failed, outcome.counters.healed, outcome.counters.unknown), (0, 0, 0));
-        assert_eq!(
-            outcome.objects[0].disposition,
-            if cancel {
-                HealObjectDisposition::Cancelled
-            } else {
-                HealObjectDisposition::Deferred {
-                    reason: HealDeferredReason::Deadline,
-                    retry_not_before: None,
-                }
-            }
-        );
-    }
-}
-
-#[tokio::test]
-async fn object_unverified_outcome_keeps_original_incarnation_without_claiming_repair() {
-    let original = Uuid::new_v4();
-    let successor = Uuid::new_v4();
-    let storage = Arc::new(MockStorage {
-        bucket_incarnation_id: Mutex::new(Some(original)),
-        bucket_incarnation_after_object_heal: Mutex::new(Some(successor)),
-        ..Default::default()
-    });
-    let task = HealTask::from_request(HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None), storage.clone());
-    task.execute().await.expect("legacy storage keeps its unverified result");
-    let outcome = task.get_outcome().await;
-    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
-    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
-    assert_eq!((outcome.counters.healed, outcome.counters.unknown), (0, 1));
-    assert_eq!(*storage.bucket_incarnation_id.lock().expect("successor"), Some(successor));
-}
-
-#[tokio::test]
-async fn object_dry_run_does_not_adopt_bucket_incarnation() {
-    let storage = Arc::new(MockStorage {
-        bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
-        ..Default::default()
-    });
-    let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
-    request.options.dry_run = true;
-    let task = HealTask::from_request(request, storage);
-    task.execute().await.expect("dry-run observation");
-    let outcome = task.get_outcome().await;
-    assert_eq!(outcome.objects.len(), 1);
-    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, None);
-    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
-    assert_eq!(outcome.counters.healed, 0);
 }
 
 #[tokio::test]
@@ -2242,7 +1990,6 @@ fn replacement_identity(
 
 #[derive(Clone)]
 enum MockHealObjectOutcome {
-    PoolMetadata { kind: EcstorePoolMetadataFailure, outer: bool },
     LegacyAmbiguous { outer: bool },
     MissingObject { outer: bool },
     MissingVersion,
@@ -2262,16 +2009,6 @@ enum MockHealObjectOutcome {
     PermanentOther(&'static str),
 }
 
-fn pool_metadata_error(kind: EcstorePoolMetadataFailure) -> Error {
-    Error::Storage(EcstoreError::other(EcstorePoolMetadataError {
-        kind,
-        operation: "bucket heal".to_owned(),
-        phase: "native_heal",
-        since: std::time::UNIX_EPOCH.into(),
-        source: None,
-    }))
-}
-
 fn unavailable_drive_heal_result(state: DriveState) -> (HealResultItem, Option<Error>) {
     (
         HealResultItem {
@@ -2443,14 +2180,6 @@ impl HealStorageAPI for MockStorage {
             .and_then(VecDeque::pop_front)
         {
             return match outcome {
-                MockHealObjectOutcome::PoolMetadata { kind, outer } => {
-                    let error = pool_metadata_error(kind);
-                    if outer {
-                        Err(error)
-                    } else {
-                        Ok((HealResultItem::default(), Some(error)))
-                    }
-                }
                 MockHealObjectOutcome::LegacyAmbiguous { outer } => {
                     let error = Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous);
                     if outer {
@@ -2527,14 +2256,6 @@ impl HealStorageAPI for MockStorage {
         }
         if let Some(outcome) = self.heal_object_outcome.lock().unwrap().take() {
             return match outcome {
-                MockHealObjectOutcome::PoolMetadata { kind, outer } => {
-                    let error = pool_metadata_error(kind);
-                    if outer {
-                        Err(error)
-                    } else {
-                        Ok((HealResultItem::default(), Some(error)))
-                    }
-                }
                 MockHealObjectOutcome::LegacyAmbiguous { outer } => {
                     let error = Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous);
                     if outer {
@@ -2666,23 +2387,14 @@ impl HealStorageAPI for MockStorage {
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
         self.bucket_heal_opts.lock().unwrap().push(*opts);
-        if let Some(error) = self
-            .typed_bucket_heal_errors
-            .lock()
-            .unwrap()
-            .get_mut(bucket)
-            .and_then(VecDeque::pop_front)
-        {
-            return Err(error);
-        }
-        if let Some(error) = self
+        if let Some(message) = self
             .bucket_heal_errors
             .lock()
             .unwrap()
             .get_mut(bucket)
             .and_then(VecDeque::pop_front)
         {
-            return Err(error);
+            return Err(Error::other(message));
         }
         Ok(HealResultItem::default())
     }
@@ -3717,70 +3429,13 @@ async fn cluster_recreate_fails_when_pool_metadata_heal_fails() {
         .await
         .expect_err("cluster recreate heal must not hide pool metadata failure");
 
-    assert!(matches!(err, Error::Other(ref message) if message == "pool metadata missing"));
+    assert!(matches!(err, Error::TaskExecutionFailed { .. }));
     assert_eq!(
         storage.heal_object_calls.lock().expect("object calls").as_slice(),
         ["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
     );
 }
 
-#[tokio::test]
-async fn cluster_pool_metadata_failures_preserve_recovery_classification_in_both_response_shapes() {
-    for outer in [false, true] {
-        for (kind, retryable) in [
-            (EcstorePoolMetadataFailure::ReadUnavailable, true),
-            (EcstorePoolMetadataFailure::TransactionUnknown, true),
-            (EcstorePoolMetadataFailure::RecoveryRequired, false),
-            (EcstorePoolMetadataFailure::FenceLost, false),
-        ] {
-            let storage = Arc::new(MockStorage {
-                listed_buckets: Mutex::new(Some(Vec::new())),
-                ..Default::default()
-            });
-            storage.heal_object_outcomes.lock().expect("object outcomes").insert(
-                POOL_META_NAME.to_string(),
-                VecDeque::from([MockHealObjectOutcome::PoolMetadata { kind, outer }]),
-            );
-            let mut request = HealRequest::new(
-                HealType::Cluster,
-                HealOptions {
-                    recreate_missing: true,
-                    timeout: None,
-                    ..Default::default()
-                },
-                HealPriority::Normal,
-            );
-            request.source = HealRequestSource::Admin;
-            let task = HealTask::from_request(request, storage.clone());
-            let error = task
-                .execute()
-                .await
-                .expect_err("pool metadata failure must reach the scheduler");
-            assert!(
-                matches!(&error, Error::Storage(error) if error.pool_metadata_failure().is_some_and(|failure| failure.kind == kind)),
-                "the original typed pool metadata state must survive either response shape: {error}"
-            );
-            assert_eq!(error.is_recoverable_heal(), retryable, "{kind:?}, outer={outer}");
-            assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), [POOL_META_NAME]);
-            if retryable {
-                let retry = task
-                    .retry_request_with_remaining_timeout()
-                    .await
-                    .expect("existing retry request");
-                assert_eq!(retry.id, task.id, "recovery retains the accepted task token");
-                HealTask::from_request(retry, storage.clone())
-                    .execute()
-                    .await
-                    .expect("the same task should complete after pool metadata recovery");
-                assert_eq!(
-                    storage.heal_object_calls.lock().expect("object calls").as_slice(),
-                    [POOL_META_NAME, POOL_META_NAME]
-                );
-            }
-        }
-    }
-}
-
 #[tokio::test]
 async fn cluster_dry_run_does_not_heal_pool_metadata() {
     let storage = Arc::new(MockStorage::default());
@@ -4144,10 +3799,7 @@ async fn test_recursive_bucket_heal_reports_typed_exhausted_and_permanent_failur
 async fn test_cluster_heal_continues_after_bucket_failure() {
     let storage = Arc::new(MockStorage {
         listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-        bucket_heal_errors: Mutex::new(HashMap::from([(
-            "bucket-a".to_string(),
-            VecDeque::from([Error::other("metadata unavailable")]),
-        )])),
+        bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])),
         ..Default::default()
     });
     let request = HealRequest::new(
@@ -4179,10 +3831,7 @@ async fn test_cluster_heal_continues_after_bucket_failure() {
 async fn test_cluster_heal_retries_only_recoverable_bucket() {
     let storage = Arc::new(MockStorage {
         listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-        bucket_heal_errors: Mutex::new(HashMap::from([(
-            "bucket-a".to_string(),
-            VecDeque::from([Error::other("lock acquisition timeout")]),
-        )])),
+        bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["lock acquisition timeout"]))])),
         ..Default::default()
     });
     let request = HealRequest::new(
@@ -4206,47 +3855,6 @@ async fn test_cluster_heal_retries_only_recoverable_bucket() {
     );
 }
 
-#[tokio::test(start_paused = true)]
-async fn cluster_bucket_heal_retries_only_recoverable_pool_metadata_failures() {
-    for (kind, retryable) in [
-        (EcstorePoolMetadataFailure::ReadUnavailable, true),
-        (EcstorePoolMetadataFailure::TransactionUnknown, true),
-        (EcstorePoolMetadataFailure::RecoveryRequired, false),
-        (EcstorePoolMetadataFailure::FenceLost, false),
-    ] {
-        let storage = Arc::new(MockStorage {
-            bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
-            bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_owned(), VecDeque::from([pool_metadata_error(kind)]))])),
-            ..Default::default()
-        });
-        let mut request = HealRequest::new(
-            HealType::Cluster,
-            HealOptions {
-                recursive: true,
-                timeout: None,
-                ..Default::default()
-            },
-            HealPriority::Normal,
-        );
-        request.source = HealRequestSource::Admin;
-        let task = HealTask::from_request(request, storage.clone());
-        let result = task.execute().await;
-        if retryable {
-            result.expect("a recoverable pool metadata state must not terminate the root heal");
-            assert_eq!(task.get_status().await, HealTaskStatus::Completed);
-            assert_eq!(
-                storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(),
-                ["bucket-a", "bucket-a"]
-            );
-        } else {
-            result.expect_err("operator recovery and lost fences must remain terminal");
-            let failure = task.take_batch_failure().await.expect("permanent cluster failure");
-            assert_eq!((failure.retryable, failure.permanent), (0, 1));
-            assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
-        }
-    }
-}
-
 #[tokio::test]
 async fn test_recursive_bucket_heal_does_not_remove_bucket_metadata() {
     let storage = Arc::new(MockStorage::default());
@@ -5009,7 +4617,7 @@ async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
     let storage = Arc::new(MockStorage {
         bucket_heal_errors: Mutex::new(HashMap::from([(
             "bucket-a".to_string(),
-            VecDeque::from([Error::other("injected bucket prepass failure")]),
+            VecDeque::from(["injected bucket prepass failure"]),
         )])),
         resume_disk: Mutex::new(Some(disk)),
         ..Default::default()

crates/heal/tests/shard_identity_receipt_test.rs

@@ -550,159 +550,3 @@ fn admin_normal_protected_repairs_report_repaired_for_data_and_parity() {
         .join()
         .expect("heal fixture result");
 }
-
-#[test]
-#[serial]
-fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
-    std::thread::Builder::new()
-        .stack_size(32 * 1024 * 1024)
-        .spawn(|| {
-            tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("mixed batch runtime")
-                .block_on(async {
-                    use rustfs_heal::heal::{
-                        manager::{HealConfig, HealManager},
-                        outcome::{HealExecutionOutcome, HealFailureClass, HealTraversalCoverage},
-                        task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
-                    };
-                    use rustfs_heal_contracts::heal_channel::HealRequestSource;
-                    use std::sync::Arc;
-
-                    let root = tempfile::tempdir().expect("mixed batch fixture");
-                    let env = TestECStoreEnv::builder().base_dir(root.path()).build().await;
-                    let bucket = "admin-mixed-batch";
-                    env.make_bucket(bucket, false).await;
-                    let set = env.ecstore.pools[0].get_disks(0);
-                    let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
-                    let mut originals = Vec::new();
-                    for (object, missing, byte) in [("healthy", 0, 0x11), ("repaired", 1, 0x52), ("failed", 3, 0xa3)] {
-                        env.ecstore
-                            .put_object(
-                                bucket,
-                                object,
-                                &mut PutObjReader::from_vec(vec![byte; 5 * 1024 * 1024 + 123]),
-                                &ObjectOptions {
-                                    shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
-                                    write_completion: WriteCompletion::TailDrained,
-                                    ..Default::default()
-                                },
-                            )
-                            .await
-                            .expect("commit complete fixture");
-                        for (slot, disk) in disks.iter().enumerate() {
-                            let meta = disk
-                                .read_version("", bucket, object, "", &ReadOptions::default())
-                                .await
-                                .expect("fixture metadata");
-                            assert_eq!(meta.erasure.data_blocks, 2, "failed fixture must retain R-1 shards");
-                            let path = env.disk_paths[slot]
-                                .join(bucket)
-                                .join(object)
-                                .join(meta.data_dir.expect("external shard").to_string())
-                                .join("part.1");
-                            let bytes = tokio::fs::read(&path).await.expect("original part");
-                            if slot < missing {
-                                tokio::fs::remove_file(&path).await.expect("inject missing part");
-                            }
-                            originals.push((object, path, bytes, slot < missing));
-                        }
-                    }
-                    let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
-                    let manager = HealManager::new(
-                        storage.clone(),
-                        Some(HealConfig {
-                            enable_auto_heal: false,
-                            ..Default::default()
-                        }),
-                    );
-                    manager.start().await.expect("start mixed batch manager");
-                    let mut request = HealRequest::new(
-                        HealType::Bucket {
-                            bucket: bucket.to_owned(),
-                        },
-                        HealOptions {
-                            recursive: true,
-                            scan_mode: HealScanMode::Deep,
-                            pool_index: Some(0),
-                            set_index: Some(0),
-                            timeout: Some(Duration::from_secs(150)),
-                            ..Default::default()
-                        },
-                        HealPriority::High,
-                    );
-                    request.source = HealRequestSource::Admin;
-                    let token = request.id.clone();
-                    manager.submit_heal_request(request).await.expect("admit original mixed task");
-                    let report = tokio::time::timeout(Duration::from_secs(150), async {
-                        loop {
-                            let report = manager.get_task_report(&token).await.expect("same-token report");
-                            if matches!(
-                                report.status,
-                                HealTaskStatus::Completed | HealTaskStatus::Failed { .. } | HealTaskStatus::Timeout
-                            ) {
-                                break report;
-                            }
-                            tokio::time::sleep(Duration::from_millis(20)).await;
-                        }
-                    })
-                    .await
-                    .expect("mixed task terminal deadline");
-                    manager.stop().await.expect("stop mixed task manager");
-                    for (object, path, original, missing) in originals {
-                        if object == "failed" && missing {
-                            assert!(!path.exists(), "below-R failure must not invent a shard");
-                        } else {
-                            assert_eq!(tokio::fs::read(&path).await.expect("preserved or repaired shard"), original);
-                        }
-                    }
-                    assert!(matches!(report.status, HealTaskStatus::Failed { .. }), "{report:?}");
-                    let outcome = report.outcome.as_ref().expect("canonical mixed outcome");
-                    assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
-                    assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
-                    assert_eq!(
-                        (
-                            outcome.counters.processed,
-                            outcome.counters.healed,
-                            outcome.counters.unchanged,
-                            outcome.counters.failed,
-                            outcome.counters.skipped,
-                            outcome.counters.unknown,
-                            outcome.counters.attempt_failures
-                        ),
-                        (3, 1, 1, 1, 0, 0, 4),
-                        "{outcome:?}"
-                    );
-                    for (object, disposition) in [
-                        ("healthy", HealObjectDisposition::VerifiedHealthy),
-                        ("repaired", HealObjectDisposition::Repaired),
-                        ("failed", HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)),
-                    ] {
-                        let receipt = outcome
-                            .objects
-                            .iter()
-                            .find(|item| item.identity.object == object)
-                            .expect("exact object receipt");
-                        assert_eq!(receipt.disposition, disposition);
-                    }
-                    drop(manager);
-                    let restarted = HealManager::new(
-                        storage,
-                        Some(HealConfig {
-                            enable_auto_heal: false,
-                            ..Default::default()
-                        }),
-                    );
-                    restarted.start().await.expect("restart with retained failed batch");
-                    let restored = restarted.get_task_report(&token).await.expect("original token after restart");
-                    assert_eq!(restored.status, report.status);
-                    assert_eq!(restored.outcome, report.outcome);
-                    assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
-                    restarted.stop().await.expect("stop restarted mixed batch manager");
-                });
-        })
-        .expect("mixed batch thread")
-        .join()
-        .expect("mixed batch result");
-}

crates/iam/src/delegation.rs

@@ -1,313 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-//! Durable restrictions owned by an authenticated user. A record is not an IAM
-//! grant or credential; issuing credentials still requires current IAM checks.
-use crate::{error::is_err_config_not_found, store::Store};
-use serde::{Deserialize, Serialize};
-
-const PREFIX: &str = "config/iam/plugin-delegations/v1";
-
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(deny_unknown_fields)]
-pub struct Restrictions {
-    #[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
-    pub version: Option<u32>,
-    #[serde(default, skip_serializing_if = "Option::is_none", deserialize_with = "present_value")]
-    pub output_bucket: Option<String>,
-    pub deployment: String,
-    pub bucket: String,
-    pub input_prefix: String,
-    pub output_prefix: String,
-    pub expires_at_unix_seconds: u64,
-}
-
-// An absent v1 field is compatible; an explicitly null v2 field is invalid.
-fn present_value<'de, T: Deserialize<'de>, D: serde::Deserializer<'de>>(deserializer: D) -> Result<Option<T>, D::Error> {
-    T::deserialize(deserializer).map(Some)
-}
-
-impl Restrictions {
-    pub fn output_bucket(&self) -> Result<&str, DelegationError> {
-        match (self.version, self.output_bucket.as_deref()) {
-            (None, None) => Ok(&self.bucket),
-            (Some(2), Some(output)) => Ok(output),
-            _ => Err(DelegationError::Invalid),
-        }
-    }
-
-    pub fn validate(&self, now: u64) -> Result<(), DelegationError> {
-        let output_bucket = self.output_bucket()?;
-        let names: [&str; 5] = [
-            &self.deployment,
-            &self.bucket,
-            &self.input_prefix,
-            &self.output_prefix,
-            output_bucket,
-        ];
-        if names
-            .iter()
-            .any(|s| s.is_empty() || s.len() > 1024 || s.contains("${") || s.chars().any(char::is_control))
-            || self.bucket.contains('/')
-            || self.bucket.contains('*')
-            || self.bucket.contains('?')
-            || output_bucket.contains('/')
-            || output_bucket.contains('*')
-            || output_bucket.contains('?')
-            || self.input_prefix.contains('*')
-            || self.input_prefix.contains('?')
-            || self.output_prefix.contains('*')
-            || self.output_prefix.contains('?')
-            || self.expires_at_unix_seconds <= now
-            || self.expires_at_unix_seconds.saturating_sub(now) > 86400
-        {
-            return Err(DelegationError::Invalid);
-        }
-        Ok(())
-    }
-}
-
-#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(deny_unknown_fields)]
-pub struct Delegation {
-    pub version: u32,
-    pub id: String,
-    pub revision: String,
-    pub owner_access_key: String,
-    pub created_at_unix_seconds: u64,
-    pub restrictions: Restrictions,
-}
-
-#[derive(Serialize, Deserialize, PartialEq, Eq)]
-#[serde(deny_unknown_fields)]
-struct Tombstone {
-    version: u32,
-    id: String,
-    revision: String,
-    owner_access_key: String,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum DelegationError {
-    #[error("invalid delegation restriction")]
-    Invalid,
-    #[error("delegation not found")]
-    NotFound,
-    #[error("delegation owner rejected")]
-    OwnerRejected,
-    #[error("delegation unavailable")]
-    Unavailable,
-    #[error("delegation revoked")]
-    Revoked,
-    #[error("delegation expired")]
-    Expired,
-}
-
-fn record_path(id: &str) -> Result<String, DelegationError> {
-    if id.len() != 32 || !id.bytes().all(|c| c.is_ascii_hexdigit()) {
-        return Err(DelegationError::Invalid);
-    }
-    Ok(format!("{PREFIX}/{id}/record.json"))
-}
-
-async fn load<T: Store>(store: &T, id: &str) -> Result<Delegation, DelegationError> {
-    let record: Delegation = store.load_iam_config_strict(record_path(id)?).await.map_err(|error| {
-        if is_err_config_not_found(&error) {
-            DelegationError::NotFound
-        } else {
-            DelegationError::Unavailable
-        }
-    })?;
-    if !matches!(record.version, 1 | 2)
-        || record.version != record.restrictions.version.unwrap_or(1)
-        || record.id != id
-        || record.revision != id
-        || record.owner_access_key.is_empty()
-    {
-        return Err(DelegationError::Unavailable);
-    }
-    record
-        .restrictions
-        .validate(record.created_at_unix_seconds)
-        .map_err(|_| DelegationError::Unavailable)?;
-    Ok(record)
-}
-
-pub(crate) async fn create<T: Store>(
-    store: &T,
-    owner: &str,
-    restrictions: Restrictions,
-    now: u64,
-) -> Result<Delegation, DelegationError> {
-    restrictions.validate(now)?;
-    if owner.is_empty() {
-        return Err(DelegationError::OwnerRejected);
-    }
-    // IDs are generated exclusively here; no update operation accepts an ID.
-    let id = format!("{:032x}", rand::random::<u128>());
-    let path = record_path(&id)?;
-    match store.load_iam_config_strict::<Delegation>(&path).await {
-        Err(error) if is_err_config_not_found(&error) => {}
-        _ => return Err(DelegationError::Unavailable),
-    }
-    let record = Delegation {
-        version: restrictions.version.unwrap_or(1),
-        revision: id.clone(),
-        id,
-        owner_access_key: owner.to_owned(),
-        created_at_unix_seconds: now,
-        restrictions,
-    };
-    store
-        .save_iam_config(&record, path)
-        .await
-        .map_err(|_| DelegationError::Unavailable)?;
-    if load(store, &record.id).await? != record {
-        return Err(DelegationError::Unavailable);
-    }
-    Ok(record)
-}
-
-pub(crate) async fn revoke<T: Store>(store: &T, owner: &str, id: &str) -> Result<(), DelegationError> {
-    let record = load(store, id).await?;
-    if record.owner_access_key != owner {
-        return Err(DelegationError::OwnerRejected);
-    }
-    let tombstone = Tombstone {
-        version: 1,
-        id: record.id,
-        revision: record.revision,
-        owner_access_key: record.owner_access_key,
-    };
-    let path = format!("{PREFIX}/{id}/revoked.json");
-    // Every retry writes exactly the same revocation; active records never change.
-    store
-        .save_iam_config(&tombstone, &path)
-        .await
-        .map_err(|_| DelegationError::Unavailable)?;
-    let saved: Tombstone = store
-        .load_iam_config_strict(path)
-        .await
-        .map_err(|_| DelegationError::Unavailable)?;
-    if saved != tombstone {
-        return Err(DelegationError::Unavailable);
-    }
-    Ok(())
-}
-
-pub(crate) async fn read_active<T: Store>(store: &T, owner: &str, id: &str, now: u64) -> Result<Delegation, DelegationError> {
-    let record = load(store, id).await?;
-    if record.owner_access_key != owner {
-        return Err(DelegationError::OwnerRejected);
-    }
-    check_active(store, record, now).await
-}
-
-pub(crate) async fn read_owned<T: Store>(store: &T, owner: &str, id: &str) -> Result<Delegation, DelegationError> {
-    let record = load(store, id).await?;
-    if record.owner_access_key != owner {
-        return Err(DelegationError::OwnerRejected);
-    }
-    Ok(record)
-}
-
-pub(crate) async fn read_active_for_broker<T: Store>(store: &T, id: &str, now: u64) -> Result<Delegation, DelegationError> {
-    let record = load(store, id).await?;
-    check_active(store, record, now).await
-}
-async fn check_active<T: Store>(store: &T, record: Delegation, now: u64) -> Result<Delegation, DelegationError> {
-    let id = &record.id;
-    match store
-        .load_iam_config_strict::<Tombstone>(format!("{PREFIX}/{id}/revoked.json"))
-        .await
-    {
-        Ok(_) => return Err(DelegationError::Revoked),
-        Err(error) if is_err_config_not_found(&error) => {}
-        Err(_) => return Err(DelegationError::Unavailable),
-    }
-    if now >= record.restrictions.expires_at_unix_seconds {
-        return Err(DelegationError::Expired);
-    }
-    Ok(record)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn delegation_v1_json_remains_compatible() {
-        let legacy = serde_json::json!({"deployment":"runner-a","bucket":"photos",
-            "input_prefix":"inputs/","output_prefix":"outputs/","expires_at_unix_seconds":200});
-        let restriction: Restrictions = serde_json::from_value(legacy.clone()).expect("legacy restriction");
-        restriction.validate(100).expect("legacy validation");
-        assert_eq!(restriction.output_bucket().expect("v1 output"), "photos");
-        assert_eq!(serde_json::to_value(restriction).expect("legacy serialization"), legacy);
-        let record = serde_json::json!({"version":1,"id":"a".repeat(32),"revision":"a".repeat(32),
-            "owner_access_key":"owner","created_at_unix_seconds":100,"restrictions":legacy});
-        let decoded: Delegation = serde_json::from_value(record.clone()).expect("legacy record");
-        assert_eq!(serde_json::to_value(decoded).expect("legacy record serialization"), record);
-    }
-
-    #[test]
-    fn delegation_v2_requires_explicit_output_bucket() {
-        let valid = serde_json::json!({"version":2,"deployment":"runner-a","bucket":"photos",
-            "input_prefix":"inputs/","output_bucket":"artifacts","output_prefix":"outputs/",
-            "expires_at_unix_seconds":200});
-        let restriction: Restrictions = serde_json::from_value(valid.clone()).expect("v2 restriction");
-        restriction.validate(100).expect("explicit output bucket");
-        assert_eq!(restriction.output_bucket().expect("v2 output"), "artifacts");
-        assert_eq!(serde_json::to_value(restriction).expect("v2 serialization"), valid);
-        for key in ["version", "output_bucket"] {
-            let mut bad = valid.clone();
-            bad.as_object_mut().expect("object").remove(key);
-            assert!(
-                serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
-                "v2 required {key}"
-            );
-        }
-        for (key, value) in [
-            ("version", serde_json::json!(1)),
-            ("version", serde_json::json!(3)),
-            ("version", serde_json::Value::Null),
-            ("output_bucket", serde_json::Value::Null),
-            ("output_bucket", serde_json::json!("")),
-            ("output_bucket", serde_json::json!("other/*")),
-            ("output_bucket", serde_json::json!("${aws:username}")),
-            ("output_bucket", serde_json::json!("other?")),
-            ("output_bucket", serde_json::json!("other\n")),
-            ("unknown", serde_json::json!(true)),
-        ] {
-            let mut bad = valid.clone();
-            bad[key] = value;
-            assert!(
-                serde_json::from_value::<Restrictions>(bad).map_or(true, |r| r.validate(100).is_err()),
-                "invalid v2 {key}"
-            );
-        }
-    }
-
-    #[test]
-    fn restrictions_are_literal_bounded_and_expiring() {
-        let mut restriction = Restrictions {
-            version: None,
-            output_bucket: None,
-            deployment: "runner-a".into(),
-            bucket: "photos".into(),
-            input_prefix: "inputs/".into(),
-            output_prefix: "outputs/".into(),
-            expires_at_unix_seconds: 200,
-        };
-        assert!(restriction.validate(100).is_ok());
-        restriction.output_prefix = "outputs/*".into();
-        assert!(matches!(restriction.validate(100), Err(DelegationError::Invalid)));
-        restriction.output_prefix = "outputs/".into();
-        assert!(matches!(restriction.validate(200), Err(DelegationError::Invalid)));
-        restriction.expires_at_unix_seconds = 200;
-        restriction.input_prefix = "inputs/${aws:username}/".into();
-        assert!(
-            matches!(restriction.validate(100), Err(DelegationError::Invalid)),
-            "C07_LITERAL_SCOPE_BOUND: policy variables are not literal restrictions"
-        );
-        assert!(record_path("../../users").is_err());
-    }
-}

crates/iam/src/lib.rs

@@ -28,7 +28,6 @@ pub(crate) const EVENT_OIDC_DIAGNOSTICS: &str = "oidc_diagnostics";
 const EVENT_OIDC_STATE: &str = "oidc_state";
 
 pub mod cache;
-pub mod delegation;
 pub mod error;
 pub mod federation;
 pub mod keyring;

crates/iam/src/storage_api.rs

@@ -91,7 +91,6 @@ pub(crate) mod crate_boundary {
 
 pub(crate) mod object_store {
     pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
-    pub(crate) use rustfs_ecstore::api::config::com::read_config_preserve_empty_with_metadata_opts;
 }
 
 pub(crate) mod runtime {

crates/iam/src/store.rs

@@ -26,11 +26,6 @@ pub trait Store: Clone + Send + Sync + 'static {
     fn has_watcher(&self) -> bool;
     async fn save_iam_config<Item: Serialize + Send>(&self, item: Item, path: impl AsRef<str> + Send) -> Result<()>;
     async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item>;
-    /// Only confirmed storage absence may return ConfigNotFound. Corrupt or
-    /// undecryptable objects must remain errors, especially for revocations.
-    async fn load_iam_config_strict<Item: DeserializeOwned>(&self, _path: impl AsRef<str> + Send) -> Result<Item> {
-        Err(crate::error::Error::IamSysNotInitialized)
-    }
     async fn delete_iam_config(&self, path: impl AsRef<str> + Send) -> Result<()>;
 
     async fn save_user_identity(&self, name: &str, user_type: UserType, item: UserIdentity, ttl: Option<usize>) -> Result<()>;

crates/iam/src/store/object.rs

@@ -42,7 +42,6 @@ use tracing::{debug, error, warn};
 
 use crate::storage_api::object_store::{
     HTTPPreconditions, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
-    read_config_preserve_empty_with_metadata_opts,
 };
 
 pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
@@ -612,11 +611,6 @@ impl ObjectStore {
         Ok(serde_json::from_slice(&outcome.plain)?)
     }
 
-    fn decode_iam_config_strict<Item: DeserializeOwned>(data: &[u8]) -> Result<Item> {
-        let outcome = Self::decrypt_data_with_source(data)?;
-        Ok(serde_json::from_slice(&outcome.plain)?)
-    }
-
     /// Parameterized core of [`Store::load_user_identity`].
     async fn load_user_identity_with(&self, name: &str, user_type: UserType, mode: LoadMode) -> Result<UserIdentity> {
         let mut u: UserIdentity = self
@@ -830,12 +824,6 @@ impl Store for ObjectStore {
     async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
         self.load_iam_config_with(path, LoadMode::Locked).await
     }
-    async fn load_iam_config_strict<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
-        let (data, _) =
-            read_config_preserve_empty_with_metadata_opts(self.object_api.clone(), path.as_ref(), &LoadMode::Locked.read_opts())
-                .await?;
-        Self::decode_iam_config_strict(&data)
-    }
     /// Saves IAM configuration with a retry mechanism on failure.
     ///
     /// Attempts to save the IAM configuration up to 5 times if the storage layer is not ready,
@@ -1470,40 +1458,6 @@ mod tests {
         assert!(result.is_err(), "corrupt stream_io data should fail decrypt");
     }
 
-    #[test]
-    fn strict_revocation_load_keeps_real_crypto_failure_distinct_from_absence() {
-        let cred = test_cred();
-        let key = format!("{}:{}", cred.access_key, cred.secret_key);
-        let plain = br#"{"version":1,"id":"revoked"}"#;
-        let mut encrypted = rustfs_crypto::encrypt_stream_io(key.as_bytes(), plain).expect("real stored ciphertext");
-        // Exercise the exact decoder used after the actual object read.
-        assert!(ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted).is_ok());
-        encrypted[50] ^= 0xFF;
-        let error = ObjectStore::decode_iam_config_strict::<serde_json::Value>(&encrypted)
-            .expect_err("corrupt tombstone must not become missing");
-        assert!(
-            matches!(error, Error::CryptoError(_)),
-            "C07_REVOCATION_FAIL_CLOSED: preserve actual crypto error"
-        );
-        assert!(
-            !crate::error::is_err_config_not_found(&error),
-            "C07_REVOCATION_FAIL_CLOSED: corrupted revocation is not absence"
-        );
-        // The preserve-empty reader keeps an existing zero-byte object as
-        // Ok(empty); ecstore's read_config_preserve_empty_distinguishes_empty_object
-        // covers that storage boundary. Its payload must fail here, not become
-        // ConfigNotFound and reactivate a revoked delegation.
-        let empty =
-            ObjectStore::decode_iam_config_strict::<serde_json::Value>(&[]).expect_err("an existing empty tombstone is corrupt");
-        assert!(
-            !crate::error::is_err_config_not_found(&empty),
-            "C07_REVOCATION_FAIL_CLOSED: empty existing revocation is not absence"
-        );
-        let invalid_json =
-            ObjectStore::decode_iam_config_strict::<serde_json::Value>(br#"{"version": }"#).expect_err("invalid stored JSON");
-        assert!(!crate::error::is_err_config_not_found(&invalid_json));
-    }
-
     #[test]
     fn test_decrypt_data_short_data_fails() {
         let short = &[0x00u8; 40]; // less than 41-byte stream_io header, not valid JSON

crates/iam/src/sys.rs

@@ -75,7 +75,6 @@ pub const STATUS_ENABLED: &str = "enabled";
 pub const STATUS_DISABLED: &str = "disabled";
 
 pub const POLICYNAME: &str = "policy";
-pub const PLUGIN_ATTEMPT_CREDENTIAL_CLAIM: &str = "rustfs:plugin-attempt-v1";
 pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
 pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
 const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-policy";
@@ -169,12 +168,6 @@ enum PreparedServicePolicyMode {
 
 #[derive(Clone)]
 enum PreparedIamMode {
-    TerminalDeny,
-    PluginAttemptSts {
-        is_owner: bool,
-        combined_policy: Policy,
-        session_policy: PreparedSessionPolicy,
-    },
     Opa,
     Owner,
     Deny,
@@ -209,7 +202,7 @@ impl PreparedIamAuth {
     pub async fn needs_existing_object_tag_for_args(&self, args: &Args<'_>) -> bool {
         match &self.mode {
             PreparedIamMode::Opa => true,
-            PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => false,
+            PreparedIamMode::Owner | PreparedIamMode::Deny => false,
             PreparedIamMode::Regular { combined_policy } => {
                 policy_needs_existing_object_tag_for_args(combined_policy, args).await
             }
@@ -217,11 +210,6 @@ impl PreparedIamAuth {
                 combined_policy,
                 session_policy,
                 ..
-            }
-            | PreparedIamMode::PluginAttemptSts {
-                combined_policy,
-                session_policy,
-                ..
             } => {
                 policy_needs_existing_object_tag_for_args(combined_policy, args).await
                     || prepared_session_policy_needs_existing_object_tag_for_args(session_policy, args).await
@@ -251,56 +239,14 @@ impl PreparedIamAuth {
     pub fn combined_policy_for_view(&self) -> Option<&Policy> {
         match &self.mode {
             PreparedIamMode::Regular { combined_policy } => Some(combined_policy),
-            PreparedIamMode::Sts { combined_policy, .. } | PreparedIamMode::PluginAttemptSts { combined_policy, .. } => {
-                Some(combined_policy)
-            }
+            PreparedIamMode::Sts { combined_policy, .. } => Some(combined_policy),
             PreparedIamMode::ServiceAccount { combined_policy, .. } => Some(combined_policy),
-            PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny | PreparedIamMode::TerminalDeny => None,
+            PreparedIamMode::Opa | PreparedIamMode::Owner | PreparedIamMode::Deny => None,
         }
     }
 }
 
 impl<T: Store> IamSys<T> {
-    /// For the authenticated, explicitly configured broker handler only.
-    pub async fn read_active_plugin_delegation_for_broker(
-        &self,
-        id: &str,
-        now: u64,
-    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
-        crate::delegation::read_active_for_broker(&self.store.api, id, now).await
-    }
-    pub async fn create_plugin_delegation(
-        &self,
-        owner: &str,
-        restrictions: crate::delegation::Restrictions,
-        now: u64,
-    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
-        crate::delegation::create(&self.store.api, owner, restrictions, now).await
-    }
-    pub async fn revoke_plugin_delegation(
-        &self,
-        owner: &str,
-        id: &str,
-    ) -> std::result::Result<(), crate::delegation::DelegationError> {
-        crate::delegation::revoke(&self.store.api, owner, id).await
-    }
-    pub async fn read_active_plugin_delegation(
-        &self,
-        owner: &str,
-        id: &str,
-        now: u64,
-    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
-        crate::delegation::read_active(&self.store.api, owner, id, now).await
-    }
-    /// Reads an owned delegation record for historical lookups, including expired or revoked records.
-    pub async fn read_owned_plugin_delegation(
-        &self,
-        owner: &str,
-        id: &str,
-    ) -> std::result::Result<crate::delegation::Delegation, crate::delegation::DelegationError> {
-        crate::delegation::read_owned(&self.store.api, owner, id).await
-    }
-
     /// Create a new IamSys instance with the given IamCache store
     ///
     /// # Arguments
@@ -335,16 +281,6 @@ impl<T: Store> IamSys<T> {
         *guard = PolicyPluginState::Ready(client);
     }
 
-    /// Restricted credentials require the built-in parent/session intersection.
-    /// External authorization bypasses that path, so only Disabled is supported.
-    pub async fn ensure_restricted_credential_vending() -> Result<()> {
-        if matches!(Self::policy_plugin_state().await, PolicyPluginState::Disabled) {
-            Ok(())
-        } else {
-            Err(Error::IAMActionNotAllowed)
-        }
-    }
-
     pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
         let policy_plugin_state = get_policy_plugin_state();
         let guard = policy_plugin_state.read().await;
@@ -1447,50 +1383,17 @@ impl<T: Store> IamSys<T> {
     /// 1) know whether policy evaluation may need `s3:ExistingObjectTag`, and
     /// 2) evaluate with final conditions without re-merging identity policies.
     pub async fn prepare_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
-        let marked = args.claims.contains_key(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
-        if args.is_owner && !marked {
+        if args.is_owner {
             return PreparedIamAuth {
                 needs_existing_object_tag: false,
                 mode: PreparedIamMode::Owner,
             };
         }
-        // One observed state decides this request's path. Marked credentials
-        // never enter OPA, including after a restart or on another node.
-        let state = Self::policy_plugin_state().await;
-        if marked {
-            if args.is_owner
-                || args.claims.get(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM) != Some(&Value::Bool(true))
-                || !matches!(state, PolicyPluginState::Disabled)
-            {
-                return PreparedIamAuth {
-                    needs_existing_object_tag: false,
-                    mode: PreparedIamMode::TerminalDeny,
-                };
-            }
-            let prepared = self.prepare_builtin_auth(args).await;
-            let mode = match prepared.mode {
-                PreparedIamMode::Sts {
-                    is_owner,
-                    combined_policy,
-                    session_policy,
-                } => PreparedIamMode::PluginAttemptSts {
-                    is_owner,
-                    combined_policy,
-                    session_policy,
-                },
-                _ => PreparedIamMode::TerminalDeny,
-            };
-            return PreparedIamAuth {
-                needs_existing_object_tag: prepared.needs_existing_object_tag,
-                mode,
-            };
-        }
-        if let Some(prepared) = state.prepared_iam_auth() {
+
+        if let Some(prepared) = Self::policy_plugin_state().await.prepared_iam_auth() {
             return prepared;
         }
-        self.prepare_builtin_auth(args).await
-    }
-    async fn prepare_builtin_auth(&self, args: &Args<'_>) -> PreparedIamAuth {
+
         let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
             return PreparedIamAuth {
                 needs_existing_object_tag: false,
@@ -1522,20 +1425,6 @@ impl<T: Store> IamSys<T> {
     /// bucket policies on an IAM denial. An error must not become an implicit deny.
     pub async fn try_eval_prepared(&self, prepared: &PreparedIamAuth, args: &Args<'_>) -> Result<bool> {
         Ok(match &prepared.mode {
-            PreparedIamMode::TerminalDeny => return Err(Error::IAMActionNotAllowed),
-            PreparedIamMode::PluginAttemptSts {
-                is_owner,
-                combined_policy,
-                session_policy,
-            } => {
-                // A false IAM result permits bucket-policy fallback. Restricted
-                // plugin credentials must instead terminate authorization.
-                let session_allowed = evaluate_prepared_session_policy(session_policy, args).await == Some(true);
-                if !session_allowed || !(*is_owner || combined_policy.is_allowed(args).await) {
-                    return Err(Error::IAMActionNotAllowed);
-                }
-                true
-            }
             PreparedIamMode::Opa => {
                 let Some(opa_enable) = Self::get_policy_plugin_client().await else {
                     tracing::warn!("eval_prepared: OPA mode requested but plugin is unavailable");
@@ -2238,145 +2127,6 @@ mod tests {
         assert!(prepared.combined_policy_for_view().is_none());
     }
 
-    #[tokio::test]
-    #[serial]
-    async fn marked_signed_sts_denial_is_terminal_before_public_bucket_fallback() {
-        let iam = IamSys::new(IamCache::new(StsTestMockStore::new(false)).await.unwrap());
-        let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
-        let signing_key = "c07-test-token-signing-key";
-        let mut metadata=HashMap::from([
-            ("parent".into(),Value::String("sts-fallback-test-parent".into())),
-            ("exp".into(),serde_json::json!((OffsetDateTime::now_utc()+time::Duration::hours(1)).unix_timestamp())),
-            (SESSION_POLICY_NAME.into(),Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/inside"]}]}"#))),
-        ]);
-        metadata.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), Value::Bool(true));
-        let mut marked = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
-        marked.parent_user = "sts-fallback-test-parent".into();
-        let marked_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&marked.session_token, signing_key)
-            .expect("signed test credentials must decode")
-            .claims;
-        let marked_user = UserIdentity::from(marked.clone());
-        iam.store
-            .cache
-            .add_or_update_sts_account(&marked.access_key, &marked_user, OffsetDateTime::now_utc());
-        metadata.remove(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM);
-        let mut ordinary = get_new_credentials_with_metadata(&metadata, signing_key).unwrap();
-        ordinary.parent_user = marked.parent_user.clone();
-        let ordinary_claims: HashMap<String, Value> = rustfs_policy::utils::extract_claims(&ordinary.session_token, signing_key)
-            .expect("signed test credentials must decode")
-            .claims;
-        iam.store.cache.add_or_update_sts_account(
-            &ordinary.access_key,
-            &UserIdentity::from(ordinary.clone()),
-            OffsetDateTime::now_utc(),
-        );
-        let groups = None;
-        let conditions = HashMap::new();
-        let args = Args {
-            account: &marked.access_key,
-            groups: &groups,
-            action: Action::S3Action(S3Action::GetObjectAction),
-            bucket: "mybucket",
-            object: "outside",
-            conditions: &conditions,
-            is_owner: false,
-            claims: &marked_claims,
-            deny_only: false,
-        };
-        *get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
-        let denied = iam.prepare_auth(&args).await;
-        let disabled_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
-        let inside = Args {
-            object: "inside",
-            ..args.clone()
-        };
-        let allowed = iam.prepare_auth(&inside).await;
-        let inside_allowed = matches!(iam.try_eval_prepared(&allowed, &inside).await, Ok(true));
-        let unmarked_args = Args {
-            account: &ordinary.access_key,
-            claims: &ordinary_claims,
-            ..args.clone()
-        };
-        let unmarked = iam.prepare_auth(&unmarked_args).await;
-        let ordinary_false = matches!(iam.try_eval_prepared(&unmarked, &unmarked_args).await, Ok(false));
-        let public:rustfs_policy::policy::BucketPolicy=serde_json::from_value(serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]})).unwrap();
-        let public_allows = public
-            .is_allowed(&rustfs_policy::policy::BucketPolicyArgs {
-                account: &marked.access_key,
-                groups: &groups,
-                action: args.action,
-                bucket: args.bucket,
-                object: args.object,
-                conditions: &conditions,
-                is_owner: false,
-            })
-            .await;
-        IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
-            url: "http://127.0.0.1:8181/allow".into(),
-            auth_token: String::new(),
-        }))
-        .await;
-        let opa_marked = iam.prepare_auth(&args).await;
-        let ready_terminal = matches!(iam.try_eval_prepared(&opa_marked, &args).await, Err(Error::IAMActionNotAllowed));
-        let opa_ordinary = iam.prepare_auth(&unmarked_args).await;
-        let ordinary_keeps_opa = matches!(opa_ordinary.mode, PreparedIamMode::Opa);
-        // The captured Disabled preparation stays on built-in intersection even
-        // if an external policy client becomes ready before evaluation.
-        let captured_still_terminal = matches!(iam.try_eval_prepared(&denied, &args).await, Err(Error::IAMActionNotAllowed));
-        *get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
-        let pending = iam.prepare_auth(&args).await;
-        let initializing_terminal = matches!(iam.try_eval_prepared(&pending, &args).await, Err(Error::IAMActionNotAllowed));
-        *get_policy_plugin_state().write().await = PolicyPluginState::Failed;
-        let failed = iam.prepare_auth(&args).await;
-        let failed_terminal = matches!(iam.try_eval_prepared(&failed, &args).await, Err(Error::IAMActionNotAllowed));
-        *get_policy_plugin_state().write().await = previous;
-        assert!(public_allows, "C07_PUBLIC_BUCKET_CONTROL: real public policy permits outside object");
-        assert!(
-            disabled_terminal && captured_still_terminal,
-            "C07_SCOPE_TERMINAL_BOUND: bucket policy must not rescue restricted scope denial"
-        );
-        assert!(
-            ready_terminal && initializing_terminal && failed_terminal,
-            "C07_EXTERNAL_POLICY_BOUND: marked persisted STS cannot enter OPA"
-        );
-        assert!(
-            inside_allowed && ordinary_false && ordinary_keeps_opa,
-            "C07_UNMARKED_COMPAT: preserve ordinary STS and valid marked scope"
-        );
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn restricted_vending_rejects_actual_external_policy_plugin_state() {
-        let previous = IamSys::<StsTestMockStore>::policy_plugin_state().await;
-        IamSys::<StsTestMockStore>::set_policy_plugin_client(opa::AuthZPlugin::new(opa::Args {
-            url: "http://127.0.0.1:8181/v1/data/rustfs/authz/allow".into(),
-            auth_token: String::new(),
-        }))
-        .await;
-        let active_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
-            .await
-            .is_err();
-        *get_policy_plugin_state().write().await = PolicyPluginState::Initializing;
-        let initializing_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
-            .await
-            .is_err();
-        *get_policy_plugin_state().write().await = PolicyPluginState::Failed;
-        let failed_rejected = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
-            .await
-            .is_err();
-        *get_policy_plugin_state().write().await = PolicyPluginState::Disabled;
-        let disabled_allowed = IamSys::<StsTestMockStore>::ensure_restricted_credential_vending()
-            .await
-            .is_ok();
-        *get_policy_plugin_state().write().await = previous;
-        assert!(
-            active_rejected && initializing_rejected && failed_rejected,
-            "C07_EXTERNAL_POLICY_BOUND: external authorization cannot bypass minted session scope"
-        );
-        assert!(disabled_allowed);
-    }
-
     #[tokio::test]
     #[serial]
     async fn test_prepare_auth_requests_existing_object_tags_in_opa_mode() {

crates/policy/src/policy/action.rs

@@ -568,24 +568,6 @@ pub enum AdminAction {
     GetTableLifecycleAction,
     #[strum(serialize = "admin:SetTableLifecycle")]
     SetTableLifecycleAction,
-    #[strum(serialize = "admin:CreatePluginDelegation")]
-    CreatePluginDelegationAction,
-    #[strum(serialize = "admin:IssuePluginAttempt")]
-    IssuePluginAttemptAction,
-    #[strum(serialize = "admin:SubmitPluginJob")]
-    SubmitPluginJobAction,
-    #[strum(serialize = "admin:GetPluginRun")]
-    GetPluginRunAction,
-    #[strum(serialize = "admin:CancelPluginRun")]
-    CancelPluginRunAction,
-    #[strum(serialize = "admin:ListPluginRuns")]
-    ListPluginRunsAction,
-    #[strum(serialize = "admin:ReplayPluginRun")]
-    ReplayPluginRunAction,
-    #[strum(serialize = "admin:GetPluginDelegation")]
-    GetPluginDelegationAction,
-    #[strum(serialize = "admin:RevokePluginDelegation")]
-    RevokePluginDelegationAction,
     #[strum(serialize = "admin:GetTableCredentials")]
     GetTableCredentialsAction,
     #[strum(serialize = "admin:RunTableMaintenance")]
@@ -731,15 +713,6 @@ impl AdminAction {
                 | AdminAction::GetTableLifecycleAction
                 | AdminAction::SetTableLifecycleAction
                 | AdminAction::GetTableCredentialsAction
-                | AdminAction::IssuePluginAttemptAction
-                | AdminAction::SubmitPluginJobAction
-                | AdminAction::GetPluginRunAction
-                | AdminAction::CancelPluginRunAction
-                | AdminAction::ListPluginRunsAction
-                | AdminAction::ReplayPluginRunAction
-                | AdminAction::GetPluginDelegationAction
-                | AdminAction::CreatePluginDelegationAction
-                | AdminAction::RevokePluginDelegationAction
                 | AdminAction::RunTableMaintenanceAction
                 | AdminAction::GetTableMetadataLocationAction
                 | AdminAction::SetTableMetadataLocationAction
@@ -940,35 +913,6 @@ mod tests {
         assert_eq!(arr[0].as_str().unwrap(), "s3:*");
     }
 
-    #[test]
-    fn plugin_delegation_actions_are_explicit_and_unscoped() {
-        for (name, expected) in [
-            ("admin:GetPluginDelegation", AdminAction::GetPluginDelegationAction),
-            ("admin:CreatePluginDelegation", AdminAction::CreatePluginDelegationAction),
-            ("admin:RevokePluginDelegation", AdminAction::RevokePluginDelegationAction),
-            ("admin:SubmitPluginJob", AdminAction::SubmitPluginJobAction),
-            ("admin:GetPluginRun", AdminAction::GetPluginRunAction),
-            ("admin:CancelPluginRun", AdminAction::CancelPluginRunAction),
-            ("admin:ListPluginRuns", AdminAction::ListPluginRunsAction),
-            ("admin:ReplayPluginRun", AdminAction::ReplayPluginRunAction),
-        ] {
-            assert_eq!(AdminAction::try_from(name).unwrap(), expected);
-            assert!(expected.is_valid());
-            assert!(!expected.is_table_resource_scoped());
-        }
-        assert_eq!(
-            AdminAction::try_from("admin:IssuePluginAttempt").unwrap(),
-            AdminAction::IssuePluginAttemptAction
-        );
-        assert!(AdminAction::IssuePluginAttemptAction.is_valid());
-        assert!(AdminAction::SubmitPluginJobAction.is_valid());
-        assert!(AdminAction::GetPluginRunAction.is_valid());
-        assert!(AdminAction::CancelPluginRunAction.is_valid());
-        assert!(AdminAction::ListPluginRunsAction.is_valid());
-        assert!(AdminAction::ReplayPluginRunAction.is_valid());
-        assert!(AdminAction::try_from("admin:IssuePluginCredentials").is_err());
-    }
-
     #[test]
     fn test_get_replication_metrics_admin_action_is_valid() {
         assert!(AdminAction::GetReplicationMetricsAction.is_valid());

crates/scanner/src/error.rs

@@ -53,10 +53,6 @@ pub enum ScannerError {
     #[error("Remote namespace scanner disk is already active")]
     RemoteDiskBusy,
 
-    /// Node resource pressure deferred work before its first storage poll.
-    #[error("Scanner node resource admission deferred")]
-    ResourceDeferred,
-
     /// Partial data usage cache produced before the scanner stopped.
     #[error("Scanner stopped with partial data usage cache")]
     PartialCache(Box<DataUsageCache>),

crates/scanner/src/lib.rs

@@ -50,9 +50,9 @@ use storage_api::owner::{
 };
 #[cfg(test)]
 use storage_api::owner::{
-    EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreExpiryState,
-    EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
-    ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
+    EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreInstanceContext,
+    EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
+    ecstore_new_disk,
 };
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
@@ -123,7 +123,7 @@ pub use storage_api::scan::{
     SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
     SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
 };
-pub use workload_admission::{set_scanner_node_admission, set_scanner_workload_admission_snapshot_provider};
+pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
 
 static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
 static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
@@ -678,37 +678,8 @@ fn reset_tier_name_cache_for_test() {
     TIER_REGISTRY_GENERATION.store(0, Ordering::Relaxed);
 }
 
-// Only a task running an isolated unit test can override these handles.
-#[cfg(test)]
-struct ExpiryLockTestContext {
-    state: Arc<tokio::sync::RwLock<EcstoreExpiryState>>,
-    store: Option<Arc<ECStore>>,
-    completed_incarnation_reads: Arc<std::sync::atomic::AtomicUsize>,
-}
-
-#[cfg(test)]
-tokio::task_local! {
-    static EXPIRY_LOCK_TEST_CONTEXT: ExpiryLockTestContext;
-}
-
-pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo, cancellation: Option<&CancellationToken>) {
-    #[cfg(test)]
-    let state = EXPIRY_LOCK_TEST_CONTEXT
-        .try_with(|context| context.state.clone())
-        .unwrap_or_else(|_| ecstore_expiry_state_handle());
-    #[cfg(not(test))]
-    let state = ecstore_expiry_state_handle();
-    let state = match cancellation {
-        Some(cancel) => tokio::select! {
-            biased;
-            _ = cancel.cancelled() => return,
-            state = state.write() => state,
-        },
-        None => state.write().await,
-    };
-    if !cancellation.is_some_and(CancellationToken::is_cancelled) {
-        state.enqueue_free_version(oi);
-    }
+pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo) {
+    ecstore_expiry_state_handle().write().await.enqueue_free_version(oi);
 }
 
 pub(crate) async fn enqueue_runtime_newer_noncurrent(
@@ -716,50 +687,20 @@ pub(crate) async fn enqueue_runtime_newer_noncurrent(
     to_delete_objs: Vec<ObjectToDelete>,
     event: Event,
     src: &LcEventSrc,
-    cancellation: Option<&CancellationToken>,
 ) -> bool {
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return false;
-    }
-    #[cfg(test)]
-    let store = EXPIRY_LOCK_TEST_CONTEXT
-        .try_with(|context| context.store.clone())
-        .unwrap_or_else(|_| ecstore_resolve_object_store_handle());
-    #[cfg(not(test))]
-    let store = ecstore_resolve_object_store_handle();
-    let Some(store) = store else {
+    let Some(store) = ecstore_resolve_object_store_handle() else {
         return false;
     };
     let Ok(bucket_incarnation_id) = store.bucket_incarnation_id(bucket).await else {
         return false;
     };
-    #[cfg(test)]
-    let _ = EXPIRY_LOCK_TEST_CONTEXT.try_with(|context| {
-        context
-            .completed_incarnation_reads
-            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-    });
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return false;
-    }
-    #[cfg(test)]
-    let state = EXPIRY_LOCK_TEST_CONTEXT
-        .try_with(|context| context.state.clone())
-        .unwrap_or_else(|_| ecstore_expiry_state_handle());
-    #[cfg(not(test))]
-    let state = ecstore_expiry_state_handle();
-    let mut state = match cancellation {
-        Some(cancel) => tokio::select! {
-            biased;
-            _ = cancel.cancelled() => return false,
-            state = state.write() => state,
-        },
-        None => state.write().await,
-    };
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return false;
-    }
-    state.enqueue_by_newer_noncurrent(bucket, to_delete_objs, event, src, bucket_incarnation_id)
+    ecstore_expiry_state_handle().write().await.enqueue_by_newer_noncurrent(
+        bucket,
+        to_delete_objs,
+        event,
+        src,
+        bucket_incarnation_id,
+    )
 }
 
 pub(crate) async fn queue_replication_heal(

crates/scanner/src/remote_scanner/stream.rs

@@ -705,7 +705,7 @@ async fn scan_and_persist_local_bucket(
         RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement")
     })?;
     let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
-    let mut guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
+    let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
         .await
         .map_err(|err| {
             if err.is_contention() {
@@ -791,23 +791,19 @@ async fn scan_and_persist_local_bucket(
 
     let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
     let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
-    let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
     let scan_ctx = ctx.child_token();
-    let scan = crate::workload_admission::with_scanner_node_requests(
-        node_tx,
-        ScannerIODisk::nsscanner_disk(
-            disk.clone(),
-            scan_ctx.clone(),
-            budget,
-            set_disks,
-            cache,
-            None,
-            ScannerDiskScanOptions {
-                scan_mode,
-                prefix_scan_scope: None,
-                checkpoint_tx: Some(checkpoint_tx),
-            },
-        ),
+    let scan = ScannerIODisk::nsscanner_disk(
+        disk.clone(),
+        scan_ctx.clone(),
+        budget,
+        set_disks,
+        cache,
+        None,
+        ScannerDiskScanOptions {
+            scan_mode,
+            prefix_scan_scope: None,
+            checkpoint_tx: Some(checkpoint_tx),
+        },
     );
     tokio::pin!(scan);
     let fence_watch = watch_remote_scanner_request_fence(next_cycle, leader_epoch, store.clone(), NS_SCANNER_FENCE_POLL_INTERVAL);
@@ -817,27 +813,6 @@ async fn scan_and_persist_local_bucket(
     let mut checkpoint_channel_closed = false;
     let outcome = loop {
         tokio::select! {
-            request = async {
-                match node_rx.as_mut() {
-                    Some(receiver) => receiver.recv().await,
-                    None => std::future::pending().await,
-                }
-            } => {
-                let Some(request) = request else {
-                    node_rx = None;
-                    continue;
-                };
-                if let Err(error) = wait_for_remote_node_resumption(
-                    guard.resume_node_operation(set.clone(), &cache_name, source, &revisions, request),
-                    fence_watch.as_mut(),
-                ).await {
-                    scan_ctx.cancel();
-                    let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
-                    return Err(RemoteScannerServerError::retry_bucket(format!(
-                        "remote namespace scanner could not resume node admission: {error}",
-                    )));
-                }
-            }
             result = &mut scan => {
                 match result {
                     Ok(outcome) => break outcome,
@@ -1146,22 +1121,6 @@ pub(crate) async fn scan_remote_bucket(
     result
 }
 
-/// Resource waits must keep checking the persisted leader fence.
-async fn wait_for_remote_node_resumption<F, G>(resume: F, fence_watch: Pin<&mut G>) -> Result<(), StorageError>
-where
-    F: Future<Output = Result<(), StorageError>>,
-    G: Future<Output = Result<(), ScannerError>>,
-{
-    tokio::select! {
-        biased;
-        result = fence_watch => Err(match result {
-            Ok(()) => StorageError::other("remote namespace scanner fence watcher stopped unexpectedly"),
-            Err(error) => StorageError::other(error),
-        }),
-        result = resume => result,
-    }
-}
-
 fn finish_remote_scanner_stream(
     result: RemoteScannerStreamResult<RemoteScannerOutcome>,
     budget: &ScannerCycleBudget,

crates/scanner/src/remote_scanner/stream/tests.rs

@@ -1006,58 +1006,6 @@ async fn fence_watcher_stops_work_after_persisted_leader_changes() {
     assert!(err.to_string().contains("requested_epoch=9"));
 }
 
-#[tokio::test]
-async fn node_admission_wait_does_not_hide_replacement_leader() {
-    use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
-    use std::num::NonZeroU16;
-
-    let store = Arc::new(MutableCycleStateStore {
-        state: RwLock::new(crate::scanner::encode_scanner_cycle_fence_for_test(7, 9)),
-    });
-    let cache = Mutex::new(None);
-    let refresh = AsyncMutex::new(());
-    let watcher = watch_remote_scanner_request_fence_with_cache(
-        7,
-        9,
-        store.clone(),
-        Duration::from_millis(5),
-        &cache,
-        &refresh,
-        Duration::from_millis(5),
-    );
-    tokio::pin!(watcher);
-    let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one shared node unit"));
-    let storage_polled = AtomicBool::new(false);
-    {
-        let resume = async {
-            let _permit = gate.acquire().await;
-            storage_polled.store(true, Ordering::Relaxed);
-            Ok(())
-        };
-        let waiting = wait_for_remote_node_resumption(resume, watcher.as_mut());
-        tokio::pin!(waiting);
-        assert!(futures::poll!(&mut waiting).is_pending(), "no resource sample grants a node unit");
-        *store.state.write().expect("persist replacement leader") = crate::scanner::encode_scanner_cycle_fence_for_test(7, 10);
-        let error = tokio::time::timeout(Duration::from_millis(100), waiting)
-            .await
-            .expect("resource pause must not hide a changed leader")
-            .expect_err("old leader cannot resume storage work");
-        assert!(error.to_string().contains("requested_epoch=9"));
-    }
-    assert!(!storage_polled.load(Ordering::Relaxed));
-    assert_eq!(gate.snapshot().active, 0);
-    gate.update(
-        &mut FeedbackController::new(),
-        Some(FeedbackObservation {
-            generation: 1,
-            max_concurrency: 1,
-            pressure: ResourcePressure::Unknown,
-            completed_units: None,
-        }),
-    );
-    assert!(gate.try_acquire().is_some(), "failed fencing drops the old node queue ticket");
-}
-
 #[tokio::test]
 async fn concurrent_cycle_cache_misses_share_one_backend_read() {
     let request = RemoteScannerRequest(test_request(Uuid::new_v4()));

crates/scanner/src/scanner/tests.rs

@@ -5440,8 +5440,8 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
         .await
         .expect_err("truncated primary must fail in the real object reader");
     assert!(
-        matches!(error, EcstoreError::FileCorrupt),
-        "unexpected truncated-primary error: {error:?}"
+        error.to_string().contains("InlineData value out of range"),
+        "unexpected truncated-primary error: {error}"
     );
     assert_eq!(
         read_config_with_revision(store.clone(), &backup_path)

crates/scanner/src/scanner_folder.rs

@@ -21,10 +21,10 @@ use std::time::{Duration, Instant, SystemTime};
 
 use crate::ReplTargetSizeSummary;
 use crate::data_usage_define::{
-    DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageHash,
-    DataUsageHashMap, DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason,
-    MAX_DATA_USAGE_CACHE_DEPTH, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry,
-    SizeSummary, hash_path, scan_frontier_covers_path,
+    DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
+    DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH,
+    PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
+    scan_frontier_covers_path,
 };
 use crate::error::ScannerError;
 use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter};
@@ -43,7 +43,6 @@ use crate::storage_api::owner::{
 };
 #[cfg(test)]
 use crate::storage_api::owner::{EcstoreExpirationStatus as ExpirationStatus, EcstoreLifecycleRule as LifecycleRule};
-use crate::workload_admission::run_scanner_node_operation;
 use metrics::{counter, describe_counter};
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
 use rustfs_filemeta::{
@@ -796,7 +795,6 @@ pub struct FolderScanner {
     resolved_failed_object_paths: HashSet<String>,
 
     sleeper: DynamicSleeper,
-    node_admission: Option<Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
     // should_heal: Arc<dyn Fn() -> bool + Send + Sync>,
     disks: Vec<Arc<Disk>>,
     disks_quorum: usize,
@@ -830,8 +828,6 @@ pub struct FolderScanner {
     pending_size_reconciliation_truncated: bool,
     #[cfg(test)]
     list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
-    #[cfg(test)]
-    defer_abandoned_entries_until_finished: bool,
 }
 
 struct RawEnumerationProgress {
@@ -1002,13 +998,12 @@ async fn next_scanner_dir_entry(
     ctx: &CancellationToken,
     sleeper: &DynamicSleeper,
     entries: &mut ScannerDirectory,
-    admission: Option<&Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
 ) -> Result<Option<tokio::fs::DirEntry>, ScannerError> {
     if entries.batch_entries == SCANNER_DIRECTORY_BATCH_ENTRIES {
         wait_for_scanner_io(ctx, tokio::task::yield_now()).await?;
         entries.batch_entries = 0;
     }
-    let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.next_entry())).await???;
+    let entry = wait_for_scanner_io(ctx, entries.next_entry()).await??;
     if entry.is_some() {
         // Tokio prefetches bounded chunks. Pacing consumption also paces the
         // next refill, without charging an extra entry for EOF.
@@ -1608,7 +1603,7 @@ impl FolderScanner {
         let candidate_type = pending_scanner_heal_candidate_type(kind);
         let priority = request.priority;
         let scan_mode = request.scan_mode.unwrap_or(self.scan_mode);
-        let result = match send_scanner_heal_request(candidate_type, request, scanner_disk_heal_scope(&self.local_disk)).await {
+        let result = match send_scanner_heal_request(candidate_type, request).await {
             Ok(result) => result,
             Err(err) => {
                 self.update_pending_scanner_heal_after_admission(
@@ -1928,12 +1923,7 @@ impl FolderScanner {
             // directory that vanished during iteration instead of failing the
             // whole parent subtree. The bucket root (no parent) still propagates
             // NotFound so namespace removal is detected by the caller.
-            let mut dir_reader = match wait_for_scanner_io(
-                &ctx,
-                run_scanner_node_operation(self.node_admission.as_ref(), &ctx, ScannerDirectory::open(&dir_path)),
-            )
-            .await??
-            {
+            let mut dir_reader = match wait_for_scanner_io(&ctx, ScannerDirectory::open(&dir_path)).await? {
                 Ok(dir_reader) => Some(dir_reader),
                 Err(e)
                     if folder.parent.is_some()
@@ -1971,7 +1961,7 @@ impl FolderScanner {
                 // Hold at most one metadata entry until a candidate reaches
                 // EOF: a later part must not leave object accounting or ILM work.
                 let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut() {
-                    let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader, self.node_admission.as_ref()).await {
+                    let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader).await {
                         Ok(Some(entry)) => entry,
                         Ok(None) => {
                             raw_enumeration_complete = true;
@@ -2082,12 +2072,7 @@ impl FolderScanner {
 
                 // Ignore entries that disappeared during traversal or hit symlink
                 // loops, but propagate other walk errors.
-                let mut entry_type = match wait_for_scanner_io(
-                    &ctx,
-                    run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()),
-                )
-                .await??
-                {
+                let mut entry_type = match wait_for_scanner_io(&ctx, entry.file_type()).await? {
                     Ok(entry_type) => entry_type,
                     Err(e) if e.kind() == ErrorKind::NotFound => {
                         debug!(
@@ -2138,12 +2123,7 @@ impl FolderScanner {
                 let mut entry_metadata_len = None;
                 if entry_type.is_symlink() {
                     wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
-                    let metadata = match wait_for_scanner_io(
-                        &ctx,
-                        run_scanner_node_operation(self.node_admission.as_ref(), &ctx, tokio::fs::metadata(&file_path)),
-                    )
-                    .await??
-                    {
+                    let metadata = match wait_for_scanner_io(&ctx, tokio::fs::metadata(&file_path)).await? {
                         Ok(metadata) => metadata,
                         Err(e) if e.kind() == ErrorKind::NotFound => {
                             debug!(
@@ -2269,7 +2249,6 @@ impl FolderScanner {
                     lifecycle: active_life_cycle.clone(),
                     object_lock: active_object_lock.clone(),
                     replication: active_replication.clone(),
-                    heal_scope: None,
                     heal_enabled,
                     heal_bitrot: self.scan_mode == HealScanMode::Deep,
                     debug: self.data_usage_scanner_debug,
@@ -2295,30 +2274,22 @@ impl FolderScanner {
                         len
                     } else {
                         wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
-                        wait_for_scanner_io(
-                            &ctx,
-                            run_scanner_node_operation(self.node_admission.as_ref(), &ctx, tokio::fs::metadata(&item.path)),
-                        )
-                        .await??
-                        .map(|metadata| metadata.len())
-                        .unwrap_or(0)
+                        wait_for_scanner_io(&ctx, tokio::fs::metadata(&item.path))
+                            .await?
+                            .map(|metadata| metadata.len())
+                            .unwrap_or(0)
                     }
                 } else {
                     0
                 };
                 wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, budgeted_bytes)).await?;
 
-                let size_result = run_scanner_node_operation(self.node_admission.as_ref(), &ctx, async {
-                    self.object_metadata_visits.increment(1);
-                    self.local_disk
-                        .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
-                        .await
-                })
-                .await?;
-                if self.node_admission.is_some() && ctx.is_cancelled() {
-                    return Err(ScannerError::Other("Operation cancelled".to_string()));
-                }
-                let sz = match size_result {
+                self.object_metadata_visits.increment(1);
+                let sz = match self
+                    .local_disk
+                    .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
+                    .await
+                {
                     Ok(sz) => sz,
                     Err(e) => {
                         let failure_action = classify_get_size_failure(&item, &e);
@@ -2635,7 +2606,7 @@ impl FolderScanner {
                     {
                         Ok(outcome) => outcome,
                         Err(e) => {
-                            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
+                            if ctx.is_cancelled() {
                                 self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
                                     .await;
                                 return Err(e);
@@ -2876,23 +2847,19 @@ impl FolderScanner {
                 let mut seen_truncated_objects: HashSet<String> = HashSet::new();
 
                 loop {
-                    if child_ctx.is_cancelled() || (agreed_closed && partial_closed && finished_closed) {
+                    if agreed_closed && partial_closed && finished_closed {
                         break;
                     }
-                    #[cfg(test)]
-                    let defer_entries = self.defer_abandoned_entries_until_finished && !finished_closed;
-                    #[cfg(not(test))]
-                    let defer_entries = false;
 
                     select! {
-                        entry_name = agreed_rx.recv(), if !agreed_closed && !defer_entries => {
+                        entry_name = agreed_rx.recv(), if !agreed_closed => {
                             let Some(entry_name) = entry_name else {
                                 agreed_closed = true;
                                 continue;
                             };
                             (self.update_current_path)(&entry_name).await;
                         }
-                        entries = partial_rx.recv(), if !partial_closed && !defer_entries => {
+                        entries = partial_rx.recv(), if !partial_closed => {
                             let Some(entries) = entries else {
                                 partial_closed = true;
                                 continue;
@@ -3018,7 +2985,6 @@ impl FolderScanner {
                                 finished_closed = true;
                                 continue;
                             };
-                            finished_closed = true;
                             if disk_errors_are_only_missing_paths(&errs) {
                                 debug!(
                                     target: "rustfs::scanner::folder",
@@ -3040,9 +3006,7 @@ impl FolderScanner {
                                     "Scanner list_path_raw finished with disk errors"
                                 );
                             }
-                            // The producer awaited callback sends, not their
-                            // consumption. Drain the remaining candidate and
-                            // agreed frames before closing this child walk.
+                            child_ctx.cancel();
                         }
                         _ = child_ctx.cancelled() => {
                             break;
@@ -3060,10 +3024,7 @@ impl FolderScanner {
                     if into.compacted {
                         // In compacted mode child totals are accumulated directly into the parent entry.
                         let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), into));
-                        fut.await.map_err(|error| match error {
-                            ScannerError::ResourceDeferred => ScannerError::ResourceDeferred,
-                            other => ScannerError::Other(other.to_string()),
-                        })?;
+                        fut.await.map_err(|e| ScannerError::Other(e.to_string()))?;
                         self.send_update_for_entry(&this_hash, &folder.parent, into).await;
                         tokio::task::yield_now().await;
                     } else {
@@ -3082,7 +3043,7 @@ impl FolderScanner {
                             )
                             .await
                         {
-                            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
+                            if ctx.is_cancelled() {
                                 self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
                                     .await;
                                 return Err(e);
@@ -3256,7 +3217,6 @@ pub(crate) async fn scan_data_folder_scoped(
         failed_object_paths_seen: HashSet::new(),
         resolved_failed_object_paths: HashSet::new(),
         sleeper,
-        node_admission: crate::workload_admission::scanner_node_admission(),
         disks,
         disks_quorum,
         usage_updates,
@@ -3284,8 +3244,6 @@ pub(crate) async fn scan_data_folder_scoped(
         pending_size_reconciliation_truncated: false,
         #[cfg(test)]
         list_path_raw_options_observer: None,
-        #[cfg(test)]
-        defer_abandoned_entries_until_finished: false,
     };
 
     let now = FolderScanner::now_secs();
@@ -3375,7 +3333,7 @@ pub(crate) async fn scan_data_folder_scoped(
             }
         }
         Err(e) => {
-            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
+            if ctx.is_cancelled() {
                 let root_hash = hash_path(&bucket_name);
                 let root_has_progress = data_usage_root_has_progress(&root);
                 let pending_heals_changed = scanner.pending_heals_changed;
@@ -3450,19 +3408,6 @@ pub(crate) async fn scan_data_folder_scoped(
                 close_disk_guard.close().await;
                 return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
             }
-            if matches!(&e, ScannerError::ResourceDeferred) {
-                // Admission pressure is not disk failure or an empty census.
-                // Keep the previous tree when no new useful progress exists.
-                let mut partial_cache = scanner.old_cache;
-                if partial_cache.cache.is_empty() {
-                    partial_cache
-                        .cache
-                        .insert(hash_path(&bucket_name).0, DataUsageEntry::default());
-                }
-                partial_cache.info.snapshot_complete = false;
-                close_disk_guard.close().await;
-                return Err(ScannerError::PartialCache(Box::new(partial_cache)));
-            }
             close_disk_guard.close().await;
             // No useful information, return original cache
             Err(e)
@@ -3473,7 +3418,6 @@ pub(crate) async fn scan_data_folder_scoped(
 mod item_actions;
 mod ledger;
 
-pub(crate) use item_actions::scanner_disk_heal_scope;
 use item_actions::*;
 pub use item_actions::{GetSizeFn, ScannerItem};
 use ledger::*;
@@ -3483,15 +3427,3 @@ pub(crate) use tests::checkpoint_fixture::run_checkpoint_fixture;
 
 #[cfg(test)]
 mod tests;
-
-#[cfg(test)]
-tokio::task_local! {
-    pub(crate) static TEST_SIZE_DISPATCH_BOUNDARIES: tokio::sync::mpsc::UnboundedSender<&'static str>;
-}
-
-#[cfg(test)]
-pub(crate) fn observe_test_size_dispatch_boundary(stage: &'static str) -> bool {
-    TEST_SIZE_DISPATCH_BOUNDARIES
-        .try_with(|sender| sender.send(stage).is_ok())
-        .unwrap_or(false)
-}

crates/scanner/src/scanner_folder/item_actions.rs

@@ -474,23 +474,10 @@ pub(super) fn record_heal_candidate_admission(
     }
 }
 
-pub(crate) fn scanner_disk_heal_scope(disk: &Disk) -> Option<DataUsageCacheSource> {
-    let location = disk.get_disk_location();
-    location.valid().then_some(())?;
-    Some(DataUsageCacheSource::new(location.pool_idx?, location.set_idx?))
-}
-
 pub(super) async fn send_scanner_heal_request(
     candidate_type: &'static str,
-    mut request: HealChannelRequest,
-    scope: Option<DataUsageCacheSource>,
+    request: HealChannelRequest,
 ) -> Result<HealAdmissionResult, ScannerError> {
-    if request.object_prefix.as_deref().is_some_and(|object| !object.is_empty()) {
-        let scope = scope
-            .ok_or_else(|| ScannerError::Other("scanner object heal requires the scanning disk's pool and set".to_string()))?;
-        request.pool_index = Some(scope.pool_index);
-        request.set_index = Some(scope.set_index);
-    }
     let priority = request.priority;
     let trace_context = scanner_heal_candidate_trace_context(&request);
     let candidate = crate::heal_completion::prepare(&request);
@@ -550,8 +537,6 @@ pub struct ScannerItem {
     pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
     pub object_lock: Option<Arc<ObjectLockConfiguration>>,
     pub replication: Option<Arc<ReplicationConfig>>,
-    /// Actual scanning-disk location; refreshed by the disk reader before actions.
-    pub heal_scope: Option<DataUsageCacheSource>,
     pub heal_enabled: bool,
     pub heal_bitrot: bool,
     pub debug: bool,
@@ -643,22 +628,6 @@ impl ScannerItem {
         tier_names: &[String],
         size_summary: &mut SizeSummary,
     ) {
-        self.apply_actions_cancellable(object_infos, lock_retention, versioning_config, tier_names, size_summary, None)
-            .await;
-    }
-
-    pub(crate) async fn apply_actions_cancellable(
-        &mut self,
-        object_infos: Vec<ObjectInfo>,
-        lock_retention: Option<Arc<ObjectLockConfiguration>>,
-        versioning_config: VersioningConfiguration,
-        tier_names: &[String],
-        size_summary: &mut SizeSummary,
-        cancellation: Option<&CancellationToken>,
-    ) {
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return;
-        }
         let object_path = self.object_path();
         if object_infos.is_empty() {
             debug!(
@@ -702,9 +671,6 @@ impl ScannerItem {
         let Some(lifecycle) = self.lifecycle.clone() else {
             let mut cumulative_size: i64 = 0;
             for (oi, resolved_size) in object_infos.iter().zip(resolved_sizes.iter()) {
-                if cancellation.is_some_and(CancellationToken::is_cancelled) {
-                    return;
-                }
                 let accounting_size = match resolved_size {
                     SizeResolution::Known { logical, .. } => *logical,
                     // A valid compressed legacy sentinel has no logical size,
@@ -712,7 +678,7 @@ impl ScannerItem {
                     // physical size is only an input to those operations; it
                     // is not folded into the logical total below.
                     SizeResolution::Unknown { physical, .. } => {
-                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                        self.heal_actions(oi, *physical, size_summary).await;
                         size_summary.actions_accounting_unknown(oi);
                         continue;
                     }
@@ -722,16 +688,13 @@ impl ScannerItem {
                     }
                 };
 
-                let size = self.heal_actions(oi, accounting_size, size_summary, cancellation).await;
+                let size = self.heal_actions(oi, accounting_size, size_summary).await;
 
                 size_summary.actions_accounting(oi, size, accounting_size);
 
                 cumulative_size = cumulative_size.saturating_add(size);
             }
 
-            if cancellation.is_some_and(CancellationToken::is_cancelled) {
-                return;
-            }
             self.alert_excessive_versions(object_infos.len(), cumulative_size);
 
             debug!(
@@ -773,10 +736,6 @@ impl ScannerItem {
             }
         };
 
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return;
-        }
-
         // Every version handed to the evaluator counts as an ILM-checked
         // version, whether or not a rule matched (NoneAction included). This is
         // reached only for buckets with lifecycle rules; it feeds
@@ -795,9 +754,6 @@ impl ScannerItem {
             .all(|candidate| Self::tier_is_known(candidate, tier_names));
         'eventLoop: {
             for (i, event) in events.iter().enumerate() {
-                if cancellation.is_some_and(CancellationToken::is_cancelled) {
-                    return;
-                }
                 let oi = &object_infos[i];
                 let known_size = resolved_sizes[i].known_size();
                 if has_corrupt_size
@@ -816,7 +772,7 @@ impl ScannerItem {
                     // accounting. Size-filtered or deferred events remain
                     // pending, so retain the version-only physical counters.
                     if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                        self.heal_actions(oi, *physical, size_summary).await;
                         size_summary.actions_accounting_unknown(oi);
                     }
                     continue;
@@ -873,7 +829,7 @@ impl ScannerItem {
                                     noncurrent_unknown.push(oi);
                                 } else {
                                     if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                                        self.heal_actions(oi, *physical, size_summary).await;
                                     }
                                     size_summary.actions_accounting_unknown(oi);
                                 }
@@ -890,7 +846,7 @@ impl ScannerItem {
                             }
                             IlmAction::NoneAction | IlmAction::ActionCount => {
                                 if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                                    self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                                    self.heal_actions(oi, *physical, size_summary).await;
                                 }
                                 size_summary.actions_accounting_unknown(oi);
                             }
@@ -908,7 +864,7 @@ impl ScannerItem {
                 // side-effecting tier actions fail closed until the registry
                 // recognizes the source again.
                 if Self::action_blocked_by_unknown_tier(event.action, oi, all_versions_known, tier_names, &event.storage_class) {
-                    size = self.heal_actions(oi, actual_size, size_summary, cancellation).await;
+                    size = self.heal_actions(oi, actual_size, size_summary).await;
                     size_summary.actions_accounting(oi, size, actual_size);
                     cumulative_size += size;
                     continue;
@@ -1017,7 +973,7 @@ impl ScannerItem {
                     }
 
                     IlmAction::NoneAction | IlmAction::ActionCount => {
-                        size = self.heal_actions(oi, actual_size, size_summary, cancellation).await;
+                        size = self.heal_actions(oi, actual_size, size_summary).await;
                     }
                 }
 
@@ -1028,9 +984,6 @@ impl ScannerItem {
             }
         }
 
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return;
-        }
         if !to_delete_objs.is_empty()
             && let Some(event) = noncurrent_events.first().cloned()
         {
@@ -1038,8 +991,7 @@ impl ScannerItem {
             let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX);
             let done_ilm = Metrics::time_ilm(action);
             let trace_started_at = trace_start_instant();
-            let queued =
-                enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner, cancellation).await;
+            let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await;
             if let Some(trace_started_at) = trace_started_at {
                 let state = if queued { "queued" } else { "not_queued" };
                 trace_emit(|| {
@@ -1070,28 +1022,15 @@ impl ScannerItem {
                 }
             }
         }
-        if !cancellation.is_some_and(CancellationToken::is_cancelled) {
-            self.alert_excessive_versions(remaining_versions, cumulative_size);
-        }
+        self.alert_excessive_versions(remaining_versions, cumulative_size);
     }
 
-    pub(super) async fn heal_actions(
-        &mut self,
-        oi: &ObjectInfo,
-        actual_size: i64,
-        size_summary: &mut SizeSummary,
-        cancellation: Option<&CancellationToken>,
-    ) -> i64 {
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return actual_size;
-        }
+    pub(super) async fn heal_actions(&mut self, oi: &ObjectInfo, actual_size: i64, size_summary: &mut SizeSummary) -> i64 {
         if self.heal_enabled {
             self.enqueue_heal(oi).await;
         }
 
-        if !cancellation.is_some_and(CancellationToken::is_cancelled) {
-            self.heal_replication(oi, size_summary).await;
-        }
+        self.heal_replication(oi, size_summary).await;
 
         actual_size
     }
@@ -1222,7 +1161,6 @@ impl ScannerItem {
                 scan_mode,
                 HealChannelPriority::Low,
             ),
-            self.heal_scope,
         )
         .await;
 
@@ -1384,7 +1322,6 @@ mod tests {
             lifecycle: None,
             object_lock: None,
             replication: None,
-            heal_scope: None,
             heal_enabled: false,
             heal_bitrot: false,
             debug: false,

crates/scanner/src/scanner_folder/tests.rs

@@ -30,9 +30,7 @@ use std::sync::Mutex;
 pub(super) mod checkpoint_fixture;
 pub(super) mod enumeration_restart;
 mod incremental_enumeration;
-mod listing_completion;
 mod metadata_visits;
-mod node_admission;
 mod usage_updates;
 
 /// Reset the process-global alert cooldown map; test-only.
@@ -339,10 +337,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         .await
         .expect("failed to create test directory");
 
-    let mut endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
-    endpoint.set_pool_index(0);
-    endpoint.set_set_index(0);
-    endpoint.set_disk_index(0);
+    let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
     let disk = new_disk(
         &endpoint,
         &DiskOption {
@@ -369,7 +364,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         failed_object_paths_seen: HashSet::new(),
         resolved_failed_object_paths: HashSet::new(),
         sleeper: SCANNER_SLEEPER.clone(),
-        node_admission: None,
         disks: Vec::new(),
         disks_quorum: 0,
         usage_updates: None,
@@ -400,7 +394,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         pending_size_reconciliation_scopes: HashSet::new(),
         pending_size_reconciliation_truncated: false,
         list_path_raw_options_observer: None,
-        defer_abandoned_entries_until_finished: false,
     };
 
     (scanner, temp_dir)
@@ -769,7 +762,6 @@ fn test_classify_get_size_failure_marks_metadata_heal_object_path() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -801,7 +793,6 @@ fn test_classify_get_size_failure_records_transient_metadata_error_without_heal(
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -858,7 +849,6 @@ async fn test_heal_replication_skips_only_unversioned_data_versions() {
         lifecycle: None,
         object_lock: None,
         replication: Some(Arc::new(ReplicationConfig::new(None, None))),
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -2261,7 +2251,6 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: true,
         heal_bitrot: true,
         debug: false,
@@ -2273,7 +2262,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
     };
     let mut size_summary = SizeSummary::default();
 
-    let size = item.heal_actions(&object_info, 123, &mut size_summary, None).await;
+    let size = item.heal_actions(&object_info, 123, &mut size_summary).await;
     assert_eq!(size, 123);
 }
 
@@ -2502,7 +2491,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
     let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
     let heal_starts = Arc::new(AtomicUsize::new(0));
     let heal_starts_clone = heal_starts.clone();
-    let healed_versions = Arc::new(Mutex::new(Vec::new()));
+    let healed_versions = Arc::new(Mutex::new(Vec::<Option<String>>::new()));
     let healed_versions_clone = healed_versions.clone();
     let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels()
         .expect("heal channels should initialize once for scanner tests");
@@ -2514,7 +2503,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
                 healed_versions_clone
                     .lock()
                     .expect("heal version capture lock should not be poisoned")
-                    .push((request.object_prefix, request.object_version_id, request.pool_index, request.set_index));
+                    .push(request.object_version_id);
                 let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt {
                     result: HealAdmissionResult::Accepted,
                     task_id: request.id,
@@ -2595,90 +2584,9 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
             .lock()
             .expect("heal version capture lock should not be poisoned")
             .iter()
-            .any(|(_, version, _, _)| version.as_deref() == Some(orphan_version_text.as_str())),
+            .any(|version| version.as_deref() == Some(orphan_version_text.as_str())),
         "sub-quorum orphan version must be submitted as an exact heal candidate"
     );
-    assert!(
-        healed_versions
-            .lock()
-            .expect("heal scope capture lock should not be poisoned")
-            .iter()
-            .filter(|(object, _, _, _)| object.is_some())
-            .all(|(_, _, pool, set)| (*pool, *set) == (Some(0), Some(0))),
-        "high-priority discovered objects must retain the scanning disk's scope"
-    );
-    let bucket_admission = scanner
-        .send_required_scanner_heal_request(
-            PendingScannerHealKind::Bucket,
-            bucket.to_string(),
-            None,
-            None,
-            build_bucket_heal_request(bucket.to_string(), HealChannelPriority::High),
-        )
-        .await
-        .expect("submit bucket request");
-    assert_eq!(bucket_admission, HealAdmissionResult::Accepted);
-    assert_eq!(
-        healed_versions.lock().expect("read captured bucket scope").last(),
-        Some(&(None, None, None, None)),
-        "bucket healing must retain its cross-pool scope"
-    );
-    scanner.new_cache.info.name = bucket.to_string();
-    scanner.new_cache.info.pending_heals = vec![pending_heal(
-        PendingScannerHealKind::Object,
-        bucket,
-        Some("pending-object"),
-        Some(&orphan_version_text),
-        0,
-        1,
-    )];
-    scanner
-        .retry_pending_scanner_heals()
-        .await
-        .expect("retry persisted object heal");
-    assert_eq!(
-        healed_versions.lock().expect("read retried object scope").last(),
-        Some(&(Some("pending-object".to_string()), Some(orphan_version_text.clone()), Some(0), Some(0))),
-        "persisted retries must bind the current scanning disk"
-    );
-    let unscoped_endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("parse uninitialized endpoint");
-    scanner.local_disk = new_disk(
-        &unscoped_endpoint,
-        &DiskOption {
-            cleanup: false,
-            health_check: false,
-        },
-    )
-    .await
-    .expect("open disk without initialized location");
-    let request_count = healed_versions.lock().expect("count admitted requests").len();
-    let deferred = scanner
-        .send_required_scanner_heal_request(
-            PendingScannerHealKind::Object,
-            bucket.to_string(),
-            Some("unscoped-object".to_string()),
-            Some(orphan_version_text.clone()),
-            build_object_heal_request(
-                bucket.to_string(),
-                "unscoped-object".to_string(),
-                Some(orphan_version_text),
-                HealScanMode::Deep,
-                HealChannelPriority::High,
-            ),
-        )
-        .await
-        .expect("defer object with unknown disk location");
-    assert_eq!(deferred, HealAdmissionResult::Full);
-    assert_eq!(healed_versions.lock().expect("count unchanged admissions").len(), request_count);
-    assert!(
-        scanner
-            .new_cache
-            .info
-            .pending_heals
-            .iter()
-            .any(|entry| entry.object.as_deref() == Some("unscoped-object")),
-        "an unscoped high-priority object must remain pending"
-    );
     for (path, expected) in expected_metadata {
         assert_eq!(
             tokio::fs::read(&path)

crates/scanner/src/scanner_folder/tests/listing_completion.rs

@@ -1,104 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use super::*;
-
-#[tokio::test]
-#[serial]
-async fn abandoned_listing_drains_candidate_buffered_before_finished() {
-    let (mut scanner, root) = build_test_scanner().await;
-    let _guard = TestGuard {
-        temp_dir: Some(root.clone()),
-    };
-    let bucket = "bucket";
-    let object = "abandoned/object";
-    tokio::fs::create_dir_all(root.join(bucket)).await.expect("local bucket");
-    let mut disks = vec![scanner.local_disk.clone()];
-    let mut original_metadata = Vec::new();
-    for (index, version) in [(2, 0x1935), (3, 0x1935), (4, 0x1934)] {
-        let disk_root = root.join(format!("disk-{index}"));
-        let metadata = metadata_for_object_version(bucket, object, Some(Uuid::from_u128(version)));
-        write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
-        original_metadata.push((disk_root.join(bucket).join(object).join(STORAGE_FORMAT_FILE), metadata));
-        let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("listing fixture endpoint");
-        disks.push(
-            new_disk(
-                &endpoint,
-                &DiskOption {
-                    cleanup: false,
-                    health_check: false,
-                },
-            )
-            .await
-            .expect("listing fixture disk"),
-        );
-    }
-    for cache in [&mut scanner.old_cache, &mut scanner.new_cache] {
-        cache.info.name = bucket.to_owned();
-    }
-    scanner.old_cache.replace(
-        "bucket/abandoned/object",
-        bucket,
-        DataUsageEntry {
-            objects: 1,
-            ..Default::default()
-        },
-    );
-    scanner.heal_object_select = 1;
-    scanner.disks = disks;
-    scanner.disks_quorum = 2;
-    scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
-    // Force the valid select ordering that receives terminal errors before the
-    // final candidate. One candidate frame fits in the bounded partial channel.
-    scanner.defer_abandoned_entries_until_finished = true;
-    let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels().expect("isolated heal channel");
-    let (captured_tx, mut captured_rx) = mpsc::unbounded_channel();
-    let responder = tokio::spawn(async move {
-        while let Some(command) = heal_rx.recv().await {
-            let rustfs_heal_contracts::heal_channel::HealReceiptCommand { request, response_tx } = command;
-            captured_tx
-                .send((request.object_prefix.clone(), request.object_version_id.clone()))
-                .expect("candidate capture remains open");
-            let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt {
-                result: HealAdmissionResult::Accepted,
-                task_id: request.id,
-            }));
-        }
-    });
-    let result = tokio::time::timeout(
-        Duration::from_secs(2),
-        scanner.scan_folder(
-            CancellationToken::new(),
-            CachedFolder {
-                name: bucket.to_owned(),
-                parent: None,
-                object_heal_prob_div: 1,
-            },
-            &mut DataUsageEntry::default(),
-        ),
-    )
-    .await;
-    responder.abort();
-    result
-        .expect("listing completion must close every callback channel")
-        .expect("complete folder scan");
-    let mut candidates = HashSet::new();
-    while let Ok((object, version)) = captured_rx.try_recv() {
-        if let (Some(object), Some(version)) = (object, version) {
-            candidates.insert((object, version));
-        }
-    }
-    let expected = [0x1934, 0x1935]
-        .into_iter()
-        // Raw base-object listings include the trailing separator. Preserve
-        // the producer's candidate identity when draining its callback frame.
-        .map(|version| (format!("{object}/"), Uuid::from_u128(version).to_string()))
-        .collect();
-    assert_eq!(
-        candidates, expected,
-        "terminal disk errors must not discard buffered exact-version candidates"
-    );
-    for (path, metadata) in original_metadata {
-        assert_eq!(tokio::fs::read(path).await.expect("retained metadata"), metadata);
-    }
-}

crates/scanner/src/scanner_io/cache.rs

@@ -172,55 +172,12 @@ fn record_checkpoint_save_failure(cache_name: &str, checkpoint: &DataUsageCache,
 
 #[derive(Debug)]
 pub(crate) struct ScannerCacheLockGuards {
-    scoped: Option<NamespaceLockGuard>,
+    scoped: NamespaceLockGuard,
 }
 
 impl ScannerCacheLockGuards {
     pub(crate) fn is_lock_lost(&self) -> bool {
-        self.scoped.as_ref().is_none_or(NamespaceLockGuard::is_lock_lost)
-    }
-
-    /// Suspend the same traversal while yielding its cache lease. Lock order on
-    /// resumption is node admission -> cache lease -> cache revision reads.
-    pub(crate) async fn resume_node_operation(
-        &mut self,
-        store: Arc<SetDisks>,
-        cache_name: &str,
-        source: DataUsageCacheSource,
-        revisions: &DataUsageCacheRevisions,
-        request: crate::workload_admission::ScannerNodeRequest,
-    ) -> Result<()> {
-        if self.is_lock_lost() {
-            return Err(Error::other("scanner cache lock was lost before node admission"));
-        }
-        if request.respond.is_closed() {
-            return Ok(());
-        }
-        let crate::workload_admission::ScannerNodeRequest { owner, respond, cancel } = request;
-        drop(self.scoped.take());
-        let resume = async {
-            let permit = owner.acquire().await;
-            *self = acquire_scanner_cache_locks(store.as_ref(), cache_name, source)
-                .await
-                .map_err(Error::other)?;
-            let current = crate::storage_api::ecstore_with_background_disk_io(
-                cancel.clone(),
-                DataUsageCache::read_revisions(store, cache_name),
-            )
-            .await?;
-            if self.is_lock_lost() || &current != revisions {
-                return Err(Error::other("scanner cache changed while waiting for node admission"));
-            }
-            // If the walk was dropped, send returns ownership and drops the
-            // permit without recording completed work.
-            let _ = respond.send(permit);
-            Ok(())
-        };
-        tokio::select! {
-            biased;
-            _ = cancel.cancelled() => Err(Error::other("scanner node admission cancelled")),
-            result = resume => result,
-        }
+        self.scoped.is_lock_lost()
     }
 }
 
@@ -258,15 +215,6 @@ impl std::fmt::Display for ScannerCacheLockError {
     }
 }
 
-impl std::error::Error for ScannerCacheLockError {
-    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        match self {
-            Self::Create { source, .. } => Some(source),
-            Self::Acquire { source, .. } => Some(source),
-        }
-    }
-}
-
 pub(crate) async fn acquire_scanner_cache_locks(
     store: &SetDisks,
     cache_name: &str,
@@ -289,7 +237,7 @@ pub(crate) async fn acquire_scanner_cache_locks(
             source,
         })?;
 
-    Ok(ScannerCacheLockGuards { scoped: Some(scoped) })
+    Ok(ScannerCacheLockGuards { scoped })
 }
 
 pub(super) async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)

crates/scanner/src/scanner_io/dirty_usage.rs

@@ -238,71 +238,6 @@ pub enum ScannerDurableDirtyUsageReplayScope {
     TopLevelEntries { entries: BTreeSet<String> },
 }
 
-impl ScannerDurableDirtyUsageReplayRecord {
-    /// Replays decoded producer hints with the canonical byte limit and semantic checks.
-    ///
-    /// Callers must retain their framing and owner checks. This performs the same
-    /// state update as [`replay_durable_dirty_usage_producer_record`]; it does not
-    /// establish storage-verified source authority.
-    pub fn replay(self) -> std::result::Result<ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayError> {
-        check_durable_dirty_usage_replay_bytes(&self)?;
-        replay_decoded_dirty_usage_record(&self)
-    }
-}
-
-struct ReplayByteLimit {
-    remaining: usize,
-}
-
-impl std::io::Write for ReplayByteLimit {
-    fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
-        self.remaining = self
-            .remaining
-            .checked_sub(bytes.len())
-            .ok_or_else(|| std::io::Error::other("dirty replay byte limit"))?;
-        Ok(bytes.len())
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        Ok(())
-    }
-}
-
-fn check_durable_dirty_usage_replay_bytes(
-    record: &ScannerDurableDirtyUsageReplayRecord,
-) -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> {
-    // Bound structure traversal and string scanning before counting escaped JSON.
-    // Each entry or prefix contributes at least one additional serialized byte.
-    let mut remaining = SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES;
-    let mut charge = |size: usize| -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> {
-        remaining = remaining
-            .checked_sub(size)
-            .ok_or(ScannerDurableDirtyUsageReplayError::ByteLimit)?;
-        Ok(())
-    };
-    charge(record.writer_epoch.len())?;
-    charge(record.entries.len())?;
-    for entry in &record.entries {
-        charge(entry.bucket.len())?;
-        if let ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } = &entry.scope {
-            charge(entries.len())?;
-            for entry in entries {
-                charge(entry.len())?;
-            }
-        }
-    }
-    let mut limit = ReplayByteLimit {
-        remaining: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES,
-    };
-    serde_json::to_writer(&mut limit, record).map_err(|error| {
-        if error.is_io() {
-            ScannerDurableDirtyUsageReplayError::ByteLimit
-        } else {
-            ScannerDurableDirtyUsageReplayError::InvalidJson
-        }
-    })
-}
-
 pub fn encode_durable_dirty_usage_producer_replay_record(
     entries: Vec<ScannerDurableDirtyUsageReplayEntry>,
 ) -> std::result::Result<Vec<u8>, ScannerDurableDirtyUsageReplayError> {
@@ -328,13 +263,7 @@ pub fn replay_durable_dirty_usage_producer_record(
     }
     let record: ScannerDurableDirtyUsageReplayRecord =
         serde_json::from_slice(bytes).map_err(|_| ScannerDurableDirtyUsageReplayError::InvalidJson)?;
-    replay_decoded_dirty_usage_record(&record)
-}
-
-fn replay_decoded_dirty_usage_record(
-    record: &ScannerDurableDirtyUsageReplayRecord,
-) -> std::result::Result<ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayError> {
-    let replay = validate_durable_dirty_usage_replay_record(record)?;
+    let replay = validate_durable_dirty_usage_replay_record(&record)?;
     let max_generation = replay.iter().map(|entry| entry.generation).max().unwrap_or(0);
     set_dirty_usage_generation_floor(max_generation)?;
     let pending = {
@@ -843,109 +772,6 @@ mod scoped_dirty_usage_tests {
         );
     }
 
-    fn decoded_replay_fixture() -> ScannerDurableDirtyUsageReplayRecord {
-        ScannerDurableDirtyUsageReplayRecord {
-            schema: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA,
-            cache_key_format: crate::DATA_USAGE_CACHE_KEY_FORMAT,
-            writer_epoch: "writer".to_string(),
-            entries: vec![ScannerDurableDirtyUsageReplayEntry {
-                bucket: "photos".to_string(),
-                generation: 7,
-                scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
-                    entries: BTreeSet::from(["2026".to_string(), "archive".to_string()]),
-                },
-                producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(),
-            }],
-        }
-    }
-
-    #[test]
-    #[serial]
-    fn durable_dirty_usage_decoded_replay_matches_byte_replay_at_size_boundary() {
-        let record = decoded_replay_fixture();
-        let fixed_bytes = serde_json::to_vec(&record).expect("fixture encoding").len() - record.writer_epoch.len();
-        for size in [
-            SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES - 1,
-            SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES,
-            SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1,
-        ] {
-            clear_dirty_usage_buckets_for_tests();
-            let mut record = record.clone();
-            record.writer_epoch = "x".repeat(size - fixed_bytes);
-            let bytes = serde_json::to_vec(&record).expect("bounded fixture encoding");
-            assert_eq!(bytes.len(), size);
-            let expected = replay_durable_dirty_usage_producer_record(&bytes);
-            let buckets = dirty_usage_buckets_for_tests();
-            let scopes = dirty_usage_bucket_scopes_for_tests();
-            let identities = dirty_usage_producer_identities().clone();
-            let coverage = DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire);
-
-            clear_dirty_usage_buckets_for_tests();
-            assert_eq!(record.replay(), expected, "canonical size {size}");
-            assert_eq!(dirty_usage_buckets_for_tests(), buckets);
-            assert_eq!(dirty_usage_bucket_scopes_for_tests(), scopes);
-            assert_eq!(*dirty_usage_producer_identities(), identities);
-            assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), coverage);
-        }
-        clear_dirty_usage_buckets_for_tests();
-    }
-
-    #[test]
-    fn durable_dirty_usage_decoded_size_counts_escaped_utf8() {
-        for repeats in [6000, 8000] {
-            let mut record = decoded_replay_fixture();
-            record.writer_epoch = "\"\n雪\\".repeat(repeats);
-            let bytes = serde_json::to_vec(&record).expect("escaped fixture encoding");
-            let expected = if bytes.len() > SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
-                Err(ScannerDurableDirtyUsageReplayError::ByteLimit)
-            } else {
-                Ok(())
-            };
-            assert_eq!(check_durable_dirty_usage_replay_bytes(&record), expected);
-        }
-    }
-
-    #[test]
-    #[serial]
-    fn durable_dirty_usage_decoded_replay_rejects_large_values_before_state_changes() {
-        for field in ["epoch", "prefix", "entries"] {
-            clear_dirty_usage_buckets_for_tests();
-            let before = scanner_dirty_usage_state();
-            let mut record = decoded_replay_fixture();
-            match field {
-                "epoch" => record.writer_epoch = "x".repeat(1024 * 1024),
-                "prefix" => {
-                    record.entries[0].scope = ScannerDurableDirtyUsageReplayScope::TopLevelEntries {
-                        entries: BTreeSet::from(["x".repeat(1024 * 1024)]),
-                    };
-                }
-                _ => record.entries = vec![record.entries[0].clone(); SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1],
-            }
-            assert_eq!(record.replay(), Err(ScannerDurableDirtyUsageReplayError::ByteLimit), "{field}");
-            assert_eq!(scanner_dirty_usage_state(), before);
-            assert!(dirty_usage_buckets_for_tests().is_empty());
-            assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
-            assert!(dirty_usage_producer_identities().is_empty());
-            assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
-        }
-        clear_dirty_usage_buckets_for_tests();
-    }
-
-    #[test]
-    #[serial]
-    fn durable_dirty_usage_decoded_replay_preserves_generation_exhaustion() {
-        clear_dirty_usage_buckets_for_tests();
-        let generation = DIRTY_USAGE_BUCKET_GENERATION.swap(u64::MAX, Ordering::AcqRel);
-        let result = decoded_replay_fixture().replay();
-        DIRTY_USAGE_BUCKET_GENERATION.store(generation, Ordering::Release);
-        assert_eq!(result, Err(ScannerDurableDirtyUsageReplayError::InvalidGeneration));
-        assert!(dirty_usage_buckets_for_tests().is_empty());
-        assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
-        assert!(dirty_usage_producer_identities().is_empty());
-        assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
-        clear_dirty_usage_buckets_for_tests();
-    }
-
     #[test]
     #[serial]
     fn durable_dirty_usage_replay_rejects_invalid_records_without_partial_state() {
@@ -1017,17 +843,9 @@ mod scoped_dirty_usage_tests {
 
         for (bytes, expected_error) in invalid_cases {
             clear_dirty_usage_buckets_for_tests();
-            let before = scanner_dirty_usage_state();
-            assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error.clone()));
+            assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error));
             assert!(dirty_usage_buckets_for_tests().is_empty());
             assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
-            let record: ScannerDurableDirtyUsageReplayRecord = serde_json::from_slice(&bytes).expect("typed invalid record");
-            assert_eq!(record.replay(), Err(expected_error));
-            assert_eq!(scanner_dirty_usage_state(), before);
-            assert!(dirty_usage_buckets_for_tests().is_empty());
-            assert!(dirty_usage_bucket_scopes_for_tests().is_empty());
-            assert!(dirty_usage_producer_identities().is_empty());
-            assert_eq!(DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire), 0);
         }
 
         clear_dirty_usage_buckets_for_tests();

crates/scanner/src/scanner_io/io_cache.rs

@@ -1090,7 +1090,7 @@ impl ScannerIOCache for SetDisks {
 
                     // Lock order: scanner leader fence -> set-scoped per-bucket cache lock ->
                     // cache object read/write.
-                    let mut cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
+                    let cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
                         Ok(guard) => guard,
                         Err(e) => {
                             if e.is_contention() {
@@ -1320,10 +1320,9 @@ impl ScannerIOCache for SetDisks {
                     );
 
                     let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
-                    let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
 
                     let scan_ctx = ctx_clone.child_token();
-                    let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
+                    let scan = disk.clone().nsscanner_disk(
                         scan_ctx.clone(),
                         budget_clone.clone(),
                         set_disk_inventory_clone.as_ref().clone(),
@@ -1334,7 +1333,7 @@ impl ScannerIOCache for SetDisks {
                             prefix_scan_scope,
                             checkpoint_tx: Some(checkpoint_tx),
                         },
-                    ));
+                    );
                     tokio::pin!(scan);
                     let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
                     lock_watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -1342,24 +1341,6 @@ impl ScannerIOCache for SetDisks {
                     let scan_result = loop {
                         tokio::select! {
                             result = &mut scan => break result,
-                            request = async {
-                                match node_rx.as_mut() {
-                                    Some(receiver) => receiver.recv().await,
-                                    None => std::future::pending().await,
-                                }
-                            } => {
-                                let Some(request) = request else {
-                                    node_rx = None;
-                                    continue;
-                                };
-                                if let Err(error) = cache_guard.resume_node_operation(
-                                    store_clone_clone.clone(), &cache_name, source, &revisions, request,
-                                ).await {
-                                    scan_ctx.cancel();
-                                    await_scanner_disk_shutdown(scan.as_mut()).await;
-                                    break Err(error);
-                                }
-                            }
                             _ = lock_watch.tick() => {
                                 if cache_guard.is_lock_lost() {
                                     scan_ctx.cancel();

crates/scanner/src/scanner_io/io_disk.rs

@@ -38,13 +38,6 @@ pub(super) fn tier_stats_template(tier_names: &[String]) -> HashMap<String, Tier
     tier_stats
 }
 
-fn ensure_scanner_node_operation_active(cancellation: Option<&CancellationToken>) -> Result<()> {
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return Err(StorageError::other("Scanner operation cancelled"));
-    }
-    Ok(())
-}
-
 #[async_trait::async_trait]
 impl ScannerIODisk for Disk {
     async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary> {
@@ -52,18 +45,14 @@ impl ScannerIODisk for Disk {
     }
 
     async fn get_size_with_tier_names(&self, mut item: ScannerItem, tier_names: &[String]) -> Result<SizeSummary> {
-        let cancellation = crate::workload_admission::scanner_node_operation_cancellation();
         let done_object = Metrics::time(Metric::ScanObject);
 
         if !is_xl_meta_path(&item.path) {
             return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
         }
 
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         let metadata_object_path = item.object_path();
-        let metadata_result = self.read_metadata(&item.bucket, &metadata_object_path).await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
-        let data = match metadata_result {
+        let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
             Ok(data) => data,
             Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
                 return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -97,10 +86,7 @@ impl ScannerIODisk for Disk {
         // (which used to query it a second time). On failure keep the
         // historical fallback: default configuration (versioned = false) plus
         // the warn that `apply_actions` used to emit.
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
-        let versioning_result = BucketVersioningSys::get(&item.bucket).await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
-        let versioning_config = match versioning_result {
+        let versioning_config = match BucketVersioningSys::get(&item.bucket).await {
             Ok(versioning_config) => versioning_config,
             Err(_) => {
                 warn!(
@@ -136,9 +122,7 @@ impl ScannerIODisk for Disk {
             ..Default::default()
         };
 
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         let lock_config = object_lock_config_for_scanner_item(&item).await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
 
         // Count every version this object contributes to the scan, independent
         // of any lifecycle configuration, so scan-coverage metrics stay honest
@@ -146,35 +130,17 @@ impl ScannerIODisk for Disk {
         // `object_infos`.
         global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);
 
-        // A caller-provided item or persisted usage cache cannot choose another set.
-        item.heal_scope = crate::scanner_folder::scanner_disk_heal_scope(self);
-        #[cfg(test)]
-        let _ = crate::scanner_folder::observe_test_size_dispatch_boundary("apply_actions");
-        item.apply_actions_cancellable(
-            object_infos,
-            lock_config,
-            versioning_config,
-            tier_names,
-            &mut size_summary,
-            cancellation.as_ref(),
-        )
-        .await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
+        item.apply_actions(object_infos, lock_config, versioning_config, tier_names, &mut size_summary)
+            .await;
 
         if !free_version_infos.is_empty() {
             for oi in free_version_infos {
-                ensure_scanner_node_operation_active(cancellation.as_ref())?;
                 if ScannerItem::tier_is_known(&oi, tier_names) {
-                    #[cfg(test)]
-                    if crate::scanner_folder::observe_test_size_dispatch_boundary("free_version_dispatch") {
-                        continue;
-                    }
-                    enqueue_runtime_free_version(oi, cancellation.as_ref()).await;
+                    enqueue_runtime_free_version(oi).await;
                 }
             }
         }
 
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         done_object();
 
         Ok(size_summary)

crates/scanner/src/scanner_io/tests.rs

@@ -46,9 +46,7 @@ use uuid::Uuid;
 
 mod checkpoint_coverage_identity;
 mod collector_lifecycle;
-mod expiry_lock;
 mod listing_window_scope;
-mod node_admission;
 mod partial_bucket_carry;
 mod per_bucket_certification;
 mod scoped_entry_fallback;
@@ -1772,7 +1770,6 @@ async fn scanner_item_object_lock_uses_cached_config() {
         lifecycle: None,
         object_lock: Some(cached.clone()),
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4522,7 +4519,6 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4578,7 +4574,6 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4661,7 +4656,6 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,

crates/scanner/src/scanner_io/tests/expiry_lock.rs

@@ -1,183 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::*;
-use crate::storage_api::owner::EcstoreExpiryState;
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::task::Poll;
-use std::time::Duration;
-
-// Return Some(result) if the operation ended too early, None only if it was
-// actually polled pending after the prerequisite read had finished.
-async fn observe_queued<F: Future>(
-    mut operation: Pin<&mut F>,
-    read_complete: impl Fn() -> bool,
-    observation: Duration,
-) -> std::result::Result<Option<F::Output>, tokio::time::error::Elapsed> {
-    tokio::time::timeout(
-        observation,
-        std::future::poll_fn(|cx| match operation.as_mut().poll(cx) {
-            Poll::Ready(value) => Poll::Ready(Some(value)),
-            Poll::Pending if read_complete() => Poll::Ready(None),
-            Poll::Pending => Poll::Pending,
-        }),
-    )
-    .await
-}
-
-#[tokio::test]
-#[serial]
-async fn free_version_cancelled_while_waiting_for_expiry_write_lock_never_queues() {
-    let state = EcstoreExpiryState::new_with_unconsumed_worker_channel(4);
-    let observation = Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS);
-    let context = crate::ExpiryLockTestContext {
-        state: state.clone(),
-        store: None,
-        completed_incarnation_reads: Arc::new(AtomicUsize::new(0)),
-    };
-    crate::EXPIRY_LOCK_TEST_CONTEXT
-        .scope(context, async {
-            let object = crate::ScannerObjectInfo {
-                bucket: "expiry-lock-bucket".to_string(),
-                name: "free-object".to_string(),
-                ..Default::default()
-            };
-            let cancellation = CancellationToken::new();
-            let guard = state.write().await;
-            let mut pending = Box::pin(crate::enqueue_runtime_free_version(object.clone(), Some(&cancellation)));
-            let first = observe_queued(pending.as_mut(), || true, observation).await;
-            let observed_wait = matches!(&first, Ok(None));
-            let already_finished = matches!(&first, Ok(Some(())));
-            cancellation.cancel();
-            let cancelled_while_locked = if already_finished {
-                false
-            } else {
-                tokio::time::timeout(observation, pending.as_mut()).await.is_ok()
-            };
-            let pending_while_locked = guard.pending_tasks();
-            drop(guard);
-            if !already_finished && !cancelled_while_locked {
-                pending.as_mut().await;
-            }
-            drop(pending);
-            let pending_after_cancel = state.read().await.pending_tasks();
-
-            // Positive control queues a real free-version task while using the same
-            // implementation and isolated, unconsumed worker channel.
-            let mut positive = Box::pin(crate::enqueue_runtime_free_version(object, None));
-            let positive_in_time = tokio::time::timeout(observation, positive.as_mut()).await.is_ok();
-            if !positive_in_time {
-                positive.as_mut().await;
-            }
-            drop(positive);
-            let pending_after_control = state.read().await.pending_tasks();
-            assert!(observed_wait, "must poll the real enqueue while the write lock is held");
-            assert!(cancelled_while_locked, "cancellation must finish before the lock is released");
-            assert_eq!(pending_while_locked, 0);
-            assert_eq!(pending_after_cancel, 0);
-            assert!(positive_in_time, "positive control must finish within the observation limit");
-            assert_eq!(pending_after_control, 1, "positive control must queue a task");
-        })
-        .await;
-}
-
-#[tokio::test]
-#[serial]
-async fn newer_noncurrent_cancelled_after_real_read_while_waiting_for_expiry_write_lock_never_queues() {
-    let (_temp_dir, store) = setup_local_scanner_store(1, 4).await;
-    let bucket = "expiry-lock-bucket";
-    store
-        .make_bucket(bucket, &MakeBucketOptions::default())
-        .await
-        .expect("create bucket");
-    let expected_incarnation = store.bucket_incarnation_id(bucket).await.expect("real incarnation metadata");
-    assert!(!expected_incarnation.is_nil());
-    let state = EcstoreExpiryState::new_with_unconsumed_worker_channel(4);
-    let reads = Arc::new(AtomicUsize::new(0));
-    let context = crate::ExpiryLockTestContext {
-        state: state.clone(),
-        store: Some(store.clone()),
-        completed_incarnation_reads: reads.clone(),
-    };
-    let observation = Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS);
-    crate::EXPIRY_LOCK_TEST_CONTEXT
-        .scope(context, async {
-            let versions = vec![crate::ScannerObjectToDelete {
-                object_name: "noncurrent-object".to_string(),
-                version_id: Some(Uuid::new_v4()),
-                ..Default::default()
-            }];
-            let event = crate::Event::default();
-            let src = crate::LcEventSrc::Scanner;
-            let cancellation = CancellationToken::new();
-            let guard = state.write().await;
-            let mut pending = Box::pin(crate::enqueue_runtime_newer_noncurrent(
-                bucket,
-                versions.clone(),
-                event.clone(),
-                &src,
-                Some(&cancellation),
-            ));
-            let first = observe_queued(pending.as_mut(), || reads.load(Ordering::SeqCst) == 1, observation).await;
-            let observed_wait_after_read = matches!(&first, Ok(None));
-            let mut result = match first {
-                Ok(Some(value)) => Some(value),
-                _ => None,
-            };
-            cancellation.cancel();
-            let cancelled_while_locked = if result.is_some() {
-                false
-            } else {
-                match tokio::time::timeout(observation, pending.as_mut()).await {
-                    Ok(value) => {
-                        result = Some(value);
-                        true
-                    }
-                    Err(_) => false,
-                }
-            };
-            let pending_while_locked = guard.pending_tasks();
-            drop(guard);
-            let result = match result {
-                Some(value) => value,
-                None => pending.as_mut().await,
-            };
-            drop(pending);
-            let pending_after_cancel = state.read().await.pending_tasks();
-            let mut positive = Box::pin(crate::enqueue_runtime_newer_noncurrent(bucket, versions, event, &src, None));
-            let bounded_positive = tokio::time::timeout(observation, positive.as_mut()).await;
-            let positive_in_time = bounded_positive.is_ok();
-            let positive_result = match bounded_positive {
-                Ok(value) => value,
-                Err(_) => positive.as_mut().await,
-            };
-            drop(positive);
-            let pending_after_control = state.read().await.pending_tasks();
-            assert!(
-                observed_wait_after_read,
-                "real incarnation read must finish before waiting on the held lock"
-            );
-            assert!(cancelled_while_locked, "cancellation must finish before the lock is released");
-            assert!(!result);
-            assert_eq!(pending_while_locked, 0);
-            assert_eq!(pending_after_cancel, 0);
-            assert!(positive_in_time, "positive control must finish within the observation limit");
-            assert!(positive_result, "positive control must enqueue the nonempty version batch");
-            assert_eq!(pending_after_control, 1);
-            assert_eq!(reads.load(Ordering::SeqCst), 2);
-        })
-        .await;
-}

crates/scanner/src/scanner_io/tests/node_admission.rs

@@ -1,136 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use super::*;
-use crate::workload_admission::ScannerNodeRequest;
-use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
-use std::num::NonZeroU16;
-use tokio::sync::oneshot;
-
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum ResumeCase {
-    Unchanged,
-    Updated,
-    Cancelled,
-}
-
-async fn check_cache_lease_resume(case: ResumeCase) {
-    let (_directory, store) = setup_local_scanner_store(1, 4).await;
-    let set = store.pools[0].disk_set[0].clone();
-    let source = DataUsageCacheSource::new(0, 0);
-    let cache_name = "node-resume/.usage-cache.bin";
-    let cache = DataUsageCache::default();
-    cache.save(set.clone(), cache_name).await.expect("seed cache");
-    let revisions = DataUsageCache::read_revisions(set.clone(), cache_name)
-        .await
-        .expect("initial revisions");
-    let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one shared unit"));
-    gate.update(
-        &mut FeedbackController::new(),
-        Some(FeedbackObservation {
-            generation: 1,
-            max_concurrency: 1,
-            pressure: ResourcePressure::Unknown,
-            completed_units: None,
-        }),
-    );
-    let mut held = Some(gate.acquire().await);
-    let mut guard = acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
-        .await
-        .expect("initial cache lease");
-    let ctx = CancellationToken::new();
-    let (respond, response) = oneshot::channel();
-    let request = ScannerNodeRequest {
-        owner: gate.clone(),
-        respond,
-        cancel: ctx.clone(),
-    };
-    let mut expected = revisions.clone();
-    let result = {
-        let resume = guard.resume_node_operation(set.clone(), cache_name, source, &revisions, request);
-        tokio::pin!(resume);
-        assert!(futures::poll!(&mut resume).is_pending(), "shared node is occupied");
-        let other = tokio::time::timeout(Duration::from_secs(1), acquire_scanner_cache_locks(set.as_ref(), cache_name, source))
-            .await
-            .expect("cache lock must be released before joining the node queue")
-            .expect("another cache worker can acquire the lease");
-        if case == ResumeCase::Updated {
-            let mut newer = cache;
-            newer.info.next_cycle = 1;
-            newer
-                .save(set.clone(), cache_name)
-                .await
-                .expect("another worker updates the cache");
-            expected = DataUsageCache::read_revisions(set.clone(), cache_name)
-                .await
-                .expect("new cache revisions");
-            assert_ne!(expected, revisions);
-        }
-        drop(other);
-        if case == ResumeCase::Cancelled {
-            ctx.cancel();
-        } else {
-            held.take().expect("held node unit").complete();
-        }
-        tokio::time::timeout(Duration::from_secs(2), resume)
-            .await
-            .expect("node resumption must finish or observe cancellation")
-    };
-    if case == ResumeCase::Unchanged {
-        result.expect("unchanged cache resumes the same walk");
-        let permit = response.await.expect("coordinator returns the node permit");
-        assert_eq!(gate.snapshot().active, 1);
-        assert_eq!(gate.completed(), Some(1), "admitting scanner work must not count it as completed");
-        assert!(!guard.is_lock_lost());
-        let lock = set
-            .new_ns_lock(RUSTFS_META_BUCKET, &scanner_cache_lock_resource(cache_name, source))
-            .await
-            .expect("competing cache lock");
-        assert!(
-            lock.get_write_lock_quiet(Duration::from_millis(20)).await.is_err(),
-            "cache lease must be restored before the operation resumes"
-        );
-        permit.complete();
-    } else {
-        result.expect_err("cancelled or stale work cannot resume");
-        assert!(response.await.is_err(), "failed resumption must not authorize storage work");
-        if case == ResumeCase::Cancelled {
-            assert!(guard.is_lock_lost());
-            assert_eq!(gate.snapshot().active, 1, "the running heal keeps its unit");
-            drop(held.take());
-            assert_eq!(gate.completed(), Some(0));
-        } else {
-            assert_eq!(gate.completed(), Some(1));
-        }
-    }
-    drop(guard);
-    assert_eq!(gate.snapshot().active, 0);
-    assert!(gate.try_acquire().is_some(), "dropped requests must leave no queue or capacity leak");
-    assert_eq!(
-        DataUsageCache::read_revisions(set.clone(), cache_name)
-            .await
-            .expect("final revisions"),
-        expected
-    );
-    acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
-        .await
-        .expect("final cache lease is available");
-}
-
-#[tokio::test]
-#[serial]
-async fn scanner_node_wait_yields_cache_lease_before_fifo_admission() {
-    check_cache_lease_resume(ResumeCase::Unchanged).await;
-}
-
-#[tokio::test]
-#[serial]
-async fn scanner_node_wait_rejects_cache_changed_while_unlocked() {
-    check_cache_lease_resume(ResumeCase::Updated).await;
-}
-
-#[tokio::test]
-#[serial]
-async fn scanner_node_wait_cancellation_releases_queue_without_stealing_running_unit() {
-    check_cache_lease_resume(ResumeCase::Cancelled).await;
-}

crates/scanner/src/storage_api.rs

@@ -22,8 +22,6 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::Notify;
 
 pub(crate) use rustfs_ecstore::with_background_disk_io as ecstore_with_background_disk_io;
-#[cfg(all(test, unix))]
-pub(crate) use rustfs_ecstore::with_background_resource_unit as ecstore_with_background_resource_unit;
 
 pub(crate) use s3s::dto::{
     BucketLifecycleConfiguration as EcstoreBucketLifecycleConfiguration, LifecycleRuleFilter as EcstoreLifecycleRuleFilter,
@@ -41,8 +39,6 @@ pub(crate) use rustfs_ecstore::api::data_usage::{NativeScannerPauseBacklogWriteF
 
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
-#[cfg(test)]
-pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState as EcstoreExpiryState;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
     apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
     lifecycle_version_delete_target as ecstore_lifecycle_version_delete_target,
@@ -103,10 +99,6 @@ pub(crate) use rustfs_ecstore::api::disk::{
 pub(crate) use rustfs_ecstore::api::disk::{
     DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
 };
-#[cfg(all(test, unix))]
-pub(crate) use rustfs_ecstore::api::disk::{
-    LocalPublicationPause as EcstoreLocalPublicationPause, LocalPublicationStage as EcstoreLocalPublicationStage,
-};
 pub(crate) use rustfs_ecstore::api::error::{
     Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
 };
@@ -169,17 +161,10 @@ pub(crate) mod ecstore_diagnostic {
 pub(crate) type EcstoreHealResultItem = <EcstoreStore as storage_contracts::HealOperations>::HealResultItem;
 
 pub(crate) mod owner {
-    #[cfg(test)]
-    pub(crate) use super::EcstoreExpiryState;
     #[cfg(test)]
     pub(crate) use super::ecstore_set_object_store_resolver;
     #[cfg(test)]
     pub(crate) use super::storage_contracts::DiskSetSelector;
-    #[cfg(all(test, unix))]
-    pub(crate) use super::{
-        EcstoreLocalPublicationPause, EcstoreLocalPublicationStage, ecstore_with_background_disk_io,
-        ecstore_with_background_resource_unit,
-    };
     pub(crate) use super::{
         MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
         ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,

crates/scanner/src/workload_admission.rs

@@ -12,143 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, LazyLock, OnceLock, RwLock};
+use std::sync::{Arc, LazyLock, RwLock};
 
-use rustfs_concurrency::adaptive::{NodeAdmission, NodePermit};
 use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
-use tokio::sync::{mpsc, oneshot};
-use tokio_util::sync::CancellationToken;
-
-use crate::ScannerError;
-
-static SCANNER_NODE_ADMISSION: OnceLock<Arc<NodeAdmission>> = OnceLock::new();
-
-tokio::task_local! {
-    static SCANNER_NODE_REQUESTS: ScannerNodeRequestSender;
-    static SCANNER_NODE_OPERATION_CANCEL: CancellationToken;
-}
-
-/// Cancellation for the currently admitted scanner operation, when present.
-pub(crate) fn scanner_node_operation_cancellation() -> Option<CancellationToken> {
-    SCANNER_NODE_OPERATION_CANCEL.try_with(Clone::clone).ok()
-}
-
-pub(crate) struct ScannerNodeRequest {
-    pub(crate) owner: Arc<NodeAdmission>,
-    pub(crate) respond: oneshot::Sender<NodePermit>,
-    pub(crate) cancel: CancellationToken,
-}
-
-pub(crate) type ScannerNodeRequestSender = mpsc::Sender<ScannerNodeRequest>;
-pub(crate) type ScannerNodeRequestReceiver = mpsc::Receiver<ScannerNodeRequest>;
-
-/// One suspended operation per walk; disabled admission allocates no channel.
-pub(crate) fn scanner_node_request_channel() -> (Option<ScannerNodeRequestSender>, Option<ScannerNodeRequestReceiver>) {
-    if scanner_node_admission().is_none() {
-        return (None, None);
-    }
-    let (sender, receiver) = mpsc::channel(1);
-    (Some(sender), Some(receiver))
-}
-
-pub(crate) async fn with_scanner_node_requests<F: std::future::Future>(
-    sender: Option<ScannerNodeRequestSender>,
-    operation: F,
-) -> F::Output {
-    match sender {
-        Some(sender) => SCANNER_NODE_REQUESTS.scope(sender, operation).await,
-        None => operation.await,
-    }
-}
-
-#[cfg(test)]
-tokio::task_local! {
-    pub(crate) static TEST_SCANNER_NODE_ADMISSION: Arc<NodeAdmission>;
-    pub(crate) static TEST_SCANNER_AFTER_NODE_ACQUIRE: Arc<dyn Fn() + Send + Sync>;
-}
-
-/// Install the heal manager's node owner before starting scanner workers.
-/// Refuse replacement: existing scans must not outlive an independently renewed owner.
-pub fn set_scanner_node_admission(owner: Arc<NodeAdmission>) -> bool {
-    SCANNER_NODE_ADMISSION.set(owner).is_ok()
-}
-
-pub(crate) fn scanner_node_admission() -> Option<Arc<NodeAdmission>> {
-    #[cfg(test)]
-    if let Ok(owner) = TEST_SCANNER_NODE_ADMISSION.try_with(Arc::clone) {
-        return Some(owner);
-    }
-    SCANNER_NODE_ADMISSION.get().cloned()
-}
-
-/// Try the fast path first. A scoped coordinator releases the cache lease before
-/// joining the node queue and validates the cache before returning a permit.
-/// Uncoordinated callers defer instead of parking under an unknown lock.
-pub(crate) async fn run_scanner_node_operation<T, E>(
-    owner: Option<&Arc<NodeAdmission>>,
-    cancel: &CancellationToken,
-    operation: impl std::future::Future<Output = Result<T, E>>,
-) -> Result<Result<T, E>, ScannerError> {
-    let Some(owner) = owner else {
-        return Ok(operation.await);
-    };
-    let operation = SCANNER_NODE_OPERATION_CANCEL.scope(cancel.clone(), operation);
-    if cancel.is_cancelled() {
-        return Err(ScannerError::Other("Operation cancelled".to_string()));
-    }
-    let permit = match owner.try_acquire() {
-        Some(permit) => permit,
-        None => {
-            let sender = SCANNER_NODE_REQUESTS
-                .try_with(Clone::clone)
-                .map_err(|_| ScannerError::ResourceDeferred)?;
-            let acquire = async {
-                let (respond, response) = oneshot::channel();
-                sender
-                    .send(ScannerNodeRequest {
-                        owner: owner.clone(),
-                        respond,
-                        cancel: cancel.clone(),
-                    })
-                    .await
-                    .map_err(|_| ScannerError::ResourceDeferred)?;
-                response.await.map_err(|_| ScannerError::ResourceDeferred)
-            };
-            tokio::select! {
-                biased;
-                _ = cancel.cancelled() => return Err(ScannerError::Other("Operation cancelled".to_string())),
-                permit = acquire => permit?,
-            }
-        }
-    };
-    #[cfg(test)]
-    let _ = TEST_SCANNER_AFTER_NODE_ACQUIRE.try_with(|hook| hook());
-    let operation_started = AtomicBool::new(false);
-    tokio::pin!(operation);
-    let observed = std::future::poll_fn(|context| {
-        operation_started.store(true, Ordering::Relaxed);
-        std::future::Future::poll(operation.as_mut(), context)
-    });
-    tokio::pin!(observed);
-    let result = tokio::select! {
-        biased;
-        _ = cancel.cancelled() => {
-            // The node slot belongs to the waiter, not the native operation.
-            drop(permit);
-            if !operation_started.load(Ordering::Relaxed) {
-                return Err(ScannerError::Other("Operation cancelled".to_string()));
-            }
-            // Keep polling the started operation under its existing locks.
-            return Ok(observed.await);
-        }
-        result = &mut observed => result,
-    };
-    if result.is_ok() && !cancel.is_cancelled() {
-        permit.complete();
-    }
-    Ok(result)
-}
 
 type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
 

crates/scanner/tests/lifecycle/scanner_heal_scope.rs

@@ -1,144 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use rustfs_filemeta::{FileInfo, FileMeta};
-use rustfs_heal_contracts::heal_channel::{
-    HealAdmissionReceipt, HealAdmissionResult, HealReceiptCommand, HealRequestSource, init_heal_channels,
-};
-use rustfs_scanner::DataUsageCacheSource;
-use rustfs_scanner::scanner_folder::ScannerItem;
-use rustfs_scanner::scanner_io::ScannerIODisk;
-use tokio::sync::mpsc;
-
-use super::storage_api::lifecycle::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
-
-#[test]
-fn scanner_object_requests_bind_the_actual_disk_scope() {
-    const CHILD: &str = "RUSTFS_TEST_SCANNER_SCOPE_CHILD";
-    if std::env::var_os(CHILD).is_none() {
-        let status = std::process::Command::new(std::env::current_exe().expect("locate integration test executable"))
-            .args([
-                "--exact",
-                "scanner_heal_scope::scanner_object_requests_bind_the_actual_disk_scope",
-                "--nocapture",
-            ])
-            .env(CHILD, "1")
-            .status()
-            .expect("run scanner scope regression in an isolated process");
-        assert!(status.success(), "isolated scanner scope regression failed: {status}");
-        return;
-    }
-    tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()
-        .expect("build isolated scanner runtime")
-        .block_on(check_scanner_object_scope());
-}
-
-async fn check_scanner_object_scope() {
-    let (_commands, mut receipts) = init_heal_channels().expect("initialize isolated scanner request channels");
-    let (captured, mut requests) = mpsc::unbounded_channel();
-    let responder = tokio::spawn(async move {
-        while let Some(command) = receipts.recv().await {
-            let HealReceiptCommand { request, response_tx } = command;
-            let receipt = HealAdmissionReceipt {
-                result: HealAdmissionResult::Accepted,
-                task_id: request.id.clone(),
-            };
-            captured.send(request).expect("retain scanner request");
-            let _ = response_tx.send(Ok(receipt));
-        }
-    });
-
-    for (pool, set, disk_index) in [
-        (Some(0), Some(0), Some(1)),
-        (Some(2), Some(3), Some(4)),
-        (None, Some(3), Some(4)),
-        (Some(2), None, Some(4)),
-        (Some(2), Some(3), None),
-        (None, None, None),
-    ] {
-        for version_id in [None, Some(uuid::Uuid::from_u128(42))] {
-            let root = tempfile::tempdir().expect("create scanner disk");
-            let bucket = "scanner-scope-bucket";
-            let object = "object";
-            let path = root.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
-            tokio::fs::create_dir_all(path.parent().expect("metadata parent"))
-                .await
-                .expect("create object directory");
-            let mut info = FileInfo::new(object, 4, 2);
-            info.volume = bucket.to_string();
-            info.name = object.to_string();
-            info.size = 1;
-            info.version_id = version_id;
-            info.data_dir = Some(uuid::Uuid::new_v4());
-            info.erasure.index = 1;
-            info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1).expect("valid object timestamp"));
-            info.add_object_part(1, "fixture-etag".to_string(), 1, info.mod_time, 1, None, None);
-            let mut meta = FileMeta::new();
-            meta.add_version(info).expect("encode original null version");
-            let original = meta.marshal_msg().expect("marshal original metadata");
-            tokio::fs::write(&path, &original).await.expect("write original metadata");
-            let mut endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("parse scanning disk endpoint");
-            if let Some(pool) = pool {
-                endpoint.set_pool_index(pool);
-            }
-            if let Some(set) = set {
-                endpoint.set_set_index(set);
-            }
-            if let Some(disk_index) = disk_index {
-                endpoint.set_disk_index(disk_index);
-            }
-            let disk = new_disk(
-                &endpoint,
-                &DiskOption {
-                    cleanup: false,
-                    health_check: false,
-                },
-            )
-            .await
-            .expect("open actual scanning disk");
-            let item = ScannerItem {
-                path: path.to_string_lossy().to_string(),
-                bucket: bucket.to_string(),
-                prefix: object.to_string(),
-                object_name: STORAGE_FORMAT_FILE.to_string(),
-                file_type: tokio::fs::metadata(&path).await.expect("stat original metadata").file_type(),
-                lifecycle: None,
-                object_lock: None,
-                replication: None,
-                // This foreign caller-provided scope must never reach admission.
-                heal_scope: Some(DataUsageCacheSource::new(99, 98)),
-                heal_enabled: true,
-                heal_bitrot: true,
-                debug: false,
-            };
-            disk.get_size_with_tier_names(item, &[]).await.expect("scan original object");
-            if let (Some(pool), Some(set), Some(_)) = (pool, set, disk_index) {
-                let request = requests.try_recv().expect("scanner submitted one real object request");
-                assert_eq!(request.pool_index, Some(pool));
-                assert_eq!(request.set_index, Some(set));
-                assert_eq!(request.bucket, bucket);
-                assert_eq!(request.object_prefix.as_deref(), Some(object));
-                assert_eq!(request.object_version_id, version_id.map(|value| value.to_string()));
-                assert_eq!(request.source, HealRequestSource::Scanner);
-                assert_eq!(request.no_lock, None);
-                assert_eq!(request.recreate_missing, Some(false));
-            }
-            assert!(requests.try_recv().is_err(), "no duplicate or unscoped request may be admitted");
-            assert_eq!(tokio::fs::read(&path).await.expect("read original metadata"), original);
-        }
-    }
-    responder.abort();
-}

crates/scanner/tests/lifecycle_integration_test.rs

@@ -39,9 +39,6 @@ use uuid::Uuid;
 
 mod storage_api;
 
-#[path = "lifecycle/scanner_heal_scope.rs"]
-mod scanner_heal_scope;
-
 use storage_api::lifecycle::{
     BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart,
     DeleteAfterObjectLockSnapshotBarrier, DiskOption, ECStore, EcstoreError, Endpoint, EndpointServerPools, Endpoints,
@@ -470,7 +467,6 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
         lifecycle,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -508,7 +504,6 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,

crates/scanner/tests/read_repair_scanner_queue.rs

@@ -0,0 +1,297 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! One process exercises dangling metadata, scanner disk reads, and heal admission.
+
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+use rustfs_ecstore::api::{
+    capacity::path2_bucket_object_with_base_path,
+    config::storageclass::{CLASS_STANDARD, DEFAULT_KVS, STANDARD, lookup_config},
+    disk::{DeleteOptions, DiskAPI, DiskError, ReadOptions, STORAGE_FORMAT_FILE},
+    object::{ObjectOptions, PutObjReader, WriteCompletion},
+};
+use rustfs_filemeta::FileInfo;
+use rustfs_heal::{
+    HealChannelProcessor,
+    heal::{
+        manager::{HealConfig, HealManager},
+        storage::ECStoreHealStorage,
+        task::HealTaskStatus,
+    },
+};
+use rustfs_heal_contracts::heal_channel::{
+    HealChannelCommand, HealChannelResponse, HealRequestSource, init_heal_channels, subscribe_heal_responses,
+};
+use rustfs_scanner::{scanner_folder::ScannerItem, scanner_heal_observation_status, scanner_io::ScannerIODisk};
+use rustfs_storage_api::{ObjectIO, ObjectOperations};
+use rustfs_test_utils::TestECStoreEnv;
+use time::OffsetDateTime;
+use tokio::{sync::broadcast, time::timeout};
+use uuid::Uuid;
+
+const DEADLINE: Duration = Duration::from_secs(60);
+
+async fn accepted_response(responses: &mut broadcast::Receiver<HealChannelResponse>, id: &str) {
+    let response = timeout(DEADLINE, responses.recv())
+        .await
+        .expect("heal admission response timed out")
+        .expect("heal admission response channel closed or lagged");
+    assert_eq!(response.request_id, id, "admission must belong to the forwarded request");
+    assert!(response.success, "heal admission failed: {:?}", response.error);
+    assert_eq!(response.data.as_deref(), Some(&b"admission=accepted,reason=none"[..]));
+}
+
+async fn wait_for_scanner_pending(expected: usize) {
+    let expected_candidates = u64::try_from(expected).expect("bounded scanner candidate count");
+    timeout(DEADLINE, async {
+        loop {
+            let status = scanner_heal_observation_status();
+            assert_ne!(status.state, "stopped", "scanner observer stopped");
+            assert!(status.unknown_candidates.is_empty(), "scanner observer lost a candidate: {status:?}");
+            if status.state == "active" {
+                assert_eq!(status.lost_completion_notifications, Some(0));
+                assert_eq!(status.terminal_task_observations_total, Some(0));
+                if status.retained_pending_tasks == Some(expected)
+                    && status.retained_pending_candidates == Some(expected_candidates)
+                    && status.queued_admissions == Some(0)
+                {
+                    break;
+                }
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .unwrap_or_else(|_| panic!("scanner receipt observation timed out: {:?}", scanner_heal_observation_status()));
+}
+
+#[tokio::test]
+async fn dangling_metadata_read_repair_and_scanner_have_distinct_queued_owners() {
+    // This test binary owns the process-wide channels and storage-class config.
+    temp_env::async_with_vars(
+        [
+            ("RUSTFS_STORAGE_CLASS_STANDARD", Some("EC:2")),
+            ("RUSTFS_STORAGE_CLASS_RRS", Some("EC:1")),
+        ],
+        async {
+            let root = tempfile::tempdir().expect("create isolated six-disk fixture");
+            let env = TestECStoreEnv::builder().disk_count(6).base_dir(root.path()).build().await;
+            let mut kvs = (*DEFAULT_KVS).clone();
+            kvs.insert(CLASS_STANDARD.to_owned(), "EC:2".to_owned());
+            let storage_class = lookup_config(&kvs, 6).expect("configure EC 4+2");
+            assert_eq!(storage_class.get_parity_for_sc(STANDARD), Some(2));
+            env.ecstore.set_storage_class(storage_class);
+
+            let bucket = "read-repair-scanner";
+            let object = "torn/latest";
+            env.make_bucket(bucket, true).await;
+            let sets = env.ecstore.all_set_disks();
+            assert_eq!(sets.len(), 1, "fixture must have one erasure set");
+            let set = &sets[0];
+            assert_eq!((set.pool_index, set.set_index), (0, 0));
+            let options = |seconds| ObjectOptions {
+                versioned: true,
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(seconds).expect("fixture timestamp")),
+                write_completion: WriteCompletion::TailDrained,
+                ..Default::default()
+            };
+            let v1 = set
+                .put_object(
+                    bucket,
+                    object,
+                    &mut PutObjReader::from_vec(b"version one".to_vec()),
+                    &options(1_700_000_000),
+                )
+                .await
+                .expect("write committed first version")
+                .version_id
+                .expect("first version id");
+            let v2 = set
+                .put_object(
+                    bucket,
+                    object,
+                    &mut PutObjReader::from_vec(b"version two".to_vec()),
+                    &options(1_700_000_060),
+                )
+                .await
+                .expect("write second version")
+                .version_id
+                .expect("second version id");
+            assert_ne!(v1, v2);
+
+            let disks = set.disks.read().await.clone();
+            assert_eq!(disks.len(), 6);
+            for disk in &disks[3..] {
+                let disk = disk.as_ref().expect("all fixture disks are online");
+                let mut remove = FileInfo {
+                    volume: bucket.to_owned(),
+                    name: object.to_owned(),
+                    version_id: Some(v2),
+                    ..Default::default()
+                };
+                remove.set_skip_tier_free_version();
+                disk.delete_version(bucket, object, remove, false, DeleteOptions::default())
+                    .await
+                    .expect("remove second version from one fixture disk");
+            }
+            for (index, disk) in disks.iter().enumerate() {
+                let disk = disk.as_ref().expect("all fixture disks are online");
+                disk.read_version("", bucket, object, &v1.to_string(), &ReadOptions::default())
+                    .await
+                    .expect("first version remains on every disk");
+                let second = disk
+                    .read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
+                    .await;
+                if index < 3 {
+                    let info = second.expect("second version remains on three disks");
+                    assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (4, 2));
+                } else {
+                    let error = second.expect_err("second version must be absent from this disk");
+                    assert!(DiskError::is_err_version_not_found(&error), "unexpected disk error: {error:?}");
+                }
+            }
+
+            let manager = Arc::new(HealManager::new(
+                Arc::new(ECStoreHealStorage::new(env.ecstore.clone())),
+                Some(HealConfig {
+                    enable_auto_heal: false,
+                    low_priority_merge_enable: true,
+                    mainline_throttle_enable: false,
+                    queue_size: 16,
+                    ..Default::default()
+                }),
+            ));
+            // No manager scheduler runs: accepted requests remain in the real queue.
+            let (mut commands, mut receipts) = init_heal_channels().expect("initialize process-local heal channels");
+            let (command_tx, command_rx) = tokio::sync::mpsc::unbounded_channel();
+            let (receipt_tx, receipt_rx) = tokio::sync::mpsc::unbounded_channel();
+            let processor_manager = manager.clone();
+            let processor = tokio::spawn(async move {
+                HealChannelProcessor::new(processor_manager)
+                    .start_with_receipts(command_rx, receipt_rx)
+                    .await
+            });
+            let mut responses = subscribe_heal_responses();
+
+            // The dangling-latest read uses submit_read_repair_heal, whose MRF intent is None.
+            let tagged = set
+                .put_object_tags(bucket, object, "stage=committed", &ObjectOptions::default())
+                .await
+                .expect("latest tagging skips the dangling second version");
+            assert_eq!(tagged.version_id, Some(v1));
+
+            // Read-repair submits asynchronously, so wait for its actual request and admission.
+            let command = timeout(DEADLINE, commands.recv())
+                .await
+                .expect("read-repair request timed out")
+                .expect("legacy heal command channel closed");
+            let read_repair_id = match &command {
+                HealChannelCommand::Start { request, .. } => {
+                    assert_eq!(request.source, HealRequestSource::ReadRepair);
+                    assert_eq!(request.bucket, bucket);
+                    assert_eq!(request.object_prefix.as_deref(), Some(object));
+                    assert_eq!(request.object_version_id.as_deref(), Some(v2.to_string().as_str()));
+                    assert_eq!(request.pool_index, Some(set.pool_index));
+                    assert_eq!(request.set_index, Some(set.set_index));
+                    request.id.clone()
+                }
+                _ => panic!("expected read-repair start command"),
+            };
+            command_tx.send(command).expect("forward original read-repair command once");
+            accepted_response(&mut responses, &read_repair_id).await;
+
+            // This isolated process has made no scanner admission before the disk read.
+            let initial = scanner_heal_observation_status();
+            assert_eq!(initial.state, "not_started");
+            assert_eq!(initial.retained_pending_tasks, None);
+            assert_eq!(initial.retained_pending_candidates, None);
+
+            let scan_disk = disks[0].as_ref().expect("first fixture disk").clone();
+            let disk_path = scan_disk.path();
+            let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
+            let (_, scanner_path) = path2_bucket_object_with_base_path(
+                disk_path.to_str().expect("fixture disk path is UTF-8"),
+                metadata_path.to_str().expect("fixture metadata path is UTF-8"),
+            );
+            let scanner = ScannerItem {
+                path: scanner_path,
+                bucket: bucket.to_owned(),
+                prefix: object.to_owned(),
+                object_name: STORAGE_FORMAT_FILE.to_owned(),
+                file_type: tokio::fs::metadata(&metadata_path)
+                    .await
+                    .expect("stat surviving real metadata")
+                    .file_type(),
+                lifecycle: None,
+                object_lock: None,
+                replication: None,
+                heal_enabled: true,
+                heal_bitrot: false,
+                debug: false,
+            };
+            let scan = tokio::spawn(async move { scan_disk.get_size_with_tier_names(scanner, &[]).await });
+            let mut scanner_ids = HashMap::new();
+            for index in 0..2 {
+                let command = timeout(DEADLINE, receipts.recv())
+                    .await
+                    .expect("scanner request timed out")
+                    .expect("scanner heal receipt channel closed");
+                let request = &command.request;
+                assert_eq!(request.source, HealRequestSource::Scanner);
+                assert_eq!(request.bucket, bucket);
+                assert_eq!(request.object_prefix.as_deref(), Some(object));
+                assert_eq!((request.pool_index, request.set_index), (None, None));
+                let version = Uuid::parse_str(request.object_version_id.as_deref().expect("scanner version id"))
+                    .expect("scanner version id is a UUID");
+                assert!(version == v1 || version == v2, "unexpected scanner version {version}");
+                let id = request.id.clone();
+                assert!(scanner_ids.insert(version, id.clone()).is_none(), "duplicate scanner version");
+                receipt_tx.send(command).expect("forward original scanner command once");
+                accepted_response(&mut responses, &id).await;
+                // Broadcast precedes the oneshot; wait until Scanner consumes each receipt.
+                wait_for_scanner_pending(index + 1).await;
+            }
+            timeout(DEADLINE, scan)
+                .await
+                .expect("scanner disk read timed out")
+                .expect("scanner task failed")
+                .expect("scanner read real object metadata");
+            assert!(
+                matches!(receipts.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Empty)),
+                "scanner submitted an unexpected extra request"
+            );
+            assert!(
+                matches!(commands.try_recv(), Err(tokio::sync::mpsc::error::TryRecvError::Empty)),
+                "an unexpected legacy request was submitted"
+            );
+            assert_eq!(scanner_ids.len(), 2);
+            let scanner_v2_id = scanner_ids.get(&v2).expect("scanner submitted the dangling version");
+            assert_ne!(scanner_v2_id, &read_repair_id);
+            assert_eq!(manager.get_queue_length().await, 3);
+            for id in scanner_ids.values().chain(std::iter::once(&read_repair_id)) {
+                assert_eq!(manager.get_task_status(id).await.expect("queued owner status"), HealTaskStatus::Pending);
+            }
+
+            drop(command_tx);
+            drop(receipt_tx);
+            timeout(DEADLINE, processor)
+                .await
+                .expect("heal channel processor shutdown timed out")
+                .expect("heal channel processor task failed")
+                .expect("heal channel processor stopped");
+        },
+    )
+    .await;
+}

crates/targets/src/runtime/mod.rs

@@ -1471,7 +1471,6 @@ mod tests {
             inner: QueueStore<QueuedPayload>,
             fail_put_failed_raw: Arc<std::sync::atomic::AtomicBool>,
             fail_del: Arc<std::sync::atomic::AtomicBool>,
-            delete_attempts: Arc<Semaphore>,
         }
 
         impl Store<QueuedPayload> for FailingFailedStore {
@@ -1500,13 +1499,10 @@ mod tests {
                 self.inner.get_raw(key)
             }
             fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
-                let result = if self.fail_del.load(Ordering::SeqCst) {
-                    Err(StoreError::Internal("injected del failure".to_string()))
-                } else {
-                    self.inner.del(key)
-                };
-                self.delete_attempts.add_permits(1);
-                result
+                if self.fail_del.load(Ordering::SeqCst) {
+                    return Err(StoreError::Internal("injected del failure".to_string()));
+                }
+                self.inner.del(key)
             }
             fn delete(&self) -> Result<(), Self::Error> {
                 self.inner.delete()
@@ -1990,7 +1986,6 @@ mod tests {
                 inner: inner.clone(),
                 fail_put_failed_raw: fail_flag.clone(),
                 fail_del: Arc::new(std::sync::atomic::AtomicBool::new(false)),
-                delete_attempts: Arc::new(Semaphore::new(0)),
             };
             // The target's terminal move writes through the rejecting failed handle, so the injected
             // failure is what the move hits.
@@ -2047,7 +2042,6 @@ mod tests {
                 inner: inner.clone(),
                 fail_put_failed_raw: Arc::new(std::sync::atomic::AtomicBool::new(false)),
                 fail_del: Arc::new(std::sync::atomic::AtomicBool::new(true)),
-                delete_attempts: Arc::new(Semaphore::new(0)),
             };
             let target = Arc::new(ProgrammedTarget::new(
                 Some(TargetError::JetStreamPublish {
@@ -2059,15 +2053,8 @@ mod tests {
             let tally = Arc::new(ReplayTally::default());
             let (cancel_tx, worker) = spawn_failing_store_worker(&store, target, tally.clone());
 
-            // Observe repeated durable overwrites without auto-advancing through thousands of
-            // real filesystem writes. Keep the original virtual observation bound on progress.
-            tokio::time::timeout(Duration::from_secs(300), store.delete_attempts.acquire_many(8))
-                .await
-                .expect("replay should retry the failing delete eight times")
-                .expect("delete progress remains open")
-                .forget();
-            cancel_tx.send(()).await.expect("replay worker should remain cancellable");
-            worker.await.expect("replay worker should drain after cancellation");
+            // Many scans run while the live delete keeps failing.
+            tokio::time::sleep(Duration::from_secs(300)).await;
             assert_eq!(
                 tally.permanent_failure.load(Ordering::SeqCst),
                 0,
@@ -2076,6 +2063,8 @@ mod tests {
             assert_eq!(inner.len(), 1, "the live entry stays for the next scan");
             assert_eq!(inner.failed_len(), 1, "the durable failed copy landed exactly once");
 
+            let _ = cancel_tx.send(()).await;
+            let _ = worker.await;
             let _ = inner.delete();
         }
 
@@ -2095,7 +2084,6 @@ mod tests {
                 inner: store.clone(),
                 fail_put_failed_raw: Arc::new(std::sync::atomic::AtomicBool::new(false)),
                 fail_del: Arc::new(std::sync::atomic::AtomicBool::new(false)),
-                delete_attempts: Arc::new(Semaphore::new(0)),
             };
             let target = Arc::new(ProgrammedTarget::new(
                 Some(TargetError::JetStreamPublish {

docs/architecture/README.md

@@ -50,7 +50,6 @@ Required headings and strings in these files are asserted by `scripts/check_arch
 | [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
 | [scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md) | changing what makes a scanner cycle `Complete`, `Superseded`, or converged, adding or re-classifying a dirty-usage producer, deciding whether a background writeback may invalidate a cycle, changing scanner worker concurrency or throttling, or activating scoped dirty acknowledgements |
 | [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
-| [plugin-delegation.md](plugin-delegation.md) | changing original-user plugin delegation, attempt credentials, or Runner task APIs |
 | [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
 | [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
 | [remote-credential-sealing-adr.md](remote-credential-sealing-adr.md) | adding, reading, or persisting a stored remote credential (replication target, remote tier, on-demand migration source), or needing the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another |

docs/architecture/heal-terminal-reports.md

@@ -4,10 +4,6 @@ Admin heal tokens retain their terminal status for ten minutes from the original
 
 ## Publication and Recovery
 
-Authenticated token queries first inspect the receiving node's in-memory automatic tasks. An original Scanner, MRF, automatic-Heal, or read-repair token can return its path-bound snapshot on that node, including the retained outcome and incremental cursor. The task source and original ID remain attached to the completion snapshot; aliases cannot enter this local query path. The local lookup never reads administrator recovery records.
-
-Administrator tokens and unknown local identities continue through the cluster coordinator. Starting and cancelling tasks retain their coordinator route. Querying another node does not discover or aggregate a remote automatic task, and this path does not extend automatic-task retention across process restart.
-
 The original schema-1 `terminal-root-heal-<token>.json` remains the commit marker. Its separate `heal-terminal-report-<token>.json` report has its own schema version and embeds the exact terminal identity. Both files belong to the same coordinator disk as the pending intent.
 
 Publication writes the report through the storage owner's conditional file update, writes the terminal marker, and then conditionally removes the pending intent. Recovery never treats a report without its marker as committed. A committed terminal continues to suppress stale pending work even when its report is unreadable. An initial publication failure leaves the existing responsibility available for recovery.

docs/architecture/plugin-delegation.md

@@ -1,142 +0,0 @@
-# Plugin delegation
-
-**Use this when:** changing plugin delegation registration, revocation, broker identity, or the scope of attempt credentials.
-
-## Integration boundary
-
-Runner owns scheduling, leases, container execution and result publication. RustFS provides original-user delegation, scoped temporary credentials and authenticated task API forwarding. Inputs and outputs use ordinary versioned S3 buckets; event-driven jobs use the existing bucket notification/Webhook path. Managed artifact sealing, inventory, garbage collection and backup retention require separate storage contracts and are outside this integration.
-
-Set `RUSTFS_PLUGIN_RUNNER_URL` to the Controller HTTPS URL ending in `/v1/ingress/manual/jobs`, and `RUSTFS_PLUGIN_RUNNER_DEPLOYMENT` to the delegation deployment. Forwarding requires the published outbound mTLS client identity and uses the published trust roots when configured. Connections are reused within a TLS generation; publishing new TLS material replaces the pool, and removing the client identity rejects new forwarding requests.
-
-## Authority
-
-A delegation is an immutable restriction record owned by an ordinary IAM user. It is neither a permission grant nor a credential. Creating it requires that user's real signed request and `admin:CreatePluginDelegation`; reading and revoking require the corresponding `admin:GetPluginDelegation` and `admin:RevokePluginDelegation` actions. Root, temporary credentials and service accounts cannot own records.
-
-The record binds a deployment, an input bucket, literal input and output prefixes, and an expiry of at most 24 hours. The v1 profile writes to that same bucket; v2 explicitly names the output bucket. Wildcards, control characters and IAM variable expressions are rejected. Its random ID is also its immutable revision. There is no update operation.
-
-The Controller is responsible for validating accepted task identity, current lease, epoch, node session and control generation before requesting credentials. RustFS authenticates the configured broker and enforces the delegation and current parent IAM. RustFS does not independently verify a remote Controller lease.
-
-```mermaid
-sequenceDiagram
-    participant U as Original IAM user
-    participant C as Controller broker
-    participant R as RustFS
-    participant P as Plugin
-    U->>R: Signed delegation registration
-    R-->>U: Immutable ID and revision
-    C->>C: Validate task and current lease
-    C->>R: Signed issue-attempt request
-    R->>R: Broker mapping + active record + current IAM
-    R->>R: Persist scoped STS and recheck authority
-    R-->>C: Short-lived credentials
-    C-->>P: Attempt credentials
-    P->>R: Exact version GET / attempt-prefix PUT
-    R->>R: Current parent IAM intersected with session policy
-```
-
-## Persistence and revocation
-
-`crates/iam/src/delegation.rs` owns record validation and immutable revocation tombstones. Records use the existing IAM store under `config/iam/plugin-delegations/v1`. Writes require a successful readback. Revocation is idempotent and durable; it does not delete or overwrite the restriction record.
-
-`ObjectStore::load_iam_config_strict` distinguishes a missing object from an empty object, invalid JSON or failed decryption. Only a genuinely absent tombstone permits an active record. Corrupt or unavailable storage rejects access rather than treating revocation as absent. Existing non-delegation readers keep their compatibility behavior.
-
-Revocation prevents future issuance. It does not promise immediate invalidation of credentials already delivered; those remain subject to their expiration and current parent IAM. Registration, IAM changes and issuance are not one atomic transaction. Issuance rechecks after persistence and deletes the generated STS account if authority changed. Failed persistence or invalidation acknowledgment is reported as unknown issuance, never as safe retry permission.
-
-Manual job submission reads the active delegation before calling the Runner. A submission that passed this check is in flight and may be admitted if revocation races with the outbound request; revocation prevents submissions whose authority check occurs afterward, but does not cancel an admitted job. Callers must treat a successful submit receipt as accepted work even if the delegation is revoked later. After an ambiguous timeout or service error, retry the same idempotency key while the delegation remains active; after revocation or expiry, query status instead of retrying submission.
-
-`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/query` accepts `{version:1,run_id}` and requires an ordinary IAM user with `admin:GetPluginRun` who owns the delegation in `{id}`. It reads the immutable owner record without checking expiry or revocation, then queries Runner using the record's delegation ID and revision and the authenticated access key. The result can contain only that caller's run status and committed manifest; it does not perform object reads or reauthorize object access. The Runner response must match the requested run ID and the strict status schema before RustFS returns it.
-
-`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/cancel` accepts the same request and requires `admin:CancelPluginRun` plus ownership of `{id}`. Expiry and revocation do not block cancelling an existing run. Runner scopes the run to that owner; missing and other-owner runs share the same not-found response, while successful and failed runs return conflict. Success returns `{version:1,run_id,state:"CANCELLED",cleanup_pending}` and does not imply that asynchronous cleanup has finished.
-
-`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/page` accepts `{version:1,limit,cursor?}` with a limit from 1 to 100 and requires `admin:ListPluginRuns` plus ownership of `{id}`. Expiry and revocation do not hide historical runs. RustFS supplies signer and delegation identity from authenticated IAM state; the Runner page is scoped to the exact caller and deployment. The strict response contains run summaries and an opaque-to-clients stable cursor; callers pass the returned cursor unchanged until it is null. RustFS bounds request/response sizes and rejects malformed, unordered, over-limit, or identity-invalid summaries.
-
-`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/history` accepts `{version:1,run_id}` and requires `admin:ListPluginRuns` plus ownership of `{id}`. RustFS derives signer and delegation identity from the authenticated user and IAM record, then proxies Runner's bounded history response. The response contains at most the latest 32 committed state events in Runner's history-ID order and a `has_older` flag; timestamps are display data and are not used to reorder events. Missing and foreign runs share the same not-found response. RustFS rejects malformed, empty, over-limit, unexpected-state, or mismatched-run responses and bounds both request and response bodies.
-
-`POST /rustfs/admin/v3/plugin-delegations/{id}/runs/replay` accepts `{version:1,origin_run_id,idempotency_key,deadline_unix_micros}` and requires `admin:ReplayPluginRun`, current IAM authorization, ownership, and an active unexpired delegation. Caller identity fields are not accepted; RustFS derives signer and delegation revision from the authenticated request and IAM record. Runner accepts only an owned failed origin, preserves its fixed plugin/configuration/input version, and requires a new deadline no later than its configured run timeout. RustFS requires a positive Unix-microsecond deadline and verifies the strict receipt echoes it. Missing or foreign origins share not-found behavior; conflicts map to an operation conflict. Requests and receipts are bounded, and responses use `Cache-Control: no-store`.
-
-## Broker profile
-
-`rustfs/src/admin/handlers/plugin_attempt.rs` implements `POST /rustfs/admin/v3/plugin-delegations/{id}/issue-attempt`. The signer must be an ordinary IAM user with `admin:IssuePluginAttempt`, independently listed in `RUSTFS_PLUGIN_BROKERS`:
-
-```json
-{"broker-access-key":"deployment-name"}
-```
-
-This configuration contains public access keys, not secrets. It is immutable per process. Missing or invalid configuration disables broker issuance. The broker is a trusted Controller identity; ordinary users cannot substitute their own request fields for Controller validation.
-
-The request carries `tenant`, `run_id`, `attempt_id`, `revision`, `deployment`, `input_bucket`, `input_key`, `input_version`, `output_bucket`, `output_prefix` and `ttl_seconds`. IDs must be non-nil. Both profiles accept one exact input version and an output prefix equal to:
-
-```text
-<delegation-output-prefix><lowercase-hex-tenant>/<run-uuid>/<attempt-uuid>/
-```
-
-The session policy permits only `s3:GetObjectVersion` for that input with an exact `s3:VersionId`, and `s3:PutObject` beneath that attempt prefix. Credential lifetime is at most 900 seconds and cannot exceed delegation expiry. The Controller must additionally cap it to its validated lease and hard deadline.
-
-The original user's current IAM remains the parent permission boundary on every S3 request. The server's signing key only seals the token; it is not the plugin's authorization identity. Broker identity and original user identity are separate.
-
-External policy plugins must be disabled: their authorization path does not enforce this built-in session-policy boundary. Initializing, failed and ready plugin states all reject issuance. Attempt tokens carry a signed, persisted marker: after restart or on another node, marked credentials are accepted only by built-in STS authorization. A denied marked request returns a terminal error, so public bucket policies cannot bypass its session scope. The first profile also rejects parent policies containing conditions, because a broker request cannot reconstruct the original user's IP, MFA or other request context. It does not silently replace that context with the broker's.
-
-## Response and failures
-
-Successful issuance returns `version`, `delegation_id`, `delegation_revision`, `owner_access_key`, `access_key_id`, `secret_access_key`, `session_token` and `expires_at_unix_seconds`, with `Cache-Control: no-store`. The Controller verifies the returned delegation and owner against the accepted caller before delivery. Credentials must never enter logs, metrics or task result manifests.
-
-Missing records and records owned by a different user have the same read response. Revoked or expired records reject access. Storage failures remain errors. Post-persistence failures carry static `error_code`, `issued` and `issuance_unknown` facts without credentials; callers must reconcile an unknown outcome rather than automatically mint another credential.
-
-The route/action inventory is maintained in `rustfs/src/admin/route_policy.rs`. This contract does not imply event capture, physical container cleanup, restart reconciliation or upstream release acceptance.
-
-
-## Cross-bucket restriction profile v2
-
-The existing registration endpoint accepts this explicit v2 restriction:
-
-```json
-{
-  "version": 2,
-  "deployment": "runner-production",
-  "bucket": "source-images",
-  "input_prefix": "inputs/",
-  "output_bucket": "plugin-artifacts",
-  "output_prefix": "outputs/",
-  "expires_at_unix_seconds": 1893456000
-}
-```
-
-The example expiry must be replaced by a future value within the existing
-24-hour limit. `bucket` still identifies the input bucket. The existing
-`issue-attempt` request and credential response do not change: its
-`input_bucket` must equal `bucket`, and `output_bucket` must equal the explicit
-v2 output bucket. Neither a different bucket nor a sibling attempt prefix can
-be supplied by the broker. The exact input version and deterministic output
-prefix checks remain in force.
-
-A v1 restriction has neither `version` nor `output_bucket`; its JSON fields
-and same-bucket behavior are unchanged. V2 requires both `version: 2` and a
-nonempty, bounded, literal `output_bucket`. Missing pairs, explicit nulls,
-unknown versions and unknown fields are rejected rather than downgraded to v1.
-The immutable record envelope carries version 1 or 2 to match its restriction
-profile. Both remain in the existing IAM record namespace; no data migration
-or mutable record update is introduced. Readers reject mismatched envelope
-and restriction versions. Older readers reject v2, so deploy compatible
-readers before creating v2 records.
-
-V2 does not grant the original user additional permissions. Issuance retains
-its current-parent checks and post-persistence recheck/invalidation, and each
-S3 request still requires both the original user's current IAM policy and the
-attempt session policy to allow it. An explicit parent Deny or removal of
-output permission prevents writes even with an already issued token. No
-administrator credential substitutes for missing parent permission.
-
-Focused tests are `delegation_v1_json_remains_compatible`,
-`delegation_v2_requires_explicit_output_bucket`,
-`attempt_v2_policy_keeps_input_version_and_output_bucket` and
-`delegation_v2_persists_and_vends_with_real_parent_intersection`.
-The last uses the actual issuance handler, persistent IAM and four-disk
-storage, including a real input version and a third bucket allowed by the
-parent but forbidden by the delegation. Removing only the output-bucket
-comparison must fail `A03_CROSS_BUCKET_OUTPUT_BOUND`; parent intersection is
-checked independently with issued temporary credentials. These are internal
-handler/IAM tests, not signed HTTP or complete cross-bucket S3 execution.
-
-This profile adds no bucket-creation API, Runner binding support, credential
-endpoint or GC permission. Coordinating Runner's pinned binding with its
-result storage bucket remains a separate integration requirement.

docs/operations/scanner-runtime-controls.md

@@ -529,21 +529,17 @@ The OTLP exporter records these node-local histograms for completed manager admi
 
 The two new histograms aggregate all observed admission sources and results without per-request labels. An ordinary uninterrupted admission partitions the phase into wait and hold. Admin forceStart releases these guards while cancelling overlaps, then reacquires them: the cancellation gap is included only in the original total phase, and the second acquisition contributes to wait. Cancellation subroutine work is outside these outer admission guard intervals. Recording happens after all three guards are released. They measure manager admission ownership, not namespace/object-lock waiting during repair. The status field `maxLockPhaseMicros` remains a manager-lifetime maximum, not a p95 sample. Keep same-window histogram counts and bucket resolution when deriving percentiles; missing observations do not mean zero latency. These metrics do not establish performance acceptance by themselves.
 
-## Experimental shared node admission
+## Experimental heal node admission
 
-`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback shared by the heal manager’s four object-heal entry points and local Scanner namespace reads and object accounting. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. This does not yet control the abandoned-child streaming list coordinator, bucket/format repair or physical devices, and it is not a replacement for existing worker, disk or byte budgets.
+`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback for the four object-heal entry points owned by the heal manager. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. This does not yet control scanner walks, bucket/format repair or physical devices, and it is not a replacement for existing worker, disk or byte budgets.
 
 The node samples its effective CPU quota/cpuset, ancestor memory headroom, CPU usage/throttling and CPU/memory/I/O pressure once per second. Sampling uses one blocking worker at a time. Resource reads retain their real start/end times: reads spanning over 20 ms or older than one second after all inputs return cannot renew admission. Counter rates use both read brackets conservatively. The concurrency cap is at most the rounded-up effective CPU count and 256 scheduling units; existing operator limits still apply independently. At most 1% memory headroom blocks new work. This is a pressure safeguard, not a memory reservation or a claim that concurrent repair buffers fit in the remaining bytes.
 
 Each complete sample leases admission for three seconds. Missing resource caps block new work; missing or low-confidence feedback retains at most one maintenance unit. CPU use above 90%, throttling or pressure above 1% of elapsed time, memory headroom at most 10%, foreground errors/cancellations, a foreground queue or a comparable foreground p99 regression reduce concurrency immediately. Recovery requires CPU use at most 70%, throttling/stalls at most 0.1% of elapsed time, at least 20% memory headroom, fresh foreground evidence and actual successful work. The existing controller probes one additional unit only after eight consecutive healthy seconds and cooldown. Limits affect new admission and do not revoke active work.
 
-The Scanner captures the same node owner once per bucket walk. Directory open/refill, entry type and size lookup, and object accounting try admission before their first storage poll; each unit ends when its operation future finishes. On contention, local and remote coordinators suspend the same traversal, release the bucket cache lease, and join the shared FIFO queue. They reacquire the cache lease and verify both cache revisions before resuming. A changed cache rejects the suspended result. Cancellation also drops the queued request; the existing cycle budget still bounds the walk. This retains directory and recursion state while allowing Heal to advance at capacity one, without repeatedly replaying a partially classified directory.
-
-Calls without a coordinating cache lease retain immediate resource deferral and useful partial-cache handling. Existing remote single-session ownership and local-disk admission shares remain in force. Disabled admission creates no request channel. Enabled walks use one bounded channel, with a response allocation and cache revision reads only on contention; the uncontended per-entry path adds no global lookup or task. The streaming abandoned-child coordinator requires a separate ownership boundary: retaining a node unit while it waits for a bounded-channel consumer that needs the same unit would deadlock at capacity one.
-
 Foreground comparison uses the existing complete response-body windows and shared admission snapshots. Each active class needs at least 100 completions and a finite p99 interval; this permits the percentile rank but is not a statistical confidence guarantee. Comparisons require the same active classes and rates within 80–125% of a retained reference. A changed workload resets recovery. The reference is an observed workload window, not an independent background-off baseline. A p99 interval can justify stability only when its upper bound is within 5% of the reference lower bound; a lower bound above 105% of the reference upper bound is pressure. Overlapping/uncertain comparisons cannot justify growth. Missing queues, stale or repeated cuts, uncertain cut boundaries and low counts are not idle evidence.
 
-Heal admission waits at most 250 ms before returning the existing typed transient deferral, before polling storage or taking namespace locks. No-lock callers never wait. Queued read repair uses the same node gate; direct foreground ECStore calls remain outside this heal-manager adapter. Cancellation releases the non-cloneable node scheduling unit. Native work retains separate physical-I/O units, and unsuccessful/cancelled/dry-run work cannot increment the node's completed-work counter. This scheduling counter is not an authoritative heal receipt and cannot discharge repair responsibility.
+Admission waits at most 250 ms before returning the existing typed transient deferral, before polling storage or taking namespace locks. No-lock callers never wait. Queued read repair uses the same node gate; direct foreground ECStore calls remain outside this heal-manager adapter. Cancellation releases the non-cloneable node scheduling unit. Native work retains separate physical-I/O units, and unsuccessful/cancelled/dry-run work cannot increment the node's completed-work counter. This scheduling counter is not an authoritative heal receipt and cannot discharge repair responsibility.
 
 The `rustfs_background_node_admission_limit` and `rustfs_background_node_admission_active` gauges describe the last published sample, not a live reservation proof. A blocked sampler can leave these exports stale even though the admission lease has expired. Full disk-owner feedback, shared scanner scheduling, escaping-buffer byte ownership and the matched-release SSD matrix/hot-shrink/72-hour soak remain required before enabling this by default or claiming optimal performance.
 

docs/testing/README.md

@@ -34,8 +34,6 @@ The [Pool layout compatibility reference](pool-layout-compatibility.md) defines
 
 The [Scanner/Heal topology protection gates](scanner-heal-topology-gates.md) map G-EXP, G-REB and G-DEC and their fault windows to the `scanner_heal_topology_` cases, their recorded inventory, and the basis of their timeouts.
 
-The [immutable validation candidate reference](validation-candidates.md) describes isolated package builds and exact candidate resolution for the current auto-testing fleet.
-
 The [distributed multipart RPC delay gates](multipart-rpc-delay.md) define authenticated method-entry barriers, latency and peer-loss oracles, and the boundary between hook evidence and production incident attribution.
 
 ## Naming conventions

docs/testing/validation-candidates.md

@@ -1,52 +0,0 @@
-# Immutable validation candidates
-
-Canonical validation on the current auto-testing fleet consumes one exact
-package, source commit and candidate-bound tool build. A successful package
-build is provenance evidence, not a canonical test PASS.
-
-The scheduled Functional Chain continues to resolve successful Nightly GNU
-Build attempts from `main`. Optional manual validation can also resolve a
-validation package built without publishing nightly aliases or release assets.
-
-## Build a validation package
-
-Dispatch **RustFS Functional Chain** from the desired driver revision with
-only `validation_package_sha` set to the full 40-character RustFS source SHA.
-Keep candidate, case, helper, tools and cleanup inputs empty or false. The
-request validator rejects mixed package/fleet execution before building.
-
-The package job calls `rustfs-validation-package.yml`, checks out the requested
-source, confirms its Git SHA and builds the ordinary release binary and DEB.
-This job uses build runners and does not acquire or mutate the Heal fleet.
-The same workflow can be dispatched directly with its `source_sha` input when
-it is available on the repository's default branch.
-
-Publication requires complete R2 configuration, conditional `PutObject` with
-`If-None-Match: *`, and a public download whose SHA-256 matches the local DEB.
-Failure emits no candidate manifest. An existing object is never overwritten.
-The only object path is:
-
-```text
-artifacts/rustfs/packages/validation/runs/<run>/<attempt>/<package-sha256>/rustfs.deb
-```
-
-The schema-3 artifact `validation-candidate-<run>-<attempt>` records the
-workflow SHA independently of the exact source SHA and source ref, the
-`validation` channel, build run/attempt, immutable URL and package checksum.
-The resolver verifies the successful manual run, repository/workflow identity,
-exact artifact name and digest, archive members, schema and immutable URL.
-The ordinary scheduled chain does not enable validation package resolution.
-
-## Validate the candidate
-
-After the package run succeeds, dispatch Functional Chain with its exact
-`build_run_id` and `build_run_attempt`, a canonical `validation_case`, and an
-exact `auto_testing_sha`. Heal also requires a new tool build verified against
-that source and helper revision. Reassess source-bound contracts and native
-prerequisites when the source changes; an older tool artifact does not certify
-a new candidate.
-
-The fleet lane retains the shared environment lock, per-case evidence and
-cleanup gate. Record new results separately. A package build, native test,
-diagnostic replay, clean recovery or partial topology result cannot replace
-complete original source assertions or rewrite a retained canonical FAIL.

rustfs/src/admin/handlers/heal.rs

@@ -1051,21 +1051,6 @@ async fn route_cluster_heal_control(
         })
 }
 
-async fn query_heal_status_on_owner<L, C, CF>(
-    local: L,
-    coordinator: C,
-) -> S3Result<rustfs_heal_contracts::heal_channel::HealChannelResponse>
-where
-    L: Future<Output = S3Result<Option<rustfs_heal_contracts::heal_channel::HealChannelResponse>>>,
-    C: FnOnce() -> CF,
-    CF: Future<Output = S3Result<rustfs_heal_contracts::heal_channel::HealChannelResponse>>,
-{
-    match local.await? {
-        Some(response) => Ok(response),
-        None => coordinator().await,
-    }
-}
-
 async fn execute_after_heal_start_preflight<P, PF, E, EF, T>(
     endpoints: &EndpointServerPools,
     options: &HealOpts,
@@ -1529,41 +1514,20 @@ impl Operation for HealHandler {
         let heal_path = encode_heal_control_path(&hip.bucket, &hip.obj_prefix);
         if !hip.client_token.is_empty() && !hip.force_start && !hip.force_stop {
             let client_token = hip.client_token.clone();
-            // Automatic workers run on their owning node. Only an exact
-            // path/token/source snapshot can answer locally; Admin tokens and
-            // unknown identities retain the coordinator authority boundary.
-            let response = query_heal_status_on_owner(
-                async {
-                    let Some(manager) = rustfs_heal::get_heal_manager() else {
-                        return Ok(None);
-                    };
-                    let processor = rustfs_heal::HealChannelProcessor::new(manager.clone());
-                    timeout(
-                        PEER_HEAL_STATUS_TIMEOUT,
-                        processor.execute_local_automatic_query(&heal_path, &client_token, hip.since_seq),
-                    )
-                    .await
-                    .map_err(|_| cluster_heal_control_unavailable("local_task_query_expired"))?
-                    .map_err(|_| cluster_heal_control_unavailable("local_task_query_failed"))
-                },
-                || async {
-                    let request_id = uuid::Uuid::new_v4().to_string();
-                    let context = app_context
-                        .clone()
-                        .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?;
-                    let route = prepare_heal_control_route(&context)?;
-                    let envelope = rustfs_protos::heal_control::Envelope::query(
-                        request_id.clone(),
-                        new_heal_control_metadata(&route)?,
-                        heal_path.clone(),
-                        client_token.clone(),
-                        hip.since_seq,
-                    )
-                    .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?;
-                    submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token.clone()).await
-                },
+            let request_id = uuid::Uuid::new_v4().to_string();
+            let context = app_context
+                .clone()
+                .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?;
+            let route = prepare_heal_control_route(&context)?;
+            let envelope = rustfs_protos::heal_control::Envelope::query(
+                request_id.clone(),
+                new_heal_control_metadata(&route)?,
+                heal_path,
+                client_token.clone(),
+                hip.since_seq,
             )
-            .await?;
+            .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?;
+            let response = submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token).await?;
             if !response.success {
                 return Err(s3_error!(
                     InternalError,
@@ -2031,54 +1995,6 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use uuid::Uuid;
 
-    #[tokio::test]
-    async fn local_automatic_heal_query_preserves_owner_and_coordinator_errors() {
-        let response = || rustfs_heal_contracts::heal_channel::HealChannelResponse {
-            request_id: "original-token".to_string(),
-            success: true,
-            data: Some(b"retained-owner-report".to_vec()),
-            error: None,
-        };
-        let called = AtomicBool::new(false);
-        let local = super::query_heal_status_on_owner(async { Ok(Some(response())) }, || async {
-            called.store(true, Ordering::SeqCst);
-            Ok(response())
-        })
-        .await
-        .expect("local automatic owner response");
-        assert!(!called.load(Ordering::SeqCst));
-        assert_eq!(local.request_id, "original-token");
-        assert_eq!(local.data.as_deref(), Some(b"retained-owner-report".as_slice()));
-
-        let fallback = super::query_heal_status_on_owner(async { Ok(None) }, || async {
-            called.store(true, Ordering::SeqCst);
-            Ok(response())
-        })
-        .await
-        .expect("unknown local identity uses coordinator");
-        assert!(called.swap(false, Ordering::SeqCst));
-        assert_eq!(fallback.data, local.data);
-
-        let error = super::query_heal_status_on_owner(
-            async { Err(super::admin_error(S3ErrorCode::InternalError, "local lookup failed")) },
-            || async {
-                called.store(true, Ordering::SeqCst);
-                Ok(response())
-            },
-        )
-        .await
-        .expect_err("local failure must not become coordinator success");
-        assert_eq!(error.code(), &S3ErrorCode::InternalError);
-        assert!(!called.load(Ordering::SeqCst));
-
-        let error = super::query_heal_status_on_owner(async { Ok(None) }, || async {
-            Err(super::admin_error(S3ErrorCode::ServiceUnavailable, "coordinator unavailable"))
-        })
-        .await
-        .expect_err("coordinator error must remain visible");
-        assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
-    }
-
     #[test]
     fn mrf_legacy_only_refresh_accepts_an_unobserved_incarnation() {
         for action in ["refresh", "recheck", "acceptUnverifiedRisk"] {

rustfs/src/admin/handlers/mod.rs

@@ -52,8 +52,6 @@ pub mod object_data_cache;
 pub mod object_zip_download;
 pub mod oidc;
 pub mod on_demand_migration;
-pub mod plugin_attempt;
-pub mod plugin_delegation;
 pub mod plugins_catalog;
 pub mod plugins_instances;
 pub mod policies;

rustfs/src/admin/handlers/plugin_attempt.rs

@@ -1,568 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-//! Restricted vending for explicitly configured brokers, never AssumeRole chaining.
-use super::plugin_delegation::{owner, response, storage_error};
-use crate::{
-    admin::{
-        router::{AdminOperation, Operation, S3Router},
-        runtime_sources,
-        storage_api::s3::{self, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result},
-    },
-    auth::check_key_valid_with_context,
-    server::ADMIN_PREFIX,
-};
-use http::{Method, StatusCode};
-use matchit::Params;
-use rustfs_iam::{
-    delegation::Delegation,
-    sys::{PLUGIN_ATTEMPT_CREDENTIAL_CLAIM, SESSION_POLICY_NAME},
-};
-use rustfs_policy::{
-    auth::get_new_credentials_with_metadata,
-    policy::{
-        Args, Policy,
-        action::{Action, AdminAction, S3Action},
-    },
-};
-use serde::{Deserialize, Serialize};
-use std::{
-    collections::HashMap,
-    sync::OnceLock,
-    time::{Duration, SystemTime, UNIX_EPOCH},
-};
-use time::OffsetDateTime;
-
-// Operator configuration contains public signer access keys, not credentials.
-// Empty/invalid configuration disables the endpoint. It is immutable per process.
-static BROKERS: OnceLock<Option<HashMap<String, String>>> = OnceLock::new();
-fn configured_deployment(key: &str) -> Option<&'static str> {
-    BROKERS
-        .get_or_init(|| {
-            let raw = std::env::var("RUSTFS_PLUGIN_BROKERS").ok()?;
-            let map: HashMap<String, String> = serde_json::from_str(&raw).ok()?;
-            if map.is_empty()
-                || map
-                    .iter()
-                    .any(|(k, v)| k.is_empty() || v.is_empty() || k.len() > 1024 || v.len() > 1024)
-            {
-                return None;
-            }
-            Some(map)
-        })
-        .as_ref()?
-        .get(key)
-        .map(String::as_str)
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct Attempt {
-    tenant: String,
-    run_id: uuid::Uuid,
-    attempt_id: uuid::Uuid,
-    revision: String,
-    deployment: String,
-    input_bucket: String,
-    input_key: String,
-    input_version: String,
-    output_bucket: String,
-    output_prefix: String,
-    ttl_seconds: u64,
-}
-#[derive(Serialize)]
-struct Issued {
-    version: u32,
-    delegation_id: String,
-    delegation_revision: String,
-    owner_access_key: String,
-    access_key_id: String,
-    secret_access_key: String,
-    session_token: String,
-    expires_at_unix_seconds: i64,
-}
-
-pub fn register_plugin_attempt_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/issue-attempt"),
-        AdminOperation(&Issue),
-    )?;
-    Ok(())
-}
-fn now() -> S3Result<u64> {
-    SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .map(|d| d.as_secs())
-        .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))
-}
-fn literal(value: &str) -> bool {
-    !value.is_empty()
-        && value.len() <= 1024
-        && !value.contains("${")
-        && !value.chars().any(|c| c.is_control() || matches!(c, '*' | '?'))
-}
-fn session_policy(record: &Delegation, attempt: &Attempt) -> S3Result<Policy> {
-    let restriction = &record.restrictions;
-    // Stored records from earlier versions must obey the same literal rules.
-    restriction.validate(record.created_at_unix_seconds).map_err(storage_error)?;
-    let exact_output = format!(
-        "{}{}/{}/{}/",
-        restriction.output_prefix,
-        hex_simd::encode_to_string(attempt.tenant.as_bytes(), hex_simd::AsciiCase::Lower),
-        attempt.run_id,
-        attempt.attempt_id
-    );
-    if attempt.tenant.is_empty()
-        || attempt.tenant.len() > 1024
-        || attempt.run_id.is_nil()
-        || attempt.attempt_id.is_nil()
-        || attempt.output_prefix != exact_output
-        || attempt.revision != record.revision
-        || attempt.deployment != restriction.deployment
-        || attempt.input_bucket != restriction.bucket
-        || attempt.output_bucket != restriction.output_bucket().map_err(storage_error)?
-        || !literal(&attempt.input_key)
-        || !attempt.input_key.starts_with(&restriction.input_prefix)
-        || !literal(&attempt.input_version)
-        || attempt.input_version == "null"
-        || !literal(&attempt.output_prefix)
-        || !attempt.output_prefix.starts_with(&restriction.output_prefix)
-        || attempt.output_prefix == restriction.output_prefix
-        || !attempt.output_prefix.ends_with('/')
-        || !(1..=900).contains(&attempt.ttl_seconds)
-    {
-        return Err(s3::error(S3ErrorCode::AccessDenied, "attempt restriction rejected"));
-    }
-    let value = serde_json::json!({"Version":"2012-10-17","Statement":[
-        {"Effect":"Allow","Action":["s3:GetObjectVersion"],"Resource":[format!("arn:aws:s3:::{}/{}",attempt.input_bucket,attempt.input_key)],"Condition":{"StringEquals":{"s3:VersionId":attempt.input_version}}},
-        {"Effect":"Allow","Action":["s3:PutObject"],"Resource":[format!("arn:aws:s3:::{}/{}*",attempt.output_bucket,attempt.output_prefix)]}
-    ]});
-    // Condition operators borrow their keys while the policy is decoded.
-    let bytes = serde_json::to_vec(&value).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))?;
-    serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))
-}
-
-fn issuance_error(code: &'static str, issued: bool, unknown: bool) -> S3Result<S3Response<(StatusCode, Body)>> {
-    let mut answer = response(&serde_json::json!({"error_code":code,"issued":issued,"issuance_unknown":unknown}))?;
-    answer.output.0 = StatusCode::SERVICE_UNAVAILABLE;
-    Ok(answer)
-}
-fn same_parent_authority(before_names: &[String], before_policy: &Policy, after_names: &[String], after_policy: &Policy) -> bool {
-    fn names(names: &[String]) -> Vec<&str> {
-        let mut names: Vec<_> = names.iter().map(String::as_str).collect();
-        names.sort_unstable();
-        names.dedup();
-        names
-    }
-    fn statements(policy: &Policy) -> Option<Vec<Vec<u8>>> {
-        let mut statements = policy
-            .statements
-            .iter()
-            .map(serde_json::to_vec)
-            .collect::<Result<Vec<_>, _>>()
-            .ok()?;
-        statements.sort_unstable();
-        Some(statements)
-    }
-
-    names(before_names) == names(after_names)
-        && matches!(
-            (statements(before_policy), statements(after_policy)),
-            (Some(before), Some(after)) if before == after
-        )
-}
-async fn require_builtin_authorization() -> S3Result<()> {
-    rustfs_iam::sys::IamSys::<rustfs_iam::store::object::ObjectStore>::ensure_restricted_credential_vending()
-        .await
-        .map_err(|_| s3::error(S3ErrorCode::AccessDenied, "external policy plugin unsupported for attempt credentials"))
-}
-
-struct Issue;
-#[async_trait::async_trait]
-impl Operation for Issue {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        require_builtin_authorization().await?;
-        let (iam, broker) = owner(&req, AdminAction::IssuePluginAttemptAction).await?;
-        let deployment =
-            configured_deployment(&broker).ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "broker deployment rejected"))?;
-        let context = runtime_sources::app_context_from_req(&req)
-            .ok_or_else(|| s3::error(S3ErrorCode::InternalError, "application context unavailable"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "attempt body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "attempt body unavailable"))?;
-        let attempt: Attempt =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid attempt request"))?;
-        if attempt.deployment != deployment {
-            return Err(s3::error(S3ErrorCode::AccessDenied, "broker deployment rejected"));
-        }
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let record = iam
-            .read_active_plugin_delegation_for_broker(id, now()?)
-            .await
-            .map_err(storage_error)?;
-        let policy = session_policy(&record, &attempt)?;
-        let (principal, root) = check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
-        if root || principal.is_temp() || principal.is_service_account() {
-            return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
-        }
-        // A broker request cannot reconstruct the original user's IP/MFA/tag
-        // conditions. This first profile rejects conditional parent policies.
-        let policies = iam
-            .policy_db_get(&principal.access_key, &principal.groups)
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InternalError, "parent policy unavailable"))?;
-        let parent_policy = iam.get_combined_policy(&policies).await;
-        if policies.is_empty() || parent_policy.statements.iter().any(|s| !s.conditions.is_empty()) {
-            return Err(s3::error(S3ErrorCode::AccessDenied, "parent condition context unsupported"));
-        }
-        let mut conditions = HashMap::new();
-        conditions.insert("s3:VersionId".to_owned(), vec![attempt.input_version.clone()]);
-        if !iam
-            .is_allowed(&Args {
-                account: &principal.access_key,
-                groups: &principal.groups,
-                action: Action::S3Action(S3Action::GetObjectVersionAction),
-                bucket: &attempt.input_bucket,
-                object: &attempt.input_key,
-                conditions: &conditions,
-                is_owner: false,
-                claims: principal.claims_or_empty(),
-                deny_only: false,
-            })
-            .await
-        {
-            return Err(s3::error(S3ErrorCode::AccessDenied, "parent input rejected"));
-        }
-        let expiration = now()?
-            .checked_add(attempt.ttl_seconds)
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?
-            .min(record.restrictions.expires_at_unix_seconds);
-        let expires_at = OffsetDateTime::from_unix_timestamp(
-            i64::try_from(expiration).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?,
-        )
-        .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid expiration"))?;
-        let mut claims = principal.claims.clone().unwrap_or_default();
-        claims.insert(PLUGIN_ATTEMPT_CREDENTIAL_CLAIM.into(), serde_json::Value::Bool(true));
-        claims.insert("exp".into(), serde_json::json!(expires_at.unix_timestamp()));
-        claims.insert("parent".into(), serde_json::json!(principal.access_key));
-        let bytes =
-            serde_json::to_vec(&policy).map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt policy unavailable"))?;
-        claims.insert(
-            SESSION_POLICY_NAME.into(),
-            serde_json::json!(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&bytes)),
-        );
-        let fresh = iam
-            .read_active_plugin_delegation_for_broker(id, now()?)
-            .await
-            .map_err(storage_error)?;
-        if fresh != record {
-            return Err(s3::error(S3ErrorCode::AccessDenied, "attempt authority changed"));
-        }
-        owner(&req, AdminAction::IssuePluginAttemptAction).await?;
-        require_builtin_authorization().await?;
-        let signing = context
-            .action_credentials()
-            .get()
-            .ok_or_else(|| s3::error(S3ErrorCode::InternalError, "token signer unavailable"))?;
-        // The server key seals the token only. Authorization above uses the
-        // original user's current IAM, and S3 continues its parent intersection.
-        let mut credential = get_new_credentials_with_metadata(&claims, &signing.secret_key)
-            .map_err(|_| s3::error(S3ErrorCode::InternalError, "attempt credential unavailable"))?;
-        credential.parent_user = principal.access_key.clone();
-        if iam.set_temp_user(&credential.access_key, &credential, None).await.is_err() {
-            return issuance_error("PLUGIN_ATTEMPT_ISSUANCE_UNKNOWN", false, true);
-        }
-        // No atomic IAM/delegation transaction spans persistence. Recheck and
-        // invalidate the generated credential if any authority changed.
-        let valid = async {
-            require_builtin_authorization().await?;
-            let after = iam
-                .read_active_plugin_delegation_for_broker(id, now()?)
-                .await
-                .map_err(storage_error)?;
-            let (current, current_root) =
-                check_key_valid_with_context("", &record.owner_access_key, Some(context.as_ref())).await?;
-            let names = iam
-                .policy_db_get(&current.access_key, &current.groups)
-                .await
-                .map_err(|_| s3::error(S3ErrorCode::InternalError, "parent policy unavailable"))?;
-            let current_policy = iam.get_combined_policy(&names).await;
-            if after != record
-                || current_root
-                || current.is_temp()
-                || current.is_service_account()
-                || current.secret_key != principal.secret_key
-                || !same_parent_authority(&policies, &parent_policy, &names, &current_policy)
-            {
-                return Err(s3::error(S3ErrorCode::AccessDenied, "attempt authority changed"));
-            }
-            // Re-evaluate the broker against its real signed request context.
-            owner(&req, AdminAction::IssuePluginAttemptAction).await?;
-            Ok::<(), S3Error>(())
-        }
-        .await;
-        if valid.is_err() {
-            if iam.delete_temp_account(&credential.access_key, true).await.is_err() {
-                return issuance_error("PLUGIN_ATTEMPT_INVALIDATION_UNKNOWN", true, true);
-            }
-            return issuance_error("PLUGIN_ATTEMPT_POST_ISSUANCE_REJECTED", true, false);
-        }
-        response(&Issued {
-            version: 1,
-            delegation_id: record.id,
-            delegation_revision: record.revision,
-            owner_access_key: record.owner_access_key,
-            access_key_id: credential.access_key,
-            secret_access_key: credential.secret_key,
-            session_token: credential.session_token,
-            expires_at_unix_seconds: expires_at.unix_timestamp(),
-        })
-    }
-}
-
-#[cfg(test)]
-#[path = "plugin_attempt_v2_tests.rs"]
-mod v2_tests;
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn parent_authority_recheck_ignores_policy_order_but_detects_changes() {
-        let first: Policy = serde_json::from_value(serde_json::json!({
-            "Version":"2012-10-17",
-            "Statement":[
-                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
-                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"}
-            ]
-        }))
-        .unwrap();
-        let reordered: Policy = serde_json::from_value(serde_json::json!({
-            "Version":"2012-10-17",
-            "Statement":[
-                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"},
-                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"}
-            ]
-        }))
-        .unwrap();
-        let before_names = vec!["owner-policy".to_owned(), "bucket-policy".to_owned()];
-        let after_names = vec!["bucket-policy".to_owned(), "owner-policy".to_owned()];
-        assert!(same_parent_authority(&before_names, &first, &after_names, &reordered));
-
-        let changed: Policy = serde_json::from_value(serde_json::json!({
-            "Version":"2012-10-17",
-            "Statement":[
-                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
-                {"Effect":"Allow","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::photos/output/*"}
-            ]
-        }))
-        .unwrap();
-        assert!(!same_parent_authority(&before_names, &first, &after_names, &changed));
-        let changed_resource: Policy = serde_json::from_value(serde_json::json!({
-            "Version":"2012-10-17",
-            "Statement":[
-                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
-                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/private/*"}
-            ]
-        }))
-        .unwrap();
-        assert!(!same_parent_authority(&before_names, &first, &after_names, &changed_resource));
-        let added_deny: Policy = serde_json::from_value(serde_json::json!({
-            "Version":"2012-10-17",
-            "Statement":[
-                {"Effect":"Deny","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/private/*"},
-                {"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/input/*"},
-                {"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::photos/output/*"}
-            ]
-        }))
-        .unwrap();
-        assert!(!same_parent_authority(&before_names, &first, &after_names, &added_deny));
-        assert!(!same_parent_authority(&before_names, &first, &["other-policy".to_owned()], &first));
-    }
-    #[tokio::test]
-    async fn attempt_policy_keeps_exact_version_and_narrow_output() {
-        let record = Delegation {
-            version: 1,
-            id: "a".repeat(32),
-            revision: "a".repeat(32),
-            owner_access_key: "user".into(),
-            created_at_unix_seconds: 1,
-            restrictions: rustfs_iam::delegation::Restrictions {
-                version: None,
-                output_bucket: None,
-                deployment: "runner".into(),
-                bucket: "photos".into(),
-                input_prefix: "input/".into(),
-                output_prefix: "output/".into(),
-                expires_at_unix_seconds: 1000,
-            },
-        };
-        let run_id = uuid::Uuid::new_v4();
-        let attempt_id = uuid::Uuid::new_v4();
-        let output_prefix = format!(
-            "output/{}/{}/{}/",
-            hex_simd::encode_to_string(b"tenant", hex_simd::AsciiCase::Lower),
-            run_id,
-            attempt_id
-        );
-        let mut attempt = Attempt {
-            tenant: "tenant".into(),
-            run_id,
-            attempt_id,
-            revision: record.revision.clone(),
-            deployment: "runner".into(),
-            input_bucket: "photos".into(),
-            input_key: "input/photo.png".into(),
-            input_version: "version-1".into(),
-            output_bucket: "photos".into(),
-            output_prefix: output_prefix.clone(),
-            ttl_seconds: 60,
-        };
-        let policy = session_policy(&record, &attempt).unwrap();
-        let scoped_output = format!("{output_prefix}result");
-        let groups = None;
-        let claims = HashMap::new();
-        for (action, object, version, expected) in [
-            (S3Action::GetObjectVersionAction, "input/photo.png", Some("version-1"), true),
-            (S3Action::GetObjectVersionAction, "input/photo.png", Some("version-2"), false),
-            (S3Action::GetObjectVersionAction, "input/photo.png", None, false),
-            (S3Action::GetObjectAction, "input/photo.png", None, false),
-            (S3Action::PutObjectAction, scoped_output.as_str(), None, true),
-            (S3Action::PutObjectAction, "outside/result", None, false),
-            (S3Action::PutObjectAction, "output/result", None, false),
-        ] {
-            let conditions = version
-                .map(|value| HashMap::from([("versionid".into(), vec![value.into()])]))
-                .unwrap_or_default();
-            assert_eq!(
-                policy
-                    .is_allowed(&Args {
-                        account: "user",
-                        groups: &groups,
-                        action: Action::S3Action(action),
-                        bucket: "photos",
-                        object,
-                        conditions: &conditions,
-                        is_owner: false,
-                        claims: &claims,
-                        deny_only: false,
-                    })
-                    .await,
-                expected,
-                "attempt policy action={action:?} object={object} version={version:?}"
-            );
-        }
-        attempt.output_bucket = "artifacts".into();
-        assert!(
-            session_policy(&record, &attempt).is_err(),
-            "v1 still requires the input bucket for output"
-        );
-        attempt.output_bucket = "photos".into();
-        attempt.output_prefix = "output/".into();
-        assert!(session_policy(&record, &attempt).is_err());
-        attempt.output_prefix = output_prefix;
-        attempt.input_key = "outside/photo.png".into();
-        assert!(session_policy(&record, &attempt).is_err());
-    }
-    #[tokio::test]
-    async fn attempt_v2_policy_keeps_input_version_and_output_bucket() {
-        let record: Delegation = serde_json::from_value(serde_json::json!({
-            "version":2,"id":"a".repeat(32),"revision":"a".repeat(32),"owner_access_key":"user",
-            "created_at_unix_seconds":1,"restrictions":{"version":2,"deployment":"runner","bucket":"photos",
-            "input_prefix":"input/","output_bucket":"artifacts","output_prefix":"output/","expires_at_unix_seconds":1000}
-        }))
-        .expect("explicit v2 delegation");
-        let run_id = uuid::Uuid::new_v4();
-        let attempt_id = uuid::Uuid::new_v4();
-        let output = format!("output/74656e616e74/{run_id}/{attempt_id}/");
-        let mut attempt = Attempt {
-            tenant: "tenant".into(),
-            run_id,
-            attempt_id,
-            revision: record.revision.clone(),
-            deployment: "runner".into(),
-            input_bucket: "photos".into(),
-            input_key: "input/photo.png".into(),
-            input_version: "version-1".into(),
-            output_bucket: "artifacts".into(),
-            output_prefix: output.clone(),
-            ttl_seconds: 60,
-        };
-        let policy = session_policy(&record, &attempt).expect("v2 scope");
-        let result = format!("{output}result");
-        for (action, bucket, object, version, expected) in [
-            (S3Action::GetObjectVersionAction, "photos", "input/photo.png", Some("version-1"), true),
-            (S3Action::GetObjectVersionAction, "photos", "input/photo.png", Some("version-2"), false),
-            (S3Action::GetObjectVersionAction, "artifacts", "input/photo.png", Some("version-1"), false),
-            (S3Action::PutObjectAction, "artifacts", result.as_str(), None, true),
-            (S3Action::PutObjectAction, "photos", result.as_str(), None, false),
-            (S3Action::PutObjectAction, "third", result.as_str(), None, false),
-            (S3Action::PutObjectAction, "artifacts", "output/other/result", None, false),
-        ] {
-            let conditions = version
-                .map(|v| HashMap::from([("versionid".into(), vec![v.into()])]))
-                .unwrap_or_default();
-            assert_eq!(
-                policy
-                    .is_allowed(&Args {
-                        account: "user",
-                        groups: &None,
-                        action: Action::S3Action(action),
-                        bucket,
-                        object,
-                        conditions: &conditions,
-                        claims: &HashMap::new(),
-                        is_owner: false,
-                        deny_only: false
-                    })
-                    .await,
-                expected,
-                "A03_CROSS_BUCKET_SCOPE: bucket={bucket} object={object}"
-            );
-        }
-        attempt.output_bucket = "third".into();
-        assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_OUTPUT_BOUND");
-        attempt.output_bucket = "photos".into();
-        assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_OUTPUT_BOUND");
-        attempt.output_bucket = "artifacts".into();
-        attempt.input_bucket = "artifacts".into();
-        assert!(session_policy(&record, &attempt).is_err(), "A03_CROSS_BUCKET_INPUT_BOUND");
-    }
-
-    #[tokio::test]
-    async fn actual_policy_variable_resolution_is_rejected_by_literal_scope() {
-        let resource = rustfs_policy::policy::resource::Resource::S3("photos/input/${aws:username}.png".into());
-        let conditions = HashMap::from([("username".into(), vec!["alice".into()])]);
-        let resolver =
-            rustfs_policy::policy::variables::VariableResolver::new(rustfs_policy::policy::variables::VariableContext {
-                username: Some("alice".into()),
-                ..Default::default()
-            });
-        assert!(
-            resource
-                .is_match_with_resolver("photos/input/alice.png", &conditions, Some(&resolver))
-                .await,
-            "C07_LITERAL_SCOPE_BOUND: actual policy engine substitutes resource variables"
-        );
-        assert!(
-            !literal("input/${aws:username}.png"),
-            "C07_LITERAL_SCOPE_BOUND: requested resources must stay literal"
-        );
-        let restriction = rustfs_iam::delegation::Restrictions {
-            version: None,
-            output_bucket: None,
-            deployment: "runner".into(),
-            bucket: "photos".into(),
-            input_prefix: "input/${aws:username}/".into(),
-            output_prefix: "output/".into(),
-            expires_at_unix_seconds: 200,
-        };
-        assert!(
-            restriction.validate(100).is_err(),
-            "C07_LITERAL_SCOPE_BOUND: legacy stored variable restrictions are rejected"
-        );
-    }
-}

rustfs/src/admin/handlers/plugin_attempt_v2_tests.rs

@@ -1,365 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-//! Real handler, persisted IAM and four-disk storage; this is not a signed HTTP/S3 end-to-end test.
-use super::*;
-use crate::admin::{
-    runtime_sources::{AppContext, IamInterface, KmsInterface, ServerContextSlot},
-    storage_api::{
-        ECStore,
-        contract::bucket::{BucketOperations as _, MakeBucketOptions},
-        contract::object::ObjectIO as _,
-    },
-};
-use crate::storage::storage_api::{StorageObjectOptions, StoragePutObjReader};
-use http_body_util::BodyExt as _;
-use rustfs_iam::{
-    store::{
-        Store as _, UserType,
-        object::{IAM_CONFIG_PREFIX, ObjectStore},
-    },
-    sys::IamSys,
-};
-use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
-use serde_json::{Value, json};
-use std::sync::Arc;
-
-const PARENT: &str = "cross-parent";
-const BROKER: &str = "cross-broker";
-const SECRET: &str = "cross-bucket-test-secret";
-const ROOT: &str = "cross-root";
-const ROOT_SECRET: &str = "cross-bucket-root-test-secret";
-
-struct RequestIam(Arc<IamSys<ObjectStore>>);
-impl IamInterface for RequestIam {
-    fn handle(&self) -> Arc<IamSys<ObjectStore>> {
-        self.0.clone()
-    }
-    fn is_ready(&self) -> bool {
-        true
-    }
-}
-struct RequestKms;
-impl KmsInterface for RequestKms {
-    fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
-        Arc::new(rustfs_kms::KmsServiceManager::new())
-    }
-}
-struct Fixture {
-    _root: tempfile::TempDir,
-    store: Arc<ECStore>,
-    iam: Arc<IamSys<ObjectStore>>,
-    context: Arc<AppContext>,
-    input_version: String,
-}
-impl Fixture {
-    async fn new() -> Self {
-        let root = tempfile::tempdir().expect("cross-bucket storage root");
-        let env = rustfs_test_utils::TestECStoreEnv::builder()
-            .base_dir(root.path())
-            .disk_count(4)
-            .build()
-            .await;
-        assert_eq!(env.disk_paths.len(), 4);
-        let store = env.ecstore;
-        for bucket in ["photos", "artifacts", "third"] {
-            store
-                .make_bucket(
-                    bucket,
-                    &MakeBucketOptions {
-                        versioning_enabled: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("real versioned bucket");
-        }
-        let object = store
-            .put_object(
-                "photos",
-                "input/photo.png",
-                &mut StoragePutObjReader::from_vec(vec![0xA5; 273]),
-                &StorageObjectOptions {
-                    versioned: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("real input version");
-        let input_version = object.version_id.expect("exact input version").to_string();
-        ObjectStore::new(store.clone())
-            .save_iam_config(json!({"version":1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
-            .await
-            .expect("persistent IAM format");
-        let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("real IAM");
-        let context = Arc::new(AppContext::new(store.clone(), Arc::new(RequestIam(iam.clone())), Arc::new(RequestKms)));
-        assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
-            access_key: ROOT.into(),
-            secret_key: ROOT_SECRET.into(),
-            status: "on".into(),
-            ..Default::default()
-        }));
-        let fixture = Self {
-            _root: root,
-            store,
-            iam,
-            context,
-            input_version,
-        };
-        fixture.user(PARENT, Self::parent_policy(true, false)).await;
-        fixture
-            .user(
-                BROKER,
-                json!({"Version":"2012-10-17","Statement":[{
-            "Effect":"Allow","Action":"admin:IssuePluginAttempt"}]}),
-            )
-            .await;
-        fixture
-    }
-    fn parent_policy(output: bool, deny: bool) -> Value {
-        let mut resources = vec!["arn:aws:s3:::photos/output/*", "arn:aws:s3:::third/output/*"];
-        if output {
-            resources.push("arn:aws:s3:::artifacts/output/*");
-        }
-        let mut statements = vec![
-            json!({"Effect":"Allow","Action":"s3:GetObjectVersion","Resource":"arn:aws:s3:::photos/input/*"}),
-            json!({"Effect":"Allow","Action":"s3:PutObject","Resource":resources}),
-        ];
-        if deny {
-            statements.push(json!({"Effect":"Deny","Action":"s3:PutObject","Resource":"arn:aws:s3:::artifacts/*"}));
-        }
-        json!({"Version":"2012-10-17","Statement":statements})
-    }
-    async fn policy(&self, name: &str, value: Value) {
-        let policy = Policy::parse_config(&serde_json::to_vec(&value).expect("policy JSON")).expect("real IAM policy");
-        self.iam.set_policy(name, policy).await.expect("persist policy");
-    }
-    async fn user(&self, name: &str, policy: Value) {
-        self.iam
-            .create_user(
-                name,
-                &AddOrUpdateUserReq {
-                    secret_key: SECRET.into(),
-                    policy: None,
-                    status: AccountStatus::Enabled,
-                },
-            )
-            .await
-            .expect("create real user");
-        self.policy(name, policy).await;
-        self.iam
-            .policy_db_set(name, UserType::Reg, false, name)
-            .await
-            .expect("attach actual IAM policy");
-    }
-    fn request(&self, body: &Value) -> S3Request<Body> {
-        let slot = ServerContextSlot::new();
-        assert!(slot.install(self.context.clone()));
-        let mut req = S3Request {
-            input: Body::from(serde_json::to_vec(body).expect("attempt JSON")),
-            method: Method::POST,
-            uri: http::Uri::default(),
-            headers: http::HeaderMap::new(),
-            extensions: http::Extensions::new(),
-            credentials: None,
-            region: None,
-            service: None,
-            trailing_headers: None,
-        };
-        req.extensions.insert(slot);
-        req.method = Method::POST;
-        req.credentials = Some(s3::auth::Credentials {
-            access_key: BROKER.into(),
-            secret_key: s3::auth::SecretKey::from(SECRET.to_owned()),
-        });
-        req
-    }
-    async fn issue(&self, record: &Delegation, body: &Value) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let mut router = matchit::Router::new();
-        router.insert("/{id}", ()).expect("parameter route");
-        let path = format!("/{}", record.id);
-        Issue
-            .call(self.request(body), router.at(&path).expect("actual delegation ID").params)
-            .await
-    }
-    async fn allowed(
-        &self,
-        credential: &rustfs_credentials::Credentials,
-        action: S3Action,
-        bucket: &str,
-        object: &str,
-        version: Option<&str>,
-    ) -> bool {
-        let conditions = version
-            .map(|v| HashMap::from([("versionid".into(), vec![v.into()])]))
-            .unwrap_or_default();
-        self.iam
-            .is_allowed(&Args {
-                account: &credential.access_key,
-                groups: &credential.groups,
-                action: Action::S3Action(action),
-                bucket,
-                object,
-                conditions: &conditions,
-                claims: credential.claims_or_empty(),
-                is_owner: false,
-                deny_only: false,
-            })
-            .await
-    }
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-#[serial_test::serial]
-async fn delegation_v2_persists_and_vends_with_real_parent_intersection() {
-    temp_env::async_with_vars(
-        [
-            ("RUSTFS_PLUGIN_BROKERS", Some(r#"{"cross-broker":"runner"}"#)),
-            ("RUSTFS_SKIP_BACKGROUND_TASK", Some("1")),
-        ],
-        async {
-            let fixture = Fixture::new().await;
-            let now = now().expect("clock");
-            let legacy = json!({"deployment":"runner","bucket":"photos","input_prefix":"input/",
-            "output_prefix":"output/","expires_at_unix_seconds":now+3600});
-            let v1 = fixture
-                .iam
-                .create_plugin_delegation(PARENT, serde_json::from_value(legacy.clone()).expect("v1"), now)
-                .await
-                .expect("persist legacy delegation");
-            assert_eq!(v1.version, 1);
-            assert_eq!(serde_json::to_value(&v1.restrictions).expect("v1 wire"), legacy);
-            let mut v2 = legacy;
-            v2["version"] = json!(2);
-            v2["output_bucket"] = json!("artifacts");
-            let record = fixture
-                .iam
-                .create_plugin_delegation(PARENT, serde_json::from_value(v2.clone()).expect("v2"), now)
-                .await
-                .expect("persist v2 delegation");
-            assert_eq!(record.version, 2);
-            let loaded = fixture
-                .iam
-                .read_active_plugin_delegation(PARENT, &record.id, now)
-                .await
-                .expect("strict stored v2");
-            assert!(loaded == record, "same persisted v2 identity");
-            assert_eq!(serde_json::to_value(&loaded.restrictions).expect("v2 wire"), v2);
-            // The record envelope and restriction profile cannot disagree on a persisted read.
-            let path = format!("config/iam/plugin-delegations/v1/{}/record.json", record.id);
-            let mut corrupt = serde_json::to_value(&record).expect("record JSON");
-            corrupt["version"] = json!(1);
-            ObjectStore::new(fixture.store.clone())
-                .save_iam_config(corrupt, &path)
-                .await
-                .expect("inject incompatible stored version");
-            assert!(
-                fixture
-                    .iam
-                    .read_active_plugin_delegation(PARENT, &record.id, now)
-                    .await
-                    .is_err(),
-                "mixed v1/v2 stored profiles fail closed"
-            );
-            ObjectStore::new(fixture.store.clone())
-                .save_iam_config(&record, &path)
-                .await
-                .expect("restore exact stored record");
-            let run = uuid::Uuid::new_v4();
-            let attempt = uuid::Uuid::new_v4();
-            let prefix = format!("output/74656e616e74/{run}/{attempt}/");
-            let body = json!({"tenant":"tenant","run_id":run,"attempt_id":attempt,"revision":record.revision,
-            "deployment":"runner","input_bucket":"photos","input_key":"input/photo.png","input_version":fixture.input_version,
-            "output_bucket":"artifacts","output_prefix":prefix,"ttl_seconds":60});
-            let loaded_v1 = fixture
-                .iam
-                .read_active_plugin_delegation(PARENT, &v1.id, now)
-                .await
-                .expect("read legacy record");
-            assert!(loaded_v1 == v1, "legacy persistent identity");
-            let mut legacy_body = body.clone();
-            legacy_body["revision"] = json!(v1.revision);
-            legacy_body["output_bucket"] = json!("photos");
-            let legacy_response = fixture.issue(&v1, &legacy_body).await.expect("legacy same-bucket issuance");
-            assert_eq!(legacy_response.output.0, StatusCode::OK);
-            let mut wrong = body.clone();
-            wrong["output_bucket"] = json!("third");
-            match fixture.issue(&record, &wrong).await {
-                Err(error) => assert_eq!(error.code(), &S3ErrorCode::AccessDenied),
-                Ok(_) => panic!("A03_CROSS_BUCKET_OUTPUT_BOUND"),
-            }
-            let response = fixture.issue(&record, &body).await.expect("actual v2 issuance");
-            assert_eq!(response.output.0, StatusCode::OK);
-            assert_eq!(response.headers[http::header::CACHE_CONTROL], "no-store");
-            let issued: Value =
-                serde_json::from_slice(&response.output.1.collect().await.expect("issued body").to_bytes()).expect("issued JSON");
-            let (credential, root) = check_key_valid_with_context(
-                issued["session_token"].as_str().expect("token"),
-                issued["access_key_id"].as_str().expect("access key"),
-                Some(fixture.context.as_ref()),
-            )
-            .await
-            .expect("persisted actual temporary identity");
-            assert!(!root && credential.is_temp());
-            let output = format!("{prefix}image.png");
-            for (action, bucket, object, version, expected) in [
-                (
-                    S3Action::GetObjectVersionAction,
-                    "photos",
-                    "input/photo.png",
-                    Some(fixture.input_version.as_str()),
-                    true,
-                ),
-                (
-                    S3Action::GetObjectVersionAction,
-                    "photos",
-                    "input/photo.png",
-                    Some("wrong-version"),
-                    false,
-                ),
-                (S3Action::GetObjectAction, "photos", "input/photo.png", None, false),
-                (S3Action::PutObjectAction, "artifacts", output.as_str(), None, true),
-                (S3Action::PutObjectAction, "photos", output.as_str(), None, false),
-                (S3Action::PutObjectAction, "third", output.as_str(), None, false),
-                (S3Action::PutObjectAction, "artifacts", "output/other/image.png", None, false),
-            ] {
-                assert_eq!(
-                    fixture.allowed(&credential, action, bucket, object, version).await,
-                    expected,
-                    "A03_CROSS_BUCKET_PARENT_INTERSECTION: bucket={bucket} object={object}"
-                );
-            }
-            fixture.policy(PARENT, Fixture::parent_policy(true, true)).await;
-            assert!(
-                !fixture
-                    .allowed(&credential, S3Action::PutObjectAction, "artifacts", &output, None)
-                    .await,
-                "explicit parent Deny remains effective"
-            );
-            fixture.policy(PARENT, Fixture::parent_policy(false, false)).await;
-            assert!(
-                !fixture
-                    .allowed(&credential, S3Action::PutObjectAction, "artifacts", &output, None)
-                    .await,
-                "removing parent output permission remains effective"
-            );
-            assert!(
-                fixture
-                    .allowed(
-                        &credential,
-                        S3Action::GetObjectVersionAction,
-                        "photos",
-                        "input/photo.png",
-                        Some(&fixture.input_version)
-                    )
-                    .await
-            );
-            fixture
-                .iam
-                .revoke_plugin_delegation(PARENT, &record.id)
-                .await
-                .expect("revoke v2");
-            assert!(fixture.issue(&record, &body).await.is_err(), "revoked v2 cannot issue again");
-        },
-    )
-    .await;
-}

rustfs/src/admin/handlers/plugin_delegation.rs

@@ -1,624 +0,0 @@
-// Copyright 2026 RustFS Team
-// SPDX-License-Identifier: Apache-2.0
-
-//! Restriction registration only. These endpoints do not vend credentials.
-use crate::admin::{
-    auth::{AdminResourceScope, validate_admin_action_with_bucket_object_for_iam},
-    plugin_runner::{self, ObjectReference, SubmitRequest},
-    router::{AdminOperation, Operation, S3Router},
-    runtime_sources,
-    storage_api::s3::{self, Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result},
-};
-use crate::{
-    auth::{check_key_valid_with_context, get_session_token},
-    server::{ADMIN_PREFIX, RemoteAddr},
-};
-use http::{HeaderMap, HeaderValue, Method, StatusCode};
-use matchit::Params;
-use rustfs_iam::delegation::{DelegationError, Restrictions};
-use rustfs_policy::policy::action::{Action, AdminAction};
-use serde::Deserialize;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-
-pub fn register_plugin_delegation_route(router: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
-    router.insert(Method::POST, &format!("{ADMIN_PREFIX}/v3/plugin-delegations"), AdminOperation(&Create))?;
-    router.insert(Method::GET, &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}"), AdminOperation(&Get))?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/revoke"),
-        AdminOperation(&Revoke),
-    )?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/submit"),
-        AdminOperation(&Submit),
-    )?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/query"),
-        AdminOperation(&QueryRun),
-    )?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/history"),
-        AdminOperation(&HistoryRun),
-    )?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/cancel"),
-        AdminOperation(&CancelRun),
-    )?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/page"),
-        AdminOperation(&PageRuns),
-    )?;
-    router.insert(
-        Method::POST,
-        &format!("{ADMIN_PREFIX}/v3/plugin-delegations/{{id}}/runs/replay"),
-        AdminOperation(&ReplayRun),
-    )?;
-    Ok(())
-}
-
-pub(super) fn storage_error(error: DelegationError) -> S3Error {
-    match error {
-        DelegationError::Invalid => s3::error(S3ErrorCode::InvalidRequest, "invalid delegation restriction"),
-        DelegationError::NotFound | DelegationError::OwnerRejected => s3::error(S3ErrorCode::NoSuchKey, "delegation not found"),
-        DelegationError::Unavailable => s3::error(S3ErrorCode::InternalError, "delegation unavailable"),
-        _ => s3::error(S3ErrorCode::AccessDenied, "delegation rejected"),
-    }
-}
-
-pub(super) async fn owner(
-    req: &S3Request<Body>,
-    action: AdminAction,
-) -> S3Result<(std::sync::Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>, String)> {
-    let signed = req
-        .credentials
-        .as_ref()
-        .ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?;
-    if get_session_token(&req.uri, &req.headers).is_some() {
-        return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
-    }
-    let context = runtime_sources::app_context_from_req(req)
-        .ok_or_else(|| s3::error(S3ErrorCode::InternalError, "application context unavailable"))?;
-    let (credentials, root) = check_key_valid_with_context("", &signed.access_key, Some(context.as_ref())).await?;
-    if root || credentials.is_temp() || credentials.is_service_account() {
-        return Err(s3::error(S3ErrorCode::AccessDenied, "original user required"));
-    }
-    if !context.iam().is_ready() {
-        return Err(s3::error(S3ErrorCode::InternalError, "iam unavailable"));
-    }
-    let iam = context.iam().handle();
-    validate_admin_action_with_bucket_object_for_iam(
-        iam.clone(),
-        &req.headers,
-        &credentials,
-        false,
-        Action::AdminAction(action),
-        req.extensions.get::<Option<RemoteAddr>>().and_then(|a| a.map(|a| a.0)),
-        AdminResourceScope::bucket_object("", ""),
-    )
-    .await?;
-    Ok((iam, credentials.access_key))
-}
-
-pub(super) fn response<T: serde::Serialize>(value: &T) -> S3Result<S3Response<(StatusCode, Body)>> {
-    let bytes =
-        serde_json::to_vec(value).map_err(|_| s3::error(S3ErrorCode::InternalError, "delegation response unavailable"))?;
-    let mut headers = HeaderMap::new();
-    headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
-    headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
-    Ok(S3Response::with_headers((StatusCode::OK, Body::from(bytes)), headers))
-}
-
-struct Create;
-#[async_trait::async_trait]
-impl Operation for Create {
-    async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, owner) = owner(&req, AdminAction::CreatePluginDelegationAction).await?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "delegation body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "delegation body unavailable"))?;
-        let restriction: Restrictions = serde_json::from_slice(&bytes)
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid delegation restriction"))?;
-        let now = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
-            .as_secs();
-        let record = iam
-            .create_plugin_delegation(&owner, restriction, now)
-            .await
-            .map_err(storage_error)?;
-        response(&record)
-    }
-}
-
-struct Get;
-#[async_trait::async_trait]
-impl Operation for Get {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, owner) = owner(&req, AdminAction::GetPluginDelegationAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let now = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
-            .as_secs();
-        let record = iam
-            .read_active_plugin_delegation(&owner, id, now)
-            .await
-            .map_err(storage_error)?;
-        response(&record)
-    }
-}
-
-struct Revoke;
-#[async_trait::async_trait]
-impl Operation for Revoke {
-    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, owner) = owner(&req, AdminAction::RevokePluginDelegationAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        iam.revoke_plugin_delegation(&owner, id).await.map_err(storage_error)?;
-        response(&serde_json::json!({"revoked":true,"id":id}))
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct SubmitInput {
-    version: u32,
-    instance_id: String,
-    idempotency_key: String,
-    object: SubmitObject,
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct SubmitObject {
-    bucket: String,
-    key: String,
-    version_id: String,
-}
-
-struct Submit;
-#[async_trait::async_trait]
-impl Operation for Submit {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, signer) = owner(&req, AdminAction::SubmitPluginJobAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "submit body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "submit body unavailable"))?;
-        let input: SubmitInput =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid submit request"))?;
-        if input.version != 1
-            || !plugin_runner::literal(&input.instance_id)
-            || !plugin_runner::valid_idempotency_key(&input.idempotency_key)
-        {
-            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid submit request"));
-        }
-        let now = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
-            .as_secs();
-        let record = iam
-            .read_active_plugin_delegation(&signer, id, now)
-            .await
-            .map_err(storage_error)?;
-        let deployment = plugin_runner::configured_deployment()?;
-        if !plugin_runner::validate_delegation_scope(
-            &record,
-            &signer,
-            id,
-            &deployment,
-            ObjectReference {
-                bucket: &input.object.bucket,
-                key: &input.object.key,
-                version_id: &input.object.version_id,
-            },
-            now,
-        ) {
-            return Err(s3::error(S3ErrorCode::AccessDenied, "submit restriction rejected"));
-        }
-        let receipt = plugin_runner::submit(&SubmitRequest {
-            version: input.version,
-            instance_id: &input.instance_id,
-            idempotency_key: &input.idempotency_key,
-            object: ObjectReference {
-                bucket: &input.object.bucket,
-                key: &input.object.key,
-                version_id: &input.object.version_id,
-            },
-            delegation_id: &record.id,
-            delegation_revision: &record.revision,
-            actual_signer_access_key: &signer,
-        })
-        .await?;
-        response(&receipt)
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct QueryRunInput {
-    version: u32,
-    run_id: uuid::Uuid,
-}
-
-struct QueryRun;
-#[async_trait::async_trait]
-impl Operation for QueryRun {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, signer) = owner(&req, AdminAction::GetPluginRunAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "query body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "query body unavailable"))?;
-        let input: QueryRunInput =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid query request"))?;
-        if input.version != 1 || input.run_id.is_nil() {
-            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid query request"));
-        }
-        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
-        let result = plugin_runner::query(&plugin_runner::QueryRequest {
-            version: input.version,
-            run_id: input.run_id,
-            delegation_id: &record.id,
-            delegation_revision: &record.revision,
-            actual_signer_access_key: &signer,
-        })
-        .await?;
-        response(&result)
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct HistoryRunInput {
-    version: u32,
-    run_id: uuid::Uuid,
-}
-
-fn valid_history_run_input(input: &HistoryRunInput) -> bool {
-    input.version == 1 && !input.run_id.is_nil()
-}
-
-struct HistoryRun;
-#[async_trait::async_trait]
-impl Operation for HistoryRun {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, signer) = owner(&req, AdminAction::ListPluginRunsAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "history body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "history body unavailable"))?;
-        let input: HistoryRunInput =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid history request"))?;
-        if !valid_history_run_input(&input) {
-            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid history request"));
-        }
-        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
-        let result = plugin_runner::history(&plugin_runner::HistoryRequest {
-            version: input.version,
-            run_id: input.run_id,
-            delegation_id: &record.id,
-            delegation_revision: &record.revision,
-            actual_signer_access_key: &signer,
-        })
-        .await?;
-        response(&result)
-    }
-}
-
-struct CancelRun;
-#[async_trait::async_trait]
-impl Operation for CancelRun {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, signer) = owner(&req, AdminAction::CancelPluginRunAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "cancel body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "cancel body unavailable"))?;
-        let input: QueryRunInput =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid cancel request"))?;
-        if input.version != 1 || input.run_id.is_nil() {
-            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid cancel request"));
-        }
-        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
-        let result = plugin_runner::cancel(&plugin_runner::CancelRequest {
-            version: input.version,
-            run_id: input.run_id,
-            delegation_id: &record.id,
-            delegation_revision: &record.revision,
-            actual_signer_access_key: &signer,
-        })
-        .await?;
-        response(&result)
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct PageRunsInput {
-    version: u32,
-    limit: u32,
-    #[serde(default)]
-    cursor: Option<plugin_runner::RunPageCursor>,
-}
-
-fn valid_page_runs_input(input: &PageRunsInput) -> bool {
-    input.version == 1 && (1..=100).contains(&input.limit) && !input.cursor.is_some_and(|cursor| cursor.run_id.is_nil())
-}
-
-struct PageRuns;
-#[async_trait::async_trait]
-impl Operation for PageRuns {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, signer) = owner(&req, AdminAction::ListPluginRunsAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "page body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "page body unavailable"))?;
-        let input: PageRunsInput =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid page request"))?;
-        if !valid_page_runs_input(&input) {
-            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid page request"));
-        }
-        let record = iam.read_owned_plugin_delegation(&signer, id).await.map_err(storage_error)?;
-        let result = plugin_runner::page(&plugin_runner::PageRequest {
-            version: input.version,
-            actual_signer_access_key: &signer,
-            delegation_id: &record.id,
-            delegation_revision: &record.revision,
-            limit: input.limit,
-            cursor: input.cursor,
-        })
-        .await?;
-        response(&result)
-    }
-}
-
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct ReplayRunInput {
-    version: u32,
-    origin_run_id: uuid::Uuid,
-    idempotency_key: String,
-    deadline_unix_micros: i64,
-}
-
-fn valid_replay_run_input(input: &ReplayRunInput) -> bool {
-    input.version == 1
-        && !input.origin_run_id.is_nil()
-        && plugin_runner::valid_idempotency_key(&input.idempotency_key)
-        && input.deadline_unix_micros > 0
-}
-
-struct ReplayRun;
-#[async_trait::async_trait]
-impl Operation for ReplayRun {
-    async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
-        let (iam, signer) = owner(&req, AdminAction::ReplayPluginRunAction).await?;
-        let id = params
-            .get("id")
-            .ok_or_else(|| s3::error(S3ErrorCode::InvalidRequest, "delegation id required"))?;
-        let bytes = tokio::time::timeout(Duration::from_secs(10), req.input.store_all_limited(16384))
-            .await
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "replay body unavailable"))?
-            .map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "replay body unavailable"))?;
-        let input: ReplayRunInput =
-            serde_json::from_slice(&bytes).map_err(|_| s3::error(S3ErrorCode::InvalidRequest, "invalid replay request"))?;
-        if !valid_replay_run_input(&input) {
-            return Err(s3::error(S3ErrorCode::InvalidRequest, "invalid replay request"));
-        }
-        let now = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .map_err(|_| s3::error(S3ErrorCode::InternalError, "clock unavailable"))?
-            .as_secs();
-        let record = iam
-            .read_active_plugin_delegation(&signer, id, now)
-            .await
-            .map_err(storage_error)?;
-        let result = plugin_runner::replay(&plugin_runner::ReplayRequest {
-            version: input.version,
-            origin_run_id: input.origin_run_id,
-            idempotency_key: &input.idempotency_key,
-            deadline_unix_micros: input.deadline_unix_micros,
-            actual_signer_access_key: &signer,
-            delegation_id: &record.id,
-            delegation_revision: &record.revision,
-        })
-        .await?;
-        response(&result)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[tokio::test]
-    async fn delegation_mutations_require_authenticated_original_user() {
-        for operation in 0..8 {
-            let req = S3Request {
-                input: Body::from("{}".to_owned()),
-                method: Method::POST,
-                uri: http::Uri::from_static("/rustfs/admin/v3/plugin-delegations"),
-                headers: HeaderMap::new(),
-                extensions: http::Extensions::new(),
-                credentials: None,
-                region: None,
-                service: None,
-                trailing_headers: None,
-            };
-            let result = if operation == 0 {
-                Get.call(req, Params::new()).await
-            } else if operation == 1 {
-                Revoke.call(req, Params::new()).await
-            } else if operation == 2 {
-                Submit.call(req, Params::new()).await
-            } else if operation == 3 {
-                QueryRun.call(req, Params::new()).await
-            } else if operation == 4 {
-                HistoryRun.call(req, Params::new()).await
-            } else if operation == 5 {
-                CancelRun.call(req, Params::new()).await
-            } else if operation == 6 {
-                PageRuns.call(req, Params::new()).await
-            } else if operation == 7 {
-                ReplayRun.call(req, Params::new()).await
-            } else {
-                Create.call(req, Params::new()).await
-            };
-            let error = result.expect_err("unsigned mutations must not reach persistence");
-            assert_eq!(error.code(), &S3ErrorCode::AccessDenied);
-        }
-    }
-
-    #[test]
-    fn submit_dto_rejects_client_authority_fields_and_null_versions() {
-        assert!(serde_json::from_str::<SubmitInput>(
-            r#"{"version":1,"instance_id":"img","idempotency_key":"k","object":{"bucket":"b","key":"k","version_id":"v"},"actual_signer_access_key":"forged"}"#
-        )
-        .is_err());
-        let input: SubmitInput = serde_json::from_str(
-            r#"{"version":1,"instance_id":"img","idempotency_key":"k","object":{"bucket":"b","key":"a/x","version_id":"null"}}"#,
-        )
-        .unwrap();
-        let record = rustfs_iam::delegation::Delegation {
-            version: 1,
-            id: "a".repeat(32),
-            revision: "a".repeat(32),
-            owner_access_key: "owner".into(),
-            created_at_unix_seconds: 1,
-            restrictions: Restrictions {
-                version: None,
-                output_bucket: None,
-                deployment: "prod".into(),
-                bucket: "b".into(),
-                input_prefix: "a/".into(),
-                output_prefix: "out/".into(),
-                expires_at_unix_seconds: 100,
-            },
-        };
-        assert!(!plugin_runner::validate_delegation_scope(
-            &record,
-            "owner",
-            &record.id,
-            "prod",
-            ObjectReference {
-                bucket: &input.object.bucket,
-                key: &input.object.key,
-                version_id: &input.object.version_id,
-            },
-            2,
-        ));
-    }
-
-    #[test]
-    fn query_run_dto_rejects_client_authority_fields() {
-        let run_id = uuid::Uuid::from_u128(1);
-        let input: QueryRunInput =
-            serde_json::from_str(&format!(r#"{{"version":1,"run_id":"{run_id}"}}"#)).expect("valid run query");
-        assert_eq!(input.run_id, run_id);
-        assert!(
-            serde_json::from_str::<QueryRunInput>(&format!(
-                r#"{{"version":1,"run_id":"{run_id}","actual_signer_access_key":"forged"}}"#
-            ))
-            .is_err()
-        );
-    }
-
-    #[test]
-    fn history_run_dto_is_strict_and_rejects_nil_run() {
-        let run_id = uuid::Uuid::from_u128(1);
-        let input: HistoryRunInput =
-            serde_json::from_str(&format!(r#"{{"version":1,"run_id":"{run_id}"}}"#)).expect("valid history request");
-        assert!(valid_history_run_input(&input));
-        for raw in [
-            r#"{"version":1,"run_id":"00000000-0000-0000-0000-000000000000"}"#,
-            r#"{"version":2,"run_id":"00000000-0000-0000-0000-000000000001"}"#,
-            r#"{"version":1,"run_id":"00000000-0000-0000-0000-000000000001","actual_signer_access_key":"forged"}"#,
-        ] {
-            let rejected = match serde_json::from_str::<HistoryRunInput>(raw) {
-                Ok(input) => !valid_history_run_input(&input),
-                Err(_) => true,
-            };
-            assert!(rejected, "accepted invalid history request: {raw}");
-        }
-    }
-
-    #[test]
-    fn page_runs_dto_is_strict_and_bounded() {
-        let cursor = plugin_runner::RunPageCursor {
-            created_at_unix_micros: 10,
-            run_id: uuid::Uuid::from_u128(1),
-        };
-        let input: PageRunsInput = serde_json::from_str(
-            r#"{"version":1,"limit":100,"cursor":{"created_at_unix_micros":10,"run_id":"00000000-0000-0000-0000-000000000001"}}"#,
-        )
-        .expect("valid page request");
-        assert_eq!(input.cursor, Some(cursor));
-        for raw in [
-            r#"{"version":1,"limit":0}"#,
-            r#"{"version":1,"limit":101}"#,
-            r#"{"version":1,"limit":1,"unknown":true}"#,
-            r#"{"version":1,"limit":1,"actual_signer_access_key":"forged"}"#,
-            r#"{"version":1,"limit":1,"cursor":{"created_at_unix_micros":10,"run_id":"00000000-0000-0000-0000-000000000000"}}"#,
-        ] {
-            let rejected = match serde_json::from_str::<PageRunsInput>(raw) {
-                Ok(input) => !valid_page_runs_input(&input),
-                Err(_) => true,
-            };
-            assert!(rejected, "accepted invalid page request: {raw}");
-        }
-    }
-
-    #[test]
-    fn replay_run_dto_is_strict_and_bounded() {
-        let run_id = uuid::Uuid::from_u128(1);
-        let input: ReplayRunInput = serde_json::from_str(&format!(
-            r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"retry-1","deadline_unix_micros":123}}"#
-        ))
-        .expect("valid replay request");
-        assert!(valid_replay_run_input(&input));
-        for raw in [
-            r#"{"version":1,"origin_run_id":"00000000-0000-0000-0000-000000000000","idempotency_key":"k","deadline_unix_micros":123}"#.to_string(),
-            format!(r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"","deadline_unix_micros":123}}"#),
-            format!(r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"k","deadline_unix_micros":0}}"#),
-            format!(
-                r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"k","deadline_unix_micros":123,"actual_signer_access_key":"forged"}}"#
-            ),
-            format!(
-                r#"{{"version":1,"origin_run_id":"{run_id}","idempotency_key":"{}","deadline_unix_micros":123}}"#,
-                "x".repeat(1025)
-            ),
-        ] {
-            let rejected = match serde_json::from_str::<ReplayRunInput>(&raw) {
-                Ok(input) => !valid_replay_run_input(&input),
-                Err(_) => true,
-            };
-            assert!(rejected, "accepted invalid replay request: {raw}");
-        }
-    }
-}

rustfs/src/admin/mod.rs

@@ -17,7 +17,6 @@ mod auth;
 pub mod console;
 pub mod handlers;
 mod plugin_contract;
-pub(crate) mod plugin_runner;
 pub(crate) mod replication_metrics_wire;
 // Contract inventory is validated by tests before later runtime integration.
 pub(crate) mod route_policy;
@@ -65,8 +64,6 @@ pub fn make_admin_route(
 fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
     health::register_health_route(r)?;
     sts::register_admin_auth_route(r)?;
-    handlers::plugin_delegation::register_plugin_delegation_route(r)?;
-    handlers::plugin_attempt::register_plugin_attempt_route(r)?;
 
     account::register_account_route(r)?;
     mfa::register_mfa_route(r)?;

rustfs/src/admin/route_policy.rs

@@ -25,15 +25,6 @@ const TLS_CERTIFICATE_PUBLISH: AdminActionRef = AdminActionRef::new("TLSCertific
 const TLS_CERTIFICATE_STATUS: AdminActionRef = AdminActionRef::new("TLSCertificateStatusAdminAction");
 const CONSOLE_LOG: AdminActionRef = AdminActionRef::new("ConsoleLogAdminAction");
 const COMMIT_TABLE: AdminActionRef = AdminActionRef::new("CommitTableAction");
-const ISSUE_PLUGIN_ATTEMPT: AdminActionRef = AdminActionRef::new("IssuePluginAttemptAction");
-const SUBMIT_PLUGIN_JOB: AdminActionRef = AdminActionRef::new("SubmitPluginJobAction");
-const GET_PLUGIN_RUN: AdminActionRef = AdminActionRef::new("GetPluginRunAction");
-const CANCEL_PLUGIN_RUN: AdminActionRef = AdminActionRef::new("CancelPluginRunAction");
-const LIST_PLUGIN_RUNS: AdminActionRef = AdminActionRef::new("ListPluginRunsAction");
-const REPLAY_PLUGIN_RUN: AdminActionRef = AdminActionRef::new("ReplayPluginRunAction");
-const GET_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("GetPluginDelegationAction");
-const CREATE_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("CreatePluginDelegationAction");
-const REVOKE_PLUGIN_DELEGATION: AdminActionRef = AdminActionRef::new("RevokePluginDelegationAction");
 const CREATE_POLICY: AdminActionRef = AdminActionRef::new("CreatePolicyAdminAction");
 const CREATE_USER: AdminActionRef = AdminActionRef::new("CreateUserAdminAction");
 const CREATE_SERVICE_ACCOUNT: AdminActionRef = AdminActionRef::new("CreateServiceAccountAdminAction");
@@ -665,66 +656,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
         SERVER_INFO,
         RouteRiskLevel::Sensitive,
     ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/issue-attempt",
-        ISSUE_PLUGIN_ATTEMPT,
-        RouteRiskLevel::High,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/submit",
-        SUBMIT_PLUGIN_JOB,
-        RouteRiskLevel::High,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/runs/query",
-        GET_PLUGIN_RUN,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/runs/history",
-        LIST_PLUGIN_RUNS,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/runs/cancel",
-        CANCEL_PLUGIN_RUN,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/runs/page",
-        LIST_PLUGIN_RUNS,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/runs/replay",
-        REPLAY_PLUGIN_RUN,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Get,
-        "/rustfs/admin/v3/plugin-delegations/{id}",
-        GET_PLUGIN_DELEGATION,
-        RouteRiskLevel::Sensitive,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations",
-        CREATE_PLUGIN_DELEGATION,
-        RouteRiskLevel::High,
-    ),
-    admin(
-        HttpMethod::Post,
-        "/rustfs/admin/v3/plugin-delegations/{id}/revoke",
-        REVOKE_PLUGIN_DELEGATION,
-        RouteRiskLevel::High,
-    ),
     admin(
         HttpMethod::Get,
         "/rustfs/admin/v4/plugins/catalog",
@@ -2002,72 +1933,6 @@ mod tests {
         assert!(validate_admin_route_policy_specs().is_ok());
     }
 
-    #[test]
-    fn plugin_submit_route_is_registered_as_high_risk_authenticated_action() {
-        let path = "/rustfs/admin/v3/plugin-delegations/{id}/submit";
-        assert_action(HttpMethod::Post, path, SUBMIT_PLUGIN_JOB);
-        let spec = ADMIN_ROUTE_POLICY_SPECS
-            .iter()
-            .find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
-            .expect("submit route policy");
-        assert_eq!(spec.risk_level(), RouteRiskLevel::High);
-    }
-
-    #[test]
-    fn plugin_run_query_route_uses_dedicated_authenticated_action() {
-        let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/query";
-        assert_action(HttpMethod::Post, path, GET_PLUGIN_RUN);
-        let spec = ADMIN_ROUTE_POLICY_SPECS
-            .iter()
-            .find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
-            .expect("plugin run query route policy");
-        assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
-    }
-
-    #[test]
-    fn plugin_run_history_route_uses_list_action() {
-        let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/history";
-        assert_action(HttpMethod::Post, path, LIST_PLUGIN_RUNS);
-        let spec = ADMIN_ROUTE_POLICY_SPECS
-            .iter()
-            .find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
-            .expect("plugin run history route policy");
-        assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
-    }
-
-    #[test]
-    fn plugin_run_cancel_route_uses_dedicated_authenticated_action() {
-        let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/cancel";
-        assert_action(HttpMethod::Post, path, CANCEL_PLUGIN_RUN);
-        let spec = ADMIN_ROUTE_POLICY_SPECS
-            .iter()
-            .find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
-            .expect("plugin run cancel route policy");
-        assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
-    }
-
-    #[test]
-    fn plugin_run_page_route_uses_dedicated_authenticated_action() {
-        let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/page";
-        assert_action(HttpMethod::Post, path, LIST_PLUGIN_RUNS);
-        let spec = ADMIN_ROUTE_POLICY_SPECS
-            .iter()
-            .find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
-            .expect("plugin run page route policy");
-        assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
-    }
-
-    #[test]
-    fn plugin_run_replay_route_uses_dedicated_authenticated_action() {
-        let path = "/rustfs/admin/v3/plugin-delegations/{id}/runs/replay";
-        assert_action(HttpMethod::Post, path, REPLAY_PLUGIN_RUN);
-        let spec = ADMIN_ROUTE_POLICY_SPECS
-            .iter()
-            .find(|spec| spec.method() == HttpMethod::Post && spec.path() == path)
-            .expect("plugin run replay route policy");
-        assert_eq!(spec.risk_level(), RouteRiskLevel::Sensitive);
-    }
-
     #[test]
     fn route_policy_inventory_has_no_internal_overlap() {
         let mut seen = BTreeSet::new();

rustfs/src/admin/route_registration_test.rs

@@ -116,48 +116,6 @@ fn registered_admin_router() -> S3Router<AdminOperation> {
 
 fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
     vec![
-        admin_route(Method::POST, "/v3/plugin-delegations"),
-        admin_route_sample(Method::GET, "/v3/plugin-delegations/{id}", "/v3/plugin-delegations/test-delegation"),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/revoke",
-            "/v3/plugin-delegations/test-delegation/revoke",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/submit",
-            "/v3/plugin-delegations/test-delegation/submit",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/runs/query",
-            "/v3/plugin-delegations/test-delegation/runs/query",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/runs/history",
-            "/v3/plugin-delegations/test-delegation/runs/history",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/runs/cancel",
-            "/v3/plugin-delegations/test-delegation/runs/cancel",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/runs/page",
-            "/v3/plugin-delegations/test-delegation/runs/page",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/runs/replay",
-            "/v3/plugin-delegations/test-delegation/runs/replay",
-        ),
-        admin_route_sample(
-            Method::POST,
-            "/v3/plugin-delegations/{id}/issue-attempt",
-            "/v3/plugin-delegations/test-delegation/issue-attempt",
-        ),
         route(Method::GET, HEALTH_PREFIX),
         route(Method::HEAD, HEALTH_PREFIX),
         route(Method::GET, HEALTH_READY_PATH),

rustfs/src/app/object/get.rs

@@ -3141,10 +3141,7 @@ impl DefaultObjectUsecase {
             // resolver; only the response headers, authorization and audit
             // summary are derived here, without a second KMS unwrap.
             Some(headers) => (
-                headers
-                    .sse_customer_algorithm
-                    .is_none()
-                    .then_some(headers.server_side_encryption),
+                Some(headers.server_side_encryption),
                 headers.sse_customer_algorithm,
                 headers.sse_customer_key_md5,
                 headers.ssekms_key_id,

rustfs/src/app/object/head.rs

@@ -462,13 +462,12 @@ impl DefaultObjectUsecase {
         })?;
 
         let metadata_map = info.user_defined.clone();
+        let server_side_encryption = metadata_map
+            .get("x-amz-server-side-encryption")
+            .map(|v| ServerSideEncryption::from(v.clone()));
         let sse_customer_algorithm = metadata_map
             .get("x-amz-server-side-encryption-customer-algorithm")
             .map(|v| SSECustomerAlgorithm::from(v.clone()));
-        let server_side_encryption = metadata_map
-            .get("x-amz-server-side-encryption")
-            .filter(|_| sse_customer_algorithm.is_none())
-            .map(|v| ServerSideEncryption::from(v.clone()));
         let sse_customer_key_md5 = metadata_map.get("x-amz-server-side-encryption-customer-key-md5").cloned();
         let sse_kms_key_id = metadata_map.get("x-amz-server-side-encryption-aws-kms-key-id").cloned();
         let storage_class = response_storage_class(&info, &metadata_map);

rustfs/src/app/object/put.rs

@@ -1435,10 +1435,7 @@ impl DefaultObjectUsecase {
 
         let output = PutObjectOutput {
             e_tag,
-            server_side_encryption: committed
-                .effective_sse
-                .clone()
-                .filter(|_| committed.sse_customer_algorithm.is_none()),
+            server_side_encryption: committed.effective_sse.clone(),
             sse_customer_algorithm: committed.sse_customer_algorithm.clone(),
             sse_customer_key_md5: committed.sse_customer_key_md5.clone(),
             ssekms_key_id: committed.effective_kms_key_id.clone(),
@@ -3967,12 +3964,6 @@ mod tests {
     async fn sse_c_put_is_accepted_on_a_bucket_with_default_encryption() {
         use md5::{Digest as _, Md5};
 
-        let context = crate::storage::storage_api::bootstrap_instance_ctx();
-        assert!(
-            context.object_encryption_resolver().is_some(),
-            "real SSE-C reads need the production encryption resolver"
-        );
-
         for (algorithm, kms_key_id, prefix) in [
             (ServerSideEncryption::AES256, None, "ssec-over-aes256-default"),
             (ServerSideEncryption::AWS_KMS, Some("bucket-key"), "ssec-over-kms-default"),
@@ -3994,69 +3985,14 @@ mod tests {
                 .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
                 .sse_customer_algorithm(Some("AES256".to_string()))
                 .sse_customer_key(Some(base64_simd::STANDARD.encode_to_string(customer_key)))
-                .sse_customer_key_md5(Some(key_md5.clone()))
+                .sse_customer_key_md5(Some(key_md5))
                 .build()
                 .expect("SSE-C PUT input must build");
 
-            let usecase = DefaultObjectUsecase::from_global();
-            let response = usecase
+            DefaultObjectUsecase::from_global()
                 .execute_put_object(&FS::new(), build_request(input, Method::PUT))
                 .await
                 .unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}"));
-            assert_eq!(response.output.server_side_encryption, None);
-            assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256"));
-            assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
-
-            let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key);
-            let mut read_headers = HeaderMap::new();
-            read_headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
-            read_headers.insert(
-                "x-amz-server-side-encryption-customer-key",
-                HeaderValue::from_str(&encoded_key).expect("encoded customer key is a valid header"),
-            );
-            read_headers.insert(
-                "x-amz-server-side-encryption-customer-key-md5",
-                HeaderValue::from_str(&key_md5).expect("customer key MD5 is a valid header"),
-            );
-            let head_input = HeadObjectInput::builder()
-                .bucket(bucket.clone())
-                .key("ledger.csv".to_string())
-                .sse_customer_algorithm(Some("AES256".to_string()))
-                .sse_customer_key(Some(encoded_key.clone()))
-                .sse_customer_key_md5(Some(key_md5.clone()))
-                .build()
-                .expect("SSE-C HEAD input must build");
-            let mut head_request = build_request(head_input, Method::HEAD);
-            head_request.headers = read_headers.clone();
-            let head = Box::pin(usecase.execute_head_object(head_request))
-                .await
-                .expect("matching SSE-C key must authorize HEAD");
-            assert_eq!(head.output.server_side_encryption, None);
-            assert_eq!(head.output.sse_customer_algorithm.as_deref(), Some("AES256"));
-            assert_eq!(head.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
-
-            let get_input = GetObjectInput::builder()
-                .bucket(bucket.clone())
-                .key("ledger.csv".to_string())
-                .sse_customer_algorithm(Some("AES256".to_string()))
-                .sse_customer_key(Some(encoded_key))
-                .sse_customer_key_md5(Some(key_md5.clone()))
-                .build()
-                .expect("SSE-C GET input must build");
-            let mut get_request = build_request(get_input, Method::GET);
-            get_request.headers = read_headers;
-            let get = Box::pin(usecase.execute_get_object(get_request))
-                .await
-                .expect("matching SSE-C key must authorize GET");
-            assert_eq!(get.output.server_side_encryption, None);
-            assert_eq!(get.output.sse_customer_algorithm.as_deref(), Some("AES256"));
-            assert_eq!(get.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
-            let mut body = get.output.body.expect("SSE-C GET must include the decrypted body");
-            let mut plaintext = Vec::new();
-            while let Some(chunk) = body.next().await {
-                plaintext.extend_from_slice(&chunk.expect("SSE-C body must decrypt completely"));
-            }
-            assert_eq!(plaintext, payload, "header classification must preserve the complete decrypted payload");
 
             let stored = store
                 .get_object_info(&bucket, "ledger.csv", &ObjectOptions::default())

rustfs/src/error.rs

@@ -310,7 +310,6 @@ impl std::error::Error for ApiError {
 pub(crate) struct ApiErrorDiagnostic {
     pub storage_code: Option<StorageErrorCode>,
     pub io_kind: Option<std::io::ErrorKind>,
-    pub io_os_error: Option<i32>,
     pub rpc_code: Option<tonic::Code>,
     pub pool_metadata: Option<PoolMetadataDiagnostic>,
     pub truncated: bool,
@@ -334,16 +333,10 @@ impl From<&PoolMetadataError> for PoolMetadataDiagnostic {
     }
 }
 
-impl ApiErrorDiagnostic {
-    pub(crate) fn from_error(error: &(dyn std::error::Error + 'static)) -> Self {
+impl ApiError {
+    pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic {
         let mut diagnostic = ApiErrorDiagnostic::default();
-        // S3Error retains its payload through an inherent accessor; its
-        // std::error::Error implementation does not expose that source.
-        let mut current = if let Some(error) = error.downcast_ref::<S3Error>() {
-            error.source().map(|source| source as &(dyn std::error::Error + 'static))
-        } else {
-            std::error::Error::source(error)
-        };
+        let mut current = std::error::Error::source(self);
         for _ in 0..16 {
             let Some(error) = current else {
                 return diagnostic;
@@ -361,11 +354,8 @@ impl ApiErrorDiagnostic {
             }
             current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
                 diagnostic.io_kind = Some(io.kind());
-                diagnostic.io_os_error = io.raw_os_error();
                 // io::Error::source can skip the wrapped error itself.
                 io.get_ref().map(|source| source as &(dyn std::error::Error + 'static))
-            } else if let Some(error) = error.downcast_ref::<S3Error>() {
-                error.source().map(|source| source as &(dyn std::error::Error + 'static))
             } else {
                 error.source()
             };
@@ -373,12 +363,6 @@ impl ApiErrorDiagnostic {
         diagnostic.truncated = current.is_some();
         diagnostic
     }
-}
-
-impl ApiError {
-    pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic {
-        ApiErrorDiagnostic::from_error(self)
-    }
 
     /// Access-denied error with the exact message emitted by the authorization
     /// paths in `storage::access`; callers there match on the code only.
@@ -571,7 +555,6 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
     let mut body_size_exceeded = false;
     let mut aws_chunked_error = None;
     let mut incomplete_body = false;
-    let mut rpc_unavailable = false;
     let digest = error_chain_find(err, |error| {
         let upload_error = error.downcast_ref::<UploadStreamError>();
         if matches!(upload_error, Some(UploadStreamError::Sha256Mismatch)) || error.is::<rustfs_rio::Sha256Mismatch>() {
@@ -581,9 +564,6 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
             return Some(S3ErrorCode::BadDigest);
         }
 
-        rpc_unavailable |= error
-            .downcast_ref::<tonic::Status>()
-            .is_some_and(|status| status.code() == tonic::Code::Unavailable);
         upload_limit_exceeded |= error.is::<UploadLimitExceeded>();
         server_source_failed |= error.is::<ServerSideSourceReadError>();
         client_body_timed_out |= error.is::<ClientBodyReadTimeout>();
@@ -611,8 +591,6 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
             Some(code)
         } else if incomplete_body {
             Some(S3ErrorCode::IncompleteBody)
-        } else if rpc_unavailable {
-            Some(S3ErrorCode::ServiceUnavailable)
         } else {
             None
         }
@@ -1263,35 +1241,6 @@ mod tests {
         assert!(!format!("{:?}", error.diagnostic()).contains("secret"));
     }
 
-    #[test]
-    fn api_error_diagnostic_survives_s3_conversion_without_changing_response() {
-        let api = ApiError::from(StorageError::Io(IoError::from_raw_os_error(30)));
-        let before = api.diagnostic();
-        assert_eq!(before.io_os_error, Some(30));
-        let mut error: S3Error = api.into();
-        error.set_request_id("server-owned-request");
-        let code = error.code().clone();
-        let status = error.status_code();
-        let message = error.message().map(str::to_owned);
-        assert_eq!(super::ApiErrorDiagnostic::from_error(&error), before);
-        assert_eq!(error.code(), &code);
-        assert_eq!(error.status_code(), status);
-        assert_eq!(error.message(), message.as_deref());
-        assert_eq!(error.request_id(), Some("server-owned-request"));
-        assert!(error.source().is_some());
-        assert!(std::error::Error::source(&error).is_none());
-
-        let error: S3Error = ApiError::from(StorageError::Io(IoError::other(StorageError::ErasureReadQuorum))).into();
-        assert_eq!(
-            super::ApiErrorDiagnostic::from_error(&error).storage_code,
-            Some(StorageErrorCode::ErasureReadQuorum)
-        );
-
-        let nested: S3Error = ApiError::from(StorageError::ErasureReadQuorum).into();
-        let api = ApiError::from(StorageError::Io(IoError::other(nested)));
-        assert_eq!(api.diagnostic().storage_code, Some(StorageErrorCode::ErasureReadQuorum));
-    }
-
     #[test]
     fn api_error_diagnostic_bounds_cyclic_error_chains() {
         #[derive(Debug)]
@@ -1313,11 +1262,6 @@ mod tests {
         };
         assert!(error.diagnostic().truncated);
         assert!(!format!("{:?}", error.diagnostic()).contains("secret"));
-        let error: S3Error = error.into();
-        let diagnostic = super::ApiErrorDiagnostic::from_error(&error);
-        assert!(diagnostic.truncated);
-        assert!(!format!("{diagnostic:?}").contains("secret"));
-
         let error = S3Error::from(ApiError {
             code: S3ErrorCode::ServiceUnavailable,
             message: "safe".into(),
@@ -1628,85 +1572,6 @@ mod tests {
         assert!(error_chain_has_type::<AwsChunkedStreamError>(&error));
     }
 
-    #[test]
-    fn rpc_unavailable_preserves_service_refusal_across_io_boundaries() {
-        let status = || tonic::Status::unavailable("private transport detail must not reach the S3 response");
-        let storage = || StorageError::from(status());
-        let original = storage();
-        let cloned = original.clone();
-        for error in [
-            ApiError::from(original),
-            ApiError::from(cloned),
-            ApiError::from(StorageError::Io(IoError::other(IoError::other(status())))),
-            ApiError::from(IoError::other(status())),
-            ApiError::from(IoError::other(storage())),
-        ] {
-            assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
-            assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable));
-            assert_eq!(error.diagnostic().rpc_code, Some(tonic::Code::Unavailable));
-            let s3_error = S3Error::from(error);
-            assert_eq!(s3_error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
-        }
-    }
-
-    #[test]
-    fn rpc_unavailable_mapping_does_not_match_other_codes_or_untyped_messages() {
-        for code in [
-            tonic::Code::Cancelled,
-            tonic::Code::Unknown,
-            tonic::Code::InvalidArgument,
-            tonic::Code::DeadlineExceeded,
-            tonic::Code::NotFound,
-            tonic::Code::AlreadyExists,
-            tonic::Code::PermissionDenied,
-            tonic::Code::ResourceExhausted,
-            tonic::Code::FailedPrecondition,
-            tonic::Code::Aborted,
-            tonic::Code::OutOfRange,
-            tonic::Code::Unimplemented,
-            tonic::Code::Internal,
-            tonic::Code::DataLoss,
-            tonic::Code::Unauthenticated,
-        ] {
-            for error in [
-                ApiError::from(StorageError::from(tonic::Status::new(code, "rpc unavailable"))),
-                ApiError::from(IoError::other(tonic::Status::new(code, "rpc unavailable"))),
-            ] {
-                assert_eq!(error.code, S3ErrorCode::InternalError, "{code:?}");
-            }
-        }
-        for error in [
-            ApiError::from(StorageError::Io(IoError::other("status: Unavailable"))),
-            ApiError::from(IoError::other("status: Unavailable")),
-        ] {
-            assert_eq!(error.code, S3ErrorCode::InternalError);
-        }
-    }
-
-    #[test]
-    fn rpc_unavailable_mapping_keeps_client_digest_error_precedence() {
-        for sha256 in [false, true] {
-            let source = || {
-                let mut status = tonic::Status::unavailable("private transport failure");
-                if sha256 {
-                    status.set_source(std::sync::Arc::new(UploadStreamError::Sha256Mismatch));
-                } else {
-                    status.set_source(std::sync::Arc::new(rustfs_rio::BadDigest {
-                        expected_md5: "expected".into(),
-                        calculated_md5: "calculated".into(),
-                    }));
-                }
-                status
-            };
-            for error in [
-                ApiError::from(StorageError::Io(IoError::other(source()))),
-                ApiError::from(IoError::other(source())),
-            ] {
-                assert_eq!(error.code.as_str(), if sha256 { CONTENT_SHA256_MISMATCH_CODE } else { "BadDigest" });
-            }
-        }
-    }
-
     #[test]
     fn body_size_limit_exceeded_maps_to_entity_too_large_across_io_boundaries() {
         // Shape observed in production (issue #7596):

rustfs/src/startup_background.rs

@@ -81,14 +81,6 @@ pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Resu
         let admission = None;
         let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()).with_admission(admission.clone()));
         init_heal_manager_with_workload_provider(heal_storage, None, Some(workload_provider.clone())).await?;
-        if let Some(admission) = &admission
-            && !rustfs_scanner::set_scanner_node_admission(admission.clone())
-        {
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::AlreadyExists,
-                "scanner node admission is already initialized",
-            ));
-        }
         #[cfg(any(target_os = "linux", all(test, unix)))]
         if let Some(admission) = admission {
             crate::background_admission::start(admission, background_cancel.clone(), workload_provider);

rustfs/src/storage/access.rs

@@ -64,13 +64,6 @@ use std::sync::OnceLock;
 use time::OffsetDateTime;
 use url::{Url, form_urlencoded};
 
-fn iam_authorization_error(err: IamError) -> ApiError {
-    match err {
-        IamError::IAMActionNotAllowed => ApiError::access_denied(),
-        other => ApiError::from(other),
-    }
-}
-
 const EVENT_OBJECT_TAG_AUTHORIZATION: &str = "object_tag_authorization";
 const LOG_COMPONENT_ACCESS: &str = "storage_access";
 const LOG_SUBSYSTEM_AUTHORIZATION: &str = "authorization";
@@ -1490,7 +1483,7 @@ async fn authorize_request_with_location_constraint<T>(
             let allowed = iam_store
                 .try_eval_prepared(&prepared, &final_args)
                 .await
-                .map_err(iam_authorization_error)?;
+                .map_err(ApiError::from)?;
             if !allowed
                 && matches!(
                     action,
@@ -1512,7 +1505,7 @@ async fn authorize_request_with_location_constraint<T>(
                 if !iam_store
                     .try_eval_prepared(&prepared, &final_args)
                     .await
-                    .map_err(iam_authorization_error)?
+                    .map_err(ApiError::from)?
                 {
                     return Err(denial.deny("iam_explicit_deny", action));
                 }
@@ -3625,14 +3618,6 @@ mod tests {
     use std::{collections::HashMap, sync::Arc};
     use time::OffsetDateTime;
 
-    #[test]
-    fn terminal_iam_denial_is_access_denied_and_backend_failure_stays_internal() {
-        let denied = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IAMActionNotAllowed));
-        assert_eq!(denied.code(), &S3ErrorCode::AccessDenied);
-        let unavailable = s3s::S3Error::from(super::iam_authorization_error(rustfs_iam::error::Error::IamSysNotInitialized));
-        assert_eq!(unavailable.code(), &S3ErrorCode::InternalError);
-    }
-
     struct UnreadyIam;
 
     impl IamInterface for UnreadyIam {

rustfs/src/storage/ecfs.rs

@@ -22,7 +22,7 @@ use super::{
     update_bucket_metadata_config_if_incarnation,
 };
 use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
-use crate::error::{ApiError, ApiErrorDiagnostic};
+use crate::error::ApiError;
 use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header};
 use crate::storage::helper::OperationHelper;
 use crate::storage::options::get_opts;
@@ -60,32 +60,6 @@ const LOG_COMPONENT_STORAGE: &str = "storage";
 const LOG_SUBSYSTEM_OBJECT: &str = "object";
 const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
 const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
-const EVENT_OBJECT_INTERNAL_ERROR: &str = "object_internal_error";
-
-fn log_object_internal_error(operation: &'static str, request_id: Option<&str>, error: &S3Error) {
-    if error.code() != &S3ErrorCode::InternalError {
-        return;
-    }
-    // Generated S3 adapters serialize handler errors without retaining their
-    // typed sources in logs. Keep only bounded classifications at this boundary.
-    let diagnostic = ApiErrorDiagnostic::from_error(error);
-    error!(
-        event = EVENT_OBJECT_INTERNAL_ERROR,
-        component = LOG_COMPONENT_STORAGE,
-        subsystem = LOG_SUBSYSTEM_OBJECT,
-        result = "error",
-        operation,
-        request_id = request_id.unwrap_or("unavailable"),
-        storage_error_code = ?diagnostic.storage_code,
-        io_error_kind = ?diagnostic.io_kind,
-        io_os_error = diagnostic.io_os_error,
-        rpc_error_code = ?diagnostic.rpc_code,
-        pool_metadata_reason = diagnostic.pool_metadata.map(|context| context.reason),
-        pool_metadata_phase = diagnostic.pool_metadata.map(|context| context.phase),
-        source_chain_truncated = diagnostic.truncated,
-        "Object request returned an internal error"
-    );
-}
 
 use crate::app::storage_api::object_usecase::bucket::replication::{
     OperatorRuleContract, ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication,
@@ -671,16 +645,8 @@ impl S3 for FS {
     #[instrument(level = "debug", skip(self, req))]
     async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
         crate::hp_guard!("S3::delete_object");
-        let request_id = req
-            .extensions
-            .get::<super::request_context::RequestContext>()
-            .map(|ctx| ctx.request_id.clone());
         let usecase = s3_api::object_usecase_for(self);
-        let result = Box::pin(usecase.execute_delete_object(req)).await;
-        if let Err(error) = &result {
-            log_object_internal_error("DeleteObject", request_id.as_deref(), error);
-        }
-        result
+        Box::pin(usecase.execute_delete_object(req)).await
     }
 
     #[instrument(level = "debug", skip(self))]
@@ -998,16 +964,8 @@ impl S3 for FS {
     )]
     async fn get_object(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
         crate::hp_guard!("S3::get_object");
-        let request_id = req
-            .extensions
-            .get::<super::request_context::RequestContext>()
-            .map(|ctx| ctx.request_id.clone());
         let usecase = s3_api::object_usecase_for(self);
-        let result = usecase.execute_get_object(req).await;
-        if let Err(error) = &result {
-            log_object_internal_error("GetObject", request_id.as_deref(), error);
-        }
-        result
+        usecase.execute_get_object(req).await
     }
 
     async fn get_object_acl(&self, req: S3Request<GetObjectAclInput>) -> S3Result<S3Response<GetObjectAclOutput>> {
@@ -2115,73 +2073,6 @@ mod tests {
     use s3s::{S3, S3Error, S3ErrorCode, S3Request};
     use std::sync::atomic::Ordering;
 
-    #[test]
-    fn object_internal_error_logs_only_correlated_bounded_classifications() {
-        #[derive(Clone, Default)]
-        struct Writer(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
-        impl std::io::Write for Writer {
-            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
-                self.0.lock().expect("log buffer").extend_from_slice(bytes);
-                Ok(bytes.len())
-            }
-            fn flush(&mut self) -> std::io::Result<()> {
-                Ok(())
-            }
-        }
-        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Writer {
-            type Writer = Self;
-            fn make_writer(&'a self) -> Self::Writer {
-                self.clone()
-            }
-        }
-        let writer = Writer::default();
-        let subscriber = tracing_subscriber::fmt()
-            .json()
-            .without_time()
-            .with_writer(writer.clone())
-            .finish();
-        let mut status = tonic::Status::internal("secret source message");
-        status
-            .metadata_mut()
-            .insert("authorization", "secret-token".parse().expect("metadata"));
-        let error: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(status)).into();
-        let original_code = error.code().clone();
-        let original_status = error.status_code();
-        let original_message = error.message().map(str::to_owned);
-        tracing::subscriber::with_default(subscriber, || {
-            super::log_object_internal_error("GetObject", Some("server-owned-request"), &error);
-            let mut opaque = S3Error::with_message(S3ErrorCode::InternalError, "secret outer message");
-            opaque.set_source(Box::new(std::io::Error::other("secret inner message")));
-            super::log_object_internal_error("DeleteObject", None, &opaque);
-            let unavailable: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(
-                tonic::Status::unavailable("secret transport message"),
-            ))
-            .into();
-            assert_eq!(unavailable.code(), &S3ErrorCode::ServiceUnavailable);
-            assert_eq!(unavailable.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
-            super::log_object_internal_error("GetObject", None, &unavailable);
-        });
-        let output = String::from_utf8(writer.0.lock().expect("log buffer").clone()).expect("UTF8 log");
-        assert!(!output.contains("secret"));
-        assert!(!output.contains("authorization"));
-        let rows = output
-            .lines()
-            .map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON log"))
-            .collect::<Vec<_>>();
-        assert_eq!(rows.len(), 2);
-        for row in &rows {
-            assert_eq!(row["fields"]["event"], "object_internal_error");
-            assert_eq!(row["fields"]["source_chain_truncated"], false);
-        }
-        assert_eq!(rows[0]["fields"]["request_id"], "server-owned-request");
-        assert_eq!(rows[0]["fields"]["rpc_error_code"], "Some(Internal)");
-        assert_eq!(rows[1]["fields"]["request_id"], "unavailable");
-        assert_eq!(rows[1]["fields"]["io_error_kind"], "Some(Other)");
-        assert_eq!(error.code(), &original_code);
-        assert_eq!(error.status_code(), original_status);
-        assert_eq!(error.message(), original_message.as_deref());
-    }
-
     #[test]
     fn website_config_rejects_missing_index_and_invalid_redirect_rules() {
         assert!(validate_website_configuration(&WebsiteConfiguration::default()).is_err());

rustfs/src/storage/scanner_dirty_journal.rs

@@ -222,12 +222,12 @@ impl JournalState {
         Some(entries)
     }
 
-    fn from_replay_record(record: &ScannerDurableDirtyUsageReplayRecord) -> Option<Self> {
+    fn from_replay_record(record: ScannerDurableDirtyUsageReplayRecord) -> Option<Self> {
         if record.entries.is_empty() || record.entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES {
             return None;
         }
         let mut state = JournalState::default();
-        for entry in &record.entries {
+        for entry in record.entries {
             if entry.bucket.is_empty()
                 || entry.bucket.contains(['/', '\\', '\0'])
                 || entry.bucket == "."
@@ -242,7 +242,7 @@ impl JournalState {
             {
                 return None;
             }
-            let scope = match &entry.scope {
+            let scope = match entry.scope {
                 ScannerDurableDirtyUsageReplayScope::WholeBucket => JournalScope::WholeBucket,
                 ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } => {
                     if entries.is_empty() || entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES {
@@ -254,18 +254,18 @@ impl JournalState {
                     {
                         return None;
                     }
-                    JournalScope::TopLevelEntries(entries.clone())
+                    JournalScope::TopLevelEntries(entries)
                 }
             };
             state.scope_bytes += journal_scope_bytes(&scope);
             if state
                 .buckets
                 .insert(
-                    entry.bucket.clone(),
+                    entry.bucket,
                     JournalBucketState {
                         generation: entry.generation,
                         scope,
-                        producers: entry.producers.clone(),
+                        producers: entry.producers,
                     },
                 )
                 .is_some()
@@ -338,7 +338,7 @@ async fn replay_durable_dirty_usage_journal(store: Arc<ECStore>, owner: &str) ->
             ..Default::default()
         };
     };
-    match replay.replay() {
+    match rustfs_scanner::replay_durable_dirty_usage_producer_record(&replay) {
         Ok(replayed) => {
             debug!(
                 event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL,
@@ -368,7 +368,7 @@ async fn replay_durable_dirty_usage_journal(store: Arc<ECStore>, owner: &str) ->
     }
 }
 
-fn decode_owned_replay_record(owner: &str, bytes: &[u8]) -> Option<(JournalState, ScannerDurableDirtyUsageReplayRecord)> {
+fn decode_owned_replay_record(owner: &str, bytes: &[u8]) -> Option<(JournalState, Vec<u8>)> {
     if bytes.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
         return None;
     }
@@ -377,16 +377,13 @@ fn decode_owned_replay_record(owner: &str, bytes: &[u8]) -> Option<(JournalState
         return None;
     }
     let replay = record.replay?;
-    let state = JournalState::from_replay_record(&replay)?;
-    Some((state, replay))
+    let bytes = serde_json::to_vec(&replay).ok()?;
+    let state = JournalState::from_replay_record(replay)?;
+    Some((state, bytes))
 }
 
 fn encode_owned_replay_record(owner: &str, state: &JournalState) -> Result<Vec<u8>, serde_json::Error> {
     let mut entries = state.replay_entries();
-    let mut prefix = Vec::with_capacity(owner.len() + 32);
-    prefix.extend_from_slice(b"{\"owner\":");
-    serde_json::to_writer(&mut prefix, owner)?;
-    prefix.extend_from_slice(b",\"replay\":");
     for compact in [false, true] {
         if let Some(entries) = &mut entries {
             if compact {
@@ -394,12 +391,13 @@ fn encode_owned_replay_record(owner: &str, state: &JournalState) -> Result<Vec<u
                     entry.scope = ScannerDurableDirtyUsageReplayScope::WholeBucket;
                 }
             }
-            if let Ok(replay) = rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries.clone()) {
-                // This producer already validated and serialized the complete frame.
-                let mut bytes = prefix.clone();
-                bytes.reserve(replay.len() + 1);
-                bytes.extend_from_slice(&replay);
-                bytes.push(b'}');
+            if let Ok(replay) = rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries.clone())
+                && let Ok(replay) = serde_json::from_slice(&replay)
+            {
+                let bytes = serde_json::to_vec(&OwnedReplayRecord {
+                    owner: owner.to_string(),
+                    replay: Some(replay),
+                })?;
                 if bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
                     return Ok(bytes);
                 }
@@ -407,8 +405,10 @@ fn encode_owned_replay_record(owner: &str, state: &JournalState) -> Result<Vec<u
         }
     }
     // Never label a truncated record as complete producer coverage.
-    prefix.extend_from_slice(b"null}");
-    Ok(prefix)
+    serde_json::to_vec(&OwnedReplayRecord {
+        owner: owner.to_string(),
+        replay: None,
+    })
 }
 
 async fn run_durable_dirty_usage_journal(store: Arc<ECStore>, owner: String, shared: Weak<SharedJournal>, changed: Arc<Notify>) {
@@ -575,84 +575,6 @@ mod tests {
         ENV.get_or_init(|| rustfs_test_utils::TestECStoreEnv::builder().build()).await
     }
 
-    fn legacy_owned_encoding(owner: &str, state: &JournalState) -> Vec<u8> {
-        let mut entries = state.replay_entries();
-        for compact in [false, true] {
-            if let Some(entries) = &mut entries {
-                if compact {
-                    for entry in entries.iter_mut() {
-                        entry.scope = ScannerDurableDirtyUsageReplayScope::WholeBucket;
-                    }
-                }
-                if let Ok(replay) = rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries.clone())
-                    && let Ok(replay) = serde_json::from_slice(&replay)
-                {
-                    let bytes = serde_json::to_vec(&OwnedReplayRecord {
-                        owner: owner.to_string(),
-                        replay: Some(replay),
-                    })
-                    .expect("legacy typed wrapper");
-                    if bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES {
-                        return bytes;
-                    }
-                }
-            }
-        }
-        serde_json::to_vec(&OwnedReplayRecord {
-            owner: owner.to_string(),
-            replay: None,
-        })
-        .expect("legacy unknown marker")
-    }
-
-    #[test]
-    fn owned_encoding_reuses_the_producer_frame_without_changing_wire_bytes() {
-        let mut scoped = JournalState::default();
-        for producer in SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION {
-            scoped.record_mutation("bucket-a".to_string(), "quote\"雪/object", 17, producer);
-        }
-        scoped.record_mutation("bucket-b".to_string(), "object", 9, SegmentInvalidationProducerIdentity::PutObject);
-        let unknown = JournalState {
-            invalidated: true,
-            ..Default::default()
-        };
-        for owner in ["owner", "quote\"slash\\newline\n", "所有者", "\0"] {
-            for state in [&scoped, &unknown, &JournalState::default()] {
-                let expected = legacy_owned_encoding(owner, state);
-                let actual = encode_owned_replay_record(owner, state).expect("complete owned encoding");
-                assert_eq!(actual, expected, "legacy bytes for owner {owner:?}");
-            }
-        }
-    }
-
-    #[test]
-    fn owned_unknown_marker_keeps_the_pinned_legacy_wire_fixture() {
-        let bytes = encode_owned_replay_record("owner", &JournalState::default()).expect("unknown marker");
-        assert_eq!(bytes, br#"{"owner":"owner","replay":null}"#);
-        assert!(decode_owned_replay_record("owner", &bytes).is_none());
-    }
-
-    #[test]
-    fn owned_replay_keeps_the_outer_json_recursion_boundary() {
-        // Ignored scope fields still consume the complete owned frame's depth
-        // budget. Parsing replay from a fresh JSON root must not bypass it.
-        for (depth, accepted) in [(122, true), (123, false)] {
-            let bytes = format!(
-                r#"{{"owner":"owner","replay":{{"schema":1,"cache_key_format":1,"writer_epoch":"epoch","entries":[{{"bucket":"bucket","generation":7,"scope":{{"type":"whole_bucket","ignored":{}0{}}},"producers":["put_object"]}}]}}}}"#,
-                "[".repeat(depth),
-                "]".repeat(depth),
-            );
-            assert!(bytes.len() < DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
-            let decoded = decode_owned_replay_record("owner", bytes.as_bytes());
-            assert_eq!(decoded.is_some(), accepted, "owned frame depth {depth}");
-            if let Some((state, _)) = decoded {
-                assert_eq!(state.buckets.len(), 1);
-                assert_eq!(state.buckets["bucket"].generation, 7);
-                assert_eq!(state.buckets["bucket"].scope, super::JournalScope::WholeBucket);
-            }
-        }
-    }
-
     #[test]
     fn journal_owner_is_stable_and_never_defaults_to_a_shared_identity() {
         assert!(durable_dirty_usage_owner("").is_none());
@@ -732,7 +654,6 @@ mod tests {
         }
         assert!(state.scope_bytes <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
         let bytes = encode_owned_replay_record(&owner, &state).expect("bounded record");
-        assert_eq!(bytes, legacy_owned_encoding(&owner, &state), "legacy compacted bytes");
         assert!(bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
         let (decoded, _) = decode_owned_replay_record(&owner, &bytes).expect("compacted replay");
         assert_eq!(decoded.buckets.len(), 3);
@@ -752,7 +673,6 @@ mod tests {
         assert!(!state.invalidated);
         assert_eq!(state.buckets.len(), 1024);
         let bytes = encode_owned_replay_record(&owner, &state).expect("unverified marker");
-        assert_eq!(bytes, legacy_owned_encoding(&owner, &state), "legacy unknown marker bytes");
         assert!(bytes.len() <= DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES);
         let record: OwnedReplayRecord = serde_json::from_slice(&bytes).expect("marker");
         assert!(record.replay.is_none());
@@ -1064,7 +984,7 @@ mod tests {
                 producers: BTreeSet::from([SegmentInvalidationProducerIdentity::Replication]),
             }],
         };
-        let mut state = JournalState::from_replay_record(&record).expect("valid replay record should hydrate");
+        let mut state = JournalState::from_replay_record(record).expect("valid replay record should hydrate");
         state.record_mutation(
             "videos".to_string(),
             "clips/object",
@@ -1091,7 +1011,7 @@ mod tests {
                 producers: BTreeSet::from([SegmentInvalidationProducerIdentity::Replication]),
             }],
         };
-        let mut state = JournalState::from_replay_record(&record).expect("valid replay record should hydrate");
+        let mut state = JournalState::from_replay_record(record).expect("valid replay record should hydrate");
         state.record_mutation(
             "photos".to_string(),
             "2026/object",

scripts/error-other-format-baseline.txt

@@ -62,7 +62,7 @@
 1|crates/ecstore/src/set_disk/ops/heal.rs
 8|crates/ecstore/src/set_disk/ops/multipart.rs
 2|crates/ecstore/src/set_disk/ops/object.rs
-1|crates/ecstore/src/set_disk/read.rs
+3|crates/ecstore/src/set_disk/read.rs
 5|crates/ecstore/src/store/bucket.rs
 1|crates/ecstore/src/store/heal_walk.rs
 10|crates/ecstore/src/store/init.rs