test: tighten panic isolation classification

.config/make/tests.mak

@@ -64,8 +64,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py

.github/workflows/README.md

@@ -16,7 +16,7 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
 | `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
 | `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
 | `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
-| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | Docker workloads with a measured resource or environment requirement beyond the hosted runner |
+| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
 | `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
 
 The self-hosted pods and the hosted image are both Ubuntu 24.04, so a binary built on one runs on the other; `ci.yml` builds the debug binary on `sm-standard-4` and runs it on both pools.
@@ -27,15 +27,11 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
 
 1. **Disk.** It restores or produces the workspace `target/`: `cargo build` or `cargo test` with `--all-targets`, `cargo nextest run --all`, or any reader of the `ci-dev` / `ci-feat-*` caches. The workspace test build leaves about 64 GB in `target/`; a hosted runner has about 14 GB.
 2. **Critical-path CPU.** It is a build that several downstream lanes wait on and would blow its timeout at 4 vCPU (`build-rustfs-debug-binary` gates the e2e and s3-tests lanes).
-3. **Hardware or environment.** Native aarch64 or access to the lab machines. Docker alone is not a reason: hosted Ubuntu ships Docker and supports the QEMU and Buildx setup actions.
+3. **Hardware.** Native aarch64, Docker-in-Docker, or the lab machines.
 4. **Cache ownership.** It reads a `rust-cache` key that a self-hosted job writes (see the cache rule below).
 
 Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
 
-The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
-
-Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
-
 A hosted runner is also the only choice for a few things the pods cannot do: `mount -o loop` and `mount -t tmpfs` fail inside the ARC pods, and a container runtime cannot guarantee that `io_uring_setup` is allowed. `uring-integration` and `e2e-distributed` are on `ubuntu-latest` for that reason, not for cost.
 
 ## Cache rule

.github/workflows/docker.yml

@@ -337,7 +337,7 @@ jobs:
     name: Build Docker Images
     needs: build-check
     if: needs.build-check.outputs.should_build == 'true'
-    runs-on: ubuntu-latest
+    runs-on: dind-sm-standard-2
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/runner-hygiene.yml

@@ -44,7 +44,7 @@ jobs:
   check-ephemerality:
     if: github.repository == 'rustfs/rustfs'
     name: Check runner ephemerality
-    runs-on: ubuntu-latest
+    runs-on: sm-standard-2
     timeout-minutes: 15
     steps:
       - name: Checkout repository

.github/workflows/rustfs-functional-chain.yml

@@ -20,91 +20,13 @@ on:
   workflow_dispatch:
     inputs:
       build_run_id:
-        description: Successful nightly or validation package run; empty for package-only build
+        description: Successful nightly build run on main
         type: string
-        required: false
-        default: ''
+        required: true
       build_run_attempt:
         description: Exact successful build attempt
         type: string
-        required: false
-        default: ''
-      validation_package_sha:
-        description: Exact source SHA for an isolated package-only build
-        type: string
-        required: false
-        default: ''
-      validation_case:
-        description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain
-        type: string
-        required: false
-        default: ''
-      i05_diagnostic_branch:
-        description: I05 diagnostic-only branch; queued-successor requires the exact native evidence bundle
-        type: choice
-        options: [timeout, queued-successor]
-        required: false
-        default: timeout
-      auto_testing_sha:
-        description: Exact auto-testing commit for a validation case
-        type: string
-        required: false
-        default: ''
-      validation_retain_evidence_run_id:
-        description: Upload existing persistent evidence from one interrupted validation run without fleet access
-        type: string
-        required: false
-        default: ''
-      validation_retain_evidence_run_attempt:
-        description: Exact interrupted run attempt to retain
-        type: string
-        required: false
-        default: '1'
-      validation_runner_inspect_only:
-        description: Inspect GitHub runner registration and queued jobs on a hosted runner without fleet access
-        type: boolean
-        required: false
-        default: false
-      validation_tools_only:
-        description: Build candidate-bound validation tools without running a fleet case
-        type: boolean
-        required: false
-        default: false
-      validation_native_tools:
-        description: Include exact-source native regressions in a tools-only build
-        type: boolean
-        required: false
-        default: false
-      validation_shared_tools:
-        description: Include the controlled C10 shared-reference encoder in a tools-only build
-        type: boolean
-        required: false
-        default: false
-      validation_manager_tools:
-        description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build
-        type: boolean
-        required: false
-        default: false
-      validation_cleanup_run_id:
-        description: Failed validation run whose supported retained environment needs evidence-based finalization
-        type: string
-        required: false
-        default: ''
-      validation_cleanup_retained_evidence:
-        description: Recover from verified persistent runner-local evidence when the failed run uploaded no artifact
-        type: boolean
-        required: false
-        default: false
-      validation_tools_run_id:
-        description: Successful tool build run; required by Heal physical metadata checks
-        type: string
-        required: false
-        default: ''
-      validation_tools_run_attempt:
-        description: Exact tool build attempt
-        type: string
-        required: false
-        default: '1'
+        required: true
   workflow_run:
     workflows: ["Nightly GNU Build"]
     types: [completed]
@@ -114,104 +36,13 @@ permissions:
   actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only && 'rustfs-functional-validation-runner-inspection' || github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }}
+  group: rustfs-functional-chain-runs
   cancel-in-progress: false
 
 jobs:
-  validation-runner-inspection:
-    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-      - name: Inspect shared runner registration and queue without fleet access
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}
-        run: >-
-          python3 -B scripts/inspect_validation_runner.py
-          --output "${RUNNER_TEMP}/validation-runner-inspection/report.json"
-      - name: Upload read-only runner inspection including incomplete reports
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-runner-inspection-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-runner-inspection/report.json
-          if-no-files-found: error
-  validation-evidence-retention:
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id != '' }}
-    # The original interrupted evidence exists only on this long-lived runner.
-    runs-on: smoke-testing
-    timeout-minutes: 15
-    concurrency:
-      group: rustfs-shared-functional-tests-v2
-      cancel-in-progress: false
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-      - name: Verify existing interrupted evidence without changing it
-        env:
-          GH_TOKEN: ${{ github.token }}
-          EVIDENCE_RETAIN_INPUTS: ${{ toJSON(inputs) }}
-        run: >-
-          python3 -B scripts/retain_validation_evidence.py
-          --report "${RUNNER_TEMP}/validation-evidence-retention/report.json"
-      - name: Upload admitted original evidence under collection provenance
-        if: ${{ env.VALIDATION_RETAIN_ROOT != '' }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-retained-${{ env.VALIDATION_RETAIN_SOURCE_RUN }}-${{ env.VALIDATION_RETAIN_SOURCE_ATTEMPT }}-by-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.VALIDATION_RETAIN_ROOT }}
-          if-no-files-found: error
-          include-hidden-files: true
-      - name: Upload retention receipt including incomplete reports
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-evidence-retention-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-evidence-retention/report.json
-          if-no-files-found: error
-  validation-package-request:
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - name: Validate exclusive package-only request
-        env:
-          PACKAGE_INPUTS: ${{ toJSON(inputs) }}
-        run: |
-          python3 - <<'PY'
-          import json
-          import os
-          import re
-          value = json.loads(os.environ["PACKAGE_INPUTS"])
-          if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None:
-              raise ValueError("package build requires an exact source SHA")
-          exclusive = ("validation_retain_evidence_run_id", "validation_runner_inspect_only", "build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha",
-                       "validation_tools_only", "validation_native_tools", "validation_shared_tools",
-                       "validation_manager_tools", "validation_cleanup_run_id", "validation_cleanup_retained_evidence", "validation_tools_run_id")
-          if any(value.get(name) for name in exclusive):
-              raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution")
-          PY
-
-  validation-package:
-    needs: validation-package-request
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
-    uses: ./.github/workflows/rustfs-validation-package.yml
-    with:
-      source_sha: ${{ inputs.validation_package_sha }}
-    secrets:
-      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
-      R2_BUCKET: ${{ secrets.R2_BUCKET }}
-
   prepare:
-    if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && !inputs.validation_cleanup_retained_evidence && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }}
-    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
+    runs-on: smoke-testing
     timeout-minutes: 10
     outputs:
       manifest: ${{ steps.candidate.outputs.manifest }}
@@ -258,395 +89,6 @@ jobs:
             echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
           fi
 
-  validation:
-    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
-    runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }}
-    timeout-minutes: 90
-    outputs:
-      source_sha: ${{ steps.candidate.outputs.source_sha }}
-    concurrency:
-      group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }}
-      cancel-in-progress: false
-    env:
-      RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }}
-      RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }}
-      RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }}
-      RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }}
-      RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }}
-    steps:
-      - name: Checkout candidate resolver
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize validation evidence
-        env:
-          VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}
-        run: |
-          set -euo pipefail
-          umask 077
-          python3 -B scripts/initialize_validation_evidence.py
-
-      - name: Resolve immutable candidate
-        id: candidate
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          BUILD_RUN_ID: ${{ inputs.build_run_id }}
-          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
-          CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
-          SELECTED_CASE: ${{ inputs.validation_case }}
-          TOOLS_ONLY: ${{ inputs.validation_tools_only }}
-          NATIVE_TOOLS: ${{ inputs.validation_native_tools }}
-          SHARED_TOOLS: ${{ inputs.validation_shared_tools }}
-          MANAGER_TOOLS: ${{ inputs.validation_manager_tools }}
-          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
-        run: |
-          set -euo pipefail
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          import re
-          from scripts.resolve_functional_candidate import resolve
-
-          revision = os.environ["AUTO_TESTING_SHA"]
-          cleanup = os.environ["CLEANUP_RUN_ID"]
-          if os.environ["CLEANUP_RETAINED_EVIDENCE"] == "true" and not cleanup:
-              raise ValueError("persistent recovery input requires one exact cleanup run")
-          if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("tools-only execution cannot select a fleet case or cleanup")
-          if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup")
-          if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup")
-          if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
-              raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
-          if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
-              raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
-          diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "timeout"
-          if diagnostic_branch not in {"timeout", "queued-successor"}:
-              raise ValueError("I05 diagnostic branch is not registered")
-          if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort":
-              raise ValueError("queued-successor is available only for the I05 diagnostic case")
-          if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
-              raise ValueError("validation requires an exact auto-testing commit SHA")
-          candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True)
-          manifest = candidate["manifest"]
-          root = Path(os.environ["VALIDATION_ROOT"])
-          (root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n")
-          (root / "candidate.json").write_text(json.dumps({
-              "package_url": manifest["package_url"],
-              "package_sha256": manifest["package_sha256"],
-              "source_sha": manifest["source_sha"],
-          }, sort_keys=True) + "\n")
-          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
-              output.write("source_sha=" + manifest["source_sha"] + "\n")
-          PY
-
-      - name: Checkout exact auto-testing revision
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Prepare validation Python runtime
-        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
-        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
-        with:
-          python-version: "3.12"
-
-      - name: Install validation Python client
-        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
-        run: |
-          set -euo pipefail
-          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          test ! -e "${validation_python}"
-          python3 -m venv "${validation_python}"
-          "${validation_python}/bin/python" -m pip install \
-            -r auto-testing/scripts/validation_ilm/requirements.txt
-          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
-
-      - name: Validate Heal tool build identity
-        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }}
-          TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }}
-        run: |
-          set -euo pipefail
-          [[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
-          [[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
-          gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \
-            > "${VALIDATION_ROOT}/tool-run.json"
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text())
-          if (value.get("conclusion") != "success" or value.get("status") != "completed"
-                  or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"])
-                  or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}):
-              raise ValueError("Heal tool run is not an exact successful validation build attempt")
-          PY
-
-      - name: Download pinned Heal metadata inspector
-        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify Heal metadata inspector
-        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
-        run: |
-          set -euo pipefail
-          TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          python3 -B auto-testing/scripts/validation_heal/inspector.py \
-            --directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            > "${VALIDATION_ROOT}/tool-manifest.json"
-          printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Select exact-source race native evidence
-        id: race_native
-        if: ${{ inputs.validation_case == 'heal.race-unversioned-overwrite' || inputs.validation_case == 'heal.race-versioned-put' || inputs.validation_case == 'heal.race-version-delete' || inputs.validation_case == 'heal.race-markers-and-batch-delete' || inputs.validation_case == 'heal.race-multipart-complete-abort' || inputs.validation_case == 'heal.admin-nolock-safety' || inputs.validation_case == 'heal.race-copy-and-metadata-update' }}
-        env:
-          CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }}
-          VALIDATION_CASE_ID: ${{ inputs.validation_case }}
-        run: |
-          set -euo pipefail
-          required=false
-          if [ -f auto-testing/scripts/validation_heal/race_native.py ]; then
-            status=0
-            python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
-              --commit "${CANDIDATE_SHA}" --case-id "${VALIDATION_CASE_ID}" || status=$?
-            if [ "${status}" -eq 0 ]; then
-              required=true
-            elif [ "${status}" -ne 2 ]; then
-              exit "${status}"
-            fi
-          fi
-          reviewed=false
-          case "${CANDIDATE_SHA}:${VALIDATION_CASE_ID}" in
-            6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca:heal.race-unversioned-overwrite | \
-            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-unversioned-overwrite | \
-            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-versioned-put | \
-            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-version-delete | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-unversioned-overwrite | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-versioned-put | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-version-delete | \
-            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-markers-and-batch-delete | \
-            757b751482bb970bc321e92059690a7698554358:heal.admin-nolock-safety | \
-            757b751482bb970bc321e92059690a7698554358:heal.race-multipart-complete-abort | \
-            757b751482bb970bc321e92059690a7698554358:heal.race-copy-and-metadata-update) reviewed=true ;;
-          esac
-          if [ "${reviewed}" = true ] && [ "${required}" != true ]; then
-            printf 'Race candidate requires its reviewed native collector before fleet execution\n' >&2
-            exit 1
-          fi
-          printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
-
-      - name: Download pinned native storage evidence
-        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor') }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify C07 native source and exact test evidence
-        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/authority.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            > "${VALIDATION_ROOT}/native-authority-manifest.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Verify C11 physical storage and exact final-rename barrier
-        if: ${{ inputs.validation_case == 'heal.bucket-incarnation-recreation' }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/incarnation_native.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${{ inputs.auto_testing_sha }}" \
-            > "${VALIDATION_ROOT}/native-incarnation-manifest.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Verify exact-source race native and runtime evidence
-        if: ${{ steps.race_native.outputs.required == 'true' }}
-        env:
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          VALIDATION_CASE_ID: ${{ inputs.validation_case }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${AUTO_TESTING_SHA}" --case-id "${VALIDATION_CASE_ID}" \
-            > "${VALIDATION_ROOT}/native-race-manifest.json"
-          runtime_driver=race.py
-          if [ "${VALIDATION_CASE_ID}" = 'heal.admin-nolock-safety' ]; then
-            runtime_driver=nolock.py
-          fi
-          VALIDATION_HEAL_NATIVE_DIRECTORY="${NATIVE_DIRECTORY}" \
-            python3 -B "auto-testing/scripts/validation_heal/${runtime_driver}" preflight \
-              --candidate "${VALIDATION_ROOT}/candidate.json" --testing-commit "${AUTO_TESTING_SHA}" \
-              --case-id "${VALIDATION_CASE_ID}" \
-              > "${VALIDATION_ROOT}/native-race-runtime-preflight.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Verify I05 queued-successor native prerequisite
-        if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor' }}
-        env:
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-        run: |
-          set -euo pipefail
-          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
-            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
-            > "${VALIDATION_ROOT}/native-i05-prerequisite.json"
-          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Download pinned manager persistence and incarnation evidence
-        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify exact manager native prerequisites before fleet mutation
-        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
-        env:
-          VALIDATION_CASE: ${{ inputs.validation_case }}
-        run: |
-          set -euo pipefail
-          MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \
-            --directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json"
-          printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Download pinned C10 shared-reference encoder
-        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          run-id: ${{ inputs.validation_tools_run_id }}
-          name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
-          path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
-
-      - name: Verify C10 exact-source codec and original tests
-        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
-        run: |
-          set -euo pipefail
-          SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json"
-          python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \
-            --directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
-            > "${VALIDATION_ROOT}/shared-fixture-manifest.json"
-          printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
-
-      - name: Run selected canonical validation case
-        if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
-        env:
-          VALIDATION_CASE: ${{ inputs.validation_case }}
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          ./auto-testing/scripts/run_validation_case.sh \
-            "${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \
-            "${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run"
-
-      - name: Inspect current Heal fleet resources without mutation
-        if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }}
-        env:
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          python3 -B auto-testing/scripts/validation_heal/environment.py \
-            --candidate "${VALIDATION_ROOT}/candidate.json" \
-            --testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment"
-
-      - name: Audit and finalize retained validation environment
-        if: ${{ inputs.validation_cleanup_run_id != '' }}
-        env:
-          GH_TOKEN: ${{ github.token }}
-          FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
-          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-          CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
-            --json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json"
-          python3 - <<'PY'
-          import json
-          import os
-          from pathlib import Path
-          run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text())
-          if (run["status"] != "completed" or run["conclusion"] not in {"failure", "cancelled"}
-              or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]):
-              raise ValueError("cleanup source must be a completed failed or cancelled Functional Chain run")
-          PY
-          if [ "${CLEANUP_RETAINED_EVIDENCE}" = true ]; then
-            python3 -B scripts/prepare_retained_validation_recovery.py \
-              --report "${VALIDATION_ROOT}/retained-recovery-input.json"
-          else
-            gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
-              --pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact"
-          fi
-          python3 -B auto-testing/scripts/validation_heal/recover_environment.py \
-            --failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \
-            --candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery"
-
-      - name: Show validation report
-        if: ${{ always() }}
-        run: |
-          if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then
-            cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          fi
-
-      - name: Upload validation evidence
-        if: ${{ always() && env.VALIDATION_ROOT != '' }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.VALIDATION_ROOT }}
-          if-no-files-found: error
-
-  validation-tools:
-    needs: [validation]
-    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }}
-    uses: ./.github/workflows/rustfs-validation-tools.yml
-    with:
-      candidate_sha: ${{ needs.validation.outputs.source_sha }}
-      auto_testing_sha: ${{ inputs.auto_testing_sha }}
-      build_native_heal: ${{ inputs.validation_native_tools }}
-      build_shared_heal: ${{ inputs.validation_shared_tools }}
-      build_manager_native: ${{ inputs.validation_manager_tools }}
-    secrets: inherit
-
   upgrade:
     needs: [prepare]
     if: ${{ always() && needs.prepare.result == 'success' }}
@@ -746,7 +188,7 @@ jobs:
   complete-chain:
     needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, fault-tolerance, table, performance]
     if: ${{ always() && needs.prepare.result != 'skipped' }}
-    runs-on: ubuntu-latest
+    runs-on: smoke-testing
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/rustfs-performance-test.yml

@@ -90,7 +90,7 @@ env:
 jobs:
   runner-check:
     if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
-    runs-on: ubuntu-latest
+    runs-on: sm-standard-2
     timeout-minutes: 5
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7

.github/workflows/rustfs-validation-package.yml

@@ -1,244 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-
-name: RustFS Validation Package
-
-on:
-  workflow_call:
-    inputs:
-      source_sha:
-        description: Exact RustFS commit SHA to build
-        required: true
-        type: string
-    secrets:
-      R2_ACCESS_KEY_ID:
-        required: true
-      R2_SECRET_ACCESS_KEY:
-        required: true
-      R2_ENDPOINT:
-        required: true
-      R2_BUCKET:
-        required: true
-  workflow_dispatch:
-    inputs:
-      source_sha:
-        description: Exact RustFS commit SHA to build
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-concurrency:
-  group: rustfs-validation-package-${{ inputs.source_sha }}
-  cancel-in-progress: false
-
-jobs:
-  resolve-source:
-    if: github.repository == 'rustfs/rustfs'
-    name: Resolve exact validation source
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    outputs:
-      source_sha: ${{ steps.source.outputs.sha }}
-      source_ref: ${{ steps.source.outputs.ref }}
-    steps:
-      - name: Validate requested source
-        id: source
-        env:
-          REQUESTED_SHA: ${{ inputs.source_sha }}
-        run: |
-          set -euo pipefail
-          [[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")"
-          trap 'rm -rf "${source_directory}"' EXIT
-          git init -q "${source_directory}"
-          git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}"
-          test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"
-          echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
-          echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
-
-  build:
-    needs: resolve-source
-    name: Build immutable validation DEB
-    runs-on: ubuntu-latest
-    timeout-minutes: 150
-    env:
-      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
-      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
-    steps:
-      - name: Checkout exact source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-          ref: ${{ needs.resolve-source.outputs.source_sha }}
-
-      - name: Confirm checked out source
-        env:
-          EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }}
-        run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
-
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
-        with:
-          cache-shared-key: validation-package-hosted-x86_64-unknown-linux-gnu
-          cache-save-if: "false"
-          install-build-packaging-tools: "false"
-          install-test-tools: "false"
-
-      - name: Build RustFS
-        run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
-
-      - name: Build validation DEB
-        shell: bash
-        run: |
-          set -euo pipefail
-          command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot
-          package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}"
-          package_file="${RUNNER_TEMP}/validation-package/rustfs.deb"
-          mkdir -p "${RUNNER_TEMP}/validation-package"
-          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
-          test -x "${binary}"
-          mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \
-            "${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs"
-          cp "${binary}" "${package_dir}/usr/bin/rustfs"
-          chmod 755 "${package_dir}/usr/bin/rustfs"
-          cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/"
-          cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF'
-          # RustFS Environment Configuration
-          # RUSTFS_ACCESS_KEY=<changeme>
-          # RUSTFS_SECRET_KEY=<changeme>
-          # RUSTFS_VOLUMES=""
-          # RUSTFS_ADDRESS=":9000"
-          ENVEOF
-          cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF'
-          Package: rustfs
-          Version: 0.0.0~validation
-          Section: utils
-          Priority: optional
-          Architecture: amd64
-          Depends: libc6 (>= 2.31)
-          Maintainer: RustFS Team <support@rustfs.com>
-          Description: RustFS immutable validation package
-           Candidate package for RustFS validation only.
-          CONTROLEOF
-          printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles"
-          ./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst"
-          ./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm"
-          ./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm"
-          chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm"
-          cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/"
-          fakeroot dpkg-deb --build "${package_dir}" "${package_file}"
-          test -s "${package_file}"
-
-      - name: Upload opaque validation package
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-package/rustfs.deb
-          if-no-files-found: error
-
-  publish:
-    needs: [resolve-source, build]
-    name: Publish immutable validation DEB
-    # Candidate-controlled build scripts never run on this fresh hosted runner.
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    env:
-      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
-      VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}
-    steps:
-      - name: Download opaque validation package
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ runner.temp }}/validation-package-input
-
-      - name: Validate and stage opaque package bytes
-        shell: bash
-        run: |
-          set -euo pipefail
-          python3 -I - <<'PYTHON'
-          import os
-          from pathlib import Path
-          import shutil
-          import stat
-
-          root = Path(os.environ["RUNNER_TEMP"])
-          incoming = root / "validation-package-input"
-          if incoming.is_symlink() or not incoming.is_dir() or sorted(p.name for p in incoming.iterdir()) != ["rustfs.deb"]:
-              raise SystemExit("Validation artifact must contain exactly one rustfs.deb file")
-          fd = os.open(incoming / "rustfs.deb", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
-          with os.fdopen(fd, "rb") as source:
-              info = os.fstat(source.fileno())
-              if not stat.S_ISREG(info.st_mode) or info.st_size == 0:
-                  raise SystemExit("Validation package must be a nonempty regular file")
-              staged = root / "validation-publication"
-              staged.mkdir(mode=0o700)
-              with (staged / "rustfs.deb").open("xb") as target:
-                  shutil.copyfileobj(source, target)
-          PYTHON
-
-      - name: Prepare trusted publication tools
-        id: tools
-        env:
-          AWS_EC2_METADATA_DISABLED: true
-        shell: bash
-        run: |
-          set -euo pipefail
-          AWS_CLI=aws
-          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y -qq python3-venv
-            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")"
-            python3 -m venv "${AWS_CLI_DIR}"
-            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
-            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
-          fi
-          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
-          echo "aws_cli=${AWS_CLI}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload validation package to immutable R2 path
-        id: publish
-        env:
-          PACKAGE_FILE: ${{ runner.temp }}/validation-publication/rustfs.deb
-          AWS_CLI: ${{ steps.tools.outputs.aws_cli }}
-          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
-          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
-          R2_BUCKET: ${{ secrets.R2_BUCKET }}
-          AWS_EC2_METADATA_DISABLED: true
-        shell: bash
-        run: |
-          set -euo pipefail
-          if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then
-            echo "Validation package publication requires complete R2 configuration" >&2
-            exit 1
-          fi
-          [[ "${VALIDATION_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          test "${VALIDATION_SOURCE_REF}" = "${VALIDATION_SOURCE_SHA}"
-          test -s "${PACKAGE_FILE}" && test ! -L "${PACKAGE_FILE}"
-          export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto
-          package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)"
-          key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb"
-          url="https://dl.rustfs.com/${key}"
-          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \
-            --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
-          downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)"
-          test "${downloaded_sha256}" = "${package_sha256}"
-          manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \
-            --arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \
-            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
-            --arg package_url "${url}" --arg package_sha256 "${package_sha256}" \
-            '{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}"
-          echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload validation candidate manifest
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ steps.publish.outputs.candidate_file }}
-          if-no-files-found: error

.github/workflows/rustfs-validation-tools.yml

@@ -1,343 +0,0 @@
-# Build source-bound helpers separately so live validation needs only artifacts.
-name: RustFS Validation Tools
-
-on:
-  workflow_call:
-    inputs:
-      candidate_sha:
-        type: string
-        required: true
-      auto_testing_sha:
-        type: string
-        required: true
-      build_native_heal:
-        type: boolean
-        required: false
-        default: false
-      build_shared_heal:
-        type: boolean
-        required: false
-        default: false
-      build_manager_native:
-        type: boolean
-        required: false
-        default: false
-    secrets:
-      PF_TESTING_GH_TOKEN:
-        required: true
-  workflow_dispatch:
-    inputs:
-      candidate_sha:
-        description: Exact RustFS source SHA from the validation candidate manifest
-        type: string
-        required: true
-      auto_testing_sha:
-        description: Exact auto-testing SHA containing the Heal metadata inspector
-        type: string
-        required: true
-      build_native_heal:
-        description: Also compile and run separate exact-source native Heal regressions
-        type: boolean
-        required: false
-        default: false
-      build_shared_heal:
-        description: Build the controlled C10 shared-reference fixture encoder
-        type: boolean
-        required: false
-        default: false
-      build_manager_native:
-        description: Build and run exact candidate checkpoint/MRF/incarnation regressions
-        type: boolean
-        required: false
-        default: false
-
-permissions:
-  contents: read
-
-jobs:
-  manager-native-regressions:
-    if: ${{ inputs.build_manager_native }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 180
-    env:
-      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
-      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-    steps:
-      - name: Validate immutable manager native inputs
-        run: |
-          set -euo pipefail
-          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
-
-      - name: Checkout exact manager candidate source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          ref: ${{ inputs.candidate_sha }}
-          path: candidate
-          persist-credentials: false
-
-      - name: Checkout exact manager collector source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Install manager native build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
-          df -h .
-
-      - name: Build exact unmodified candidate manager test executable
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          python3 -B auto-testing/scripts/validation_heal/manager_native.py build \
-            --source candidate --commit "${CANDIDATE_SHA}" --out manager-tools
-
-      - name: Run exact manager regressions independently
-        run: |
-          set -euo pipefail
-          python3 -B auto-testing/scripts/validation_heal/manager_native.py run \
-            --directory manager-tools --out manager-evidence
-
-      - name: Upload manager executable identities and original diagnostics
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            manager-tools/heal-manager-native-tests
-            manager-tools/*.json
-            manager-tools/*.jsonl
-            manager-tools/*.stderr
-            manager-tools/*.log
-            manager-evidence/
-          if-no-files-found: warn
-
-  native-heal-regressions:
-    if: ${{ inputs.build_native_heal }}
-    runs-on: ubuntu-22.04
-    timeout-minutes: 180
-    env:
-      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
-      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-    steps:
-      - name: Validate immutable native source inputs
-        run: |
-          set -euo pipefail
-          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
-
-      - name: Checkout exact native candidate source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          ref: ${{ inputs.candidate_sha }}
-          path: candidate
-          persist-credentials: false
-
-      - name: Checkout exact native collector source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Install native build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
-          df -h .
-
-      - name: Build and enumerate the unmodified candidate test executable
-        run: |
-          set -euo pipefail
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          use_race_native=false
-          if [[ -f auto-testing/scripts/validation_heal/race_native.py ]]; then
-            support_rc=0
-            python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
-              --commit "${CANDIDATE_SHA}" || support_rc=$?
-            case "${support_rc}" in
-              0) use_race_native=true ;;
-              2) ;; # The original exact-source collector handles legacy candidates.
-              *) exit "${support_rc}" ;;
-            esac
-          fi
-          if [[ "${use_race_native}" == true ]]; then
-            python3 -B auto-testing/scripts/validation_heal/race_native.py build \
-              --source candidate --commit "${CANDIDATE_SHA}" --testing-commit "${AUTO_TESTING_SHA}" --out native-tools
-          else
-            python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \
-              --source candidate --commit "${CANDIDATE_SHA}" --out native-tools
-          fi
-
-      - name: Run exact native regressions separately from fleet validation
-        run: |
-          set -euo pipefail
-          if [[ -f native-tools/race-native.json ]]; then
-            test ! -e native-tools/native-regressions.json
-            python3 -B auto-testing/scripts/validation_heal/race_native.py run \
-              --directory native-tools --out native-evidence
-          else
-            python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \
-              --directory native-tools --out native-evidence
-          fi
-
-      - name: Run existing F07 directory fsync failure regressions
-        if: ${{ inputs.candidate_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
-        run: |
-          set -euo pipefail
-          # Older immutable harness pins retain their original native suite;
-          # absence of this additive collector must never fabricate its proof.
-          if [[ ! -f auto-testing/scripts/validation_heal/persistence_native.py ]]; then
-            mkdir native-persistence-evidence
-            echo '{"complete":false,"reason":"pinned harness predates F07 directory-fsync collector","requiredNativeCoverageComplete":false,"canonicalCasePassClaimed":false}' \
-              > native-persistence-evidence/not-available.json
-            exit 0
-          fi
-          python3 -B auto-testing/scripts/validation_heal/persistence_native.py run \
-            --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
-            --testing-commit "${AUTO_TESTING_SHA}"
-          python3 -B auto-testing/scripts/validation_heal/persistence_native.py verify \
-            --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
-            --testing-commit "${AUTO_TESTING_SHA}"
-
-      - name: Upload native executable identities and original diagnostics
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            native-tools/ecstore-native-tests
-            native-tools/ecstore-race-native-tests
-            native-tools/*.json
-            native-tools/*.jsonl
-            native-tools/*.stderr
-            native-tools/*.log
-            native-evidence/
-            native-persistence-evidence/
-          if-no-files-found: warn
-
-  heal-metadata-inspector:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    env:
-      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
-      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
-    steps:
-      - name: Validate immutable source inputs
-        run: |
-          set -euo pipefail
-          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
-          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
-
-      - name: Checkout exact candidate source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          ref: ${{ inputs.candidate_sha }}
-          path: candidate
-          persist-credentials: false
-
-      - name: Checkout exact inspector source
-        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          repository: rustfs/auto-testing
-          ref: ${{ inputs.auto_testing_sha }}
-          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          path: auto-testing
-          persist-credentials: false
-
-      - name: Build the read-only candidate metadata decoder
-        run: |
-          set -euo pipefail
-          test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}"
-          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
-          df -h .
-          python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \
-            --source candidate --commit "${CANDIDATE_SHA}" --out inspector-build
-          # Seed the helper's separate graph from the candidate's frozen graph.
-          # Cargo adds the helper package; both lock identities are retained.
-          cp candidate/Cargo.lock inspector-build/Cargo.lock
-          cargo build --release --manifest-path inspector-build/Cargo.toml
-          python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \
-            --binary inspector-build/target/release/heal-meta-inspect --source candidate
-          mkdir validation-tools
-          cp inspector-build/target/release/heal-meta-inspect validation-tools/
-          cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock
-          python3 - <<'PY'
-          import hashlib
-          import json
-          import os
-          from pathlib import Path
-          import subprocess
-
-          source = json.loads(Path("inspector-build/source.json").read_text())
-          binary = Path("validation-tools/heal-meta-inspect")
-          source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"],
-                        binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
-                        inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(),
-                        toolchain=subprocess.check_output(["rustc", "-vV"], text=True),
-                        target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"],
-                        runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
-          Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n")
-          PY
-          git -C candidate diff --exit-code
-
-      - name: Build and test the controlled C10 metadata encoder
-        if: ${{ inputs.build_shared_heal }}
-        run: |
-          set -euo pipefail
-          python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \
-            --source candidate --commit "${CANDIDATE_SHA}" --out shared-build
-          cp candidate/Cargo.lock shared-build/Cargo.lock
-          export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target"
-          cargo build --release --manifest-path shared-build/Cargo.toml
-          mkdir shared-tools
-          if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \
-            > shared-tools/shared-fixture-tests.log 2>&1; then
-            cat shared-tools/shared-fixture-tests.log
-            exit 1
-          fi
-          cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/
-          cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock
-          python3 - <<'PY'
-          import hashlib
-          import json
-          import os
-          from pathlib import Path
-          import subprocess
-
-          source = json.loads(Path("shared-build/source.json").read_text())
-          source.update(schemaVersion=1, built=True, sourceUnmodified=True,
-              candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(),
-              testingCommit=os.environ["AUTO_TESTING_SHA"],
-              binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(),
-              sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(),
-              toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu",
-              runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
-          Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n")
-          PY
-          git -C candidate diff --exit-code
-
-      - name: Upload controlled C10 fixture encoder evidence
-        if: ${{ always() && inputs.build_shared_heal }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
-          path: shared-tools/
-          if-no-files-found: warn
-
-      - name: Upload immutable validation tools
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
-          path: validation-tools/
-          if-no-files-found: error

crates/concurrency/Cargo.toml

@@ -26,7 +26,7 @@ rustfs-io-core = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 
 # Async runtime
-tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "time"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
 
 # Logging
 tracing = { workspace = true }

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -1086,9 +1086,8 @@ impl ExpiryState {
         }))
     }
 
-    #[cfg(any(test, feature = "test-util"))]
-    #[doc(hidden)]
-    pub fn new_with_unconsumed_worker_channel(capacity: usize) -> Arc<RwLock<Self>> {
+    #[cfg(test)]
+    fn new_with_unconsumed_worker_channel(capacity: usize) -> Arc<RwLock<Self>> {
         let (tx, rx) = mpsc::channel(capacity);
         Arc::new(RwLock::new(Self {
             tasks_tx: vec![tx],
@@ -3517,21 +3516,13 @@ async fn stale_upload_current_size_with_opts(
         .await
         .ok()?;
 
-    stale_upload_size_from_parts(&parts)
-}
-
-fn stale_upload_size_from_parts(parts: &crate::storage_api_contracts::multipart::ListPartsInfo) -> Option<usize> {
-    if parts.is_truncated {
-        return None;
-    }
-    parts.parts.iter().try_fold(0usize, |total, part| {
-        // ListParts represents a quorum-confirmed missing part with an empty ETag.
-        if part.etag.as_deref().is_none_or(str::is_empty) {
-            return None;
-        }
-        let actual_size = usize::try_from(part.actual_size.max(0)).ok()?;
-        total.checked_add(part.size.max(actual_size))
-    })
+    Some(
+        parts
+            .parts
+            .iter()
+            .map(|part| part.actual_size.max(part.size as i64).max(0) as usize)
+            .sum(),
+    )
 }
 
 async fn stale_upload_lifecycle_due(
@@ -3548,43 +3539,18 @@ async fn stale_upload_lifecycle_due(
     let bucket = metadata.get(RUSTFS_MULTIPART_BUCKET_KEY)?;
     let object = metadata.get(RUSTFS_MULTIPART_OBJECT_KEY)?;
 
-    let mut lifecycle = match metadata_boundary::get_lifecycle_config(bucket).await {
+    let lifecycle = match metadata_boundary::get_lifecycle_config(bucket).await {
         Ok((lifecycle, _)) => lifecycle,
         Err(_) => return None,
     };
 
-    let has_size_filter = |filter: &s3s::dto::LifecycleRuleFilter| {
-        filter.object_size_greater_than.is_some()
-            || filter.object_size_less_than.is_some()
-            || filter
-                .and
-                .as_ref()
-                .is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
-    };
-    let needs_size = lifecycle
-        .rules
-        .iter()
-        .any(|rule| rule.abort_incomplete_multipart_upload.is_some() && rule.filter.as_ref().is_some_and(has_size_filter));
-    let size = if needs_size {
-        match stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock).await {
-            Some(size) => size,
-            None => {
-                // An unknown size cannot authorize a size-filtered abort.
-                // Unrelated rules can still be evaluated independently.
-                lifecycle
-                    .rules
-                    .retain(|rule| !rule.filter.as_ref().is_some_and(has_size_filter));
-                0
-            }
-        }
-    } else {
-        0
-    };
     let object_opts = ObjectOpts {
         name: object.clone(),
         user_tags: metadata.get(AMZ_OBJECT_TAGGING).cloned().unwrap_or_default(),
         mod_time: Some(initiated),
-        size,
+        size: stale_upload_current_size_with_opts(set, metadata, upload_dir, no_lock)
+            .await
+            .unwrap_or_default(),
         is_latest: true,
         delete_marker: false,
         user_defined: metadata.clone(),
@@ -3837,8 +3803,7 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
     for candidate in candidates.into_values() {
         let upload_dir = candidate.path.rsplit('/').next().unwrap_or_default().to_string();
         let mut due = stale_upload_default_due(candidate.initiated, default_expiry);
-        if now < due
-            && let Some(metadata) = candidate.metadata.as_ref()
+        if let Some(metadata) = candidate.metadata.as_ref()
             && let Some(lifecycle_due) = stale_upload_lifecycle_due(set, metadata, candidate.initiated, &upload_dir, false).await
             && lifecycle_due < due
         {
@@ -3867,9 +3832,8 @@ async fn cleanup_stale_multipart_uploads_in_set(set: &Arc<SetDisks>, now: Offset
         let current_metadata = cleanup_guard.file_info().metadata.clone();
         let current_initiated = initiated_from_upload_dir(&upload_dir, cleanup_guard.file_info().mod_time);
         let mut current_due = stale_upload_default_due(current_initiated, default_expiry);
-        if now < current_due
-            && let Some(lifecycle_due) =
-                stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
+        if let Some(lifecycle_due) =
+            stale_upload_lifecycle_due(set, &current_metadata, current_initiated, &upload_dir, true).await
             && lifecycle_due < current_due
         {
             current_due = lifecycle_due;
@@ -15904,261 +15868,6 @@ mod tests {
         assert!(ecstore.is_ok(), "fresh local ECStore boot should succeed, got {ecstore:?}");
     }
 
-    #[tokio::test(flavor = "current_thread")]
-    #[serial]
-    async fn stale_multipart_cleanup_preserves_partial_metadata_without_upload_part_warning() {
-        use tracing::instrument::WithSubscriber as _;
-        #[derive(Clone, Default)]
-        struct CapturedLogs(Arc<StdMutex<Vec<u8>>>);
-        struct LogWriter(Arc<StdMutex<Vec<u8>>>);
-        impl std::io::Write for LogWriter {
-            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
-                self.0.lock().unwrap().extend_from_slice(bytes);
-                Ok(bytes.len())
-            }
-            fn flush(&mut self) -> std::io::Result<()> {
-                Ok(())
-            }
-        }
-        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
-            type Writer = LogWriter;
-            fn make_writer(&'a self) -> Self::Writer {
-                LogWriter(self.0.clone())
-            }
-        }
-
-        let (paths, ecstore) = setup_test_env().await;
-        let bucket = format!("stale-partial-{}", Uuid::new_v4().simple());
-        create_test_bucket(&ecstore, &bucket).await;
-        for copies in [1, 2] {
-            let object = format!("partial-{copies}.bin");
-            let initiated = OffsetDateTime::now_utc() - time::Duration::hours(30);
-            let upload = ecstore
-                .new_multipart_upload(
-                    &bucket,
-                    &object,
-                    &ObjectOptions {
-                        mod_time: Some(initiated),
-                        max_parity: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("multipart upload should be created");
-            let decoded = base64_simd::URL_SAFE_NO_PAD
-                .decode_to_vec(upload.upload_id.as_bytes())
-                .unwrap();
-            let decoded = String::from_utf8(decoded).unwrap();
-            let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload id");
-            let upload_path = format!("{}/{upload_dir}", multipart_sha_dir(&bucket, &object));
-            let physical_paths: Vec<_> = paths
-                .iter()
-                .map(|path| path.join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path))
-                .collect();
-            let original: Vec<_> = physical_paths
-                .iter()
-                .take(copies)
-                .map(|path| std::fs::read(path.join(STORAGE_FORMAT_FILE)).unwrap())
-                .collect();
-            let info = super::get_file_info(
-                &original[0],
-                RUSTFS_META_MULTIPART_BUCKET,
-                &upload_path,
-                "",
-                super::FileInfoOpts {
-                    data: false,
-                    include_free_versions: false,
-                    include_part_checksums: false,
-                },
-            )
-            .unwrap();
-            assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (2, 2));
-            for path in physical_paths.iter().skip(copies) {
-                fs::remove_dir_all(path).await.expect("remove selected metadata copies");
-            }
-            if copies == 2 {
-                ecstore
-                    .get_multipart_info(&bucket, &object, &upload.upload_id, &ObjectOptions::default())
-                    .await
-                    .expect("two metadata copies must meet read quorum");
-            }
-
-            let logs = CapturedLogs::default();
-            let subscriber = tracing_subscriber::fmt()
-                .with_max_level(tracing::Level::DEBUG)
-                .with_writer(logs.clone())
-                .with_ansi(false)
-                .without_time()
-                .finish();
-            cleanup_stale_multipart_uploads_once_at(
-                ecstore.clone(),
-                OffsetDateTime::now_utc(),
-                StdDuration::from_secs(24 * 60 * 60),
-            )
-            .with_subscriber(subscriber)
-            .await;
-
-            for (path, expected) in physical_paths.iter().zip(&original) {
-                assert_eq!(fs::read(path.join(STORAGE_FORMAT_FILE)).await.unwrap(), *expected);
-            }
-            let log_output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
-            assert!(
-                log_output.contains(&upload_path),
-                "missing background diagnostic for {upload_path}: {log_output}"
-            );
-            assert!(log_output.contains("multipart_cleanup_lock_or_recheck_failed"), "{log_output}");
-            assert!(!log_output.contains("op=\"upload_part\""), "unexpected upload-part warning: {log_output}");
-            for path in physical_paths.iter().take(copies) {
-                fs::remove_dir_all(path).await.unwrap();
-            }
-        }
-    }
-
-    #[tokio::test(flavor = "current_thread")]
-    #[serial]
-    async fn stale_multipart_cleanup_lists_parts_only_when_size_is_needed() {
-        use std::sync::atomic::{AtomicUsize, Ordering};
-        use tracing::instrument::WithSubscriber as _;
-        use tracing_subscriber::prelude::*;
-
-        struct PartListCounter(Arc<AtomicUsize>);
-        impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for PartListCounter {
-            fn on_new_span(
-                &self,
-                attrs: &tracing::span::Attributes<'_>,
-                _id: &tracing::span::Id,
-                _ctx: tracing_subscriber::layer::Context<'_, S>,
-            ) {
-                if attrs.metadata().name() == "list_object_parts" {
-                    self.0.fetch_add(1, Ordering::Relaxed);
-                }
-            }
-        }
-
-        let (_, ecstore) = setup_test_env().await;
-        let bucket = format!("stale-list-parts-{}", Uuid::new_v4().simple());
-        create_test_bucket(&ecstore, &bucket).await;
-        set_abort_incomplete_lifecycle(&bucket, "logs/", 0).await;
-        let now = OffsetDateTime::now_utc();
-        for (object, age, size_filter, should_list) in [
-            ("logs/default-expired", time::Duration::hours(30), false, false),
-            ("logs/lifecycle-only", time::Duration::minutes(5), false, false),
-            ("logs/size-filter", time::Duration::minutes(5), true, true),
-        ] {
-            if size_filter {
-                set_abort_incomplete_lifecycle_with_size(&bucket, "logs/", 0, 0).await;
-            }
-            let upload = ecstore
-                .new_multipart_upload(
-                    &bucket,
-                    object,
-                    &ObjectOptions {
-                        mod_time: Some(now - age),
-                        ..Default::default()
-                    },
-                )
-                .await
-                .expect("create multipart upload");
-            if size_filter {
-                let mut data = PutObjReader::from_vec(vec![1, 2, 3, 4]);
-                ecstore
-                    .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
-                    .await
-                    .expect("upload a part for size filtering");
-            }
-            let calls = Arc::new(AtomicUsize::new(0));
-            let subscriber = tracing_subscriber::registry().with(PartListCounter(Arc::clone(&calls)));
-            cleanup_stale_multipart_uploads_once_at(ecstore.clone(), now, StdDuration::from_secs(24 * 60 * 60))
-                .with_subscriber(subscriber)
-                .await;
-            let listed = calls.load(Ordering::Relaxed);
-            assert_eq!(listed > 0, should_list, "unexpected ListParts calls for {object}: {listed}");
-            let result = ecstore
-                .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
-                .await;
-            assert!(
-                matches!(result, Err(ref err) if is_err_invalid_upload_id(err)),
-                "upload must be removed: {result:?}"
-            );
-        }
-    }
-
-    #[test]
-    fn stale_multipart_cleanup_rejects_incomplete_or_overflowing_part_sizes() {
-        use crate::storage_api_contracts::multipart::{ListPartsInfo, PartInfo};
-
-        let part = |size, actual_size, etag: &str| PartInfo {
-            size,
-            actual_size,
-            etag: Some(etag.to_owned()),
-            ..Default::default()
-        };
-        let mut parts = ListPartsInfo {
-            parts: vec![part(4, 6, "valid")],
-            ..Default::default()
-        };
-        assert_eq!(super::stale_upload_size_from_parts(&parts), Some(6));
-        parts.is_truncated = true;
-        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
-        parts.is_truncated = false;
-        parts.parts = vec![part(0, 0, "")];
-        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
-        parts.parts = vec![part(usize::MAX, 0, "valid"), part(1, 1, "valid")];
-        assert_eq!(super::stale_upload_size_from_parts(&parts), None);
-    }
-
-    #[tokio::test]
-    #[serial]
-    async fn stale_multipart_cleanup_handles_unknown_size_without_false_abort() {
-        let (_, ecstore) = setup_test_env().await;
-        let bucket = format!("stale-unknown-size-{}", Uuid::new_v4().simple());
-        let object = "logs/object";
-        create_test_bucket(&ecstore, &bucket).await;
-        let size_rule = r#"<Rule><ID>size</ID><Status>Enabled</Status><Filter><ObjectSizeLessThan>5</ObjectSizeLessThan></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule>"#;
-        let independent_rule = r#"<Rule><ID>prefix</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><AbortIncompleteMultipartUpload><DaysAfterInitiation>0</DaysAfterInitiation></AbortIncompleteMultipartUpload></Rule>"#;
-        let lifecycle = |rules: &str| format!("<LifecycleConfiguration>{rules}</LifecycleConfiguration>").into_bytes();
-        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle(size_rule))
-            .await
-            .expect("install upper-size lifecycle rule");
-        let metadata = HashMap::from([
-            (super::RUSTFS_MULTIPART_BUCKET_KEY.to_owned(), bucket.clone()),
-            (super::RUSTFS_MULTIPART_OBJECT_KEY.to_owned(), object.to_owned()),
-        ]);
-        let initiated = OffsetDateTime::now_utc() - time::Duration::hours(1);
-        let set = &ecstore.pools[0].disk_set[0];
-        let upload = ecstore
-            .new_multipart_upload(&bucket, object, &ObjectOptions::default())
-            .await
-            .expect("create empty multipart upload");
-        let decoded = base64_simd::URL_SAFE_NO_PAD
-            .decode_to_vec(upload.upload_id.as_bytes())
-            .expect("decode upload ID");
-        let decoded = String::from_utf8(decoded).expect("upload ID must be UTF-8");
-        let (_, upload_dir) = decoded.split_once('.').expect("deployment-prefixed upload ID");
-        assert!(
-            super::stale_upload_lifecycle_due(set, &metadata, initiated, upload_dir, false)
-                .await
-                .is_some(),
-            "a known empty upload must match the upper-size rule"
-        );
-        let missing_upload = format!("{}x{}", Uuid::new_v4(), initiated.unix_timestamp_nanos());
-        assert!(
-            super::stale_upload_lifecycle_due(set, &metadata, initiated, &missing_upload, false)
-                .await
-                .is_none(),
-            "an unreadable size must not match the upper-size rule"
-        );
-        metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle(&format!("{size_rule}{independent_rule}")))
-            .await
-            .expect("install size-independent lifecycle rule");
-        assert!(
-            super::stale_upload_lifecycle_due(set, &metadata, initiated, &missing_upload, false)
-                .await
-                .is_some(),
-            "a size-independent rule remains eligible when size is unknown"
-        );
-    }
-
     #[tokio::test]
     #[serial]
     async fn stale_multipart_cleanup_uses_default_expiry_without_lifecycle() {

crates/ecstore/src/bucket/metadata_sys.rs

@@ -3186,17 +3186,9 @@ pub mod test_support {
     use crate::store::init_local_disks_with_instance_ctx;
 
     pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
-        isolated_store_over_temp_disks_with_count(4).await
-    }
-
-    pub async fn isolated_store_over_temp_disks_with_count(disk_count: usize) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
-        assert!(
-            matches!(disk_count, 4 | 16),
-            "isolated store fixture requires a registered logical set geometry"
-        );
-        let mut dirs = Vec::with_capacity(disk_count);
-        let mut endpoints = Vec::with_capacity(disk_count);
-        for disk_idx in 0..disk_count {
+        let mut dirs = Vec::with_capacity(4);
+        let mut endpoints = Vec::with_capacity(4);
+        for disk_idx in 0..4 {
             let dir = tempfile::tempdir().expect("tempdir should be created");
             let mut endpoint =
                 Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
@@ -3209,7 +3201,7 @@ pub mod test_support {
         let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
             legacy: false,
             set_count: 1,
-            drives_per_set: disk_count,
+            drives_per_set: 4,
             endpoints: Endpoints::from(endpoints),
             cmd_line: "metadata-sys-cache-test".to_string(),
             platform: "test".to_string(),

crates/ecstore/src/disk/local.rs

@@ -8450,23 +8450,6 @@ impl LocalDisk {
                     return Err(std::io::Error::other("metadata read worker cancelled"));
                 }
                 let _io_permit = io_permit;
-                #[cfg(all(any(test, feature = "test-util"), not(windows)))]
-                {
-                    let opened = root.open_readonly(&path);
-                    if opened.is_ok() {
-                        os::prepared_publication_test_hooks::run(
-                            os::prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
-                            &path,
-                        );
-                    }
-                    let result = read_fn(opened);
-                    os::prepared_publication_test_hooks::run(
-                        os::prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
-                        &path,
-                    );
-                    Ok(result)
-                }
-                #[cfg(not(all(any(test, feature = "test-util"), not(windows))))]
                 Ok(read_fn(root.open_readonly(&path)))
             })
             .await

crates/ecstore/src/disk/os.rs

@@ -333,8 +333,6 @@ pub(crate) mod prepared_publication_test_hooks {
         PreparedRename,
         Rename,
         Remove,
-        MetadataReadAfterOpen,
-        MetadataReadAfterRead,
         #[cfg(test)]
         Rollback,
         #[cfg(test)]
@@ -459,8 +457,6 @@ pub enum LocalPublicationStage {
     PreparedRename,
     Rename,
     Remove,
-    MetadataReadAfterOpen,
-    MetadataReadAfterRead,
 }
 
 #[cfg(all(feature = "test-util", not(windows)))]
@@ -473,8 +469,6 @@ impl LocalPublicationPause {
             LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
             LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
             LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
-            LocalPublicationStage::MetadataReadAfterOpen => prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
-            LocalPublicationStage::MetadataReadAfterRead => prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
         };
         let (entered_tx, entered) = oneshot::channel();
         let (release, release_rx) = std::sync::mpsc::channel::<()>();

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -603,10 +603,6 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
     request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string);
     // A degraded read's surviving minority is not a recreate instruction; the
     // locked storage read decides whether the target is absent (backlog#2900).
-    // Reader failures include checksum damage that metadata-only healing cannot detect.
-    if matches!(reason, "missing_shards" | "decode_error") {
-        request.scan_mode = Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
-    }
 
     let request_id = request.id.clone();
     let bucket = bucket.to_string();
@@ -13831,61 +13827,6 @@ mod tests {
         release_read_repair_heal_reservation(&key).await;
     }
 
-    #[tokio::test]
-    async fn read_repair_reader_failure_requests_deep_scan_for_the_exact_version() {
-        fn captured() -> &'static std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>> {
-            static REQUEST: OnceLock<std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>>> =
-                OnceLock::new();
-            REQUEST.get_or_init(|| std::sync::Mutex::new(None))
-        }
-        fn capture(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
-            *captured().lock().expect("read-repair request capture must be available") = Some(request);
-            Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
-        }
-
-        for (reason, expected_scan) in [
-            ("missing_shards", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
-            ("decode_error", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
-            ("metadata_read_error", None),
-        ] {
-            let object = format!("reader-failure-{}", Uuid::new_v4());
-            let version = Uuid::new_v4().to_string();
-            submit_read_repair_heal_with_submitter(
-                ReadRepairHealSubmission {
-                    bucket: "reader-failure",
-                    object: &object,
-                    version_id: Some(&version),
-                    pool_index: 1,
-                    set_index: 2,
-                    part_number: Some(1),
-                    reason,
-                    mrf_intent: None,
-                },
-                capture,
-            )
-            .await;
-            let request = tokio::time::timeout(Duration::from_secs(5), async {
-                loop {
-                    if let Some(request) = captured().lock().expect("captured request must be readable").take() {
-                        break request;
-                    }
-                    tokio::task::yield_now().await;
-                }
-            })
-            .await
-            .expect("the actual asynchronous admission must be captured");
-            assert_eq!(request.scan_mode, expected_scan, "reader damage needs payload verification");
-            assert_eq!(request.source, HealRequestSource::ReadRepair);
-            assert_eq!(request.bucket, "reader-failure");
-            assert_eq!(request.object_prefix.as_deref(), Some(object.as_str()));
-            assert_eq!(request.object_version_id.as_deref(), Some(version.as_str()));
-            assert_eq!((request.pool_index, request.set_index), (Some(1), Some(2)));
-            assert_eq!(request.recreate_missing, None, "storage evidence decides absence");
-            let key = ReadRepairHealCacheKey::new("reader-failure", &object, Some(&version), 1, 2);
-            release_read_repair_heal_reservation(&key).await;
-        }
-    }
-
     // ========================================================================
     // backlog#898 — groups A/C: pure old-data-dir cleanup classification.
     // ========================================================================

crates/ecstore/src/set_disk/mod.rs

@@ -889,8 +889,6 @@ pub(crate) use ops::heal::AbsenceProofRequest;
 #[cfg(test)]
 pub(crate) use ops::heal::DanglingDeleteFailure;
 pub(crate) use ops::heal::HealedObjectAbsence;
-#[cfg(test)]
-pub(crate) use ops::heal::ReadRepairPauseScope;
 
 #[cfg(test)]
 pub(crate) use ops::hermetic_set_disks_isolated;
@@ -4247,23 +4245,14 @@ pub(super) struct MultipartWriteQuorumContext<'a> {
     part_number: Option<usize>,
 }
 
-fn record_multipart_write_quorum_failure(
-    stage: &'static str,
-    errs: &[Option<DiskError>],
-    write_quorum: usize,
-) -> crate::disk::error_reduce::WriteQuorumFailureSummary {
-    let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
-    runtime_sources::record_erasure_write_quorum_failure(stage, summary.dominant_error_label);
-    summary
-}
-
 fn log_multipart_write_quorum_failure(
     context: MultipartWriteQuorumContext<'_>,
     errs: &[Option<DiskError>],
     write_quorum: usize,
     returned_error: &DiskError,
 ) {
-    let summary = record_multipart_write_quorum_failure(context.stage, errs, write_quorum);
+    let summary = build_write_quorum_failure_summary(errs, OBJECT_OP_IGNORED_ERRS, write_quorum);
+    runtime_sources::record_erasure_write_quorum_failure(context.stage, summary.dominant_error_label);
     warn!(
         target: "rustfs_ecstore::set_disk",
         event = EVENT_SET_DISK_MULTIPART,

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -44,9 +44,9 @@ use super::super::{
     create_bitrot_writer, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map,
     get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
     log_multipart_write_quorum_failure, parts_after_marker, path_join_buf, prepare_complete_multipart_parts,
-    record_compression_total_memory, record_multipart_write_quorum_failure, reduce_read_quorum_errs, reduce_write_quorum_errs,
-    remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size,
-    strip_internal_multipart_metadata, to_object_err, warn,
+    record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
+    restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, strip_internal_multipart_metadata,
+    to_object_err, warn,
 };
 use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
 #[cfg(test)]
@@ -233,19 +233,6 @@ fn admitted_multipart_size(current: u64, candidate: u64, limit: u64) -> Result<u
     Ok(total)
 }
 
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum MultipartMetadataQuorum {
-    Read,
-    Write,
-    Cleanup,
-}
-
-impl From<bool> for MultipartMetadataQuorum {
-    fn from(write: bool) -> Self {
-        if write { Self::Write } else { Self::Read }
-    }
-}
-
 pub(crate) struct StaleMultipartCleanupGuard {
     file_info: FileInfo,
     upload_path: String,
@@ -1152,7 +1139,7 @@ impl SetDisks {
             }
             let upload_id = runtime_sources::deployment_upload_id(raw_upload_id);
             let file_info = match self
-                .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, MultipartMetadataQuorum::Read, true)
+                .check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, false, true)
                 .await
             {
                 Ok((file_info, _)) => file_info,
@@ -1239,7 +1226,7 @@ impl SetDisks {
             .acquire_write_lock_diag("abort_data_movement_multipart", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
             .await?;
         let file_info = match self
-            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, MultipartMetadataQuorum::Write, true)
+            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, true, true)
             .await
         {
             Ok((file_info, _)) => file_info,
@@ -1346,7 +1333,7 @@ impl SetDisks {
         data_movement: bool,
     ) -> Result<(FileInfo, Vec<FileInfo>)> {
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, data_movement)?;
-        self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write.into(), false)
+        self.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, write, false)
             .await
     }
 
@@ -1386,7 +1373,7 @@ impl SetDisks {
         object: &str,
         upload_id: &str,
         upload_id_path: &str,
-        quorum_mode: MultipartMetadataQuorum,
+        write: bool,
         include_recovery: bool,
     ) -> Result<(FileInfo, Vec<FileInfo>)> {
         // Slow metadata I/O must not hold the topology registry's read lock.
@@ -1502,26 +1489,22 @@ impl SetDisks {
         }
 
         let mut quorum = read_quorum as usize;
-        if quorum_mode != MultipartMetadataQuorum::Read {
+        if write {
             quorum = write_quorum as usize;
 
             if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
-                if quorum_mode == MultipartMetadataQuorum::Write {
-                    log_multipart_write_quorum_failure(
-                        MultipartWriteQuorumContext {
-                            stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
-                            bucket,
-                            object,
-                            upload_id,
-                            part_number: None,
-                        },
-                        &errs,
-                        quorum,
-                        &err,
-                    );
-                } else {
-                    record_multipart_write_quorum_failure(MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, &errs, quorum);
-                }
+                log_multipart_write_quorum_failure(
+                    MultipartWriteQuorumContext {
+                        stage: MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
+                        bucket,
+                        object,
+                        upload_id,
+                        part_number: None,
+                    },
+                    &errs,
+                    quorum,
+                    &err,
+                );
                 return Err(map_error(err));
             }
         } else if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, quorum) {
@@ -1540,7 +1523,7 @@ impl SetDisks {
             .acquire_write_lock_diag("stale_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, upload_path)
             .await?;
         let (file_info, _) = self
-            .check_multipart_upload_path_exists("", "", "", upload_path, MultipartMetadataQuorum::Cleanup, true)
+            .check_multipart_upload_path_exists("", "", "", upload_path, true, true)
             .await?;
         let write_quorum = file_info.write_quorum(self.default_write_quorum());
         Ok(StaleMultipartCleanupGuard {
@@ -3049,7 +3032,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         // its full write quorum and the upload's incarnation/lifecycle fences.
         let upload_id_path = Self::validated_multipart_upload_path(bucket, object, upload_id, opts.data_movement)?;
         let (fi, _) = self
-            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, MultipartMetadataQuorum::Read, true)
+            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
             .await?;
         ensure_data_movement_upload_access(&fi, bucket, object, upload_id, opts)?;
         ensure_multipart_bucket_incarnation(&self.ctx, &fi, bucket, object, upload_id, opts.expected_bucket_incarnation_id)
@@ -3209,14 +3192,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     }
                     SourceRecovery::Missing => {
                         return match self
-                            .check_multipart_upload_path_exists(
-                                bucket,
-                                object,
-                                upload_id,
-                                &upload_id_path,
-                                MultipartMetadataQuorum::Read,
-                                true,
-                            )
+                            .check_multipart_upload_path_exists(bucket, object, upload_id, &upload_id_path, false, true)
                             .await
                         {
                             Ok(_) => Err(Error::ErasureWriteQuorum),
@@ -11157,7 +11133,7 @@ mod tests {
             *disk = None;
         }
         let (info, _) = set
-            .check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+            .check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
             .await
             .expect("four primary plus four recovery copies reach metadata read quorum");
         assert_eq!(info.name, path);
@@ -11178,7 +11154,7 @@ mod tests {
             *disk = None;
         }
         assert!(
-            set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+            set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
                 .await
                 .is_err(),
             "two files on each of four disks must not count as eight voters"
@@ -14495,7 +14471,7 @@ mod tests {
                     object,
                     &u_new,
                     &SetDisks::get_upload_id_dir(bucket, object, &u_new),
-                    MultipartMetadataQuorum::Write,
+                    true,
                     true,
                 )
                 .await
@@ -14540,7 +14516,7 @@ mod tests {
                 object,
                 &upload,
                 &SetDisks::get_upload_id_dir(bucket, object, &upload),
-                MultipartMetadataQuorum::Write,
+                true,
                 true,
             )
             .await
@@ -14623,7 +14599,7 @@ mod tests {
                     assert_completion_files_preserved(&disk_stores, RUSTFS_META_MULTIPART_BUCKET, &source_files).await;
                     set_disks
                         .check_multipart_upload_path_exists(
-                            bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), MultipartMetadataQuorum::Write, true,
+                            bucket, object, &u_new, &SetDisks::get_upload_id_dir(bucket, object, &u_new), true, true,
                         )
                         .await
                         .expect("upload authority must survive the post-commit crash");

crates/ecstore/src/set_disk/ops/multipart_abort_tests.rs

@@ -507,7 +507,7 @@ mod native_abort_closure_tests {
         let path = SetDisks::get_upload_id_dir(bucket, object, &upload);
         pin_historical_unmarked_upload(&disks, bucket, &path).await;
         let before = originals(&disks, &path).await;
-        set.check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+        set.check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
             .await
             .expect("readonly legacy cleanup identity lookup");
         assert_eq!(
@@ -786,7 +786,7 @@ mod native_abort_closure_tests {
             "known orphan native evidence remains unknown: {inspected:?}"
         );
         let readonly = set
-            .check_multipart_upload_path_exists(bucket, object, &upload, &path, MultipartMetadataQuorum::Read, true)
+            .check_multipart_upload_path_exists(bucket, object, &upload, &path, false, true)
             .await;
         assert!(
             matches!(

crates/ecstore/src/set_disk/ops/object.rs

@@ -8177,23 +8177,13 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
     }
 }
 
-#[cfg(test)]
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum DeleteObjectCommitPause {
-    BeforeNamespace,
-    BeforeCommit,
-    AfterPublish,
-}
-
 #[cfg(test)]
 struct DeleteObjectCommitBarrierState {
     bucket: String,
     object: String,
-    pause: DeleteObjectCommitPause,
+    pause_after_publish: bool,
     arrived: tokio::sync::Notify,
     release: tokio::sync::Notify,
-    namespace_pending: tokio::sync::Notify,
-    namespace_acquired: std::sync::atomic::AtomicBool,
 }
 
 #[cfg(test)]
@@ -8208,27 +8198,21 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
 #[cfg(test)]
 impl DeleteObjectCommitBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeCommit)
-    }
-
-    pub(crate) fn install_before_namespace(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeNamespace)
+        Self::install_with_mode(bucket, object, false)
     }
 
     #[cfg(feature = "test-util")]
     pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::AfterPublish)
+        Self::install_with_mode(bucket, object, true)
     }
 
-    fn install_with_mode(bucket: &str, object: &str, pause: DeleteObjectCommitPause) -> Self {
+    fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self {
         let state = Arc::new(DeleteObjectCommitBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
-            pause,
+            pause_after_publish,
             arrived: tokio::sync::Notify::new(),
             release: tokio::sync::Notify::new(),
-            namespace_pending: tokio::sync::Notify::new(),
-            namespace_acquired: std::sync::atomic::AtomicBool::new(false),
         });
         let mut slot = DELETE_OBJECT_COMMIT_BARRIER
             .get_or_init(|| std::sync::Mutex::new(None))
@@ -8248,19 +8232,6 @@ impl DeleteObjectCommitBarrier {
     pub(crate) fn release(&self) {
         self.state.release.notify_one();
     }
-
-    pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
-        assert_eq!(self.state.pause, DeleteObjectCommitPause::BeforeNamespace);
-        let pending = self.state.namespace_pending.notified();
-        self.release();
-        tokio::time::timeout(Duration::from_secs(5), pending)
-            .await
-            .expect("delete object must poll its actual namespace acquisition to Pending");
-    }
-
-    pub(crate) fn namespace_acquired(&self) -> bool {
-        self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
-    }
 }
 
 #[cfg(test)]
@@ -8277,43 +8248,6 @@ impl Drop for DeleteObjectCommitBarrier {
     }
 }
 
-#[cfg(test)]
-async fn pause_delete_object_before_namespace(bucket: &str, object: &str) {
-    let barrier = DELETE_OBJECT_COMMIT_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("delete object commit barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
-        })
-        .cloned();
-    if let Some(barrier) = barrier {
-        barrier.arrived.notify_one();
-        barrier.release.notified().await;
-    }
-}
-
-#[cfg(test)]
-fn notify_delete_object_namespace(bucket: &str, object: &str, acquired: bool) {
-    let barrier = DELETE_OBJECT_COMMIT_BARRIER
-        .get_or_init(|| std::sync::Mutex::new(None))
-        .lock()
-        .expect("delete object commit barrier mutex should not poison")
-        .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
-        })
-        .cloned();
-    if let Some(barrier) = barrier {
-        if acquired {
-            barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
-        } else {
-            barrier.namespace_pending.notify_one();
-        }
-    }
-}
-
 #[cfg(test)]
 async fn pause_delete_object_commit(bucket: &str, object: &str) {
     let barrier = DELETE_OBJECT_COMMIT_BARRIER
@@ -8321,9 +8255,7 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) {
         .lock()
         .expect("delete object commit barrier mutex should not poison")
         .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeCommit
-        })
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish)
         .cloned();
     if let Some(barrier) = barrier {
         barrier.arrived.notify_one();
@@ -8338,9 +8270,7 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
         .lock()
         .expect("delete object commit barrier mutex should not poison")
         .as_ref()
-        .filter(|barrier| {
-            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::AfterPublish
-        })
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish)
         .cloned();
     if let Some(barrier) = barrier {
         barrier.arrived.notify_one();
@@ -8909,7 +8839,7 @@ impl SetDisks {
         // quorum, failing write quorum on update_object_meta (backlog#872).
         let read_opts = opts.as_read_opts();
         let (mut fi, _, disks) = self
-            .get_object_fileinfo_for_metadata_update(bucket, object, &read_opts)
+            .get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
             .await?
             .into_owned();
 
@@ -9364,56 +9294,36 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let src_version_id = src_opts.version_id.as_deref().unwrap_or_default();
-        let quorum = Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count);
-        let selection = match &quorum {
-            Ok((read_quorum, write_quorum)) => {
-                let read_quorum = usize::try_from(*read_quorum)
-                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![src_bucket, src_object]))?;
-                let write_quorum = usize::try_from(*write_quorum)
-                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![src_bucket, src_object]))?;
-                Self::select_fileinfo_for_read(src_bucket, &disks, &metas, &errs, src_version_id, read_quorum, write_quorum)
-                    .map(|(online_disks, fi)| (online_disks, fi, write_quorum))
-            }
-            Err(err) => Err(err.clone()),
-        };
-        let (online_disks, mut fi, write_quorum) = match selection {
-            Ok(selection) => selection,
+        let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
+            Ok((r, w)) => (
+                usize::try_from(r)
+                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![src_bucket, src_object]))?,
+                usize::try_from(w)
+                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![src_bucket, src_object]))?,
+            ),
             Err(mut err) => {
-                // The merged latest version can be one a failed PUT left below
-                // quorum; resolve it like a latest GET before the dangling
-                // cleanup below can report a key with a committed version absent.
-                if let Some(snapshot) = self
-                    .select_latest_past_dangling_versions(src_bucket, src_object, src_opts, true, &disks, &err)
-                    .await
+                if err == DiskError::ErasureReadQuorum
+                    && !src_bucket.starts_with(RUSTFS_META_BUCKET)
+                    && self
+                        .delete_if_dangling(src_bucket, src_object, &metas, &errs, &HashMap::new(), src_opts.clone())
+                        .await
+                        .is_ok()
                 {
-                    // A version-only copy writes the served version's own
-                    // copies, so they and its write quorum replace the latest
-                    // read's.
-                    let (fi, parts_metadata, online_disks) = snapshot.into_owned();
-                    metas = parts_metadata;
-                    let write_quorum = fi.write_quorum(self.default_write_quorum());
-                    (online_disks, fi, write_quorum)
-                } else {
-                    if quorum.is_err()
-                        && err == DiskError::ErasureReadQuorum
-                        && !src_bucket.starts_with(RUSTFS_META_BUCKET)
-                        && self
-                            .delete_if_dangling(src_bucket, src_object, &metas, &errs, &HashMap::new(), src_opts.clone())
-                            .await
-                            .is_ok()
-                    {
-                        if src_opts.version_id.is_some() {
-                            err = DiskError::FileVersionNotFound
-                        } else {
-                            err = DiskError::FileNotFound
-                        }
+                    if src_opts.version_id.is_some() {
+                        err = DiskError::FileVersionNotFound
+                    } else {
+                        err = DiskError::FileNotFound
                     }
-                    return Err(to_object_err(err.into(), vec![src_bucket, src_object]));
                 }
+                return Err(to_object_err(err.into(), vec![src_bucket, src_object]));
             }
         };
 
+        let src_version_id = src_opts.version_id.as_deref().unwrap_or_default();
+        let (online_disks, mut fi) =
+            Self::select_fileinfo_for_read(src_bucket, &disks, &metas, &errs, src_version_id, read_quorum, write_quorum)
+                .map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
+
         if fi.deleted {
             if src_opts.version_id.is_none() {
                 return Err(to_object_err(Error::FileNotFound, vec![src_bucket, src_object]));
@@ -9551,16 +9461,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 }
             }
 
-            // A version-only copy writes and then reverts on a quorum miss;
-            // refuse it before writing when the served version's disks cannot
-            // reach its write quorum even if every write succeeds.
-            let reachable = online_disks
-                .iter()
-                .map(|disk| disk.is_none().then_some(DiskError::DiskNotFound))
-                .collect::<Vec<_>>();
-            if let Some(err) = reduce_write_quorum_errs(&reachable, OBJECT_OP_IGNORED_ERRS, write_quorum) {
-                return Err(to_object_err(err.into(), vec![src_bucket, src_object]));
-            }
             Self::write_unique_file_info(&online_disks, "", src_bucket, src_object, &metas, write_quorum)
                 .await
                 .map_err(|e| to_object_err(e.into(), vec![src_bucket, src_object]))?;
@@ -10629,20 +10529,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         // Guard lock for single object delete
         let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock {
-            #[cfg(test)]
-            let guard = {
-                pause_delete_object_before_namespace(bucket, object).await;
-                let guard = self
-                    .acquire_write_lock_diag_with_pending_hook("delete_object", bucket, object, || {
-                        notify_delete_object_namespace(bucket, object, false);
-                    })
-                    .await?;
-                notify_delete_object_namespace(bucket, object, true);
-                guard
-            };
-            #[cfg(not(test))]
-            let guard = self.acquire_write_lock_diag("delete_object", bucket, object).await?;
-            Some(Arc::new(guard))
+            Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?))
         } else {
             None
         };
@@ -11260,63 +11147,45 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         };
 
-        let version_id = opts.version_id.as_deref().unwrap_or_default();
-        let quorum = Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count);
-        let selection = match &quorum {
-            Ok((read_quorum, write_quorum)) => {
-                let read_quorum = usize::try_from(*read_quorum)
-                    .map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
-                let write_quorum = usize::try_from(*write_quorum)
-                    .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
-                Self::select_fileinfo_for_read(bucket, &disks, &metas, &errs, version_id, read_quorum, write_quorum)
-            }
-            Err(err) => Err(err.clone()),
-        };
-        let (online_disks, mut fi) = match selection {
-            Ok(selection) => selection,
+        let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&metas, &errs, self.default_parity_count) {
+            Ok((read_quorum, write_quorum)) => (read_quorum, write_quorum),
             Err(mut err) => {
-                // The merged latest version can be one a failed PUT left below
-                // quorum; resolve it like a latest GET before the dangling
-                // cleanup below can report a key with a committed version absent.
-                // A replication terminal CAS stays side-effect free: the walk
-                // queues heal for the versions it skips, so it does not run.
-                let resolved = if opts.replication_status_writeback.is_none() {
-                    self.select_latest_past_dangling_versions(bucket, object, opts, false, &disks, &err)
-                        .await
-                } else {
-                    None
-                };
-                if let Some(snapshot) = resolved {
-                    let (fi, _, online_disks) = snapshot.into_owned();
-                    (online_disks, fi)
-                } else {
-                    // A replication terminal CAS is a validation/write-back
-                    // attempt, never a heal operation. In particular,
-                    // ValidateOnly must remain zero-write, and a worker whose
-                    // outer replication lease is already lost must not delete
-                    // the surviving xl.meta copies before the namespace fence
-                    // below can reject it. Leave dangling cleanup to
-                    // heal/scanner.
-                    if quorum.is_err()
-                        && opts.replication_status_writeback.is_none()
-                        && err == DiskError::ErasureReadQuorum
-                        && !bucket.starts_with(RUSTFS_META_BUCKET)
-                        && self
-                            .delete_if_dangling(bucket, object, &metas, &errs, &HashMap::new(), opts.clone())
-                            .await
-                            .is_ok()
-                    {
-                        if opts.version_id.is_some() {
-                            err = DiskError::FileVersionNotFound
-                        } else {
-                            err = DiskError::FileNotFound
-                        }
-                    }
+                // A replication terminal CAS is a validation/write-back
+                // attempt, never a heal operation. In particular,
+                // ValidateOnly must remain zero-write, and a worker whose
+                // outer replication lease is already lost must not delete the
+                // surviving xl.meta copies before the namespace fence below
+                // can reject it. Leave dangling cleanup to heal/scanner.
+                if opts.replication_status_writeback.is_some() {
                     return Err(to_object_err(err.into(), vec![bucket, object]));
                 }
+                if err == DiskError::ErasureReadQuorum
+                    && !bucket.starts_with(RUSTFS_META_BUCKET)
+                    && self
+                        .delete_if_dangling(bucket, object, &metas, &errs, &HashMap::new(), opts.clone())
+                        .await
+                        .is_ok()
+                {
+                    if opts.version_id.is_some() {
+                        err = DiskError::FileVersionNotFound
+                    } else {
+                        err = DiskError::FileNotFound
+                    }
+                }
+                return Err(to_object_err(err.into(), vec![bucket, object]));
             }
         };
 
+        let read_quorum =
+            usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
+        let write_quorum = usize::try_from(write_quorum)
+            .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
+
+        let version_id = opts.version_id.as_deref().unwrap_or_default();
+        let (online_disks, mut fi) =
+            Self::select_fileinfo_for_read(bucket, &disks, &metas, &errs, version_id, read_quorum, write_quorum)
+                .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
+
         if fi.deleted {
             return Err(to_object_err(Error::MethodNotAllowed, vec![bucket, object]));
         }
@@ -21567,206 +21436,6 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
-    #[tokio::test]
-    async fn deep_heal_preserves_committed_generation_during_old_data_cleanup() {
-        use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
-
-        for (disk_count, parity, read_repair) in [(4, 2, false), (4, 2, true), (16, 4, false), (16, 4, true)] {
-            let (dirs, disks, set) = super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated(
-                disk_count, 0, parity,
-            )
-            .await;
-            let bucket = "native-heal-commit-cleanup";
-            let object = format!("generation-{disk_count}-{parity}-{read_repair}.bin");
-            for disk in &disks {
-                disk.make_volume(bucket).await.expect("create actual fixture volume");
-            }
-            let options = ObjectOptions {
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
-                ..Default::default()
-            };
-            set.put_object(bucket, &object, &mut PutObjReader::from_vec(vec![0x17; TEST_OBJECT_SIZE]), &options)
-                .await
-                .expect("acknowledge original generation on every disk");
-            let old = disks[0]
-                .read_version("", bucket, &object, "", &ReadOptions::default())
-                .await
-                .expect("read actual original generation");
-            let old_dir = old.data_dir.expect("original external data directory");
-            let old_part = dirs[0]
-                .path()
-                .join(bucket)
-                .join(&object)
-                .join(old_dir.to_string())
-                .join("part.1");
-            assert!(tokio::fs::try_exists(&old_part).await.expect("original part exists"));
-
-            // Cleanup slots follow the writer's shuffled erasure distribution,
-            // not the physical fixture order. Pause every real cleanup slot so
-            // no physical old shard can disappear before the proof below.
-            let barriers: Vec<_> = (0..disk_count)
-                .map(|slot| rename_fanout_barrier::arm(&object, slot, rename_fanout_barrier::PHASE_CLEANUP))
-                .collect();
-            let writer_set = Arc::clone(&set);
-            let writer_object = object.clone();
-            let writer_options = options.clone();
-            let payload = vec![0xa3; TEST_OBJECT_SIZE];
-            let writer_payload = payload.clone();
-            let writer = tokio::spawn(async move {
-                writer_set
-                    .put_object(bucket, &writer_object, &mut PutObjReader::from_vec(writer_payload), &writer_options)
-                    .await
-            });
-            tokio::time::timeout(Duration::from_secs(30), async {
-                for barrier in &barriers {
-                    barrier.wait_until_paused().await;
-                }
-            })
-            .await
-            .expect("overwrite commits before its real old-data cleanup pause");
-            assert!(!writer.is_finished(), "tail-drained ACK must remain pending at cleanup");
-            assert!(
-                tokio::fs::try_exists(&old_part)
-                    .await
-                    .expect("old source still present at pause")
-            );
-
-            let mut committed = Vec::new();
-            let mut healthy_metadata = Vec::new();
-            for disk in &disks {
-                let info = disk
-                    .read_version("", bucket, &object, "", &ReadOptions::default())
-                    .await
-                    .expect("read independently committed physical generation without GET");
-                assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (disk_count - parity, parity));
-                assert!(info.data.is_none(), "use actual external parts");
-                assert_ne!(info.data_dir, Some(old_dir));
-                committed.push(info);
-                healthy_metadata.push(
-                    disk.read_all(bucket, &format!("{object}/xl.meta"))
-                        .await
-                        .expect("committed raw metadata"),
-                );
-            }
-            let new_dir = committed[0].data_dir.expect("new committed data directory");
-            assert!(committed.iter().all(|info| info.data_dir == Some(new_dir)));
-            let new_part = dirs[0]
-                .path()
-                .join(bucket)
-                .join(&object)
-                .join(new_dir.to_string())
-                .join("part.1");
-            let expected_part = tokio::fs::read(&new_part).await.expect("exact new source bytes");
-            let mut committed_parts = Vec::new();
-            for dir in &dirs {
-                let original = dir.path().join(bucket).join(&object).join(old_dir.to_string()).join("part.1");
-                assert!(
-                    tokio::fs::try_exists(original)
-                        .await
-                        .expect("every old cleanup slot is paused")
-                );
-                committed_parts.push(
-                    tokio::fs::read(dir.path().join(bucket).join(&object).join(new_dir.to_string()).join("part.1"))
-                        .await
-                        .expect("every new encoded shard before fault"),
-                );
-            }
-            tokio::fs::remove_file(&new_part)
-                .await
-                .expect("remove only the newly committed target shard");
-            println!(
-                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} old_cleanup_paused new_generation_committed target_part_missing"
-            );
-
-            let (result, error) = tokio::time::timeout(
-                Duration::from_secs(30),
-                set.heal_object(
-                    bucket,
-                    &object,
-                    "",
-                    &HealOpts {
-                        scan_mode: HealScanMode::Deep,
-                        read_repair,
-                        ..Default::default()
-                    },
-                ),
-            )
-            .await
-            .expect("real locked Heal must progress during old cleanup")
-            .expect("complete actual deep Heal");
-            assert!(error.is_none(), "new generation must remain repairable: {error:?}");
-            assert_eq!(result.drives_healed(), Some(1));
-            // Shared data-phase repair deliberately does not certify full integrity.
-            assert_eq!(result.integrity_verified, !read_repair);
-            assert_eq!(tokio::fs::read(&new_part).await.expect("rebuilt part before any GET"), expected_part);
-            let repaired = disks[0]
-                .read_version("", bucket, &object, "", &ReadOptions::default())
-                .await
-                .expect("read actual repaired generation");
-            assert_eq!(repaired.data_dir, Some(new_dir));
-            assert_eq!(repaired.mod_time, committed[0].mod_time);
-            for (disk, expected) in disks.iter().zip(&healthy_metadata).skip(1) {
-                assert_eq!(
-                    disk.read_all(bucket, &format!("{object}/xl.meta"))
-                        .await
-                        .expect("healthy raw metadata after Heal"),
-                    *expected
-                );
-            }
-            println!(
-                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} locked_heal_completed new_generation_part_rebuilt_before_get"
-            );
-
-            for barrier in &barriers {
-                barrier.release();
-            }
-            let acknowledged = tokio::time::timeout(Duration::from_secs(30), writer)
-                .await
-                .expect("cleanup drains")
-                .expect("writer joins")
-                .expect("overwrite ACK remains successful");
-            assert_eq!(acknowledged.size, i64::try_from(payload.len()).expect("fixture size fits i64"));
-            drop(barriers);
-            for (slot, dir) in dirs.iter().enumerate() {
-                let root = dir.path().join(bucket).join(&object);
-                assert!(
-                    !tokio::fs::try_exists(root.join(old_dir.to_string()))
-                        .await
-                        .expect("old cleanup completed")
-                );
-                assert_eq!(
-                    tokio::fs::read(root.join(new_dir.to_string()).join("part.1"))
-                        .await
-                        .expect("new shard survives cleanup before GET"),
-                    committed_parts[slot],
-                    "old cleanup must preserve every exact committed encoded shard"
-                );
-            }
-            let mut reader = set
-                .get_object_reader(bucket, &object, None, HeaderMap::new(), &options)
-                .await
-                .expect("read final acknowledged generation");
-            let mut body = Vec::new();
-            reader
-                .stream
-                .read_to_end(&mut body)
-                .await
-                .expect("read complete final payload");
-            assert_eq!(body, payload);
-            for disk in &disks {
-                let final_info = disk
-                    .read_version("", bucket, &object, "", &ReadOptions::default())
-                    .await
-                    .expect("final physical generation");
-                assert_eq!(final_info.data_dir, Some(new_dir), "old cleanup must not revert the healed generation");
-            }
-            println!(
-                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} overwrite_ack_preserved final_physical_generation_and_full_body_verified"
-            );
-        }
-    }
-
     #[tokio::test]
     async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/set_disk/read.rs

@@ -278,10 +278,6 @@ const MAX_DANGLING_LATEST_VERSIONS: usize = 8;
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum DanglingRead {
     KeepQuorumError,
-    /// Metadata updates of the latest version keep the quorum error but, like
-    /// object reads, look past newer versions proven dangling
-    /// (`read_latest_past_dangling_versions`).
-    SkipDanglingVersions,
     ReportNotFound,
 }
 
@@ -733,10 +729,9 @@ impl SetDisks {
     }
 
     /// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
-    /// quorum fanout. Read-before-write callers must use this or
-    /// `get_object_fileinfo_for_metadata_update`: the returned online-disk set
-    /// is the write target, and the early-stop subset would fail write quorum
-    /// (backlog#872 regression fix).
+    /// quorum fanout. Read-before-write callers (object tagging) must use this:
+    /// the returned online-disk set is the write target, and the early-stop
+    /// subset would fail write quorum (backlog#872 regression fix).
     #[tracing::instrument(level = "debug", skip(self))]
     pub(in crate::set_disk) async fn get_object_fileinfo_gated(
         &self,
@@ -758,64 +753,6 @@ impl SetDisks {
         .await
     }
 
-    /// The full-fanout read behind a metadata update such as object tagging.
-    /// A latest read whose newest versions never committed resolves to the
-    /// version a latest GET serves, while a dangling key keeps the quorum
-    /// error. The returned online disks are the write target.
-    #[tracing::instrument(level = "debug", skip(self))]
-    pub(in crate::set_disk) async fn get_object_fileinfo_for_metadata_update(
-        &self,
-        bucket: &str,
-        object: &str,
-        opts: &ObjectOptions,
-    ) -> Result<GetObjectFileInfo> {
-        self.get_object_fileinfo_gated_inner(
-            bucket,
-            object,
-            opts,
-            false,
-            false,
-            MetadataReadPurpose::FullLayout,
-            DanglingRead::SkipDanglingVersions,
-        )
-        .await
-    }
-
-    /// Recovers a failed latest selection in read-before-write callers that
-    /// read and select metadata themselves (`put_object_metadata` and the
-    /// CopyObject source) the way a latest GET does. `None` keeps the caller's
-    /// error: a failure other than read quorum, an explicit version, the
-    /// system bucket's commit-proof rule, or a walk that finds no readable
-    /// version or cannot read every drive. The selected metadata keeps its
-    /// part checksums because a version-only copy carries them into the new
-    /// version.
-    pub(in crate::set_disk) async fn select_latest_past_dangling_versions(
-        &self,
-        bucket: &str,
-        object: &str,
-        opts: &ObjectOptions,
-        read_data: bool,
-        disks: &[Option<DiskStore>],
-        err: &DiskError,
-    ) -> Option<GetObjectFileInfo> {
-        if *err != DiskError::ErasureReadQuorum || opts.version_id.is_some() || crate::bucket::utils::is_meta_bucketname(bucket) {
-            return None;
-        }
-        let opts = ObjectOptions {
-            include_part_checksums: true,
-            ..opts.clone()
-        };
-        Box::pin(self.read_latest_past_dangling_versions(
-            bucket,
-            object,
-            &opts,
-            read_data,
-            MetadataReadPurpose::FullLayout,
-            disks,
-        ))
-        .await
-    }
-
     #[allow(clippy::too_many_arguments)]
     async fn get_object_fileinfo_gated_inner(
         &self,
@@ -1041,7 +978,7 @@ impl SetDisks {
                 Err(err) => {
                     if err == DiskError::ErasureReadQuorum
                         && vid.is_empty()
-                        && dangling_read != DanglingRead::KeepQuorumError
+                        && dangling_read == DanglingRead::ReportNotFound
                         && metadata_fanout_complete
                         && !crate::bucket::utils::is_meta_bucketname(bucket)
                         && let Some(snapshot) =
@@ -1818,10 +1755,7 @@ impl SetDisks {
                         "Create bitrot reader failed read quorum"
                     );
                     record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
-                    // The metadata selected this object, but the part cannot
-                    // be reconstructed. A dominant disk I/O/not-found error
-                    // must not erase the object's read-quorum failure.
-                    return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+                    return Err(to_object_err(read_err.into(), vec![bucket, object]));
                 }
                 let reason = GetObjectFailureReason::ReadQuorum;
                 error!(
@@ -1846,7 +1780,7 @@ impl SetDisks {
                     "Create bitrot reader did not have enough disks"
                 );
                 record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
-                return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+                return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
             }
 
             // Check if we have missing shards even though we can read successfully
@@ -2381,10 +2315,10 @@ impl SetDisks {
             if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
                 let reason = classify_disk_error(&read_err);
                 record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
-                return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+                return Err(to_object_err(read_err.into(), vec![bucket, object]));
             }
             record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
-            return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
+            return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
         }
 
         let missing_shards = reader_setup.completed_failed_shards();
@@ -8195,105 +8129,6 @@ mod tests {
         (data_dir, files)
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn external_missing_shards_keep_typed_read_quorum() {
-        use tokio::io::AsyncReadExt;
-
-        let payload = vec![0x37; 256 * 1024];
-        for readable_shards in [1, 2, 4] {
-            let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
-            let mut fi = codec_streaming_test_fileinfo(i64::try_from(payload.len()).expect("fixture size fits i64"), 1);
-            fi.erasure.data_blocks = erasure.data_shards;
-            fi.erasure.parity_blocks = erasure.parity_shards;
-            fi.erasure.block_size = erasure.block_size;
-            fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
-            let (dirs, disks) = local_test_disks(erasure.total_shard_count(), CODEC_STREAMING_TEST_BUCKET).await;
-            let (data_dir, files) = codec_streaming_external_files(&erasure, &payload, &disks).await;
-            fi.data_dir = Some(data_dir);
-            for directory in dirs.iter().skip(readable_shards) {
-                tokio::fs::remove_file(
-                    directory
-                        .path()
-                        .join(CODEC_STREAMING_TEST_BUCKET)
-                        .join(CODEC_STREAMING_TEST_OBJECT)
-                        .join(data_dir.to_string())
-                        .join("part.1"),
-                )
-                .await
-                .expect("remove real part while retaining its selected metadata");
-            }
-
-            let mut output = Vec::new();
-            let legacy = SetDisks::get_object_with_fileinfo(
-                CODEC_STREAMING_TEST_BUCKET,
-                CODEC_STREAMING_TEST_OBJECT,
-                Arc::new(ErasureCache::new()),
-                0,
-                fi.size,
-                &mut output,
-                fi.clone(),
-                files.clone(),
-                &disks,
-                0,
-                0,
-                false,
-                true,
-                false,
-                false,
-                GET_OBJECT_PATH_LEGACY_DUPLEX,
-                "plain_single_part",
-                "le_1mib",
-            )
-            .await;
-            let streaming = SetDisks::get_object_mid_size_reader_with_fileinfo(
-                CODEC_STREAMING_TEST_BUCKET,
-                CODEC_STREAMING_TEST_OBJECT,
-                Arc::new(ErasureCache::new()),
-                &fi,
-                &files,
-                &disks,
-                0,
-                0,
-                false,
-                true,
-                "plain_single_part",
-                "le_1mib",
-                false,
-            )
-            .await;
-            if readable_shards < erasure.data_shards {
-                for error in [
-                    legacy.expect_err("legacy read must reject quorum minus one"),
-                    streaming.err().expect("streaming setup must reject quorum minus one"),
-                ] {
-                    assert_eq!(
-                        error,
-                        Error::InsufficientReadQuorum(
-                            CODEC_STREAMING_TEST_BUCKET.to_owned(),
-                            CODEC_STREAMING_TEST_OBJECT.to_owned(),
-                        )
-                    );
-                }
-                assert!(output.is_empty(), "quorum rejection must precede all response bytes");
-            } else {
-                legacy.expect("legacy reader retains reconstructable quorum");
-                assert_eq!(output, payload, "legacy read returns every original byte at quorum");
-                let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) =
-                    streaming.expect("streaming quorum remains readable")
-                else {
-                    panic!("mid-size quorum must yield an actual reader");
-                };
-                let mut decoded = Vec::new();
-                reader
-                    .read_to_end(&mut decoded)
-                    .await
-                    .expect("streaming reader returns the complete body");
-                assert_eq!(decoded, payload, "streaming read returns every original byte at quorum");
-            }
-        }
-    }
-
     #[test]
     #[serial]
     fn payload_read_repair_is_suppressed_only_for_read_only_requests() {
@@ -10095,7 +9930,6 @@ mod dangling_read_tests {
     use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
     use crate::storage_api_contracts::object::ObjectOperations;
     use http::HeaderMap;
-    use std::collections::HashMap;
     use std::path::PathBuf;
     use tempfile::TempDir;
     use tokio::io::AsyncReadExt;
@@ -10617,353 +10451,6 @@ mod dangling_read_tests {
         assert_ne!(latest.version_id, Some(v2));
     }
 
-    /// Version one on every drive under a version two that a failed PUT left
-    /// on drives `0..3`, below the read quorum of four.
-    async fn put_committed_below_dangling(set: &SetDisks, bucket: &str, object: &str) -> (Uuid, Uuid) {
-        let v1 = put(set, bucket, object, b"version one", true)
-            .await
-            .version_id
-            .expect("versioned PUT should assign a version id");
-        let v2 = put(set, bucket, object, b"version two", true)
-            .await
-            .version_id
-            .expect("versioned PUT should assign a version id");
-        drop_version_from(set, bucket, object, v2, 3..DRIVES).await;
-        (v1, v2)
-    }
-
-    /// Each drive's copy of `version_id`, or `None` where the drive lacks it.
-    async fn version_metadata(
-        set: &SetDisks,
-        bucket: &str,
-        object: &str,
-        version_id: Uuid,
-    ) -> Vec<Option<HashMap<String, String>>> {
-        let disks = set.disks.read().await.clone();
-        let mut copies = Vec::with_capacity(disks.len());
-        for disk in &disks {
-            let disk = disk.as_ref().expect("every drive should be online");
-            copies.push(
-                disk.read_version("", bucket, object, &version_id.to_string(), &ReadOptions::default())
-                    .await
-                    .ok()
-                    .map(|fi| fi.metadata),
-            );
-        }
-        copies
-    }
-
-    #[tokio::test]
-    async fn latest_tagging_skips_dangling_versions_above_a_committed_version() {
-        let bucket = "dangling-latest-tagging";
-        let object = "torn/latest";
-        let (_dirs, set) = dangling_test_set(bucket).await;
-        let (v1, v2) = put_committed_below_dangling(&set, bucket, object).await;
-        let tag_of = |metadata: &Option<HashMap<String, String>>| {
-            metadata
-                .as_ref()
-                .and_then(|metadata| metadata.get(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING).cloned())
-        };
-
-        let tagged = set
-            .put_object_tags(bucket, object, "stage=committed", &ObjectOptions::default())
-            .await
-            .expect("PutObjectTagging must update the version a latest GET serves");
-        assert_eq!(tagged.version_id, Some(v1));
-        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
-            assert_eq!(
-                tag_of(metadata).as_deref(),
-                Some("stage=committed"),
-                "drive {disk} must carry the tag on version one"
-            );
-        }
-        for (disk, metadata) in version_metadata(&set, bucket, object, v2).await.iter().enumerate() {
-            assert_eq!(tag_of(metadata), None, "drive {disk} must leave the dangling version untouched");
-        }
-        assert_eq!(*head(&set, bucket, object, None).await.expect("latest HEAD").user_tags, "stage=committed");
-
-        let untagged = set
-            .delete_object_tags(bucket, object, &ObjectOptions::default())
-            .await
-            .expect("DeleteObjectTagging must update the version a latest GET serves");
-        assert_eq!(untagged.version_id, Some(v1));
-        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
-            assert_eq!(tag_of(metadata).as_deref(), Some(""), "drive {disk} must clear the tag on version one");
-        }
-    }
-
-    #[tokio::test]
-    async fn latest_metadata_update_skips_dangling_versions_above_a_committed_version() {
-        let bucket = "dangling-latest-metadata";
-        let object = "torn/latest";
-        let (_dirs, set) = dangling_test_set(bucket).await;
-        let (v1, v2) = put_committed_below_dangling(&set, bucket, object).await;
-
-        let updated = set
-            .put_object_metadata(
-                bucket,
-                object,
-                &ObjectOptions {
-                    versioned: true,
-                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "updated".to_string())])),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("a latest metadata update must target the version a latest GET serves");
-        assert_eq!(updated.version_id, Some(v1));
-        assert!(updated.is_latest, "the updated version is the latest one");
-        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
-            let metadata = metadata.as_ref().expect("version one is on every drive");
-            assert_eq!(metadata.get("x-amz-meta-probe").map(String::as_str), Some("updated"), "drive {disk}");
-        }
-        for (disk, metadata) in version_metadata(&set, bucket, object, v2).await.iter().enumerate() {
-            assert!(
-                metadata
-                    .as_ref()
-                    .is_none_or(|metadata| !metadata.contains_key("x-amz-meta-probe")),
-                "drive {disk} must leave the dangling version untouched"
-            );
-        }
-    }
-
-    #[tokio::test]
-    async fn latest_copy_source_skips_dangling_versions_above_a_committed_version() {
-        let bucket = "dangling-latest-copy";
-        let (_dirs, set) = dangling_test_set(bucket).await;
-
-        // A metadata-only self-copy rewrites the latest version in place.
-        let object = "torn/in-place";
-        let (v1, _) = put_committed_below_dangling(&set, bucket, object).await;
-        let mut source = head(&set, bucket, object, None).await.expect("latest HEAD");
-        assert_eq!(source.version_id, Some(v1));
-        source.metadata_only = true;
-        source.user_defined = Arc::new(HashMap::from([("x-amz-meta-copied".to_string(), "in-place".to_string())]));
-        let copied = set
-            .copy_object(
-                bucket,
-                object,
-                bucket,
-                object,
-                &mut source,
-                &ObjectOptions::default(),
-                &ObjectOptions::default(),
-            )
-            .await
-            .expect("a CopyObject source without a version id must resolve like a latest GET");
-        assert_eq!(copied.version_id, Some(v1));
-        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
-            let metadata = metadata.as_ref().expect("version one is on every drive");
-            assert_eq!(metadata.get("x-amz-meta-copied").map(String::as_str), Some("in-place"), "drive {disk}");
-        }
-
-        // A version-only self-copy commits a new version that shares the
-        // served version's data.
-        let object = "torn/new-version";
-        let (_, v2) = put_committed_below_dangling(&set, bucket, object).await;
-        let v3 = Uuid::new_v4();
-        let mut source = head(&set, bucket, object, None).await.expect("latest HEAD");
-        source.version_only = true;
-        let copied = set
-            .copy_object(
-                bucket,
-                object,
-                bucket,
-                object,
-                &mut source,
-                &ObjectOptions {
-                    versioned: true,
-                    ..Default::default()
-                },
-                &ObjectOptions {
-                    version_id: Some(v3.to_string()),
-                    versioned: true,
-                    object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
-                        crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-                    ))),
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("a version-only CopyObject source without a version id must resolve like a latest GET");
-        assert_eq!(copied.version_id, Some(v3));
-        let latest = head(&set, bucket, object, None).await.expect("latest HEAD");
-        assert_eq!(latest.version_id, Some(v3));
-        assert_eq!(get(&set, bucket, object, None).await.expect("latest GET"), b"version one");
-        assert!(
-            version_metadata(&set, bucket, object, v2).await[..3]
-                .iter()
-                .all(Option::is_some),
-            "the copy must not remove the dangling version; heal owns its cleanup"
-        );
-    }
-
-    #[tokio::test]
-    async fn latest_metadata_update_past_dangling_versions_keeps_write_quorum() {
-        // EC 3+3: read quorum three, write quorum four. Version one reached
-        // three drives, so it is readable but below write quorum; versions two
-        // and three are dangling on two drives each. A latest read serves
-        // version one, and a metadata update of it is refused before writing.
-        let bucket = "dangling-latest-write-quorum";
-        let object = "torn/latest";
-        let (_dirs, set) = dangling_test_set_with(bucket, 6, 3).await;
-        let mut versions = Vec::new();
-        for body in [b"version zero".as_slice(), b"version one", b"version two", b"version three"] {
-            versions.push(
-                put(&set, bucket, object, body, true)
-                    .await
-                    .version_id
-                    .expect("versioned PUT should assign a version id"),
-            );
-        }
-        let [_, v1, v2, v3] = versions[..] else {
-            unreachable!("four versions were written");
-        };
-        drop_version_from(&set, bucket, object, v1, 3..6).await;
-        drop_version_from(&set, bucket, object, v2, 2..6).await;
-        drop_version_from(&set, bucket, object, v3, 0..3).await;
-        drop_version_from(&set, bucket, object, v3, 5..6).await;
-
-        let latest = head(&set, bucket, object, None)
-            .await
-            .expect("version one is readable at read quorum");
-        assert_eq!(latest.version_id, Some(v1));
-
-        let is_write_quorum_error =
-            |err: &StorageError| matches!(err, StorageError::ErasureWriteQuorum | StorageError::InsufficientWriteQuorum(_, _));
-        let Err(err) = set
-            .put_object_tags(bucket, object, "stage=refused", &ObjectOptions::default())
-            .await
-        else {
-            panic!("tagging a version below write quorum must be refused");
-        };
-        assert!(is_write_quorum_error(&err), "tagging keeps the write-quorum error: {err:?}");
-        let Err(err) = set
-            .put_object_metadata(
-                bucket,
-                object,
-                &ObjectOptions {
-                    versioned: true,
-                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "refused".to_string())])),
-                    ..Default::default()
-                },
-            )
-            .await
-        else {
-            panic!("a metadata update of a version below write quorum must be refused");
-        };
-        assert!(is_write_quorum_error(&err), "the metadata update keeps the write-quorum error: {err:?}");
-        let v4 = Uuid::new_v4();
-        let mut source = head(&set, bucket, object, None).await.expect("latest HEAD");
-        source.version_only = true;
-        let Err(err) = set
-            .copy_object(
-                bucket,
-                object,
-                bucket,
-                object,
-                &mut source,
-                &ObjectOptions {
-                    versioned: true,
-                    ..Default::default()
-                },
-                &ObjectOptions {
-                    version_id: Some(v4.to_string()),
-                    versioned: true,
-                    object_lock_config_snapshot: Some(Arc::new(crate::object_api::ObjectLockConfigSnapshot::new(
-                        crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
-                    ))),
-                    ..Default::default()
-                },
-            )
-            .await
-        else {
-            panic!("a version-only copy of a version below write quorum must be refused");
-        };
-        assert!(is_write_quorum_error(&err), "the version-only copy keeps the write-quorum error: {err:?}");
-        assert!(
-            version_metadata(&set, bucket, object, v4).await.iter().all(Option::is_none),
-            "a refused version-only copy must not write the new version to any drive"
-        );
-        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
-            assert!(
-                metadata.as_ref().is_none_or(|metadata| {
-                    !metadata.contains_key(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING)
-                        && !metadata.contains_key("x-amz-meta-probe")
-                }),
-                "drive {disk} must not be written by a refused update"
-            );
-        }
-    }
-
-    #[tokio::test]
-    async fn replication_status_writeback_never_walks_past_dangling_versions() {
-        // A replication terminal CAS stays side-effect free: it keeps the
-        // read-quorum error instead of resolving the latest version, so it
-        // neither queues heal for the dangling version nor writes version one.
-        let bucket = "dangling-latest-replication-cas";
-        let object = "torn/latest";
-        let (_dirs, set) = dangling_test_set(bucket).await;
-        let (v1, _) = put_committed_below_dangling(&set, bucket, object).await;
-
-        let Err(err) = set
-            .put_object_metadata(
-                bucket,
-                object,
-                &ObjectOptions {
-                    versioned: true,
-                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "written".to_string())])),
-                    replication_status_writeback: Some(Box::new(crate::object_api::ReplicationStatusWritebackCondition {
-                        mode: crate::object_api::ReplicationStatusWritebackMode::ValidateOnly,
-                        ..Default::default()
-                    })),
-                    ..Default::default()
-                },
-            )
-            .await
-        else {
-            panic!("a replication terminal CAS must not resolve past a dangling version");
-        };
-        assert!(is_read_quorum_error(&err), "the CAS keeps the read-quorum error: {err:?}");
-        for (disk, metadata) in version_metadata(&set, bucket, object, v1).await.iter().enumerate() {
-            let metadata = metadata.as_ref().expect("version one is on every drive");
-            assert!(!metadata.contains_key("x-amz-meta-probe"), "drive {disk} must not be written");
-        }
-    }
-
-    #[tokio::test]
-    async fn latest_metadata_updates_past_dangling_versions_require_every_drive_to_answer() {
-        let bucket = "dangling-latest-update-offline";
-        let object = "torn/latest";
-        let (_dirs, set) = dangling_test_set(bucket).await;
-        put_committed_below_dangling(&set, bucket, object).await;
-        let offline = set.disks.write().await[5].take().expect("drive five should start online");
-
-        let Err(err) = set
-            .put_object_tags(bucket, object, "stage=uncertain", &ObjectOptions::default())
-            .await
-        else {
-            panic!("tagging must not resolve past a version an offline drive may commit");
-        };
-        assert!(is_read_quorum_error(&err), "tagging keeps the read-quorum error: {err:?}");
-        let Err(err) = set
-            .put_object_metadata(
-                bucket,
-                object,
-                &ObjectOptions {
-                    versioned: true,
-                    eval_metadata: Some(HashMap::from([("x-amz-meta-probe".to_string(), "uncertain".to_string())])),
-                    ..Default::default()
-                },
-            )
-            .await
-        else {
-            panic!("a metadata update must not resolve past a version an offline drive may commit");
-        };
-        assert!(is_read_quorum_error(&err), "the metadata update keeps the read-quorum error: {err:?}");
-        set.disks.write().await[5] = Some(offline);
-    }
-
     #[tokio::test]
     async fn dangling_classification_keeps_quorum_error_for_corrupt_metadata() {
         let bucket = "dangling-read-corrupt";

crates/ecstore/src/store/heal.rs

@@ -2167,210 +2167,6 @@ mod tests {
         shutdown.cancel();
     }
 
-    #[tokio::test]
-    #[serial_test::serial]
-    async fn bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit() {
-        use crate::set_disk::ReadRepairPauseScope;
-
-        let (_root, store, shutdown) = multi_pool_heal_store().await;
-        let bucket = format!("heal-commit-owner-{}", Uuid::new_v4().simple());
-        let object = "history.bin";
-        let version = Uuid::new_v4().to_string();
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("original bucket");
-        let old = store
-            .bucket_incarnation_id_from_disk(&bucket)
-            .await
-            .expect("original identity");
-        let set = store.pools[0].get_disks(0);
-        let mut reader = PutObjReader::from_vec(vec![0x17; 1024 * 1024]);
-        set.put_object(
-            &bucket,
-            object,
-            &mut reader,
-            &ObjectOptions {
-                versioned: true,
-                version_id: Some(version.clone()),
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("original acknowledged version");
-        let missing = set.disks.read().await[0].clone().expect("repair target");
-        let healthy = set.disks.read().await[1].clone().expect("healthy target");
-        let baseline = healthy
-            .read_all(&bucket, &format!("{object}/xl.meta"))
-            .await
-            .expect("healthy metadata baseline");
-        let original = healthy
-            .read_version("", &bucket, object, &version, &Default::default())
-            .await
-            .expect("original version metadata");
-        assert!(original.data.is_none(), "the commit barrier fixture must use actual external data shards");
-        missing
-            .delete(
-                &bucket,
-                object,
-                DeleteOptions {
-                    recursive: true,
-                    immediate: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect("remove one physical shard");
-
-        // The scope pauses the real repair after reconstruction, immediately
-        // before its generation check and final shard rename.
-        let (_pause, arrived, release) = ReadRepairPauseScope::install_incarnation_commit(&bucket, object);
-        let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
-        let worker_store = store.clone();
-        let worker_bucket = bucket.clone();
-        let worker_version = version.clone();
-        let opts = HealOpts {
-            pool: Some(0),
-            set: Some(0),
-            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
-            ..Default::default()
-        };
-        let mut waiter = tokio::spawn(async move {
-            worker_store
-                .run_bucket_heal_at_incarnation(&worker_bucket, old, &opts, move |store, bucket, opts| async move {
-                    let result = store.heal_object_with_proof(&bucket, object, &worker_version, &opts).await?;
-                    drained_tx
-                        .send(result)
-                        .expect("retain the actual storage result after caller cancellation");
-                    Ok(())
-                })
-                .await
-        });
-        tokio::select! {
-            paused = tokio::time::timeout(std::time::Duration::from_secs(10), arrived.notified()) => {
-                paused.expect("real missing-shard repair reaches the final commit pause");
-            }
-            completed = &mut waiter => panic!("repair ended before the actual final commit barrier: {completed:?}"),
-        }
-        assert!(
-            missing.read_xl(&bucket, object, false).await.is_err(),
-            "final shard is not published at the pause"
-        );
-        println!("incarnation_commit_barrier: final_rename_paused");
-        waiter.abort();
-        assert!(waiter.await.expect_err("caller aborted").is_cancelled());
-        let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
-        assert!(
-            futures::poll!(&mut writer).is_pending(),
-            "bucket deletion/recreation cannot acquire its generation writer while the cancelled repair still owns commit"
-        );
-        println!("incarnation_commit_barrier: caller_aborted_writer_pending");
-        release.notify_one();
-        let result = tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
-            .await
-            .expect("actual repair drains after release")
-            .expect("retained repair result");
-        assert!(
-            result.error.is_none(),
-            "physical repair must succeed before lifecycle handoff: {:?}",
-            result.error
-        );
-        let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
-            .await
-            .expect("generation writer acquires only after physical repair drains")
-            .expect("generation writer");
-        assert!(
-            missing.read_xl(&bucket, object, false).await.is_ok(),
-            "the physical shard is published before writer admission"
-        );
-        let repaired = missing
-            .read_version("", &bucket, object, &version, &Default::default())
-            .await
-            .expect("repaired exact version metadata");
-        assert_eq!(repaired.version_id, original.version_id);
-        assert_eq!(repaired.data_dir, original.data_dir);
-        let checked = missing
-            .check_parts(&bucket, object, &repaired)
-            .await
-            .expect("verify the published physical shard");
-        assert_eq!(
-            checked.results,
-            vec![crate::disk::CHECK_PART_SUCCESS],
-            "the actual published part passes bitrot verification"
-        );
-        assert_eq!(
-            healthy
-                .read_all(&bucket, &format!("{object}/xl.meta"))
-                .await
-                .expect("healthy after repair"),
-            baseline,
-            "repair does not overwrite the surviving version"
-        );
-        println!("incarnation_commit_barrier: repair_drained_writer_acquired");
-        drop(guard);
-
-        set.delete_object(
-            &bucket,
-            object,
-            ObjectOptions {
-                versioned: true,
-                version_id: Some(version.clone()),
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("normal explicit deletion of the old version");
-        store
-            .delete_bucket(&bucket, &DeleteBucketOptions::default())
-            .await
-            .expect("normal empty bucket deletion");
-        store
-            .make_bucket(&bucket, &MakeBucketOptions::default())
-            .await
-            .expect("same-name successor bucket");
-        let new = store
-            .bucket_incarnation_id_from_disk(&bucket)
-            .await
-            .expect("successor identity");
-        assert_ne!(old, new);
-        let successor_version = Uuid::new_v4().to_string();
-        let mut successor_reader = PutObjReader::from_vec(vec![0x29; 512 * 1024]);
-        set.put_object(
-            &bucket,
-            object,
-            &mut successor_reader,
-            &ObjectOptions {
-                versioned: true,
-                version_id: Some(successor_version),
-                write_completion: crate::object_api::WriteCompletion::TailDrained,
-                ..Default::default()
-            },
-        )
-        .await
-        .expect("acknowledged successor version");
-        let successor = healthy
-            .read_all(&bucket, &format!("{object}/xl.meta"))
-            .await
-            .expect("successor metadata");
-        assert!(
-            store
-                .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
-                .await
-                .is_err()
-        );
-        assert_eq!(
-            healthy
-                .read_all(&bucket, &format!("{object}/xl.meta"))
-                .await
-                .expect("successor after obsolete admission"),
-            successor,
-            "obsolete admission cannot change the same-name successor"
-        );
-        println!("incarnation_commit_barrier: successor_rejects_obsolete_admission");
-        shutdown.cancel();
-    }
-
     #[tokio::test]
     #[serial_test::serial]
     async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {

crates/filemeta/src/filemeta.rs

@@ -2429,52 +2429,6 @@ mod test {
         }
     }
 
-    #[test]
-    fn truncated_inline_xlmeta_is_file_corrupt_only_for_supported_formats() {
-        let mut meta = FileMeta::new();
-        meta.data
-            .replace("fixture", vec![0x5a; 4095])
-            .expect("seed real inline framing");
-        let inline_len = meta.data.as_slice().len();
-        let original = meta.marshal_msg().expect("serialize valid inline metadata");
-        let inline_start = original.len() - inline_len;
-        let parsed = FileMeta::load(&original).expect("complete inline metadata must decode");
-        assert_eq!(parsed.data, meta.data);
-        for cut in inline_start + 1..original.len() {
-            assert_eq!(
-                FileMeta::load(&original[..cut]).expect_err("incomplete inline framing must fail"),
-                Error::FileCorrupt,
-                "inline truncation at byte {cut} must remain repairable"
-            );
-        }
-
-        for inline_version in [0, 2] {
-            let mut unsupported = original[..original.len() / 2].to_vec();
-            unsupported[inline_start] = inline_version;
-            assert_ne!(
-                FileMeta::load(&unsupported).expect_err("malformed unsupported inline format must fail"),
-                Error::FileCorrupt,
-                "unknown inline version must not authorize repair"
-            );
-        }
-        for (header_ver, meta_ver) in [
-            (XL_HEADER_VERSION + 1, XL_META_VERSION),
-            (XL_HEADER_VERSION, XL_META_VERSION + 1),
-        ] {
-            let mut headers = Vec::new();
-            rmp::encode::write_uint(&mut headers, u64::from(header_ver)).expect("write future header");
-            rmp::encode::write_uint(&mut headers, u64::from(meta_ver)).expect("write future metadata version");
-            rmp::encode::write_uint(&mut headers, 0).expect("write empty version count");
-            let mut unsupported = build_xl_buffer(&headers);
-            unsupported.extend_from_slice(&meta.data.as_slice()[..inline_len / 2]);
-            assert_ne!(
-                FileMeta::load(&unsupported).expect_err("unsupported metadata with truncated inline data must fail"),
-                Error::FileCorrupt,
-                "future metadata schema must not authorize repair"
-            );
-        }
-    }
-
     #[test]
     fn truncated_xlmeta_index_and_format_reads_are_file_corrupt() {
         let buf = FileMeta::default()

crates/filemeta/src/filemeta/codec.rs

@@ -188,13 +188,7 @@ impl FileMeta {
         if !buf.is_empty() {
             InlineData::validate_slice(buf).map_err(|e| {
                 error!("data validation failed: {}", e);
-                // Validation only examines bytes already read. Classify damage
-                // as repairable corruption only for formats we understand.
-                if InlineData::version_ok_slice(buf) && Self::decode_xl_headers(meta).is_ok() {
-                    Error::FileCorrupt
-                } else {
-                    e
-                }
+                e
             })?;
         }
 

crates/filemeta/src/filemeta_inline.rs

@@ -168,11 +168,11 @@ impl InlineData {
         self.0.as_slice()
     }
     pub fn version_ok(&self) -> bool {
-        Self::version_ok_slice(&self.0)
-    }
+        if self.0.is_empty() {
+            return true;
+        }
 
-    pub(crate) fn version_ok_slice(raw: &[u8]) -> bool {
-        raw.first().is_none_or(|version| *version > 0 && *version <= INLINE_DATA_VER)
+        self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
     }
 
     pub fn after_version(&self) -> &[u8] {

crates/heal/src/heal/channel.rs

@@ -196,20 +196,6 @@ impl HealChannelProcessor {
             .map_err(Error::other)
     }
 
-    /// Query a local automatic task; other tokens require coordinator routing.
-    pub async fn execute_local_automatic_query(
-        &self,
-        heal_path: &str,
-        client_token: &str,
-        since_seq: Option<u64>,
-    ) -> Result<Option<HealChannelResponse>> {
-        self.heal_manager
-            .get_local_automatic_task_report(heal_path, client_token, since_seq)
-            .await?
-            .map(|report| Self::query_report_response(client_token.to_owned(), Ok(report)))
-            .transpose()
-    }
-
     /// Execute cancellation directly against the manager.
     pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
         let (response_tx, response_rx) = oneshot::channel();
@@ -477,13 +463,6 @@ impl HealChannelProcessor {
                 .await
         };
 
-        let response = Self::query_report_response(client_token, report)?;
-        let _ = response_tx.send(Ok(response.clone()));
-        self.publish_response(response);
-        Ok(())
-    }
-
-    fn query_report_response(client_token: String, report: Result<HealTaskReport>) -> Result<HealChannelResponse> {
         let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
         let settings = report
             .as_ref()
@@ -602,21 +581,27 @@ impl HealChannelProcessor {
                 0,
             ),
             Err(crate::Error::InvalidClientToken) => {
-                return Ok(HealChannelResponse {
+                let response = HealChannelResponse {
                     request_id: client_token,
                     success: false,
                     data: None,
                     error: Some("invalid heal client token".to_string()),
-                });
+                };
+                let _ = response_tx.send(Ok(response.clone()));
+                self.publish_response(response);
+                return Ok(());
             }
             Err(err) => {
                 let error_text = err.to_string();
-                return Ok(HealChannelResponse {
+                let response = HealChannelResponse {
                     request_id: client_token,
                     success: false,
                     data: None,
                     error: Some(error_text),
-                });
+                };
+                let _ = response_tx.send(Ok(response.clone()));
+                self.publish_response(response);
+                return Ok(());
             }
         };
 
@@ -633,12 +618,17 @@ impl HealChannelProcessor {
             },
         )?;
 
-        Ok(HealChannelResponse {
+        let response = HealChannelResponse {
             request_id: client_token,
             success: true,
             data: Some(data),
             error: detail,
-        })
+        };
+
+        let _ = response_tx.send(Ok(response.clone()));
+        self.publish_response(response);
+
+        Ok(())
     }
 
     /// Process cancel request

crates/heal/src/heal/manager.rs

@@ -234,8 +234,6 @@ fn record_displaced_terminal(
     request: &HealRequest,
 ) -> Arc<CompletedHealStatus> {
     let terminal = Arc::new(CompletedHealStatus {
-        task_id: request.id.clone(),
-        source: request.source,
         outcome: None,
         progress: None,
         retained_bytes: std::sync::OnceLock::new(),
@@ -671,10 +669,7 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
     if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
         return None;
     }
-    // Admin traversals already exhaust their per-object retries. Restarting
-    // the whole traversal under the same token discards those receipts and
-    // reports previously repaired objects as unchanged on the next scan.
-    if task.has_batch_failure() && (task.source == HealRequestSource::Admin || !retryable_batch_failure) {
+    if task.has_batch_failure() && !retryable_batch_failure {
         return None;
     }
 
@@ -1076,9 +1071,9 @@ pub struct HealManager {
 /// cascade without re-locking.
 enum TaskStateLookup {
     Active(Arc<HealTask>),
-    Retrying(HealTaskStatus, HealOptions, HealRequestSource),
+    Retrying(HealTaskStatus, HealOptions),
     Completed(Arc<CompletedHealStatus>),
-    Queued(HealOptions, HealRequestSource),
+    Queued(HealOptions),
     /// A durable root heal record that cannot be admitted or quarantined;
     /// it answers status queries with the reason until an operator repairs it.
     Unrunnable(String),
@@ -2591,23 +2586,6 @@ impl HealManager {
     /// completed entry. `heal_path` additionally constrains the map matches
     /// the way the `*_for_path` variants always have.
     async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
-        let state = self.lookup_memory_task_state(canonical_task_id, heal_path).await;
-        if !matches!(state, TaskStateLookup::NotFound) {
-            return Ok(state);
-        }
-        if let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
-            && heal_path.is_none_or(|path| heal_type_matches_path(&completed.heal_type, path))
-        {
-            return Ok(TaskStateLookup::Completed(Arc::new(completed)));
-        }
-        // A durable record that cannot run still answers with its reason.
-        if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
-            return Ok(TaskStateLookup::Unrunnable(reason));
-        }
-        Ok(TaskStateLookup::NotFound)
-    }
-
-    async fn lookup_memory_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
         let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
 
         {
@@ -2616,7 +2594,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|task| matches_path(&task.heal_type))
             {
-                return TaskStateLookup::Active(Arc::clone(task));
+                return Ok(TaskStateLookup::Active(Arc::clone(task)));
             }
         }
 
@@ -2626,7 +2604,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|retrying| matches_path(&retrying.request.heal_type))
             {
-                return TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone(), retrying.request.source);
+                return Ok(TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone()));
             }
         }
 
@@ -2639,7 +2617,7 @@ impl HealManager {
             prune_completed_heal_statuses(&mut completed_heals);
             if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
                 if completed_status_is_retrying(&completed.status) {
-                    return TaskStateLookup::Completed(Arc::clone(completed));
+                    return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
                 }
                 terminal_completed = Some(Arc::clone(completed));
             }
@@ -2648,7 +2626,7 @@ impl HealManager {
         {
             let queue = self.heal_queue.lock().await;
             if let Some(request) = queue.request_matching_id_and_path(canonical_task_id, heal_path) {
-                return TaskStateLookup::Queued(request.options.clone(), request.source);
+                return Ok(TaskStateLookup::Queued(request.options.clone()));
             }
         }
 
@@ -2661,10 +2639,22 @@ impl HealManager {
                 .cloned();
         }
 
-        if let Some(completed) = terminal_completed {
-            return TaskStateLookup::Completed(completed);
+        if terminal_completed.is_none()
+            && let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
+            && matches_path(&completed.heal_type)
+        {
+            terminal_completed = Some(Arc::new(completed));
         }
-        TaskStateLookup::NotFound
+
+        if let Some(completed) = terminal_completed {
+            return Ok(TaskStateLookup::Completed(completed));
+        }
+        // A durable root record that can neither run nor be quarantined still
+        // answers status queries with its reason.
+        if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
+            return Ok(TaskStateLookup::Unrunnable(reason));
+        }
+        Ok(TaskStateLookup::NotFound)
     }
 
     async fn publish_admin_terminal(
@@ -2688,8 +2678,6 @@ impl HealManager {
     ) -> Result<bool> {
         let previous = self.completed_heals.lock().await.get(task_id).cloned();
         let mut completed = CompletedHealStatus {
-            task_id: task_id.to_owned(),
-            source,
             outcome: None,
             progress: None,
             retained_bytes: std::sync::OnceLock::new(),
@@ -2725,9 +2713,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
-            TaskStateLookup::Retrying(status, _, _) => Ok(status),
+            TaskStateLookup::Retrying(status, _) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
-            TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
+            TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
             TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
             TaskStateLookup::NotFound => Err(Error::TaskNotFound {
                 task_id: task_id.to_string(),
@@ -2746,9 +2734,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
-            TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
+            TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
-            TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
+            TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
             TaskStateLookup::Unrunnable(error) => {
                 Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
             }
@@ -2762,42 +2750,6 @@ impl HealManager {
         self.get_task_report_for_path_since(heal_path, task_id, None).await
     }
 
-    /// Resolve a node-owned automatic task without granting local control over
-    /// Admin tasks. The source and report come from the same state snapshot.
-    pub async fn get_local_automatic_task_report(
-        &self,
-        heal_path: &str,
-        task_id: &str,
-        since: Option<u64>,
-    ) -> Result<Option<HealTaskReport>> {
-        let canonical_task_id = self.canonical_task_id(task_id).await;
-        if canonical_task_id != task_id {
-            return Ok(None);
-        }
-        let state = self.lookup_memory_task_state(&canonical_task_id, Some(heal_path)).await;
-        let source = match &state {
-            TaskStateLookup::Active(task) => task.source,
-            TaskStateLookup::Retrying(_, _, source) | TaskStateLookup::Queued(_, source) => *source,
-            TaskStateLookup::Completed(completed) if completed.task_id == task_id => completed.source,
-            TaskStateLookup::Completed(_) => return Ok(None),
-            TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
-        };
-        if !matches!(
-            source,
-            HealRequestSource::Scanner | HealRequestSource::Mrf | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
-        ) {
-            return Ok(None);
-        }
-        let report = match state {
-            TaskStateLookup::Active(task) => active_task_report(&task, since).await,
-            TaskStateLookup::Retrying(status, options, _) => empty_task_report_with_options(status, options),
-            TaskStateLookup::Completed(completed) => completed_task_report(&completed, since),
-            TaskStateLookup::Queued(options, _) => empty_task_report_with_options(HealTaskStatus::Pending, options),
-            TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
-        };
-        Ok(Some(report))
-    }
-
     /// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
     pub async fn get_task_report_for_path_since(
         &self,
@@ -2808,9 +2760,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
-            TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
+            TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
-            TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
+            TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
             TaskStateLookup::Unrunnable(error) => {
                 Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
             }
@@ -2834,9 +2786,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
-            TaskStateLookup::Retrying(status, _, _) => Ok(status),
+            TaskStateLookup::Retrying(status, _) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
-            TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
+            TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
             TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
             TaskStateLookup::NotFound => {
                 if self.path_has_task(heal_path).await {

crates/heal/src/heal/manager/queue.rs

@@ -109,9 +109,7 @@ pub(super) enum QueuePushOutcome {
 
 #[derive(Debug, Clone)]
 pub(super) struct CompletedHealStatus {
-    pub(super) task_id: String,
     pub(super) heal_type: HealType,
-    pub(super) source: HealRequestSource,
     /// Options used to execute the task, retained for token-scoped status.
     pub(super) options: HealOptions,
     pub(super) status: HealTaskStatus,
@@ -139,7 +137,6 @@ impl CompletedHealStatus {
     fn measure_retained_bytes(&self) -> usize {
         let mut bytes = size_of::<Self>();
         let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
-        add(self.task_id.capacity());
         add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
         match &self.heal_type {
             HealType::Cluster => {}
@@ -261,9 +258,7 @@ impl CompletedHealStatus {
             outcome.finish(Some(crate::heal::outcome::HealAbortReason::Cancelled));
         }
         let mut snapshot = Self {
-            task_id: task.id.clone(),
             heal_type: task.heal_type.clone(),
-            source: task.source,
             options: task.options.clone(),
             status,
             progress: Some(task.get_progress().await),

crates/heal/src/heal/manager/root_recovery.rs

@@ -460,8 +460,6 @@ impl RootHealTerminal {
 
     fn into_completed(self) -> CompletedHealStatus {
         CompletedHealStatus {
-            task_id: self.task_id,
-            source: HealRequestSource::Admin,
             outcome: None,
             progress: self.progress,
             retained_bytes: std::sync::OnceLock::new(),

crates/heal/src/heal/manager/tests.rs

@@ -118,8 +118,6 @@ struct MockStorage;
 
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
-        task_id: "fixture-token".to_string(),
-        source: HealRequestSource::Internal,
         outcome: None,
         heal_type: HealType::Cluster,
         options: HealOptions::default(),
@@ -3073,181 +3071,6 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
     );
 }
 
-#[tokio::test]
-async fn local_automatic_query_preserves_source_path_token_and_state() {
-    use crate::heal::channel::HealChannelProcessor;
-    use crate::heal::outcome::{
-        HealExecutionOutcome, HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome,
-        HealTraversalCoverage,
-    };
-
-    for source in [
-        HealRequestSource::Scanner,
-        HealRequestSource::Mrf,
-        HealRequestSource::AutoHeal,
-        HealRequestSource::ReadRepair,
-        HealRequestSource::Admin,
-        HealRequestSource::Internal,
-    ] {
-        for state in ["queued", "active", "retrying", "completed", "displaced"] {
-            let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
-            let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
-            let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
-            request.source = source;
-            let token = request.id.clone();
-            let task = Arc::new(HealTask::from_request(request.clone(), storage));
-            let mut snapshot = CompletedHealStatus::snapshot(&task, HealTaskStatus::Completed).await;
-            assert_eq!(snapshot.source, source, "completion must retain actual task source");
-            let mut outcome = HealTaskOutcome::default();
-            outcome.execution = HealExecutionOutcome::Completed;
-            outcome.coverage = HealTraversalCoverage::Complete;
-            outcome.record(HealObjectOutcome {
-                identity: HealObjectIdentity {
-                    kind: HealObjectKind::Object,
-                    bucket: "bucket".to_string(),
-                    object: "object".to_string(),
-                    version_id: None,
-                    bucket_incarnation_id: Some(uuid::Uuid::nil()),
-                    pool_index: Some(0),
-                    set_index: Some(0),
-                },
-                disposition: HealObjectDisposition::VerifiedHealthy,
-                detail: None,
-            });
-            snapshot.outcome = Some(Arc::new(outcome));
-            snapshot.seqed_items = vec![(
-                3,
-                HealResultItem {
-                    bucket: "bucket".to_string(),
-                    object: "object".to_string(),
-                    ..Default::default()
-                },
-            )];
-            snapshot.min_seq = 3;
-            snapshot.next_seq = 4;
-            let expected_status = match state {
-                "queued" => {
-                    manager.heal_queue.lock().await.push(request.clone());
-                    HealTaskStatus::Pending
-                }
-                "active" => {
-                    manager.active_heals.lock().await.insert(token.clone(), task);
-                    HealTaskStatus::Pending
-                }
-                "retrying" => {
-                    manager.retrying_heals.lock().await.insert(
-                        token.clone(),
-                        RetryingHeal {
-                            request: request.clone(),
-                            error: "transient".to_string(),
-                            cancel_token: CancellationToken::new(),
-                        },
-                    );
-                    HealTaskStatus::Retrying {
-                        error: "transient".to_string(),
-                        retry_attempt: request.retry_attempts,
-                    }
-                }
-                "completed" => {
-                    manager.completed_heals.lock().await.insert(token.clone(), Arc::new(snapshot));
-                    HealTaskStatus::Completed
-                }
-                "displaced" => {
-                    let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
-                    terminal.status.clone()
-                }
-                _ => unreachable!("fixed test states"),
-            };
-            let automatic = !matches!(source, HealRequestSource::Admin | HealRequestSource::Internal);
-            let report = manager
-                .get_local_automatic_task_report("bucket/object", &token, Some(2))
-                .await
-                .expect("local source lookup");
-            assert_eq!(report.is_some(), automatic, "source={source:?} state={state}");
-            if let Some(report) = report {
-                assert_eq!(report.status, expected_status);
-                if state == "completed" {
-                    let outcome = report.outcome.as_ref().expect("retained structured outcome");
-                    assert_eq!(outcome.objects.len(), 1);
-                    assert_eq!(outcome.objects[0].identity.object, "object");
-                    assert_eq!(outcome.objects[0].identity.pool_index, Some(0));
-                    assert_eq!(outcome.objects[0].identity.set_index, Some(0));
-                    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::VerifiedHealthy);
-                    assert_eq!(report.min_seq, 3);
-                    assert_eq!(report.next_seq, 4);
-                    assert_eq!(report.result_items.len(), 1);
-                    assert_eq!(report.result_items[0].object, "object");
-                }
-                let processor = HealChannelProcessor::new(manager.clone());
-                let local = processor
-                    .execute_local_automatic_query("bucket/object", &token, Some(2))
-                    .await
-                    .expect("local response")
-                    .expect("automatic task response");
-                let direct = processor
-                    .execute_query_request_since("bucket/object".to_string(), token.clone(), Some(2))
-                    .await
-                    .expect("original query response");
-                assert!(local.success);
-                assert_eq!(local.request_id, direct.request_id);
-                assert_eq!(local.data, direct.data, "same report encoding and cursor contract");
-                assert_eq!(local.error, direct.error);
-                let coordinator = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
-                let missing = HealChannelProcessor::new(coordinator)
-                    .execute_query_request("bucket/object".to_string(), token.clone())
-                    .await
-                    .expect("coordinator query");
-                let body: serde_json::Value =
-                    serde_json::from_slice(missing.data.as_deref().expect("coordinator response")).expect("coordinator JSON");
-                assert_eq!(body["summary"], "notFound", "a remote manager does not own the automatic task");
-                if state == "completed" {
-                    let completed = manager
-                        .completed_heals
-                        .lock()
-                        .await
-                        .get(&token)
-                        .cloned()
-                        .expect("original completion");
-                    manager
-                        .completed_heals
-                        .lock()
-                        .await
-                        .insert("retired-alias".to_string(), completed);
-                    assert!(
-                        manager
-                            .get_local_automatic_task_report("bucket/object", "retired-alias", None)
-                            .await
-                            .expect("retired alias lookup")
-                            .is_none()
-                    );
-                }
-            }
-            assert!(
-                manager
-                    .get_local_automatic_task_report("bucket/other", &token, None)
-                    .await
-                    .expect("wrong path lookup")
-                    .is_none()
-            );
-            assert!(
-                manager
-                    .get_local_automatic_task_report("bucket/object", "unknown", None)
-                    .await
-                    .expect("unknown token lookup")
-                    .is_none()
-            );
-            manager.insert_task_alias("admin-alias", &token).await;
-            assert!(
-                manager
-                    .get_local_automatic_task_report("bucket/object", "admin-alias", None)
-                    .await
-                    .expect("alias lookup")
-                    .is_none()
-            );
-        }
-    }
-}
-
 #[tokio::test]
 async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -4063,38 +3886,6 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab
     }
 }
 
-#[tokio::test]
-async fn admin_exhausted_batches_do_not_restart_the_traversal() {
-    for heal_type in [
-        HealType::Bucket {
-            bucket: "bucket".to_owned(),
-        },
-        HealType::Prefix {
-            bucket: "bucket".to_owned(),
-            prefix: "prefix/".to_owned(),
-        },
-        HealType::Cluster,
-    ] {
-        let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::Normal);
-        request.source = HealRequestSource::Admin;
-        let task = HealTask::from_request(request, Arc::new(MockStorage));
-        let result = Err(task
-            .record_batch_failure(BatchHealFailure {
-                scope: "completed-admin-traversal".to_owned(),
-                failed: 1,
-                retryable: 1,
-                permanent: 0,
-                first_object: "exhausted-object".to_owned(),
-                first_error: Error::Storage(EcstoreError::SlowDown).to_string(),
-            })
-            .await);
-        assert!(
-            retry_request_for_result_with_budget(&task, &result).await.is_none(),
-            "the exhausted admin batch must retain its original object receipts"
-        );
-    }
-}
-
 #[tokio::test]
 async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
     for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
@@ -4177,8 +3968,6 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: request.source,
-            task_id: request.id.clone(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type,
             options: request.options.clone(),
@@ -5027,8 +4816,6 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: request.source,
-            task_id: request.id.clone(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type.clone(),
             options: request.options.clone(),
@@ -5068,8 +4855,6 @@ async fn test_get_task_status_reads_recent_completed_status() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: HealRequestSource::Internal,
-            task_id: "completed-token".to_string(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Bucket {
                 bucket: "bucket".to_string(),
@@ -5103,8 +4888,6 @@ async fn test_get_task_report_for_path_reads_completed_items() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
-            source: HealRequestSource::Internal,
-            task_id: "completed-token".to_string(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Object {
                 bucket: "bucket".to_string(),

crates/heal/src/heal/manager/tests/root_recovery.rs

@@ -117,8 +117,6 @@ fn admin_request(heal_type: HealType) -> HealRequest {
 
 fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
-        task_id: "admin-fixture".to_string(),
-        source: HealRequestSource::Admin,
         outcome: None,
         heal_type: heal_type.clone(),
         options: HealOptions::default(),
@@ -1241,8 +1239,6 @@ async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() {
         .expect("durable bucket responsibility");
     let completed = CompletedHealStatus {
         outcome: None,
-        source: request.source,
-        task_id: request.id.clone(),
         heal_type: request.heal_type.clone(),
         options: request.options.clone(),
         status: HealTaskStatus::Completed,

crates/heal/src/heal/task.rs

@@ -37,7 +37,7 @@ use std::{
     collections::VecDeque,
     future::Future,
     sync::{
-        Arc, OnceLock,
+        Arc,
         atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
     },
     time::{Duration, Instant, SystemTime},
@@ -59,19 +59,6 @@ pub(crate) struct OutcomeFinishTestHook {
 pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
     std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
 
-#[cfg(test)]
-static STORAGE_RECEIPT_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
-    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
-
-#[cfg(test)]
-async fn pause_storage_receipt(task_id: &str) {
-    let hook = STORAGE_RECEIPT_TEST_HOOK.lock().await.clone();
-    if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
-        hook.reached.notify_one();
-        hook.release.notified().await;
-    }
-}
-
 #[cfg(test)]
 async fn pause_outcome_finish(task_id: &str) {
     let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
@@ -449,9 +436,6 @@ pub struct HealTask {
     replacement_start_retry_count: Arc<AtomicU32>,
     /// Set when the locked storage read classified the single target as absent.
     object_absent: Arc<AtomicBool>,
-    /// The first resolved incarnation belongs to this task even if storage
-    /// fails or the bucket is recreated before its terminal outcome is recorded.
-    single_object_incarnation: Arc<OnceLock<Option<Uuid>>>,
     /// Task status
     pub status: Arc<RwLock<HealTaskStatus>>,
     /// Progress tracking
@@ -524,7 +508,6 @@ impl HealTask {
             replacement_running: Arc::new(AtomicBool::new(false)),
             replacement_start_retry_count: Arc::new(AtomicU32::new(0)),
             object_absent: Arc::new(AtomicBool::new(false)),
-            single_object_incarnation: Arc::new(OnceLock::new()),
             status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
             progress: Arc::new(RwLock::new(HealProgress::new())),
             result_items: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_RETAINED_HEAL_RESULT_ITEMS))),
@@ -686,9 +669,7 @@ impl HealTask {
             } => (bucket, object, Some(version_id.as_str())),
             _ => return None,
         };
-        let mut identity = self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index);
-        identity.bucket_incarnation_id = self.single_object_incarnation.get().copied().flatten();
-        Some(identity)
+        Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
     }
 
     pub(super) async fn record_verified_storage_receipt(

crates/heal/src/heal/task/heal_bucket.rs

@@ -306,9 +306,9 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_stage(3, 3);
                 }
-                // Keep storage/quorum identity for the cluster retry classifier.
-                // The structured log above already carries the bucket context.
-                Err(e)
+                Err(Error::TaskExecutionFailed {
+                    message: format!("Failed to heal bucket {bucket}: {e}"),
+                })
             }
         }
     }
@@ -796,27 +796,21 @@ impl HealTask {
                                 match unavailable_recreate_error(&storage_result.item, &heal_opts) {
                                     Some(error) => Some(error),
                                     None => {
-                                        #[cfg(test)]
-                                        super::pause_storage_receipt(&self.id).await;
+                                        telemetry_unknown |= !increment_counter(&mut healed);
+                                        telemetry_unknown |= !add_bytes(
+                                            &mut bytes,
+                                            u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
+                                        );
                                         recorded_authoritative_outcome =
                                             self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
-                                        if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
-                                            Some(Error::TaskCancelled)
-                                        } else {
-                                            telemetry_unknown |= !increment_counter(&mut healed);
-                                            telemetry_unknown |= !add_bytes(
-                                                &mut bytes,
-                                                u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
-                                            );
-                                            if !recorded_authoritative_outcome
-                                                && !heal_opts.dry_run
-                                                && !storage_result.item.detail.is_empty()
-                                            {
-                                                detail = Some(storage_result.item.detail.clone());
-                                            }
-                                            self.record_result_item(storage_result.item).await;
-                                            None
+                                        if !recorded_authoritative_outcome
+                                            && !heal_opts.dry_run
+                                            && !storage_result.item.detail.is_empty()
+                                        {
+                                            detail = Some(storage_result.item.detail.clone());
                                         }
+                                        self.record_result_item(storage_result.item).await;
+                                        None
                                     }
                                 }
                             }

crates/heal/src/heal/task/heal_object.rs

@@ -220,11 +220,6 @@ impl HealTask {
         let mut expected_identity =
             self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
         expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
-        if matches!(self.heal_type, HealType::Object { .. }) {
-            expected_identity.bucket_incarnation_id = *self
-                .single_object_incarnation
-                .get_or_init(|| expected_identity.bucket_incarnation_id);
-        }
 
         let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
         let heal_result = if self.source == HealRequestSource::ReadRepair {

crates/heal/src/heal/task/tests.rs

@@ -305,53 +305,6 @@ mod canonical_outcome {
         assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
     }
 
-    #[tokio::test]
-    async fn bucket_cancellation_after_storage_success_keeps_typed_cancelled_responsibility() {
-        let incarnation = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            heal_object_receipts: Mutex::new(HashMap::from([(
-                "object-a".to_string(),
-                VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
-            )])),
-            bucket_incarnation_id: Mutex::new(Some(incarnation)),
-            ..Default::default()
-        });
-        let task = Arc::new(bucket_task(storage.clone()));
-        let hook = Arc::new(OutcomeFinishTestHook {
-            task_id: task.id.clone(),
-            reached: tokio::sync::Notify::new(),
-            release: tokio::sync::Notify::new(),
-        });
-        *STORAGE_RECEIPT_TEST_HOOK.lock().await = Some(hook.clone());
-        let runner = {
-            let task = task.clone();
-            tokio::spawn(async move { task.execute().await })
-        };
-        tokio::time::timeout(Duration::from_secs(10), hook.reached.notified())
-            .await
-            .expect("storage success must reach the receipt boundary");
-        assert_eq!(storage.healed_objects.lock().expect("storage mutations").as_slice(), ["object-a"]);
-        task.cancel().await.expect("cancel before receipt recording");
-        hook.release.notify_one();
-        let result = tokio::time::timeout(Duration::from_secs(10), runner)
-            .await
-            .expect("cancelled worker must stop")
-            .expect("worker must not panic");
-        STORAGE_RECEIPT_TEST_HOOK.lock().await.take();
-        assert!(matches!(result, Err(Error::TaskCancelled)));
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Cancelled));
-        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
-        assert_eq!(outcome.counters.processed, 1);
-        assert_eq!(outcome.counters.healed, 0);
-        assert_eq!(outcome.counters.unchanged, 0);
-        assert_eq!(outcome.counters.unknown, 0);
-        assert_eq!(outcome.counters.skipped, 1);
-        assert_eq!(outcome.objects.len(), 1);
-        assert_eq!(outcome.objects[0].identity.object, "object-a");
-        assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
-    }
-
     #[tokio::test]
     async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
         let expected_incarnation = Uuid::new_v4();
@@ -856,85 +809,6 @@ mod canonical_outcome {
         );
     }
 
-    #[tokio::test(start_paused = true)]
-    async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() {
-        for (failures, succeeds) in [(1, true), (4, false)] {
-            let storage = Arc::new(MockStorage {
-                listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
-                typed_bucket_heal_errors: Mutex::new(HashMap::from([(
-                    "bucket-a".to_string(),
-                    (0..failures)
-                        .map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum))
-                        .collect(),
-                )])),
-                ..Default::default()
-            });
-            let task = HealTask::from_request(
-                HealRequest::new(
-                    HealType::Cluster,
-                    HealOptions {
-                        recursive: true,
-                        timeout: None,
-                        ..Default::default()
-                    },
-                    HealPriority::Normal,
-                ),
-                storage.clone(),
-            );
-            let result = task.execute().await;
-            assert_eq!(
-                result.is_ok(),
-                succeeds,
-                "original cluster task must honor its bounded bucket retry budget"
-            );
-            let outcome = task.get_outcome().await;
-            if succeeds {
-                assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
-                assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
-            } else {
-                assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
-                assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
-                let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure");
-                assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
-            }
-            let calls = storage.bucket_heal_calls.lock().expect("original bucket calls");
-            assert_eq!(
-                calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(),
-                if succeeds { 2 } else { 4 }
-            );
-            assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited");
-        }
-    }
-
-    #[tokio::test]
-    async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() {
-        for error in [
-            Error::Storage(EcstoreError::ErasureWriteQuorum),
-            Error::Disk(DiskError::FileAccessDenied),
-        ] {
-            let retryable = error.is_recoverable_heal();
-            let storage = Arc::new(MockStorage {
-                typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])),
-                ..Default::default()
-            });
-            let task = bucket_task(storage.clone());
-            let observed = task
-                .heal_bucket("bucket-a")
-                .await
-                .expect_err("bucket prepass must retain the storage error");
-            assert!(matches!(
-                observed,
-                Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied)
-            ));
-            assert_eq!(observed.is_recoverable_heal(), retryable);
-            assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
-            assert!(
-                storage.heal_object_calls.lock().expect("object calls").is_empty(),
-                "failed prepass must not heal objects"
-            );
-        }
-    }
-
     #[tokio::test(start_paused = true)]
     async fn cancellation_and_deadline_leave_partial_coverage() {
         for cancel in [false, true] {
@@ -1693,7 +1567,6 @@ struct MockStorage {
     listed_buckets: Mutex<Option<Vec<String>>>,
     bucket_discovery_incomplete: bool,
     bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
-    typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
     bucket_heal_calls: Mutex<Vec<String>>,
     delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
     delete_recovery_calls: Mutex<Vec<String>>,
@@ -1942,127 +1815,6 @@ async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
                 | HealObjectDisposition::AuthoritativelyAbsent
         )
     }));
-    assert_eq!(outcome.objects.len(), 1);
-    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
-}
-
-#[tokio::test]
-async fn object_failure_retains_incarnation_for_inner_and_outer_storage_errors() {
-    for failure in [
-        MockHealObjectOutcome::PermissionDenied,
-        MockHealObjectOutcome::RetryableReadQuorum,
-        MockHealObjectOutcome::RetryableLockTimeout,
-        MockHealObjectOutcome::ErrOther("backend failed"),
-    ] {
-        let incarnation = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            bucket_incarnation_id: Mutex::new(Some(incarnation)),
-            heal_object_outcome: Mutex::new(Some(failure)),
-            ..Default::default()
-        });
-        let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), Some("version-a".to_owned()));
-        request.options.pool_index = Some(2);
-        request.options.set_index = Some(3);
-        let task = HealTask::from_request(request, storage);
-        task.execute()
-            .await
-            .expect_err("the failed storage operation must remain failed");
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.objects.len(), 1);
-        assert_eq!(
-            (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
-            (1, 1, 1)
-        );
-        assert_eq!((outcome.counters.healed, outcome.counters.unchanged, outcome.counters.unknown), (0, 0, 0));
-        let object = &outcome.objects[0];
-        assert_eq!(object.identity.bucket_incarnation_id, Some(incarnation));
-        assert_eq!(object.identity.bucket, "bucket-a");
-        assert_eq!(object.identity.object, "object-a");
-        assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
-        assert_eq!((object.identity.pool_index, object.identity.set_index), (Some(2), Some(3)));
-        assert!(matches!(object.disposition, HealObjectDisposition::Failed(_)));
-    }
-}
-
-#[tokio::test(start_paused = true)]
-async fn object_abort_keeps_pre_storage_incarnation_after_bucket_recreation() {
-    for cancel in [false, true] {
-        let original = Uuid::new_v4();
-        let storage = Arc::new(MockStorage {
-            bucket_incarnation_id: Mutex::new(Some(original)),
-            block_heal_object: Mutex::new(true),
-            ..Default::default()
-        });
-        let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
-        request.options.timeout = Some(Duration::from_secs(10));
-        let task = HealTask::from_request(request, storage.clone());
-        let execution = task.execute();
-        tokio::pin!(execution);
-        assert!(futures::poll!(&mut execution).is_pending());
-        assert_eq!(storage.heal_object_calls.lock().expect("storage boundary").as_slice(), ["object-a"]);
-        *storage.bucket_incarnation_id.lock().expect("recreated bucket") = Some(Uuid::new_v4());
-        if cancel {
-            task.cancel().await.expect("cancel blocked storage");
-        } else {
-            tokio::time::advance(Duration::from_secs(11)).await;
-        }
-        let result = execution.await;
-        assert!(matches!(
-            (&result, cancel),
-            (Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
-        ));
-        let outcome = task.get_outcome().await;
-        assert_eq!(outcome.objects.len(), 1);
-        assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
-        assert_eq!((outcome.counters.processed, outcome.counters.skipped), (1, 1));
-        assert_eq!((outcome.counters.failed, outcome.counters.healed, outcome.counters.unknown), (0, 0, 0));
-        assert_eq!(
-            outcome.objects[0].disposition,
-            if cancel {
-                HealObjectDisposition::Cancelled
-            } else {
-                HealObjectDisposition::Deferred {
-                    reason: HealDeferredReason::Deadline,
-                    retry_not_before: None,
-                }
-            }
-        );
-    }
-}
-
-#[tokio::test]
-async fn object_unverified_outcome_keeps_original_incarnation_without_claiming_repair() {
-    let original = Uuid::new_v4();
-    let successor = Uuid::new_v4();
-    let storage = Arc::new(MockStorage {
-        bucket_incarnation_id: Mutex::new(Some(original)),
-        bucket_incarnation_after_object_heal: Mutex::new(Some(successor)),
-        ..Default::default()
-    });
-    let task = HealTask::from_request(HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None), storage.clone());
-    task.execute().await.expect("legacy storage keeps its unverified result");
-    let outcome = task.get_outcome().await;
-    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
-    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
-    assert_eq!((outcome.counters.healed, outcome.counters.unknown), (0, 1));
-    assert_eq!(*storage.bucket_incarnation_id.lock().expect("successor"), Some(successor));
-}
-
-#[tokio::test]
-async fn object_dry_run_does_not_adopt_bucket_incarnation() {
-    let storage = Arc::new(MockStorage {
-        bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
-        ..Default::default()
-    });
-    let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
-    request.options.dry_run = true;
-    let task = HealTask::from_request(request, storage);
-    task.execute().await.expect("dry-run observation");
-    let outcome = task.get_outcome().await;
-    assert_eq!(outcome.objects.len(), 1);
-    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, None);
-    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
-    assert_eq!(outcome.counters.healed, 0);
 }
 
 #[tokio::test]
@@ -2635,15 +2387,6 @@ impl HealStorageAPI for MockStorage {
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
         self.bucket_heal_opts.lock().unwrap().push(*opts);
-        if let Some(error) = self
-            .typed_bucket_heal_errors
-            .lock()
-            .unwrap()
-            .get_mut(bucket)
-            .and_then(VecDeque::pop_front)
-        {
-            return Err(error);
-        }
         if let Some(message) = self
             .bucket_heal_errors
             .lock()

crates/heal/tests/shard_identity_receipt_test.rs

@@ -550,159 +550,3 @@ fn admin_normal_protected_repairs_report_repaired_for_data_and_parity() {
         .join()
         .expect("heal fixture result");
 }
-
-#[test]
-#[serial]
-fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
-    std::thread::Builder::new()
-        .stack_size(32 * 1024 * 1024)
-        .spawn(|| {
-            tokio::runtime::Builder::new_current_thread()
-                .enable_all()
-                .build()
-                .expect("mixed batch runtime")
-                .block_on(async {
-                    use rustfs_heal::heal::{
-                        manager::{HealConfig, HealManager},
-                        outcome::{HealExecutionOutcome, HealFailureClass, HealTraversalCoverage},
-                        task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
-                    };
-                    use rustfs_heal_contracts::heal_channel::HealRequestSource;
-                    use std::sync::Arc;
-
-                    let root = tempfile::tempdir().expect("mixed batch fixture");
-                    let env = TestECStoreEnv::builder().base_dir(root.path()).build().await;
-                    let bucket = "admin-mixed-batch";
-                    env.make_bucket(bucket, false).await;
-                    let set = env.ecstore.pools[0].get_disks(0);
-                    let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
-                    let mut originals = Vec::new();
-                    for (object, missing, byte) in [("healthy", 0, 0x11), ("repaired", 1, 0x52), ("failed", 3, 0xa3)] {
-                        env.ecstore
-                            .put_object(
-                                bucket,
-                                object,
-                                &mut PutObjReader::from_vec(vec![byte; 5 * 1024 * 1024 + 123]),
-                                &ObjectOptions {
-                                    shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
-                                    write_completion: WriteCompletion::TailDrained,
-                                    ..Default::default()
-                                },
-                            )
-                            .await
-                            .expect("commit complete fixture");
-                        for (slot, disk) in disks.iter().enumerate() {
-                            let meta = disk
-                                .read_version("", bucket, object, "", &ReadOptions::default())
-                                .await
-                                .expect("fixture metadata");
-                            assert_eq!(meta.erasure.data_blocks, 2, "failed fixture must retain R-1 shards");
-                            let path = env.disk_paths[slot]
-                                .join(bucket)
-                                .join(object)
-                                .join(meta.data_dir.expect("external shard").to_string())
-                                .join("part.1");
-                            let bytes = tokio::fs::read(&path).await.expect("original part");
-                            if slot < missing {
-                                tokio::fs::remove_file(&path).await.expect("inject missing part");
-                            }
-                            originals.push((object, path, bytes, slot < missing));
-                        }
-                    }
-                    let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
-                    let manager = HealManager::new(
-                        storage.clone(),
-                        Some(HealConfig {
-                            enable_auto_heal: false,
-                            ..Default::default()
-                        }),
-                    );
-                    manager.start().await.expect("start mixed batch manager");
-                    let mut request = HealRequest::new(
-                        HealType::Bucket {
-                            bucket: bucket.to_owned(),
-                        },
-                        HealOptions {
-                            recursive: true,
-                            scan_mode: HealScanMode::Deep,
-                            pool_index: Some(0),
-                            set_index: Some(0),
-                            timeout: Some(Duration::from_secs(150)),
-                            ..Default::default()
-                        },
-                        HealPriority::High,
-                    );
-                    request.source = HealRequestSource::Admin;
-                    let token = request.id.clone();
-                    manager.submit_heal_request(request).await.expect("admit original mixed task");
-                    let report = tokio::time::timeout(Duration::from_secs(150), async {
-                        loop {
-                            let report = manager.get_task_report(&token).await.expect("same-token report");
-                            if matches!(
-                                report.status,
-                                HealTaskStatus::Completed | HealTaskStatus::Failed { .. } | HealTaskStatus::Timeout
-                            ) {
-                                break report;
-                            }
-                            tokio::time::sleep(Duration::from_millis(20)).await;
-                        }
-                    })
-                    .await
-                    .expect("mixed task terminal deadline");
-                    manager.stop().await.expect("stop mixed task manager");
-                    for (object, path, original, missing) in originals {
-                        if object == "failed" && missing {
-                            assert!(!path.exists(), "below-R failure must not invent a shard");
-                        } else {
-                            assert_eq!(tokio::fs::read(&path).await.expect("preserved or repaired shard"), original);
-                        }
-                    }
-                    assert!(matches!(report.status, HealTaskStatus::Failed { .. }), "{report:?}");
-                    let outcome = report.outcome.as_ref().expect("canonical mixed outcome");
-                    assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
-                    assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
-                    assert_eq!(
-                        (
-                            outcome.counters.processed,
-                            outcome.counters.healed,
-                            outcome.counters.unchanged,
-                            outcome.counters.failed,
-                            outcome.counters.skipped,
-                            outcome.counters.unknown,
-                            outcome.counters.attempt_failures
-                        ),
-                        (3, 1, 1, 1, 0, 0, 4),
-                        "{outcome:?}"
-                    );
-                    for (object, disposition) in [
-                        ("healthy", HealObjectDisposition::VerifiedHealthy),
-                        ("repaired", HealObjectDisposition::Repaired),
-                        ("failed", HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)),
-                    ] {
-                        let receipt = outcome
-                            .objects
-                            .iter()
-                            .find(|item| item.identity.object == object)
-                            .expect("exact object receipt");
-                        assert_eq!(receipt.disposition, disposition);
-                    }
-                    drop(manager);
-                    let restarted = HealManager::new(
-                        storage,
-                        Some(HealConfig {
-                            enable_auto_heal: false,
-                            ..Default::default()
-                        }),
-                    );
-                    restarted.start().await.expect("restart with retained failed batch");
-                    let restored = restarted.get_task_report(&token).await.expect("original token after restart");
-                    assert_eq!(restored.status, report.status);
-                    assert_eq!(restored.outcome, report.outcome);
-                    assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
-                    restarted.stop().await.expect("stop restarted mixed batch manager");
-                });
-        })
-        .expect("mixed batch thread")
-        .join()
-        .expect("mixed batch result");
-}

crates/obs/src/telemetry/isolation_probe_tests.rs

@@ -0,0 +1,811 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Test-only diagnostic isolation prototype. Nothing here is a GET hook.
+
+use super::local::build_json_log_layer;
+use opentelemetry::{
+    Context, InstrumentationScope, KeyValue,
+    baggage::BaggageExt,
+    logs::AnyValue,
+    trace::{SpanContext, SpanId, TraceContextExt, TraceFlags, TraceId, TraceState},
+};
+use opentelemetry_appender_tracing::layer::{OpenTelemetryTracingBridge, TracingSpanAttributes};
+use opentelemetry_sdk::{
+    Resource,
+    error::OTelSdkResult,
+    logs::{LogBatch, LogExporter, SdkLogRecord, SdkLoggerProvider},
+};
+use serde_json::Value;
+use std::{
+    collections::BTreeSet,
+    io::{self, Write},
+    sync::{Arc, Mutex},
+};
+use tracing::{Dispatch, dispatcher};
+use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt};
+
+const TARGET: &str = "rustfs_isolation_probe_test";
+const FAKE_URI: &str = "/synthetic/private-object";
+const FAKE_AUTH: &str = "synthetic-private-header";
+
+#[derive(Clone, Debug, Default)]
+struct MemoryWriter(Arc<Mutex<Vec<u8>>>);
+
+impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for MemoryWriter {
+    type Writer = Self;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        self.clone()
+    }
+}
+
+impl Write for MemoryWriter {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.0.lock().expect("capture JSON output").extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Default)]
+struct MemoryExporter {
+    records: Arc<Mutex<Vec<(SdkLogRecord, InstrumentationScope)>>>,
+    resource: Arc<Mutex<Option<Resource>>>,
+}
+
+impl LogExporter for MemoryExporter {
+    async fn export(&self, batch: LogBatch<'_>) -> OTelSdkResult {
+        self.records
+            .lock()
+            .expect("capture exporter records")
+            .extend(batch.iter().map(|(record, scope)| (record.clone(), scope.clone())));
+        Ok(())
+    }
+
+    fn set_resource(&mut self, resource: &Resource) {
+        *self.resource.lock().expect("capture exporter resource") = Some(resource.clone());
+    }
+}
+
+struct Sinks {
+    dispatch: Dispatch,
+    json: MemoryWriter,
+    exporter: MemoryExporter,
+    _provider: SdkLoggerProvider,
+}
+
+impl Sinks {
+    fn new(span_list: bool, enrich_spans: bool) -> Self {
+        let json = MemoryWriter::default();
+        let exporter = MemoryExporter::default();
+        let provider = SdkLoggerProvider::builder()
+            .with_resource(Resource::builder_empty().build())
+            .with_simple_exporter(exporter.clone())
+            .build();
+        let bridge = OpenTelemetryTracingBridge::builder(&provider);
+        let bridge = if enrich_spans {
+            bridge.with_tracing_span_attributes(TracingSpanAttributes::all()).build()
+        } else {
+            bridge.build()
+        };
+        let subscriber = tracing_subscriber::registry()
+            .with(build_json_log_layer(json.clone(), false, FmtSpan::NONE, span_list))
+            .with(bridge);
+        Self {
+            dispatch: Dispatch::new(subscriber),
+            json,
+            exporter,
+            _provider: provider,
+        }
+    }
+
+    fn json_records(&self) -> Vec<Value> {
+        let bytes = self.json.0.lock().expect("capture JSON output").clone();
+        let text = String::from_utf8(bytes).expect("JSON output is UTF-8");
+        text.lines()
+            .map(|line| serde_json::from_str(line).expect("each captured record is JSON"))
+            .collect()
+    }
+
+    fn exported_records(&self) -> Vec<(SdkLogRecord, InstrumentationScope)> {
+        self.exporter.records.lock().expect("capture exporter records").clone()
+    }
+}
+
+fn fake_ambient_context() -> Context {
+    Context::new()
+        .with_remote_span_context(SpanContext::new(
+            TraceId::from(17),
+            SpanId::from(19),
+            TraceFlags::SAMPLED,
+            true,
+            TraceState::default(),
+        ))
+        .with_baggage([KeyValue::new("synthetic-secret", FAKE_AUTH)])
+}
+
+fn emit_root_event(phase: u64) {
+    tracing::info!(target: TARGET, parent: None, expected = 1024_u64, emitted = 64_u64, phase, "diagnostic fixture");
+}
+
+// This prototype is synchronous and must use a fresh, span-free dispatcher.
+fn emit_candidate(dispatch: &Dispatch, phase: u64) {
+    let _context_guard = Context::new().attach();
+    dispatcher::with_default(dispatch, || {
+        assert!(
+            Context::current().baggage().get("synthetic-secret").is_none(),
+            "ambient baggage must be hidden"
+        );
+        assert!(!Context::current().has_active_span(), "ambient OTEL span must be hidden");
+        emit_root_event(phase);
+    });
+}
+
+#[test]
+fn parent_none_alone_keeps_json_parent_and_otel_thread_context() {
+    let sinks = Sinks::new(true, true);
+    let _context_guard = fake_ambient_context().attach();
+
+    dispatcher::with_default(&sinks.dispatch, || {
+        let span = tracing::info_span!("http-request", uri = FAKE_URI, authorization = FAKE_AUTH, request_id = "synthetic-id");
+        let _span_guard = span.enter();
+        emit_root_event(0);
+    });
+
+    let json = sinks.json_records();
+    let logs = sinks.exported_records();
+    assert_eq!(json.len(), 1, "the control event must be captured");
+    assert_eq!(logs.len(), 1, "the control event must reach the exporter");
+    assert_eq!(json[0]["span"]["uri"].as_str(), Some(FAKE_URI));
+    assert_eq!(json[0]["span"]["authorization"].as_str(), Some(FAKE_AUTH));
+    let context = logs[0].0.trace_context().expect("ambient OTEL trace context is present");
+    assert_eq!(context.trace_id, TraceId::from(17));
+    assert_eq!(context.span_id, SpanId::from(19));
+}
+
+#[test]
+fn diagnostic_fixture_is_detached_for_both_sinks_and_restores_context() {
+    for span_list in [false, true] {
+        for enrich_spans in [false, true] {
+            let outer = Sinks::new(span_list, enrich_spans);
+            let isolated = Sinks::new(span_list, enrich_spans);
+            let _context_guard = fake_ambient_context().attach();
+
+            dispatcher::with_default(&outer.dispatch, || {
+                let span =
+                    tracing::info_span!("http-request", uri = FAKE_URI, authorization = FAKE_AUTH, request_id = "synthetic-id");
+                let _span_guard = span.enter();
+                for phase in [0, 1] {
+                    emit_candidate(&isolated.dispatch, phase);
+                    assert_eq!(tracing::Span::current().id(), span.id(), "restore the outer tracing span");
+                    assert_eq!(Context::current().span().span_context().trace_id(), TraceId::from(17));
+                    assert!(Context::current().baggage().get("synthetic-secret").is_some());
+                }
+            });
+
+            assert!(outer.json_records().is_empty(), "diagnostic events must not reach the outer JSON sink");
+            assert!(outer.exported_records().is_empty(), "diagnostic events must not reach the outer exporter");
+            let json = isolated.json_records();
+            let logs = isolated.exported_records();
+            assert_eq!(json.len(), 2, "both diagnostic events must reach the isolated JSON sink");
+            assert_eq!(logs.len(), 2, "both diagnostic events must reach the isolated exporter");
+
+            let json_keys: BTreeSet<&str> = [
+                "timestamp",
+                "level",
+                "target",
+                "filename",
+                "line_number",
+                "threadName",
+                "threadId",
+                "message",
+                "expected",
+                "emitted",
+                "phase",
+            ]
+            .into_iter()
+            .collect();
+            let otel_keys: BTreeSet<&str> = [
+                "code.namespace",
+                "code.filepath",
+                "code.filename",
+                "code.lineno",
+                "expected",
+                "emitted",
+                "phase",
+            ]
+            .into_iter()
+            .collect();
+            let resource = isolated.exporter.resource.lock().expect("capture exporter resource");
+            assert!(
+                resource
+                    .as_ref()
+                    .expect("resource was passed to exporter")
+                    .iter()
+                    .next()
+                    .is_none()
+            );
+            for (phase, (entry, (log, scope))) in [0_u64, 1].into_iter().zip(json.iter().zip(&logs)) {
+                let object = entry.as_object().expect("diagnostic JSON must be an object");
+                assert!(object.keys().all(|key| json_keys.contains(key.as_str())), "unexpected JSON field");
+                assert_eq!(entry["target"].as_str(), Some(TARGET));
+                assert_eq!(entry["phase"].as_u64(), Some(phase));
+                assert_eq!(entry["expected"].as_u64(), Some(1024));
+                assert_eq!(entry["emitted"].as_u64(), Some(64));
+                assert!(!entry.to_string().contains(FAKE_URI));
+                assert!(!entry.to_string().contains(FAKE_AUTH));
+
+                assert_eq!(log.target().map(|target| target.as_ref()), Some(TARGET));
+                assert!(log.trace_context().is_none(), "ambient OTEL context must not reach the exporter");
+                assert_eq!(scope.name(), "");
+                assert!(scope.version().is_none());
+                assert!(scope.schema_url().is_none());
+                assert!(scope.attributes().next().is_none());
+                assert!(
+                    log.attributes_iter().all(|(key, _)| otel_keys.contains(key.as_str())),
+                    "unexpected OTEL field"
+                );
+                let phase = i64::try_from(phase).expect("fixture phase fits into i64");
+                assert!(
+                    log.attributes_iter()
+                        .any(|(key, value)| key.as_str() == "phase" && value == &AnyValue::Int(phase))
+                );
+                assert!(
+                    log.attributes_iter()
+                        .any(|(key, value)| key.as_str() == "expected" && value == &AnyValue::Int(1024))
+                );
+                assert!(
+                    log.attributes_iter()
+                        .any(|(key, value)| key.as_str() == "emitted" && value == &AnyValue::Int(64))
+                );
+                assert!(!format!("{log:?}").contains(FAKE_URI));
+                assert!(!format!("{log:?}").contains(FAKE_AUTH));
+            }
+        }
+    }
+}
+
+mod async_isolation {
+    use super::*;
+    use std::{
+        collections::HashSet,
+        future::pending,
+        panic::{AssertUnwindSafe, catch_unwind},
+        sync::atomic::{AtomicBool, AtomicUsize, Ordering},
+        time::Duration,
+    };
+    use tracing::{Instrument, instrument::WithSubscriber};
+    use tracing_subscriber::EnvFilter;
+
+    const PARTICIPANTS: usize = 4;
+    const WAIT: Duration = Duration::from_secs(30);
+
+    fn parent_span(dispatch: &Dispatch) -> tracing::Span {
+        dispatcher::with_default(dispatch, || {
+            tracing::info_span!("http-request", uri = FAKE_URI, authorization = FAKE_AUTH, request_id = "synthetic-id")
+        })
+    }
+
+    fn observe_once(dispatch: &Dispatch, parent: &Option<tracing::span::Id>, phase: u64) {
+        let _context = fake_ambient_context().attach();
+        assert_eq!(&tracing::Span::current().id(), parent);
+        emit_candidate(dispatch, phase);
+        assert_eq!(&tracing::Span::current().id(), parent);
+        assert_eq!(Context::current().span().span_context().trace_id(), TraceId::from(17));
+        assert!(Context::current().baggage().get("synthetic-secret").is_some());
+    }
+
+    fn assert_isolated_events(sinks: &Sinks, expected_phases: &[u64]) {
+        let json = sinks.json_records();
+        let logs = sinks.exported_records();
+        assert_eq!(json.len(), expected_phases.len(), "each event must reach the isolated JSON sink");
+        assert_eq!(logs.len(), expected_phases.len(), "each event must reach the isolated exporter");
+        let json_keys: BTreeSet<&str> = [
+            "timestamp",
+            "level",
+            "target",
+            "filename",
+            "line_number",
+            "threadName",
+            "threadId",
+            "message",
+            "expected",
+            "emitted",
+            "phase",
+        ]
+        .into_iter()
+        .collect();
+        let otel_keys: BTreeSet<&str> = [
+            "code.namespace",
+            "code.filepath",
+            "code.filename",
+            "code.lineno",
+            "expected",
+            "emitted",
+            "phase",
+        ]
+        .into_iter()
+        .collect();
+        let mut json_phases = Vec::new();
+        for entry in &json {
+            let object = entry.as_object().expect("diagnostic JSON must be an object");
+            assert!(object.keys().all(|key| json_keys.contains(key.as_str())), "unexpected JSON field");
+            assert_eq!(entry["target"].as_str(), Some(TARGET));
+            assert_eq!(entry["message"].as_str(), Some("diagnostic fixture"));
+            assert_eq!(entry["expected"].as_u64(), Some(1024));
+            assert_eq!(entry["emitted"].as_u64(), Some(64));
+            json_phases.push(entry["phase"].as_u64().expect("phase is an unsigned number"));
+            assert!(!entry.to_string().contains(FAKE_URI));
+            assert!(!entry.to_string().contains(FAKE_AUTH));
+        }
+        let mut otel_phases = Vec::new();
+        for (log, scope) in &logs {
+            assert_eq!(log.target().map(|target| target.as_ref()), Some(TARGET));
+            assert!(log.trace_context().is_none());
+            assert!(log.attributes_iter().all(|(key, _)| otel_keys.contains(key.as_str())));
+            assert!(matches!(log.body(), Some(AnyValue::String(body)) if body.as_str() == "diagnostic fixture"));
+            assert!(
+                log.attributes_iter()
+                    .any(|(key, value)| key.as_str() == "expected" && value == &AnyValue::Int(1024))
+            );
+            assert!(
+                log.attributes_iter()
+                    .any(|(key, value)| key.as_str() == "emitted" && value == &AnyValue::Int(64))
+            );
+            let phase = log
+                .attributes_iter()
+                .find_map(|(key, value)| match (key.as_str(), value) {
+                    ("phase", AnyValue::Int(phase)) => u64::try_from(*phase).ok(),
+                    _ => None,
+                })
+                .expect("phase is a nonnegative OTEL integer");
+            otel_phases.push(phase);
+            assert_eq!(scope.name(), "");
+            assert!(scope.version().is_none());
+            assert!(scope.schema_url().is_none());
+            assert!(scope.attributes().next().is_none());
+            assert!(!format!("{log:?}").contains(FAKE_URI));
+            assert!(!format!("{log:?}").contains(FAKE_AUTH));
+        }
+        json_phases.sort_unstable();
+        otel_phases.sort_unstable();
+        let mut expected_phases = expected_phases.to_vec();
+        expected_phases.sort_unstable();
+        assert_eq!(json_phases, expected_phases);
+        assert_eq!(otel_phases, expected_phases);
+        let resource = sinks.exporter.resource.lock().expect("capture exporter resource");
+        assert!(resource.as_ref().expect("exporter resource exists").iter().next().is_none());
+    }
+
+    #[test]
+    fn separate_threads_keep_their_contexts_across_async_suspension() {
+        let outer = Sinks::new(true, true);
+        let isolated = Sinks::new(true, true);
+        let barrier = Arc::new(tokio::sync::Barrier::new(PARTICIPANTS));
+        let thread_ids: HashSet<_> = std::thread::scope(|scope| {
+            let mut handles = Vec::new();
+            for _ in 0..PARTICIPANTS {
+                let outer_dispatch = outer.dispatch.clone();
+                let isolated_dispatch = isolated.dispatch.clone();
+                let barrier = Arc::clone(&barrier);
+                handles.push(scope.spawn(move || {
+                    let thread_id = std::thread::current().id();
+                    let span = parent_span(&outer_dispatch);
+                    let span_id = span.id();
+                    let runtime = tokio::runtime::Builder::new_current_thread()
+                        .enable_all()
+                        .build()
+                        .expect("build a test-only runtime");
+                    runtime.block_on(
+                        async move {
+                            for phase in [0_u64, 1] {
+                                tokio::time::timeout(WAIT, barrier.wait())
+                                    .await
+                                    .expect("all threads reach the barrier");
+                                observe_once(&isolated_dispatch, &span_id, phase);
+                                tokio::task::yield_now().await;
+                            }
+                        }
+                        .instrument(span)
+                        .with_subscriber(outer_dispatch),
+                    );
+                    thread_id
+                }));
+            }
+            handles
+                .into_iter()
+                .map(|handle| handle.join().expect("thread completes"))
+                .collect()
+        });
+        assert_eq!(thread_ids.len(), PARTICIPANTS, "the fixture must use distinct OS threads");
+        assert!(outer.json_records().is_empty());
+        assert!(outer.exported_records().is_empty());
+        assert_isolated_events(&isolated, &[0, 1, 0, 1, 0, 1, 0, 1]);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn shared_runtime_reenters_outer_context_after_each_await() {
+        let outer = Sinks::new(true, true);
+        let isolated = Sinks::new(true, true);
+        let barrier = Arc::new(tokio::sync::Barrier::new(PARTICIPANTS));
+        let mut tasks = Vec::new();
+        for _ in 0..PARTICIPANTS {
+            let outer_dispatch = outer.dispatch.clone();
+            let isolated_dispatch = isolated.dispatch.clone();
+            let barrier = Arc::clone(&barrier);
+            let span = parent_span(&outer_dispatch);
+            let span_id = span.id();
+            tasks.push(tokio::spawn(
+                async move {
+                    for phase in [0_u64, 1] {
+                        tokio::time::timeout(WAIT, barrier.wait())
+                            .await
+                            .expect("all tasks reach the barrier");
+                        observe_once(&isolated_dispatch, &span_id, phase);
+                        tokio::task::yield_now().await;
+                    }
+                }
+                .instrument(span)
+                .with_subscriber(outer_dispatch),
+            ));
+        }
+        for task in tasks {
+            tokio::time::timeout(WAIT, task)
+                .await
+                .expect("task finishes")
+                .expect("task succeeds");
+        }
+        assert!(outer.json_records().is_empty());
+        assert!(outer.exported_records().is_empty());
+        assert_isolated_events(&isolated, &[0, 1, 0, 1, 0, 1, 0, 1]);
+    }
+
+    #[test]
+    fn either_isolation_step_alone_leaves_ambient_context() {
+        let outer = Sinks::new(true, true);
+        let separate = Sinks::new(true, true);
+        dispatcher::with_default(&outer.dispatch, || {
+            let span = parent_span(&outer.dispatch);
+            let _span = span.enter();
+            let _context = fake_ambient_context().attach();
+            dispatcher::with_default(&separate.dispatch, || emit_root_event(0));
+            let _empty_context = Context::new().attach();
+            emit_root_event(1);
+        });
+
+        let separate_json = separate.json_records();
+        let separate_logs = separate.exported_records();
+        assert_eq!(separate_json.len(), 1);
+        assert_eq!(separate_logs.len(), 1);
+        assert!(separate_json[0].get("span").is_none());
+        assert_eq!(
+            separate_logs[0]
+                .0
+                .trace_context()
+                .expect("ambient trace still exists")
+                .trace_id,
+            TraceId::from(17)
+        );
+
+        let outer_json = outer.json_records();
+        let outer_logs = outer.exported_records();
+        assert_eq!(outer_json.len(), 1);
+        assert_eq!(outer_logs.len(), 1);
+        assert_eq!(outer_json[0]["span"]["uri"].as_str(), Some(FAKE_URI));
+        assert_eq!(outer_json[0]["span"]["authorization"].as_str(), Some(FAKE_AUTH));
+        assert!(outer_logs[0].0.trace_context().is_none());
+    }
+
+    fn filtered_sinks(filter: &str, sensitive_resource: bool) -> Sinks {
+        let json = MemoryWriter::default();
+        let exporter = MemoryExporter::default();
+        let resource = if sensitive_resource {
+            Resource::builder_empty()
+                .with_attributes([KeyValue::new("synthetic.resource", FAKE_AUTH)])
+                .build()
+        } else {
+            Resource::builder_empty().build()
+        };
+        let provider = SdkLoggerProvider::builder()
+            .with_resource(resource)
+            .with_simple_exporter(exporter.clone())
+            .build();
+        let bridge = OpenTelemetryTracingBridge::builder(&provider)
+            .with_tracing_span_attributes(TracingSpanAttributes::all())
+            .build();
+        let subscriber = tracing_subscriber::registry()
+            .with(EnvFilter::new(filter))
+            .with(build_json_log_layer(json.clone(), false, FmtSpan::NONE, true))
+            .with(bridge);
+        Sinks {
+            dispatch: Dispatch::new(subscriber),
+            json,
+            exporter,
+            _provider: provider,
+        }
+    }
+
+    #[test]
+    fn filters_and_resources_do_not_cross_dispatchers() {
+        let outer = filtered_sinks("info,rustfs_isolation_probe_test=off", true);
+        let isolated = filtered_sinks("off,rustfs_isolation_probe_test=info", false);
+        let disabled = filtered_sinks("off", false);
+        dispatcher::with_default(&outer.dispatch, || {
+            let span = parent_span(&outer.dispatch);
+            let _span = span.enter();
+            let parent_id = span.id();
+            observe_once(&isolated.dispatch, &parent_id, 0);
+            observe_once(&disabled.dispatch, &parent_id, 1);
+            tracing::info!(target: "isolation_outer_control", value = 1_u64, "outer control");
+        });
+        let outer_json = outer.json_records();
+        let outer_logs = outer.exported_records();
+        assert_eq!(outer_json.len(), 1);
+        assert_eq!(outer_logs.len(), 1);
+        assert_eq!(outer_json[0]["target"].as_str(), Some("isolation_outer_control"));
+        assert_eq!(outer_logs[0].0.target().map(|target| target.as_ref()), Some("isolation_outer_control"));
+        assert!(
+            outer
+                .exporter
+                .resource
+                .lock()
+                .expect("capture outer resource")
+                .as_ref()
+                .expect("outer resource exists")
+                .get(&opentelemetry::Key::new("synthetic.resource"))
+                .is_some()
+        );
+        assert_isolated_events(&isolated, &[0]);
+        assert!(disabled.json_records().is_empty());
+        assert!(disabled.exported_records().is_empty());
+    }
+
+    struct CancellationProbe {
+        outer: Dispatch,
+        isolated: Dispatch,
+        drops: Arc<AtomicUsize>,
+        restored: Arc<AtomicBool>,
+    }
+
+    impl Drop for CancellationProbe {
+        fn drop(&mut self) {
+            dispatcher::with_default(&self.outer, || {
+                let span = parent_span(&self.outer);
+                let _span = span.enter();
+                let _context = fake_ambient_context().attach();
+                emit_candidate(&self.isolated, 1);
+                let restored = tracing::Span::current().id() == span.id()
+                    && Context::current().span().span_context().trace_id() == TraceId::from(17)
+                    && Context::current().baggage().get("synthetic-secret").is_some();
+                self.restored.store(restored, Ordering::SeqCst);
+            });
+            self.drops.fetch_add(1, Ordering::SeqCst);
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn cancellation_drop_emits_in_isolation_and_restores_context() {
+        let outer = Sinks::new(true, true);
+        let isolated = Sinks::new(true, true);
+        let drops = Arc::new(AtomicUsize::new(0));
+        let restored = Arc::new(AtomicBool::new(false));
+        let probe = CancellationProbe {
+            outer: outer.dispatch.clone(),
+            isolated: isolated.dispatch.clone(),
+            drops: Arc::clone(&drops),
+            restored: Arc::clone(&restored),
+        };
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let task = tokio::spawn(async move {
+            let _probe = probe;
+            started_tx.send(()).expect("signal that the cancellation fixture is active");
+            pending::<()>().await;
+        });
+        tokio::time::timeout(WAIT, started_rx)
+            .await
+            .expect("task starts")
+            .expect("start signal");
+        task.abort();
+        let error = tokio::time::timeout(WAIT, task)
+            .await
+            .expect("cancelled task finishes")
+            .expect_err("the task must be cancelled");
+        assert!(error.is_cancelled());
+        assert_eq!(drops.load(Ordering::SeqCst), 1);
+        assert!(restored.load(Ordering::SeqCst));
+        assert!(outer.json_records().is_empty());
+        assert!(outer.exported_records().is_empty());
+        assert_isolated_events(&isolated, &[1]);
+    }
+
+    #[derive(Clone)]
+    struct PanicWriter;
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for PanicWriter {
+        type Writer = Self;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            self.clone()
+        }
+    }
+
+    impl Write for PanicWriter {
+        fn write(&mut self, _: &[u8]) -> io::Result<usize> {
+            panic!("synthetic writer panic");
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn panic_during_sink_write_restores_the_outer_context() {
+        let outer = Sinks::new(true, true);
+        let panic_dispatch =
+            Dispatch::new(tracing_subscriber::registry().with(build_json_log_layer(PanicWriter, false, FmtSpan::NONE, true)));
+        dispatcher::with_default(&outer.dispatch, || {
+            let span = parent_span(&outer.dispatch);
+            let _span = span.enter();
+            let _context = fake_ambient_context().attach();
+            let panic = catch_unwind(AssertUnwindSafe(|| emit_candidate(&panic_dispatch, 0)));
+            assert!(panic.is_err(), "the fixture must panic while writing the isolated event");
+            assert_eq!(tracing::Span::current().id(), span.id());
+            assert_eq!(Context::current().span().span_context().trace_id(), TraceId::from(17));
+            assert!(Context::current().baggage().get("synthetic-secret").is_some());
+            tracing::info!(target: "isolation_outer_control", value = 1_u64, "outer control");
+        });
+        let json = outer.json_records();
+        let logs = outer.exported_records();
+        assert_eq!(json.len(), 1);
+        assert_eq!(logs.len(), 1);
+        assert_eq!(json[0]["target"].as_str(), Some("isolation_outer_control"));
+        assert_eq!(logs[0].0.target().map(|target| target.as_ref()), Some("isolation_outer_control"));
+    }
+
+    #[derive(Clone)]
+    struct ObservedPanicWriter(Arc<AtomicBool>);
+
+    impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for ObservedPanicWriter {
+        type Writer = Self;
+
+        fn make_writer(&'a self) -> Self::Writer {
+            self.clone()
+        }
+    }
+
+    impl Write for ObservedPanicWriter {
+        fn write(&mut self, _: &[u8]) -> io::Result<usize> {
+            self.0.store(true, Ordering::SeqCst);
+            panic!("synthetic writer panic");
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
+
+    fn classify_panic_isolation() -> &'static str {
+        let outer = Sinks::new(true, true);
+        let reached = Arc::new(AtomicBool::new(false));
+        let panic_dispatch = Dispatch::new(tracing_subscriber::registry().with(build_json_log_layer(
+            ObservedPanicWriter(Arc::clone(&reached)),
+            false,
+            FmtSpan::NONE,
+            true,
+        )));
+        let early_result = dispatcher::with_default(&outer.dispatch, || {
+            let span = parent_span(&outer.dispatch);
+            let _span = span.enter();
+            let _context = fake_ambient_context().attach();
+            let outer_span = span.id();
+            if outer_span.is_none() {
+                return Some("outer_span_missing");
+            }
+            let caught = catch_unwind(AssertUnwindSafe(|| emit_candidate(&panic_dispatch, 0))).is_err();
+            if !reached.load(Ordering::SeqCst) {
+                return Some("writer_not_reached");
+            }
+            if !caught {
+                return Some("writer_panic_not_caught");
+            }
+            let dispatcher_restored = dispatcher::get_default(|current| {
+                current
+                    .downcast_ref::<MemoryWriter>()
+                    .is_some_and(|writer| Arc::ptr_eq(&writer.0, &outer.json.0))
+            });
+            if !dispatcher_restored {
+                return Some("dispatcher_not_restored");
+            }
+            if tracing::Span::current().id() != outer_span {
+                return Some("tracing_not_restored");
+            }
+            let current = Context::current();
+            if current.span().span_context().trace_id() != TraceId::from(17) {
+                return Some("trace_not_restored");
+            }
+            if current.span().span_context().span_id() != SpanId::from(19) {
+                return Some("span_id_not_restored");
+            }
+            if current.baggage().get("synthetic-secret").map(|value| value.as_str()) != Some(FAKE_AUTH) {
+                return Some("baggage_not_restored");
+            }
+            tracing::info!(target: "isolation_outer_control", value = 1_u64, "outer control");
+            None
+        });
+        if let Some(class) = early_result {
+            return class;
+        }
+        let bytes = match outer.json.0.lock() {
+            Ok(bytes) => bytes.clone(),
+            Err(_) => return "outer_json_unknown",
+        };
+        let text = match std::str::from_utf8(&bytes) {
+            Ok(text) => text,
+            Err(_) => return "outer_json_unknown",
+        };
+        let json: Vec<Value> = match text.lines().map(serde_json::from_str).collect() {
+            Ok(json) => json,
+            Err(_) => return "outer_json_unknown",
+        };
+        let logs = outer.exported_records();
+        if json.len() == 2
+            && json[0]["target"].as_str() == Some(TARGET)
+            && json[1]["target"].as_str() == Some("isolation_outer_control")
+        {
+            return "stale_isolated_json";
+        }
+        if json.len() != 1 || json[0]["target"].as_str() != Some("isolation_outer_control") {
+            return "outer_json_unexpected";
+        }
+        if logs.len() != 1 || logs[0].0.target().map(|target| target.as_ref()) != Some("isolation_outer_control") {
+            return "outer_otel_unexpected";
+        }
+        "clean"
+    }
+
+    type PanicHook = Box<dyn Fn(&std::panic::PanicHookInfo<'_>) + Send + Sync + 'static>;
+
+    struct RestorePanicHook(Option<PanicHook>);
+
+    impl Drop for RestorePanicHook {
+        fn drop(&mut self) {
+            if let Some(previous) = self.0.take() {
+                std::panic::set_hook(previous);
+            }
+        }
+    }
+
+    #[test]
+    fn classify_panic_writer_isolation_without_raw_output() {
+        // This process-global hook is used only by the workflow's single exact test.
+        let hook = RestorePanicHook(Some(std::panic::take_hook()));
+        std::panic::set_hook(Box::new(|_| {
+            let _ = writeln!(std::io::stderr(), "\nRUSTFS_PANIC_HOOK=redacted");
+        }));
+        let result = catch_unwind(AssertUnwindSafe(|| {
+            let class = catch_unwind(AssertUnwindSafe(classify_panic_isolation)).unwrap_or("diagnostic_internal_panic");
+            println!("\nRUSTFS_PANIC_DIAGNOSTIC={class}");
+            assert!(class == "clean", "isolation diagnostic failed");
+        }));
+        drop(hook);
+        if let Err(payload) = result {
+            std::panic::resume_unwind(payload);
+        }
+    }
+}

crates/obs/src/telemetry/mod.rs

@@ -43,6 +43,8 @@
 pub mod dial9;
 mod filter;
 mod guard;
+#[cfg(test)]
+mod isolation_probe_tests;
 mod local;
 mod otel;
 mod recorder;

crates/scanner/src/error.rs

@@ -53,10 +53,6 @@ pub enum ScannerError {
     #[error("Remote namespace scanner disk is already active")]
     RemoteDiskBusy,
 
-    /// Node resource pressure deferred work before its first storage poll.
-    #[error("Scanner node resource admission deferred")]
-    ResourceDeferred,
-
     /// Partial data usage cache produced before the scanner stopped.
     #[error("Scanner stopped with partial data usage cache")]
     PartialCache(Box<DataUsageCache>),

crates/scanner/src/lib.rs

@@ -50,9 +50,9 @@ use storage_api::owner::{
 };
 #[cfg(test)]
 use storage_api::owner::{
-    EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreExpiryState,
-    EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
-    ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
+    EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints, EcstoreInstanceContext,
+    EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
+    ecstore_new_disk,
 };
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
@@ -123,7 +123,7 @@ pub use storage_api::scan::{
     SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
     SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
 };
-pub use workload_admission::{set_scanner_node_admission, set_scanner_workload_admission_snapshot_provider};
+pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
 
 static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
 static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
@@ -678,37 +678,8 @@ fn reset_tier_name_cache_for_test() {
     TIER_REGISTRY_GENERATION.store(0, Ordering::Relaxed);
 }
 
-// Only a task running an isolated unit test can override these handles.
-#[cfg(test)]
-struct ExpiryLockTestContext {
-    state: Arc<tokio::sync::RwLock<EcstoreExpiryState>>,
-    store: Option<Arc<ECStore>>,
-    completed_incarnation_reads: Arc<std::sync::atomic::AtomicUsize>,
-}
-
-#[cfg(test)]
-tokio::task_local! {
-    static EXPIRY_LOCK_TEST_CONTEXT: ExpiryLockTestContext;
-}
-
-pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo, cancellation: Option<&CancellationToken>) {
-    #[cfg(test)]
-    let state = EXPIRY_LOCK_TEST_CONTEXT
-        .try_with(|context| context.state.clone())
-        .unwrap_or_else(|_| ecstore_expiry_state_handle());
-    #[cfg(not(test))]
-    let state = ecstore_expiry_state_handle();
-    let state = match cancellation {
-        Some(cancel) => tokio::select! {
-            biased;
-            _ = cancel.cancelled() => return,
-            state = state.write() => state,
-        },
-        None => state.write().await,
-    };
-    if !cancellation.is_some_and(CancellationToken::is_cancelled) {
-        state.enqueue_free_version(oi);
-    }
+pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo) {
+    ecstore_expiry_state_handle().write().await.enqueue_free_version(oi);
 }
 
 pub(crate) async fn enqueue_runtime_newer_noncurrent(
@@ -716,50 +687,20 @@ pub(crate) async fn enqueue_runtime_newer_noncurrent(
     to_delete_objs: Vec<ObjectToDelete>,
     event: Event,
     src: &LcEventSrc,
-    cancellation: Option<&CancellationToken>,
 ) -> bool {
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return false;
-    }
-    #[cfg(test)]
-    let store = EXPIRY_LOCK_TEST_CONTEXT
-        .try_with(|context| context.store.clone())
-        .unwrap_or_else(|_| ecstore_resolve_object_store_handle());
-    #[cfg(not(test))]
-    let store = ecstore_resolve_object_store_handle();
-    let Some(store) = store else {
+    let Some(store) = ecstore_resolve_object_store_handle() else {
         return false;
     };
     let Ok(bucket_incarnation_id) = store.bucket_incarnation_id(bucket).await else {
         return false;
     };
-    #[cfg(test)]
-    let _ = EXPIRY_LOCK_TEST_CONTEXT.try_with(|context| {
-        context
-            .completed_incarnation_reads
-            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-    });
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return false;
-    }
-    #[cfg(test)]
-    let state = EXPIRY_LOCK_TEST_CONTEXT
-        .try_with(|context| context.state.clone())
-        .unwrap_or_else(|_| ecstore_expiry_state_handle());
-    #[cfg(not(test))]
-    let state = ecstore_expiry_state_handle();
-    let mut state = match cancellation {
-        Some(cancel) => tokio::select! {
-            biased;
-            _ = cancel.cancelled() => return false,
-            state = state.write() => state,
-        },
-        None => state.write().await,
-    };
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return false;
-    }
-    state.enqueue_by_newer_noncurrent(bucket, to_delete_objs, event, src, bucket_incarnation_id)
+    ecstore_expiry_state_handle().write().await.enqueue_by_newer_noncurrent(
+        bucket,
+        to_delete_objs,
+        event,
+        src,
+        bucket_incarnation_id,
+    )
 }
 
 pub(crate) async fn queue_replication_heal(

crates/scanner/src/remote_scanner/stream.rs

@@ -705,7 +705,7 @@ async fn scan_and_persist_local_bucket(
         RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement")
     })?;
     let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
-    let mut guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
+    let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
         .await
         .map_err(|err| {
             if err.is_contention() {
@@ -791,23 +791,19 @@ async fn scan_and_persist_local_bucket(
 
     let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
     let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
-    let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
     let scan_ctx = ctx.child_token();
-    let scan = crate::workload_admission::with_scanner_node_requests(
-        node_tx,
-        ScannerIODisk::nsscanner_disk(
-            disk.clone(),
-            scan_ctx.clone(),
-            budget,
-            set_disks,
-            cache,
-            None,
-            ScannerDiskScanOptions {
-                scan_mode,
-                prefix_scan_scope: None,
-                checkpoint_tx: Some(checkpoint_tx),
-            },
-        ),
+    let scan = ScannerIODisk::nsscanner_disk(
+        disk.clone(),
+        scan_ctx.clone(),
+        budget,
+        set_disks,
+        cache,
+        None,
+        ScannerDiskScanOptions {
+            scan_mode,
+            prefix_scan_scope: None,
+            checkpoint_tx: Some(checkpoint_tx),
+        },
     );
     tokio::pin!(scan);
     let fence_watch = watch_remote_scanner_request_fence(next_cycle, leader_epoch, store.clone(), NS_SCANNER_FENCE_POLL_INTERVAL);
@@ -817,27 +813,6 @@ async fn scan_and_persist_local_bucket(
     let mut checkpoint_channel_closed = false;
     let outcome = loop {
         tokio::select! {
-            request = async {
-                match node_rx.as_mut() {
-                    Some(receiver) => receiver.recv().await,
-                    None => std::future::pending().await,
-                }
-            } => {
-                let Some(request) = request else {
-                    node_rx = None;
-                    continue;
-                };
-                if let Err(error) = wait_for_remote_node_resumption(
-                    guard.resume_node_operation(set.clone(), &cache_name, source, &revisions, request),
-                    fence_watch.as_mut(),
-                ).await {
-                    scan_ctx.cancel();
-                    let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
-                    return Err(RemoteScannerServerError::retry_bucket(format!(
-                        "remote namespace scanner could not resume node admission: {error}",
-                    )));
-                }
-            }
             result = &mut scan => {
                 match result {
                     Ok(outcome) => break outcome,
@@ -1146,22 +1121,6 @@ pub(crate) async fn scan_remote_bucket(
     result
 }
 
-/// Resource waits must keep checking the persisted leader fence.
-async fn wait_for_remote_node_resumption<F, G>(resume: F, fence_watch: Pin<&mut G>) -> Result<(), StorageError>
-where
-    F: Future<Output = Result<(), StorageError>>,
-    G: Future<Output = Result<(), ScannerError>>,
-{
-    tokio::select! {
-        biased;
-        result = fence_watch => Err(match result {
-            Ok(()) => StorageError::other("remote namespace scanner fence watcher stopped unexpectedly"),
-            Err(error) => StorageError::other(error),
-        }),
-        result = resume => result,
-    }
-}
-
 fn finish_remote_scanner_stream(
     result: RemoteScannerStreamResult<RemoteScannerOutcome>,
     budget: &ScannerCycleBudget,

crates/scanner/src/remote_scanner/stream/tests.rs

@@ -1006,58 +1006,6 @@ async fn fence_watcher_stops_work_after_persisted_leader_changes() {
     assert!(err.to_string().contains("requested_epoch=9"));
 }
 
-#[tokio::test]
-async fn node_admission_wait_does_not_hide_replacement_leader() {
-    use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
-    use std::num::NonZeroU16;
-
-    let store = Arc::new(MutableCycleStateStore {
-        state: RwLock::new(crate::scanner::encode_scanner_cycle_fence_for_test(7, 9)),
-    });
-    let cache = Mutex::new(None);
-    let refresh = AsyncMutex::new(());
-    let watcher = watch_remote_scanner_request_fence_with_cache(
-        7,
-        9,
-        store.clone(),
-        Duration::from_millis(5),
-        &cache,
-        &refresh,
-        Duration::from_millis(5),
-    );
-    tokio::pin!(watcher);
-    let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one shared node unit"));
-    let storage_polled = AtomicBool::new(false);
-    {
-        let resume = async {
-            let _permit = gate.acquire().await;
-            storage_polled.store(true, Ordering::Relaxed);
-            Ok(())
-        };
-        let waiting = wait_for_remote_node_resumption(resume, watcher.as_mut());
-        tokio::pin!(waiting);
-        assert!(futures::poll!(&mut waiting).is_pending(), "no resource sample grants a node unit");
-        *store.state.write().expect("persist replacement leader") = crate::scanner::encode_scanner_cycle_fence_for_test(7, 10);
-        let error = tokio::time::timeout(Duration::from_millis(100), waiting)
-            .await
-            .expect("resource pause must not hide a changed leader")
-            .expect_err("old leader cannot resume storage work");
-        assert!(error.to_string().contains("requested_epoch=9"));
-    }
-    assert!(!storage_polled.load(Ordering::Relaxed));
-    assert_eq!(gate.snapshot().active, 0);
-    gate.update(
-        &mut FeedbackController::new(),
-        Some(FeedbackObservation {
-            generation: 1,
-            max_concurrency: 1,
-            pressure: ResourcePressure::Unknown,
-            completed_units: None,
-        }),
-    );
-    assert!(gate.try_acquire().is_some(), "failed fencing drops the old node queue ticket");
-}
-
 #[tokio::test]
 async fn concurrent_cycle_cache_misses_share_one_backend_read() {
     let request = RemoteScannerRequest(test_request(Uuid::new_v4()));

crates/scanner/src/scanner/tests.rs

@@ -5440,8 +5440,8 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
         .await
         .expect_err("truncated primary must fail in the real object reader");
     assert!(
-        matches!(error, EcstoreError::FileCorrupt),
-        "unexpected truncated-primary error: {error:?}"
+        error.to_string().contains("InlineData value out of range"),
+        "unexpected truncated-primary error: {error}"
     );
     assert_eq!(
         read_config_with_revision(store.clone(), &backup_path)

crates/scanner/src/scanner_folder.rs

@@ -21,10 +21,10 @@ use std::time::{Duration, Instant, SystemTime};
 
 use crate::ReplTargetSizeSummary;
 use crate::data_usage_define::{
-    DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageHash,
-    DataUsageHashMap, DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason,
-    MAX_DATA_USAGE_CACHE_DEPTH, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry,
-    SizeSummary, hash_path, scan_frontier_covers_path,
+    DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
+    DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH,
+    PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
+    scan_frontier_covers_path,
 };
 use crate::error::ScannerError;
 use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter};
@@ -43,7 +43,6 @@ use crate::storage_api::owner::{
 };
 #[cfg(test)]
 use crate::storage_api::owner::{EcstoreExpirationStatus as ExpirationStatus, EcstoreLifecycleRule as LifecycleRule};
-use crate::workload_admission::run_scanner_node_operation;
 use metrics::{counter, describe_counter};
 use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
 use rustfs_filemeta::{
@@ -796,7 +795,6 @@ pub struct FolderScanner {
     resolved_failed_object_paths: HashSet<String>,
 
     sleeper: DynamicSleeper,
-    node_admission: Option<Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
     // should_heal: Arc<dyn Fn() -> bool + Send + Sync>,
     disks: Vec<Arc<Disk>>,
     disks_quorum: usize,
@@ -830,8 +828,6 @@ pub struct FolderScanner {
     pending_size_reconciliation_truncated: bool,
     #[cfg(test)]
     list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
-    #[cfg(test)]
-    defer_abandoned_entries_until_finished: bool,
 }
 
 struct RawEnumerationProgress {
@@ -1002,13 +998,12 @@ async fn next_scanner_dir_entry(
     ctx: &CancellationToken,
     sleeper: &DynamicSleeper,
     entries: &mut ScannerDirectory,
-    admission: Option<&Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
 ) -> Result<Option<tokio::fs::DirEntry>, ScannerError> {
     if entries.batch_entries == SCANNER_DIRECTORY_BATCH_ENTRIES {
         wait_for_scanner_io(ctx, tokio::task::yield_now()).await?;
         entries.batch_entries = 0;
     }
-    let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.next_entry())).await???;
+    let entry = wait_for_scanner_io(ctx, entries.next_entry()).await??;
     if entry.is_some() {
         // Tokio prefetches bounded chunks. Pacing consumption also paces the
         // next refill, without charging an extra entry for EOF.
@@ -1608,7 +1603,7 @@ impl FolderScanner {
         let candidate_type = pending_scanner_heal_candidate_type(kind);
         let priority = request.priority;
         let scan_mode = request.scan_mode.unwrap_or(self.scan_mode);
-        let result = match send_scanner_heal_request(candidate_type, request, scanner_disk_heal_scope(&self.local_disk)).await {
+        let result = match send_scanner_heal_request(candidate_type, request).await {
             Ok(result) => result,
             Err(err) => {
                 self.update_pending_scanner_heal_after_admission(
@@ -1928,12 +1923,7 @@ impl FolderScanner {
             // directory that vanished during iteration instead of failing the
             // whole parent subtree. The bucket root (no parent) still propagates
             // NotFound so namespace removal is detected by the caller.
-            let mut dir_reader = match wait_for_scanner_io(
-                &ctx,
-                run_scanner_node_operation(self.node_admission.as_ref(), &ctx, ScannerDirectory::open(&dir_path)),
-            )
-            .await??
-            {
+            let mut dir_reader = match wait_for_scanner_io(&ctx, ScannerDirectory::open(&dir_path)).await? {
                 Ok(dir_reader) => Some(dir_reader),
                 Err(e)
                     if folder.parent.is_some()
@@ -1971,7 +1961,7 @@ impl FolderScanner {
                 // Hold at most one metadata entry until a candidate reaches
                 // EOF: a later part must not leave object accounting or ILM work.
                 let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut() {
-                    let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader, self.node_admission.as_ref()).await {
+                    let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader).await {
                         Ok(Some(entry)) => entry,
                         Ok(None) => {
                             raw_enumeration_complete = true;
@@ -2082,12 +2072,7 @@ impl FolderScanner {
 
                 // Ignore entries that disappeared during traversal or hit symlink
                 // loops, but propagate other walk errors.
-                let mut entry_type = match wait_for_scanner_io(
-                    &ctx,
-                    run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()),
-                )
-                .await??
-                {
+                let mut entry_type = match wait_for_scanner_io(&ctx, entry.file_type()).await? {
                     Ok(entry_type) => entry_type,
                     Err(e) if e.kind() == ErrorKind::NotFound => {
                         debug!(
@@ -2138,12 +2123,7 @@ impl FolderScanner {
                 let mut entry_metadata_len = None;
                 if entry_type.is_symlink() {
                     wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
-                    let metadata = match wait_for_scanner_io(
-                        &ctx,
-                        run_scanner_node_operation(self.node_admission.as_ref(), &ctx, tokio::fs::metadata(&file_path)),
-                    )
-                    .await??
-                    {
+                    let metadata = match wait_for_scanner_io(&ctx, tokio::fs::metadata(&file_path)).await? {
                         Ok(metadata) => metadata,
                         Err(e) if e.kind() == ErrorKind::NotFound => {
                             debug!(
@@ -2269,7 +2249,6 @@ impl FolderScanner {
                     lifecycle: active_life_cycle.clone(),
                     object_lock: active_object_lock.clone(),
                     replication: active_replication.clone(),
-                    heal_scope: None,
                     heal_enabled,
                     heal_bitrot: self.scan_mode == HealScanMode::Deep,
                     debug: self.data_usage_scanner_debug,
@@ -2295,30 +2274,22 @@ impl FolderScanner {
                         len
                     } else {
                         wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
-                        wait_for_scanner_io(
-                            &ctx,
-                            run_scanner_node_operation(self.node_admission.as_ref(), &ctx, tokio::fs::metadata(&item.path)),
-                        )
-                        .await??
-                        .map(|metadata| metadata.len())
-                        .unwrap_or(0)
+                        wait_for_scanner_io(&ctx, tokio::fs::metadata(&item.path))
+                            .await?
+                            .map(|metadata| metadata.len())
+                            .unwrap_or(0)
                     }
                 } else {
                     0
                 };
                 wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, budgeted_bytes)).await?;
 
-                let size_result = run_scanner_node_operation(self.node_admission.as_ref(), &ctx, async {
-                    self.object_metadata_visits.increment(1);
-                    self.local_disk
-                        .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
-                        .await
-                })
-                .await?;
-                if self.node_admission.is_some() && ctx.is_cancelled() {
-                    return Err(ScannerError::Other("Operation cancelled".to_string()));
-                }
-                let sz = match size_result {
+                self.object_metadata_visits.increment(1);
+                let sz = match self
+                    .local_disk
+                    .get_size_with_tier_names(item.clone(), &self.tier_registry.names)
+                    .await
+                {
                     Ok(sz) => sz,
                     Err(e) => {
                         let failure_action = classify_get_size_failure(&item, &e);
@@ -2635,7 +2606,7 @@ impl FolderScanner {
                     {
                         Ok(outcome) => outcome,
                         Err(e) => {
-                            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
+                            if ctx.is_cancelled() {
                                 self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
                                     .await;
                                 return Err(e);
@@ -2876,23 +2847,19 @@ impl FolderScanner {
                 let mut seen_truncated_objects: HashSet<String> = HashSet::new();
 
                 loop {
-                    if child_ctx.is_cancelled() || (agreed_closed && partial_closed && finished_closed) {
+                    if agreed_closed && partial_closed && finished_closed {
                         break;
                     }
-                    #[cfg(test)]
-                    let defer_entries = self.defer_abandoned_entries_until_finished && !finished_closed;
-                    #[cfg(not(test))]
-                    let defer_entries = false;
 
                     select! {
-                        entry_name = agreed_rx.recv(), if !agreed_closed && !defer_entries => {
+                        entry_name = agreed_rx.recv(), if !agreed_closed => {
                             let Some(entry_name) = entry_name else {
                                 agreed_closed = true;
                                 continue;
                             };
                             (self.update_current_path)(&entry_name).await;
                         }
-                        entries = partial_rx.recv(), if !partial_closed && !defer_entries => {
+                        entries = partial_rx.recv(), if !partial_closed => {
                             let Some(entries) = entries else {
                                 partial_closed = true;
                                 continue;
@@ -3018,7 +2985,6 @@ impl FolderScanner {
                                 finished_closed = true;
                                 continue;
                             };
-                            finished_closed = true;
                             if disk_errors_are_only_missing_paths(&errs) {
                                 debug!(
                                     target: "rustfs::scanner::folder",
@@ -3040,9 +3006,7 @@ impl FolderScanner {
                                     "Scanner list_path_raw finished with disk errors"
                                 );
                             }
-                            // The producer awaited callback sends, not their
-                            // consumption. Drain the remaining candidate and
-                            // agreed frames before closing this child walk.
+                            child_ctx.cancel();
                         }
                         _ = child_ctx.cancelled() => {
                             break;
@@ -3060,10 +3024,7 @@ impl FolderScanner {
                     if into.compacted {
                         // In compacted mode child totals are accumulated directly into the parent entry.
                         let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), into));
-                        fut.await.map_err(|error| match error {
-                            ScannerError::ResourceDeferred => ScannerError::ResourceDeferred,
-                            other => ScannerError::Other(other.to_string()),
-                        })?;
+                        fut.await.map_err(|e| ScannerError::Other(e.to_string()))?;
                         self.send_update_for_entry(&this_hash, &folder.parent, into).await;
                         tokio::task::yield_now().await;
                     } else {
@@ -3082,7 +3043,7 @@ impl FolderScanner {
                             )
                             .await
                         {
-                            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
+                            if ctx.is_cancelled() {
                                 self.preserve_partial_child_progress(&folder_item.parent, &h, into, &dst)
                                     .await;
                                 return Err(e);
@@ -3256,7 +3217,6 @@ pub(crate) async fn scan_data_folder_scoped(
         failed_object_paths_seen: HashSet::new(),
         resolved_failed_object_paths: HashSet::new(),
         sleeper,
-        node_admission: crate::workload_admission::scanner_node_admission(),
         disks,
         disks_quorum,
         usage_updates,
@@ -3284,8 +3244,6 @@ pub(crate) async fn scan_data_folder_scoped(
         pending_size_reconciliation_truncated: false,
         #[cfg(test)]
         list_path_raw_options_observer: None,
-        #[cfg(test)]
-        defer_abandoned_entries_until_finished: false,
     };
 
     let now = FolderScanner::now_secs();
@@ -3375,7 +3333,7 @@ pub(crate) async fn scan_data_folder_scoped(
             }
         }
         Err(e) => {
-            if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
+            if ctx.is_cancelled() {
                 let root_hash = hash_path(&bucket_name);
                 let root_has_progress = data_usage_root_has_progress(&root);
                 let pending_heals_changed = scanner.pending_heals_changed;
@@ -3450,19 +3408,6 @@ pub(crate) async fn scan_data_folder_scoped(
                 close_disk_guard.close().await;
                 return Err(ScannerError::NamespaceNotFoundCache(Box::new(partial_cache)));
             }
-            if matches!(&e, ScannerError::ResourceDeferred) {
-                // Admission pressure is not disk failure or an empty census.
-                // Keep the previous tree when no new useful progress exists.
-                let mut partial_cache = scanner.old_cache;
-                if partial_cache.cache.is_empty() {
-                    partial_cache
-                        .cache
-                        .insert(hash_path(&bucket_name).0, DataUsageEntry::default());
-                }
-                partial_cache.info.snapshot_complete = false;
-                close_disk_guard.close().await;
-                return Err(ScannerError::PartialCache(Box::new(partial_cache)));
-            }
             close_disk_guard.close().await;
             // No useful information, return original cache
             Err(e)
@@ -3473,7 +3418,6 @@ pub(crate) async fn scan_data_folder_scoped(
 mod item_actions;
 mod ledger;
 
-pub(crate) use item_actions::scanner_disk_heal_scope;
 use item_actions::*;
 pub use item_actions::{GetSizeFn, ScannerItem};
 use ledger::*;
@@ -3483,15 +3427,3 @@ pub(crate) use tests::checkpoint_fixture::run_checkpoint_fixture;
 
 #[cfg(test)]
 mod tests;
-
-#[cfg(test)]
-tokio::task_local! {
-    pub(crate) static TEST_SIZE_DISPATCH_BOUNDARIES: tokio::sync::mpsc::UnboundedSender<&'static str>;
-}
-
-#[cfg(test)]
-pub(crate) fn observe_test_size_dispatch_boundary(stage: &'static str) -> bool {
-    TEST_SIZE_DISPATCH_BOUNDARIES
-        .try_with(|sender| sender.send(stage).is_ok())
-        .unwrap_or(false)
-}

crates/scanner/src/scanner_folder/item_actions.rs

@@ -474,23 +474,10 @@ pub(super) fn record_heal_candidate_admission(
     }
 }
 
-pub(crate) fn scanner_disk_heal_scope(disk: &Disk) -> Option<DataUsageCacheSource> {
-    let location = disk.get_disk_location();
-    location.valid().then_some(())?;
-    Some(DataUsageCacheSource::new(location.pool_idx?, location.set_idx?))
-}
-
 pub(super) async fn send_scanner_heal_request(
     candidate_type: &'static str,
-    mut request: HealChannelRequest,
-    scope: Option<DataUsageCacheSource>,
+    request: HealChannelRequest,
 ) -> Result<HealAdmissionResult, ScannerError> {
-    if request.object_prefix.as_deref().is_some_and(|object| !object.is_empty()) {
-        let scope = scope
-            .ok_or_else(|| ScannerError::Other("scanner object heal requires the scanning disk's pool and set".to_string()))?;
-        request.pool_index = Some(scope.pool_index);
-        request.set_index = Some(scope.set_index);
-    }
     let priority = request.priority;
     let trace_context = scanner_heal_candidate_trace_context(&request);
     let candidate = crate::heal_completion::prepare(&request);
@@ -550,8 +537,6 @@ pub struct ScannerItem {
     pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
     pub object_lock: Option<Arc<ObjectLockConfiguration>>,
     pub replication: Option<Arc<ReplicationConfig>>,
-    /// Actual scanning-disk location; refreshed by the disk reader before actions.
-    pub heal_scope: Option<DataUsageCacheSource>,
     pub heal_enabled: bool,
     pub heal_bitrot: bool,
     pub debug: bool,
@@ -643,22 +628,6 @@ impl ScannerItem {
         tier_names: &[String],
         size_summary: &mut SizeSummary,
     ) {
-        self.apply_actions_cancellable(object_infos, lock_retention, versioning_config, tier_names, size_summary, None)
-            .await;
-    }
-
-    pub(crate) async fn apply_actions_cancellable(
-        &mut self,
-        object_infos: Vec<ObjectInfo>,
-        lock_retention: Option<Arc<ObjectLockConfiguration>>,
-        versioning_config: VersioningConfiguration,
-        tier_names: &[String],
-        size_summary: &mut SizeSummary,
-        cancellation: Option<&CancellationToken>,
-    ) {
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return;
-        }
         let object_path = self.object_path();
         if object_infos.is_empty() {
             debug!(
@@ -702,9 +671,6 @@ impl ScannerItem {
         let Some(lifecycle) = self.lifecycle.clone() else {
             let mut cumulative_size: i64 = 0;
             for (oi, resolved_size) in object_infos.iter().zip(resolved_sizes.iter()) {
-                if cancellation.is_some_and(CancellationToken::is_cancelled) {
-                    return;
-                }
                 let accounting_size = match resolved_size {
                     SizeResolution::Known { logical, .. } => *logical,
                     // A valid compressed legacy sentinel has no logical size,
@@ -712,7 +678,7 @@ impl ScannerItem {
                     // physical size is only an input to those operations; it
                     // is not folded into the logical total below.
                     SizeResolution::Unknown { physical, .. } => {
-                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                        self.heal_actions(oi, *physical, size_summary).await;
                         size_summary.actions_accounting_unknown(oi);
                         continue;
                     }
@@ -722,16 +688,13 @@ impl ScannerItem {
                     }
                 };
 
-                let size = self.heal_actions(oi, accounting_size, size_summary, cancellation).await;
+                let size = self.heal_actions(oi, accounting_size, size_summary).await;
 
                 size_summary.actions_accounting(oi, size, accounting_size);
 
                 cumulative_size = cumulative_size.saturating_add(size);
             }
 
-            if cancellation.is_some_and(CancellationToken::is_cancelled) {
-                return;
-            }
             self.alert_excessive_versions(object_infos.len(), cumulative_size);
 
             debug!(
@@ -773,10 +736,6 @@ impl ScannerItem {
             }
         };
 
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return;
-        }
-
         // Every version handed to the evaluator counts as an ILM-checked
         // version, whether or not a rule matched (NoneAction included). This is
         // reached only for buckets with lifecycle rules; it feeds
@@ -795,9 +754,6 @@ impl ScannerItem {
             .all(|candidate| Self::tier_is_known(candidate, tier_names));
         'eventLoop: {
             for (i, event) in events.iter().enumerate() {
-                if cancellation.is_some_and(CancellationToken::is_cancelled) {
-                    return;
-                }
                 let oi = &object_infos[i];
                 let known_size = resolved_sizes[i].known_size();
                 if has_corrupt_size
@@ -816,7 +772,7 @@ impl ScannerItem {
                     // accounting. Size-filtered or deferred events remain
                     // pending, so retain the version-only physical counters.
                     if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                        self.heal_actions(oi, *physical, size_summary).await;
                         size_summary.actions_accounting_unknown(oi);
                     }
                     continue;
@@ -873,7 +829,7 @@ impl ScannerItem {
                                     noncurrent_unknown.push(oi);
                                 } else {
                                     if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                                        self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                                        self.heal_actions(oi, *physical, size_summary).await;
                                     }
                                     size_summary.actions_accounting_unknown(oi);
                                 }
@@ -890,7 +846,7 @@ impl ScannerItem {
                             }
                             IlmAction::NoneAction | IlmAction::ActionCount => {
                                 if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
-                                    self.heal_actions(oi, *physical, size_summary, cancellation).await;
+                                    self.heal_actions(oi, *physical, size_summary).await;
                                 }
                                 size_summary.actions_accounting_unknown(oi);
                             }
@@ -908,7 +864,7 @@ impl ScannerItem {
                 // side-effecting tier actions fail closed until the registry
                 // recognizes the source again.
                 if Self::action_blocked_by_unknown_tier(event.action, oi, all_versions_known, tier_names, &event.storage_class) {
-                    size = self.heal_actions(oi, actual_size, size_summary, cancellation).await;
+                    size = self.heal_actions(oi, actual_size, size_summary).await;
                     size_summary.actions_accounting(oi, size, actual_size);
                     cumulative_size += size;
                     continue;
@@ -1017,7 +973,7 @@ impl ScannerItem {
                     }
 
                     IlmAction::NoneAction | IlmAction::ActionCount => {
-                        size = self.heal_actions(oi, actual_size, size_summary, cancellation).await;
+                        size = self.heal_actions(oi, actual_size, size_summary).await;
                     }
                 }
 
@@ -1028,9 +984,6 @@ impl ScannerItem {
             }
         }
 
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return;
-        }
         if !to_delete_objs.is_empty()
             && let Some(event) = noncurrent_events.first().cloned()
         {
@@ -1038,8 +991,7 @@ impl ScannerItem {
             let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX);
             let done_ilm = Metrics::time_ilm(action);
             let trace_started_at = trace_start_instant();
-            let queued =
-                enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner, cancellation).await;
+            let queued = enqueue_runtime_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await;
             if let Some(trace_started_at) = trace_started_at {
                 let state = if queued { "queued" } else { "not_queued" };
                 trace_emit(|| {
@@ -1070,28 +1022,15 @@ impl ScannerItem {
                 }
             }
         }
-        if !cancellation.is_some_and(CancellationToken::is_cancelled) {
-            self.alert_excessive_versions(remaining_versions, cumulative_size);
-        }
+        self.alert_excessive_versions(remaining_versions, cumulative_size);
     }
 
-    pub(super) async fn heal_actions(
-        &mut self,
-        oi: &ObjectInfo,
-        actual_size: i64,
-        size_summary: &mut SizeSummary,
-        cancellation: Option<&CancellationToken>,
-    ) -> i64 {
-        if cancellation.is_some_and(CancellationToken::is_cancelled) {
-            return actual_size;
-        }
+    pub(super) async fn heal_actions(&mut self, oi: &ObjectInfo, actual_size: i64, size_summary: &mut SizeSummary) -> i64 {
         if self.heal_enabled {
             self.enqueue_heal(oi).await;
         }
 
-        if !cancellation.is_some_and(CancellationToken::is_cancelled) {
-            self.heal_replication(oi, size_summary).await;
-        }
+        self.heal_replication(oi, size_summary).await;
 
         actual_size
     }
@@ -1222,7 +1161,6 @@ impl ScannerItem {
                 scan_mode,
                 HealChannelPriority::Low,
             ),
-            self.heal_scope,
         )
         .await;
 
@@ -1384,7 +1322,6 @@ mod tests {
             lifecycle: None,
             object_lock: None,
             replication: None,
-            heal_scope: None,
             heal_enabled: false,
             heal_bitrot: false,
             debug: false,

crates/scanner/src/scanner_folder/tests.rs

@@ -30,9 +30,7 @@ use std::sync::Mutex;
 pub(super) mod checkpoint_fixture;
 pub(super) mod enumeration_restart;
 mod incremental_enumeration;
-mod listing_completion;
 mod metadata_visits;
-mod node_admission;
 mod usage_updates;
 
 /// Reset the process-global alert cooldown map; test-only.
@@ -339,10 +337,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         .await
         .expect("failed to create test directory");
 
-    let mut endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
-    endpoint.set_pool_index(0);
-    endpoint.set_set_index(0);
-    endpoint.set_disk_index(0);
+    let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
     let disk = new_disk(
         &endpoint,
         &DiskOption {
@@ -369,7 +364,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         failed_object_paths_seen: HashSet::new(),
         resolved_failed_object_paths: HashSet::new(),
         sleeper: SCANNER_SLEEPER.clone(),
-        node_admission: None,
         disks: Vec::new(),
         disks_quorum: 0,
         usage_updates: None,
@@ -400,7 +394,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         pending_size_reconciliation_scopes: HashSet::new(),
         pending_size_reconciliation_truncated: false,
         list_path_raw_options_observer: None,
-        defer_abandoned_entries_until_finished: false,
     };
 
     (scanner, temp_dir)
@@ -769,7 +762,6 @@ fn test_classify_get_size_failure_marks_metadata_heal_object_path() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -801,7 +793,6 @@ fn test_classify_get_size_failure_records_transient_metadata_error_without_heal(
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -858,7 +849,6 @@ async fn test_heal_replication_skips_only_unversioned_data_versions() {
         lifecycle: None,
         object_lock: None,
         replication: Some(Arc::new(ReplicationConfig::new(None, None))),
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -2261,7 +2251,6 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: true,
         heal_bitrot: true,
         debug: false,
@@ -2273,7 +2262,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
     };
     let mut size_summary = SizeSummary::default();
 
-    let size = item.heal_actions(&object_info, 123, &mut size_summary, None).await;
+    let size = item.heal_actions(&object_info, 123, &mut size_summary).await;
     assert_eq!(size, 123);
 }
 
@@ -2502,7 +2491,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
     let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
     let heal_starts = Arc::new(AtomicUsize::new(0));
     let heal_starts_clone = heal_starts.clone();
-    let healed_versions = Arc::new(Mutex::new(Vec::new()));
+    let healed_versions = Arc::new(Mutex::new(Vec::<Option<String>>::new()));
     let healed_versions_clone = healed_versions.clone();
     let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels()
         .expect("heal channels should initialize once for scanner tests");
@@ -2514,7 +2503,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
                 healed_versions_clone
                     .lock()
                     .expect("heal version capture lock should not be poisoned")
-                    .push((request.object_prefix, request.object_version_id, request.pool_index, request.set_index));
+                    .push(request.object_version_id);
                 let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt {
                     result: HealAdmissionResult::Accepted,
                     task_id: request.id,
@@ -2595,90 +2584,9 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
             .lock()
             .expect("heal version capture lock should not be poisoned")
             .iter()
-            .any(|(_, version, _, _)| version.as_deref() == Some(orphan_version_text.as_str())),
+            .any(|version| version.as_deref() == Some(orphan_version_text.as_str())),
         "sub-quorum orphan version must be submitted as an exact heal candidate"
     );
-    assert!(
-        healed_versions
-            .lock()
-            .expect("heal scope capture lock should not be poisoned")
-            .iter()
-            .filter(|(object, _, _, _)| object.is_some())
-            .all(|(_, _, pool, set)| (*pool, *set) == (Some(0), Some(0))),
-        "high-priority discovered objects must retain the scanning disk's scope"
-    );
-    let bucket_admission = scanner
-        .send_required_scanner_heal_request(
-            PendingScannerHealKind::Bucket,
-            bucket.to_string(),
-            None,
-            None,
-            build_bucket_heal_request(bucket.to_string(), HealChannelPriority::High),
-        )
-        .await
-        .expect("submit bucket request");
-    assert_eq!(bucket_admission, HealAdmissionResult::Accepted);
-    assert_eq!(
-        healed_versions.lock().expect("read captured bucket scope").last(),
-        Some(&(None, None, None, None)),
-        "bucket healing must retain its cross-pool scope"
-    );
-    scanner.new_cache.info.name = bucket.to_string();
-    scanner.new_cache.info.pending_heals = vec![pending_heal(
-        PendingScannerHealKind::Object,
-        bucket,
-        Some("pending-object"),
-        Some(&orphan_version_text),
-        0,
-        1,
-    )];
-    scanner
-        .retry_pending_scanner_heals()
-        .await
-        .expect("retry persisted object heal");
-    assert_eq!(
-        healed_versions.lock().expect("read retried object scope").last(),
-        Some(&(Some("pending-object".to_string()), Some(orphan_version_text.clone()), Some(0), Some(0))),
-        "persisted retries must bind the current scanning disk"
-    );
-    let unscoped_endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("parse uninitialized endpoint");
-    scanner.local_disk = new_disk(
-        &unscoped_endpoint,
-        &DiskOption {
-            cleanup: false,
-            health_check: false,
-        },
-    )
-    .await
-    .expect("open disk without initialized location");
-    let request_count = healed_versions.lock().expect("count admitted requests").len();
-    let deferred = scanner
-        .send_required_scanner_heal_request(
-            PendingScannerHealKind::Object,
-            bucket.to_string(),
-            Some("unscoped-object".to_string()),
-            Some(orphan_version_text.clone()),
-            build_object_heal_request(
-                bucket.to_string(),
-                "unscoped-object".to_string(),
-                Some(orphan_version_text),
-                HealScanMode::Deep,
-                HealChannelPriority::High,
-            ),
-        )
-        .await
-        .expect("defer object with unknown disk location");
-    assert_eq!(deferred, HealAdmissionResult::Full);
-    assert_eq!(healed_versions.lock().expect("count unchanged admissions").len(), request_count);
-    assert!(
-        scanner
-            .new_cache
-            .info
-            .pending_heals
-            .iter()
-            .any(|entry| entry.object.as_deref() == Some("unscoped-object")),
-        "an unscoped high-priority object must remain pending"
-    );
     for (path, expected) in expected_metadata {
         assert_eq!(
             tokio::fs::read(&path)

crates/scanner/src/scanner_folder/tests/listing_completion.rs

@@ -1,104 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use super::*;
-
-#[tokio::test]
-#[serial]
-async fn abandoned_listing_drains_candidate_buffered_before_finished() {
-    let (mut scanner, root) = build_test_scanner().await;
-    let _guard = TestGuard {
-        temp_dir: Some(root.clone()),
-    };
-    let bucket = "bucket";
-    let object = "abandoned/object";
-    tokio::fs::create_dir_all(root.join(bucket)).await.expect("local bucket");
-    let mut disks = vec![scanner.local_disk.clone()];
-    let mut original_metadata = Vec::new();
-    for (index, version) in [(2, 0x1935), (3, 0x1935), (4, 0x1934)] {
-        let disk_root = root.join(format!("disk-{index}"));
-        let metadata = metadata_for_object_version(bucket, object, Some(Uuid::from_u128(version)));
-        write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
-        original_metadata.push((disk_root.join(bucket).join(object).join(STORAGE_FORMAT_FILE), metadata));
-        let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("listing fixture endpoint");
-        disks.push(
-            new_disk(
-                &endpoint,
-                &DiskOption {
-                    cleanup: false,
-                    health_check: false,
-                },
-            )
-            .await
-            .expect("listing fixture disk"),
-        );
-    }
-    for cache in [&mut scanner.old_cache, &mut scanner.new_cache] {
-        cache.info.name = bucket.to_owned();
-    }
-    scanner.old_cache.replace(
-        "bucket/abandoned/object",
-        bucket,
-        DataUsageEntry {
-            objects: 1,
-            ..Default::default()
-        },
-    );
-    scanner.heal_object_select = 1;
-    scanner.disks = disks;
-    scanner.disks_quorum = 2;
-    scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
-    // Force the valid select ordering that receives terminal errors before the
-    // final candidate. One candidate frame fits in the bounded partial channel.
-    scanner.defer_abandoned_entries_until_finished = true;
-    let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels().expect("isolated heal channel");
-    let (captured_tx, mut captured_rx) = mpsc::unbounded_channel();
-    let responder = tokio::spawn(async move {
-        while let Some(command) = heal_rx.recv().await {
-            let rustfs_heal_contracts::heal_channel::HealReceiptCommand { request, response_tx } = command;
-            captured_tx
-                .send((request.object_prefix.clone(), request.object_version_id.clone()))
-                .expect("candidate capture remains open");
-            let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt {
-                result: HealAdmissionResult::Accepted,
-                task_id: request.id,
-            }));
-        }
-    });
-    let result = tokio::time::timeout(
-        Duration::from_secs(2),
-        scanner.scan_folder(
-            CancellationToken::new(),
-            CachedFolder {
-                name: bucket.to_owned(),
-                parent: None,
-                object_heal_prob_div: 1,
-            },
-            &mut DataUsageEntry::default(),
-        ),
-    )
-    .await;
-    responder.abort();
-    result
-        .expect("listing completion must close every callback channel")
-        .expect("complete folder scan");
-    let mut candidates = HashSet::new();
-    while let Ok((object, version)) = captured_rx.try_recv() {
-        if let (Some(object), Some(version)) = (object, version) {
-            candidates.insert((object, version));
-        }
-    }
-    let expected = [0x1934, 0x1935]
-        .into_iter()
-        // Raw base-object listings include the trailing separator. Preserve
-        // the producer's candidate identity when draining its callback frame.
-        .map(|version| (format!("{object}/"), Uuid::from_u128(version).to_string()))
-        .collect();
-    assert_eq!(
-        candidates, expected,
-        "terminal disk errors must not discard buffered exact-version candidates"
-    );
-    for (path, metadata) in original_metadata {
-        assert_eq!(tokio::fs::read(path).await.expect("retained metadata"), metadata);
-    }
-}

crates/scanner/src/scanner_io/cache.rs

@@ -172,55 +172,12 @@ fn record_checkpoint_save_failure(cache_name: &str, checkpoint: &DataUsageCache,
 
 #[derive(Debug)]
 pub(crate) struct ScannerCacheLockGuards {
-    scoped: Option<NamespaceLockGuard>,
+    scoped: NamespaceLockGuard,
 }
 
 impl ScannerCacheLockGuards {
     pub(crate) fn is_lock_lost(&self) -> bool {
-        self.scoped.as_ref().is_none_or(NamespaceLockGuard::is_lock_lost)
-    }
-
-    /// Suspend the same traversal while yielding its cache lease. Lock order on
-    /// resumption is node admission -> cache lease -> cache revision reads.
-    pub(crate) async fn resume_node_operation(
-        &mut self,
-        store: Arc<SetDisks>,
-        cache_name: &str,
-        source: DataUsageCacheSource,
-        revisions: &DataUsageCacheRevisions,
-        request: crate::workload_admission::ScannerNodeRequest,
-    ) -> Result<()> {
-        if self.is_lock_lost() {
-            return Err(Error::other("scanner cache lock was lost before node admission"));
-        }
-        if request.respond.is_closed() {
-            return Ok(());
-        }
-        let crate::workload_admission::ScannerNodeRequest { owner, respond, cancel } = request;
-        drop(self.scoped.take());
-        let resume = async {
-            let permit = owner.acquire().await;
-            *self = acquire_scanner_cache_locks(store.as_ref(), cache_name, source)
-                .await
-                .map_err(Error::other)?;
-            let current = crate::storage_api::ecstore_with_background_disk_io(
-                cancel.clone(),
-                DataUsageCache::read_revisions(store, cache_name),
-            )
-            .await?;
-            if self.is_lock_lost() || &current != revisions {
-                return Err(Error::other("scanner cache changed while waiting for node admission"));
-            }
-            // If the walk was dropped, send returns ownership and drops the
-            // permit without recording completed work.
-            let _ = respond.send(permit);
-            Ok(())
-        };
-        tokio::select! {
-            biased;
-            _ = cancel.cancelled() => Err(Error::other("scanner node admission cancelled")),
-            result = resume => result,
-        }
+        self.scoped.is_lock_lost()
     }
 }
 
@@ -258,15 +215,6 @@ impl std::fmt::Display for ScannerCacheLockError {
     }
 }
 
-impl std::error::Error for ScannerCacheLockError {
-    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        match self {
-            Self::Create { source, .. } => Some(source),
-            Self::Acquire { source, .. } => Some(source),
-        }
-    }
-}
-
 pub(crate) async fn acquire_scanner_cache_locks(
     store: &SetDisks,
     cache_name: &str,
@@ -289,7 +237,7 @@ pub(crate) async fn acquire_scanner_cache_locks(
             source,
         })?;
 
-    Ok(ScannerCacheLockGuards { scoped: Some(scoped) })
+    Ok(ScannerCacheLockGuards { scoped })
 }
 
 pub(super) async fn await_scanner_disk_shutdown<F>(scan: Pin<&mut F>)

crates/scanner/src/scanner_io/io_cache.rs

@@ -1090,7 +1090,7 @@ impl ScannerIOCache for SetDisks {
 
                     // Lock order: scanner leader fence -> set-scoped per-bucket cache lock ->
                     // cache object read/write.
-                    let mut cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
+                    let cache_guard = match acquire_scanner_cache_locks(store_clone_clone.as_ref(), &cache_name, source).await {
                         Ok(guard) => guard,
                         Err(e) => {
                             if e.is_contention() {
@@ -1320,10 +1320,9 @@ impl ScannerIOCache for SetDisks {
                     );
 
                     let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
-                    let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
 
                     let scan_ctx = ctx_clone.child_token();
-                    let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
+                    let scan = disk.clone().nsscanner_disk(
                         scan_ctx.clone(),
                         budget_clone.clone(),
                         set_disk_inventory_clone.as_ref().clone(),
@@ -1334,7 +1333,7 @@ impl ScannerIOCache for SetDisks {
                             prefix_scan_scope,
                             checkpoint_tx: Some(checkpoint_tx),
                         },
-                    ));
+                    );
                     tokio::pin!(scan);
                     let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
                     lock_watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
@@ -1342,24 +1341,6 @@ impl ScannerIOCache for SetDisks {
                     let scan_result = loop {
                         tokio::select! {
                             result = &mut scan => break result,
-                            request = async {
-                                match node_rx.as_mut() {
-                                    Some(receiver) => receiver.recv().await,
-                                    None => std::future::pending().await,
-                                }
-                            } => {
-                                let Some(request) = request else {
-                                    node_rx = None;
-                                    continue;
-                                };
-                                if let Err(error) = cache_guard.resume_node_operation(
-                                    store_clone_clone.clone(), &cache_name, source, &revisions, request,
-                                ).await {
-                                    scan_ctx.cancel();
-                                    await_scanner_disk_shutdown(scan.as_mut()).await;
-                                    break Err(error);
-                                }
-                            }
                             _ = lock_watch.tick() => {
                                 if cache_guard.is_lock_lost() {
                                     scan_ctx.cancel();

crates/scanner/src/scanner_io/io_disk.rs

@@ -38,13 +38,6 @@ pub(super) fn tier_stats_template(tier_names: &[String]) -> HashMap<String, Tier
     tier_stats
 }
 
-fn ensure_scanner_node_operation_active(cancellation: Option<&CancellationToken>) -> Result<()> {
-    if cancellation.is_some_and(CancellationToken::is_cancelled) {
-        return Err(StorageError::other("Scanner operation cancelled"));
-    }
-    Ok(())
-}
-
 #[async_trait::async_trait]
 impl ScannerIODisk for Disk {
     async fn get_size(&self, item: ScannerItem) -> Result<SizeSummary> {
@@ -52,18 +45,14 @@ impl ScannerIODisk for Disk {
     }
 
     async fn get_size_with_tier_names(&self, mut item: ScannerItem, tier_names: &[String]) -> Result<SizeSummary> {
-        let cancellation = crate::workload_admission::scanner_node_operation_cancellation();
         let done_object = Metrics::time(Metric::ScanObject);
 
         if !is_xl_meta_path(&item.path) {
             return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
         }
 
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         let metadata_object_path = item.object_path();
-        let metadata_result = self.read_metadata(&item.bucket, &metadata_object_path).await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
-        let data = match metadata_result {
+        let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
             Ok(data) => data,
             Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
                 return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -97,10 +86,7 @@ impl ScannerIODisk for Disk {
         // (which used to query it a second time). On failure keep the
         // historical fallback: default configuration (versioned = false) plus
         // the warn that `apply_actions` used to emit.
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
-        let versioning_result = BucketVersioningSys::get(&item.bucket).await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
-        let versioning_config = match versioning_result {
+        let versioning_config = match BucketVersioningSys::get(&item.bucket).await {
             Ok(versioning_config) => versioning_config,
             Err(_) => {
                 warn!(
@@ -136,9 +122,7 @@ impl ScannerIODisk for Disk {
             ..Default::default()
         };
 
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         let lock_config = object_lock_config_for_scanner_item(&item).await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
 
         // Count every version this object contributes to the scan, independent
         // of any lifecycle configuration, so scan-coverage metrics stay honest
@@ -146,35 +130,17 @@ impl ScannerIODisk for Disk {
         // `object_infos`.
         global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);
 
-        // A caller-provided item or persisted usage cache cannot choose another set.
-        item.heal_scope = crate::scanner_folder::scanner_disk_heal_scope(self);
-        #[cfg(test)]
-        let _ = crate::scanner_folder::observe_test_size_dispatch_boundary("apply_actions");
-        item.apply_actions_cancellable(
-            object_infos,
-            lock_config,
-            versioning_config,
-            tier_names,
-            &mut size_summary,
-            cancellation.as_ref(),
-        )
-        .await;
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
+        item.apply_actions(object_infos, lock_config, versioning_config, tier_names, &mut size_summary)
+            .await;
 
         if !free_version_infos.is_empty() {
             for oi in free_version_infos {
-                ensure_scanner_node_operation_active(cancellation.as_ref())?;
                 if ScannerItem::tier_is_known(&oi, tier_names) {
-                    #[cfg(test)]
-                    if crate::scanner_folder::observe_test_size_dispatch_boundary("free_version_dispatch") {
-                        continue;
-                    }
-                    enqueue_runtime_free_version(oi, cancellation.as_ref()).await;
+                    enqueue_runtime_free_version(oi).await;
                 }
             }
         }
 
-        ensure_scanner_node_operation_active(cancellation.as_ref())?;
         done_object();
 
         Ok(size_summary)

crates/scanner/src/scanner_io/tests.rs

@@ -46,9 +46,7 @@ use uuid::Uuid;
 
 mod checkpoint_coverage_identity;
 mod collector_lifecycle;
-mod expiry_lock;
 mod listing_window_scope;
-mod node_admission;
 mod partial_bucket_carry;
 mod per_bucket_certification;
 mod scoped_entry_fallback;
@@ -1772,7 +1770,6 @@ async fn scanner_item_object_lock_uses_cached_config() {
         lifecycle: None,
         object_lock: Some(cached.clone()),
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4522,7 +4519,6 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4578,7 +4574,6 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4661,7 +4656,6 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,

crates/scanner/src/scanner_io/tests/expiry_lock.rs

@@ -1,183 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::*;
-use crate::storage_api::owner::EcstoreExpiryState;
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::task::Poll;
-use std::time::Duration;
-
-// Return Some(result) if the operation ended too early, None only if it was
-// actually polled pending after the prerequisite read had finished.
-async fn observe_queued<F: Future>(
-    mut operation: Pin<&mut F>,
-    read_complete: impl Fn() -> bool,
-    observation: Duration,
-) -> std::result::Result<Option<F::Output>, tokio::time::error::Elapsed> {
-    tokio::time::timeout(
-        observation,
-        std::future::poll_fn(|cx| match operation.as_mut().poll(cx) {
-            Poll::Ready(value) => Poll::Ready(Some(value)),
-            Poll::Pending if read_complete() => Poll::Ready(None),
-            Poll::Pending => Poll::Pending,
-        }),
-    )
-    .await
-}
-
-#[tokio::test]
-#[serial]
-async fn free_version_cancelled_while_waiting_for_expiry_write_lock_never_queues() {
-    let state = EcstoreExpiryState::new_with_unconsumed_worker_channel(4);
-    let observation = Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS);
-    let context = crate::ExpiryLockTestContext {
-        state: state.clone(),
-        store: None,
-        completed_incarnation_reads: Arc::new(AtomicUsize::new(0)),
-    };
-    crate::EXPIRY_LOCK_TEST_CONTEXT
-        .scope(context, async {
-            let object = crate::ScannerObjectInfo {
-                bucket: "expiry-lock-bucket".to_string(),
-                name: "free-object".to_string(),
-                ..Default::default()
-            };
-            let cancellation = CancellationToken::new();
-            let guard = state.write().await;
-            let mut pending = Box::pin(crate::enqueue_runtime_free_version(object.clone(), Some(&cancellation)));
-            let first = observe_queued(pending.as_mut(), || true, observation).await;
-            let observed_wait = matches!(&first, Ok(None));
-            let already_finished = matches!(&first, Ok(Some(())));
-            cancellation.cancel();
-            let cancelled_while_locked = if already_finished {
-                false
-            } else {
-                tokio::time::timeout(observation, pending.as_mut()).await.is_ok()
-            };
-            let pending_while_locked = guard.pending_tasks();
-            drop(guard);
-            if !already_finished && !cancelled_while_locked {
-                pending.as_mut().await;
-            }
-            drop(pending);
-            let pending_after_cancel = state.read().await.pending_tasks();
-
-            // Positive control queues a real free-version task while using the same
-            // implementation and isolated, unconsumed worker channel.
-            let mut positive = Box::pin(crate::enqueue_runtime_free_version(object, None));
-            let positive_in_time = tokio::time::timeout(observation, positive.as_mut()).await.is_ok();
-            if !positive_in_time {
-                positive.as_mut().await;
-            }
-            drop(positive);
-            let pending_after_control = state.read().await.pending_tasks();
-            assert!(observed_wait, "must poll the real enqueue while the write lock is held");
-            assert!(cancelled_while_locked, "cancellation must finish before the lock is released");
-            assert_eq!(pending_while_locked, 0);
-            assert_eq!(pending_after_cancel, 0);
-            assert!(positive_in_time, "positive control must finish within the observation limit");
-            assert_eq!(pending_after_control, 1, "positive control must queue a task");
-        })
-        .await;
-}
-
-#[tokio::test]
-#[serial]
-async fn newer_noncurrent_cancelled_after_real_read_while_waiting_for_expiry_write_lock_never_queues() {
-    let (_temp_dir, store) = setup_local_scanner_store(1, 4).await;
-    let bucket = "expiry-lock-bucket";
-    store
-        .make_bucket(bucket, &MakeBucketOptions::default())
-        .await
-        .expect("create bucket");
-    let expected_incarnation = store.bucket_incarnation_id(bucket).await.expect("real incarnation metadata");
-    assert!(!expected_incarnation.is_nil());
-    let state = EcstoreExpiryState::new_with_unconsumed_worker_channel(4);
-    let reads = Arc::new(AtomicUsize::new(0));
-    let context = crate::ExpiryLockTestContext {
-        state: state.clone(),
-        store: Some(store.clone()),
-        completed_incarnation_reads: reads.clone(),
-    };
-    let observation = Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS);
-    crate::EXPIRY_LOCK_TEST_CONTEXT
-        .scope(context, async {
-            let versions = vec![crate::ScannerObjectToDelete {
-                object_name: "noncurrent-object".to_string(),
-                version_id: Some(Uuid::new_v4()),
-                ..Default::default()
-            }];
-            let event = crate::Event::default();
-            let src = crate::LcEventSrc::Scanner;
-            let cancellation = CancellationToken::new();
-            let guard = state.write().await;
-            let mut pending = Box::pin(crate::enqueue_runtime_newer_noncurrent(
-                bucket,
-                versions.clone(),
-                event.clone(),
-                &src,
-                Some(&cancellation),
-            ));
-            let first = observe_queued(pending.as_mut(), || reads.load(Ordering::SeqCst) == 1, observation).await;
-            let observed_wait_after_read = matches!(&first, Ok(None));
-            let mut result = match first {
-                Ok(Some(value)) => Some(value),
-                _ => None,
-            };
-            cancellation.cancel();
-            let cancelled_while_locked = if result.is_some() {
-                false
-            } else {
-                match tokio::time::timeout(observation, pending.as_mut()).await {
-                    Ok(value) => {
-                        result = Some(value);
-                        true
-                    }
-                    Err(_) => false,
-                }
-            };
-            let pending_while_locked = guard.pending_tasks();
-            drop(guard);
-            let result = match result {
-                Some(value) => value,
-                None => pending.as_mut().await,
-            };
-            drop(pending);
-            let pending_after_cancel = state.read().await.pending_tasks();
-            let mut positive = Box::pin(crate::enqueue_runtime_newer_noncurrent(bucket, versions, event, &src, None));
-            let bounded_positive = tokio::time::timeout(observation, positive.as_mut()).await;
-            let positive_in_time = bounded_positive.is_ok();
-            let positive_result = match bounded_positive {
-                Ok(value) => value,
-                Err(_) => positive.as_mut().await,
-            };
-            drop(positive);
-            let pending_after_control = state.read().await.pending_tasks();
-            assert!(
-                observed_wait_after_read,
-                "real incarnation read must finish before waiting on the held lock"
-            );
-            assert!(cancelled_while_locked, "cancellation must finish before the lock is released");
-            assert!(!result);
-            assert_eq!(pending_while_locked, 0);
-            assert_eq!(pending_after_cancel, 0);
-            assert!(positive_in_time, "positive control must finish within the observation limit");
-            assert!(positive_result, "positive control must enqueue the nonempty version batch");
-            assert_eq!(pending_after_control, 1);
-            assert_eq!(reads.load(Ordering::SeqCst), 2);
-        })
-        .await;
-}

crates/scanner/src/scanner_io/tests/node_admission.rs

@@ -1,136 +0,0 @@
-// Copyright 2026 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use super::*;
-use crate::workload_admission::ScannerNodeRequest;
-use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
-use std::num::NonZeroU16;
-use tokio::sync::oneshot;
-
-#[derive(Clone, Copy, PartialEq, Eq)]
-enum ResumeCase {
-    Unchanged,
-    Updated,
-    Cancelled,
-}
-
-async fn check_cache_lease_resume(case: ResumeCase) {
-    let (_directory, store) = setup_local_scanner_store(1, 4).await;
-    let set = store.pools[0].disk_set[0].clone();
-    let source = DataUsageCacheSource::new(0, 0);
-    let cache_name = "node-resume/.usage-cache.bin";
-    let cache = DataUsageCache::default();
-    cache.save(set.clone(), cache_name).await.expect("seed cache");
-    let revisions = DataUsageCache::read_revisions(set.clone(), cache_name)
-        .await
-        .expect("initial revisions");
-    let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one shared unit"));
-    gate.update(
-        &mut FeedbackController::new(),
-        Some(FeedbackObservation {
-            generation: 1,
-            max_concurrency: 1,
-            pressure: ResourcePressure::Unknown,
-            completed_units: None,
-        }),
-    );
-    let mut held = Some(gate.acquire().await);
-    let mut guard = acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
-        .await
-        .expect("initial cache lease");
-    let ctx = CancellationToken::new();
-    let (respond, response) = oneshot::channel();
-    let request = ScannerNodeRequest {
-        owner: gate.clone(),
-        respond,
-        cancel: ctx.clone(),
-    };
-    let mut expected = revisions.clone();
-    let result = {
-        let resume = guard.resume_node_operation(set.clone(), cache_name, source, &revisions, request);
-        tokio::pin!(resume);
-        assert!(futures::poll!(&mut resume).is_pending(), "shared node is occupied");
-        let other = tokio::time::timeout(Duration::from_secs(1), acquire_scanner_cache_locks(set.as_ref(), cache_name, source))
-            .await
-            .expect("cache lock must be released before joining the node queue")
-            .expect("another cache worker can acquire the lease");
-        if case == ResumeCase::Updated {
-            let mut newer = cache;
-            newer.info.next_cycle = 1;
-            newer
-                .save(set.clone(), cache_name)
-                .await
-                .expect("another worker updates the cache");
-            expected = DataUsageCache::read_revisions(set.clone(), cache_name)
-                .await
-                .expect("new cache revisions");
-            assert_ne!(expected, revisions);
-        }
-        drop(other);
-        if case == ResumeCase::Cancelled {
-            ctx.cancel();
-        } else {
-            held.take().expect("held node unit").complete();
-        }
-        tokio::time::timeout(Duration::from_secs(2), resume)
-            .await
-            .expect("node resumption must finish or observe cancellation")
-    };
-    if case == ResumeCase::Unchanged {
-        result.expect("unchanged cache resumes the same walk");
-        let permit = response.await.expect("coordinator returns the node permit");
-        assert_eq!(gate.snapshot().active, 1);
-        assert_eq!(gate.completed(), Some(1), "admitting scanner work must not count it as completed");
-        assert!(!guard.is_lock_lost());
-        let lock = set
-            .new_ns_lock(RUSTFS_META_BUCKET, &scanner_cache_lock_resource(cache_name, source))
-            .await
-            .expect("competing cache lock");
-        assert!(
-            lock.get_write_lock_quiet(Duration::from_millis(20)).await.is_err(),
-            "cache lease must be restored before the operation resumes"
-        );
-        permit.complete();
-    } else {
-        result.expect_err("cancelled or stale work cannot resume");
-        assert!(response.await.is_err(), "failed resumption must not authorize storage work");
-        if case == ResumeCase::Cancelled {
-            assert!(guard.is_lock_lost());
-            assert_eq!(gate.snapshot().active, 1, "the running heal keeps its unit");
-            drop(held.take());
-            assert_eq!(gate.completed(), Some(0));
-        } else {
-            assert_eq!(gate.completed(), Some(1));
-        }
-    }
-    drop(guard);
-    assert_eq!(gate.snapshot().active, 0);
-    assert!(gate.try_acquire().is_some(), "dropped requests must leave no queue or capacity leak");
-    assert_eq!(
-        DataUsageCache::read_revisions(set.clone(), cache_name)
-            .await
-            .expect("final revisions"),
-        expected
-    );
-    acquire_scanner_cache_locks(set.as_ref(), cache_name, source)
-        .await
-        .expect("final cache lease is available");
-}
-
-#[tokio::test]
-#[serial]
-async fn scanner_node_wait_yields_cache_lease_before_fifo_admission() {
-    check_cache_lease_resume(ResumeCase::Unchanged).await;
-}
-
-#[tokio::test]
-#[serial]
-async fn scanner_node_wait_rejects_cache_changed_while_unlocked() {
-    check_cache_lease_resume(ResumeCase::Updated).await;
-}
-
-#[tokio::test]
-#[serial]
-async fn scanner_node_wait_cancellation_releases_queue_without_stealing_running_unit() {
-    check_cache_lease_resume(ResumeCase::Cancelled).await;
-}

crates/scanner/src/storage_api.rs

@@ -22,8 +22,6 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::Notify;
 
 pub(crate) use rustfs_ecstore::with_background_disk_io as ecstore_with_background_disk_io;
-#[cfg(all(test, unix))]
-pub(crate) use rustfs_ecstore::with_background_resource_unit as ecstore_with_background_resource_unit;
 
 pub(crate) use s3s::dto::{
     BucketLifecycleConfiguration as EcstoreBucketLifecycleConfiguration, LifecycleRuleFilter as EcstoreLifecycleRuleFilter,
@@ -41,8 +39,6 @@ pub(crate) use rustfs_ecstore::api::data_usage::{NativeScannerPauseBacklogWriteF
 
 pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
-#[cfg(test)]
-pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState as EcstoreExpiryState;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
     apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
     lifecycle_version_delete_target as ecstore_lifecycle_version_delete_target,
@@ -103,10 +99,6 @@ pub(crate) use rustfs_ecstore::api::disk::{
 pub(crate) use rustfs_ecstore::api::disk::{
     DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
 };
-#[cfg(all(test, unix))]
-pub(crate) use rustfs_ecstore::api::disk::{
-    LocalPublicationPause as EcstoreLocalPublicationPause, LocalPublicationStage as EcstoreLocalPublicationStage,
-};
 pub(crate) use rustfs_ecstore::api::error::{
     Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
 };
@@ -169,17 +161,10 @@ pub(crate) mod ecstore_diagnostic {
 pub(crate) type EcstoreHealResultItem = <EcstoreStore as storage_contracts::HealOperations>::HealResultItem;
 
 pub(crate) mod owner {
-    #[cfg(test)]
-    pub(crate) use super::EcstoreExpiryState;
     #[cfg(test)]
     pub(crate) use super::ecstore_set_object_store_resolver;
     #[cfg(test)]
     pub(crate) use super::storage_contracts::DiskSetSelector;
-    #[cfg(all(test, unix))]
-    pub(crate) use super::{
-        EcstoreLocalPublicationPause, EcstoreLocalPublicationStage, ecstore_with_background_disk_io,
-        ecstore_with_background_resource_unit,
-    };
     pub(crate) use super::{
         MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
         ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,

crates/scanner/src/workload_admission.rs

@@ -12,143 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, LazyLock, OnceLock, RwLock};
+use std::sync::{Arc, LazyLock, RwLock};
 
-use rustfs_concurrency::adaptive::{NodeAdmission, NodePermit};
 use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
-use tokio::sync::{mpsc, oneshot};
-use tokio_util::sync::CancellationToken;
-
-use crate::ScannerError;
-
-static SCANNER_NODE_ADMISSION: OnceLock<Arc<NodeAdmission>> = OnceLock::new();
-
-tokio::task_local! {
-    static SCANNER_NODE_REQUESTS: ScannerNodeRequestSender;
-    static SCANNER_NODE_OPERATION_CANCEL: CancellationToken;
-}
-
-/// Cancellation for the currently admitted scanner operation, when present.
-pub(crate) fn scanner_node_operation_cancellation() -> Option<CancellationToken> {
-    SCANNER_NODE_OPERATION_CANCEL.try_with(Clone::clone).ok()
-}
-
-pub(crate) struct ScannerNodeRequest {
-    pub(crate) owner: Arc<NodeAdmission>,
-    pub(crate) respond: oneshot::Sender<NodePermit>,
-    pub(crate) cancel: CancellationToken,
-}
-
-pub(crate) type ScannerNodeRequestSender = mpsc::Sender<ScannerNodeRequest>;
-pub(crate) type ScannerNodeRequestReceiver = mpsc::Receiver<ScannerNodeRequest>;
-
-/// One suspended operation per walk; disabled admission allocates no channel.
-pub(crate) fn scanner_node_request_channel() -> (Option<ScannerNodeRequestSender>, Option<ScannerNodeRequestReceiver>) {
-    if scanner_node_admission().is_none() {
-        return (None, None);
-    }
-    let (sender, receiver) = mpsc::channel(1);
-    (Some(sender), Some(receiver))
-}
-
-pub(crate) async fn with_scanner_node_requests<F: std::future::Future>(
-    sender: Option<ScannerNodeRequestSender>,
-    operation: F,
-) -> F::Output {
-    match sender {
-        Some(sender) => SCANNER_NODE_REQUESTS.scope(sender, operation).await,
-        None => operation.await,
-    }
-}
-
-#[cfg(test)]
-tokio::task_local! {
-    pub(crate) static TEST_SCANNER_NODE_ADMISSION: Arc<NodeAdmission>;
-    pub(crate) static TEST_SCANNER_AFTER_NODE_ACQUIRE: Arc<dyn Fn() + Send + Sync>;
-}
-
-/// Install the heal manager's node owner before starting scanner workers.
-/// Refuse replacement: existing scans must not outlive an independently renewed owner.
-pub fn set_scanner_node_admission(owner: Arc<NodeAdmission>) -> bool {
-    SCANNER_NODE_ADMISSION.set(owner).is_ok()
-}
-
-pub(crate) fn scanner_node_admission() -> Option<Arc<NodeAdmission>> {
-    #[cfg(test)]
-    if let Ok(owner) = TEST_SCANNER_NODE_ADMISSION.try_with(Arc::clone) {
-        return Some(owner);
-    }
-    SCANNER_NODE_ADMISSION.get().cloned()
-}
-
-/// Try the fast path first. A scoped coordinator releases the cache lease before
-/// joining the node queue and validates the cache before returning a permit.
-/// Uncoordinated callers defer instead of parking under an unknown lock.
-pub(crate) async fn run_scanner_node_operation<T, E>(
-    owner: Option<&Arc<NodeAdmission>>,
-    cancel: &CancellationToken,
-    operation: impl std::future::Future<Output = Result<T, E>>,
-) -> Result<Result<T, E>, ScannerError> {
-    let Some(owner) = owner else {
-        return Ok(operation.await);
-    };
-    let operation = SCANNER_NODE_OPERATION_CANCEL.scope(cancel.clone(), operation);
-    if cancel.is_cancelled() {
-        return Err(ScannerError::Other("Operation cancelled".to_string()));
-    }
-    let permit = match owner.try_acquire() {
-        Some(permit) => permit,
-        None => {
-            let sender = SCANNER_NODE_REQUESTS
-                .try_with(Clone::clone)
-                .map_err(|_| ScannerError::ResourceDeferred)?;
-            let acquire = async {
-                let (respond, response) = oneshot::channel();
-                sender
-                    .send(ScannerNodeRequest {
-                        owner: owner.clone(),
-                        respond,
-                        cancel: cancel.clone(),
-                    })
-                    .await
-                    .map_err(|_| ScannerError::ResourceDeferred)?;
-                response.await.map_err(|_| ScannerError::ResourceDeferred)
-            };
-            tokio::select! {
-                biased;
-                _ = cancel.cancelled() => return Err(ScannerError::Other("Operation cancelled".to_string())),
-                permit = acquire => permit?,
-            }
-        }
-    };
-    #[cfg(test)]
-    let _ = TEST_SCANNER_AFTER_NODE_ACQUIRE.try_with(|hook| hook());
-    let operation_started = AtomicBool::new(false);
-    tokio::pin!(operation);
-    let observed = std::future::poll_fn(|context| {
-        operation_started.store(true, Ordering::Relaxed);
-        std::future::Future::poll(operation.as_mut(), context)
-    });
-    tokio::pin!(observed);
-    let result = tokio::select! {
-        biased;
-        _ = cancel.cancelled() => {
-            // The node slot belongs to the waiter, not the native operation.
-            drop(permit);
-            if !operation_started.load(Ordering::Relaxed) {
-                return Err(ScannerError::Other("Operation cancelled".to_string()));
-            }
-            // Keep polling the started operation under its existing locks.
-            return Ok(observed.await);
-        }
-        result = &mut observed => result,
-    };
-    if result.is_ok() && !cancel.is_cancelled() {
-        permit.complete();
-    }
-    Ok(result)
-}
 
 type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
 

crates/scanner/tests/lifecycle/scanner_heal_scope.rs

@@ -1,144 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use rustfs_filemeta::{FileInfo, FileMeta};
-use rustfs_heal_contracts::heal_channel::{
-    HealAdmissionReceipt, HealAdmissionResult, HealReceiptCommand, HealRequestSource, init_heal_channels,
-};
-use rustfs_scanner::DataUsageCacheSource;
-use rustfs_scanner::scanner_folder::ScannerItem;
-use rustfs_scanner::scanner_io::ScannerIODisk;
-use tokio::sync::mpsc;
-
-use super::storage_api::lifecycle::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
-
-#[test]
-fn scanner_object_requests_bind_the_actual_disk_scope() {
-    const CHILD: &str = "RUSTFS_TEST_SCANNER_SCOPE_CHILD";
-    if std::env::var_os(CHILD).is_none() {
-        let status = std::process::Command::new(std::env::current_exe().expect("locate integration test executable"))
-            .args([
-                "--exact",
-                "scanner_heal_scope::scanner_object_requests_bind_the_actual_disk_scope",
-                "--nocapture",
-            ])
-            .env(CHILD, "1")
-            .status()
-            .expect("run scanner scope regression in an isolated process");
-        assert!(status.success(), "isolated scanner scope regression failed: {status}");
-        return;
-    }
-    tokio::runtime::Builder::new_current_thread()
-        .enable_all()
-        .build()
-        .expect("build isolated scanner runtime")
-        .block_on(check_scanner_object_scope());
-}
-
-async fn check_scanner_object_scope() {
-    let (_commands, mut receipts) = init_heal_channels().expect("initialize isolated scanner request channels");
-    let (captured, mut requests) = mpsc::unbounded_channel();
-    let responder = tokio::spawn(async move {
-        while let Some(command) = receipts.recv().await {
-            let HealReceiptCommand { request, response_tx } = command;
-            let receipt = HealAdmissionReceipt {
-                result: HealAdmissionResult::Accepted,
-                task_id: request.id.clone(),
-            };
-            captured.send(request).expect("retain scanner request");
-            let _ = response_tx.send(Ok(receipt));
-        }
-    });
-
-    for (pool, set, disk_index) in [
-        (Some(0), Some(0), Some(1)),
-        (Some(2), Some(3), Some(4)),
-        (None, Some(3), Some(4)),
-        (Some(2), None, Some(4)),
-        (Some(2), Some(3), None),
-        (None, None, None),
-    ] {
-        for version_id in [None, Some(uuid::Uuid::from_u128(42))] {
-            let root = tempfile::tempdir().expect("create scanner disk");
-            let bucket = "scanner-scope-bucket";
-            let object = "object";
-            let path = root.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
-            tokio::fs::create_dir_all(path.parent().expect("metadata parent"))
-                .await
-                .expect("create object directory");
-            let mut info = FileInfo::new(object, 4, 2);
-            info.volume = bucket.to_string();
-            info.name = object.to_string();
-            info.size = 1;
-            info.version_id = version_id;
-            info.data_dir = Some(uuid::Uuid::new_v4());
-            info.erasure.index = 1;
-            info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1).expect("valid object timestamp"));
-            info.add_object_part(1, "fixture-etag".to_string(), 1, info.mod_time, 1, None, None);
-            let mut meta = FileMeta::new();
-            meta.add_version(info).expect("encode original null version");
-            let original = meta.marshal_msg().expect("marshal original metadata");
-            tokio::fs::write(&path, &original).await.expect("write original metadata");
-            let mut endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("parse scanning disk endpoint");
-            if let Some(pool) = pool {
-                endpoint.set_pool_index(pool);
-            }
-            if let Some(set) = set {
-                endpoint.set_set_index(set);
-            }
-            if let Some(disk_index) = disk_index {
-                endpoint.set_disk_index(disk_index);
-            }
-            let disk = new_disk(
-                &endpoint,
-                &DiskOption {
-                    cleanup: false,
-                    health_check: false,
-                },
-            )
-            .await
-            .expect("open actual scanning disk");
-            let item = ScannerItem {
-                path: path.to_string_lossy().to_string(),
-                bucket: bucket.to_string(),
-                prefix: object.to_string(),
-                object_name: STORAGE_FORMAT_FILE.to_string(),
-                file_type: tokio::fs::metadata(&path).await.expect("stat original metadata").file_type(),
-                lifecycle: None,
-                object_lock: None,
-                replication: None,
-                // This foreign caller-provided scope must never reach admission.
-                heal_scope: Some(DataUsageCacheSource::new(99, 98)),
-                heal_enabled: true,
-                heal_bitrot: true,
-                debug: false,
-            };
-            disk.get_size_with_tier_names(item, &[]).await.expect("scan original object");
-            if let (Some(pool), Some(set), Some(_)) = (pool, set, disk_index) {
-                let request = requests.try_recv().expect("scanner submitted one real object request");
-                assert_eq!(request.pool_index, Some(pool));
-                assert_eq!(request.set_index, Some(set));
-                assert_eq!(request.bucket, bucket);
-                assert_eq!(request.object_prefix.as_deref(), Some(object));
-                assert_eq!(request.object_version_id, version_id.map(|value| value.to_string()));
-                assert_eq!(request.source, HealRequestSource::Scanner);
-                assert_eq!(request.no_lock, None);
-                assert_eq!(request.recreate_missing, Some(false));
-            }
-            assert!(requests.try_recv().is_err(), "no duplicate or unscoped request may be admitted");
-            assert_eq!(tokio::fs::read(&path).await.expect("read original metadata"), original);
-        }
-    }
-    responder.abort();
-}

crates/scanner/tests/lifecycle_integration_test.rs

@@ -39,9 +39,6 @@ use uuid::Uuid;
 
 mod storage_api;
 
-#[path = "lifecycle/scanner_heal_scope.rs"]
-mod scanner_heal_scope;
-
 use storage_api::lifecycle::{
     BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart,
     DeleteAfterObjectLockSnapshotBarrier, DiskOption, ECStore, EcstoreError, Endpoint, EndpointServerPools, Endpoints,
@@ -470,7 +467,6 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
         lifecycle,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -508,7 +504,6 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
         lifecycle: None,
         object_lock: None,
         replication: None,
-        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,

docs/architecture/erasure-coding.md

@@ -313,7 +313,7 @@ Small objects store their payload inline after the container CRC ([filemeta_inli
 ## 8. Read path and read quorum
 
 - **INVARIANT — read quorum = `data_blocks`.** `object_quorum_from_meta` returns `(read_quorum = data_blocks, write_quorum)` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)); `parity_blocks = common_parity(...)` is the parity value held by the most disks that still reaches its own read quorum. When `default_parity_count == 0`, read = write = all shards.
-- Authoritative FileInfo selection — `find_file_info_in_quorum` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)) groups valid metas by a content-identity SHA-256 (`file_info_quorum_hash`) that hashes size/flags/mod_time/transition/version_id/data_dir/parts and, for real objects, data/parity/distribution — **excluding replication-status keys** so replication noise never splits quorum. A meta counts only if its mod_time equals the common mod_time (or etag matches when mod_time is absent). The winning hash must reach quorum, else `ErasureReadQuorum`. S3 reads select at read quorum through `select_valid_fileinfo`, the same rule heal uses and MinIO's `getObjectFileInfo`: the most common mod_time wins and a tie goes to the newer one, so a write that failed after reaching exactly `data_blocks` copies is served as listing shows it, instead of `ErasureReadQuorum` until heal runs. Two identities tied at the selected mod_time (a data = parity set split in half, such as a metadata update that failed after reaching half the drives) resolve to the one on the lowest-index disk, in reads and heal alike, so a retried metadata write can repair the split. A selected version with local shards also needs as many copies as its own `data_blocks`, since delete markers and empty versions can lower the common parity's read quorum. A metadata update whose target disks cannot reach write quorum is refused before any write. A version replaced by a committed write keeps fewer than `data_blocks` copies and can never be selected again; the newer side of a failed write can be served and then lose to the older side if one of its disks fails before heal converges the set. Reads in the system bucket (`is_meta_bucketname`) keep the commit-proof rule in `select_committed_fileinfo`: when enough copies answer to prove write quorum, the latest version must hold it, and a degraded read never serves a newer version below write quorum. Pool metadata's PREPARE/COMMIT recovery and identity epochs rely on an ordinary read implying a committed record, so a split stays unreadable until native recovery resolves it. When every drive answered and an S3 latest read still cannot select a version, `read_latest_past_dangling_versions` walks the version stack newest first through exact-version reads, as MinIO's `pickLatestQuorumFilesInfo` resolves stacks: it serves the readable version with the newest selected mod_time, skips versions whose exact read proves them dangling (the NoSuchVersion an explicit GET reports) and queues heal for them, and keeps the read-quorum error on any other outcome. A failed versioned PUT that left the new version on fewer drives than its data blocks therefore no longer hides the committed version below it; ListObjectVersions can still show the dangling version as latest until heal removes it. Metadata updates of the latest version (object tagging, `put_object_metadata`) and the CopyObject source read resolve a failed selection through the same walk; when the walk finds no readable version, tagging keeps the read-quorum error while `put_object_metadata` and CopyObject keep their existing `delete_if_dangling` cleanup. Replication status write-back never walks. The update targets only the served version's disks and is refused before writing when they cannot reach its write quorum, including a version-only copy.
+- Authoritative FileInfo selection — `find_file_info_in_quorum` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)) groups valid metas by a content-identity SHA-256 (`file_info_quorum_hash`) that hashes size/flags/mod_time/transition/version_id/data_dir/parts and, for real objects, data/parity/distribution — **excluding replication-status keys** so replication noise never splits quorum. A meta counts only if its mod_time equals the common mod_time (or etag matches when mod_time is absent). The winning hash must reach quorum, else `ErasureReadQuorum`. S3 reads select at read quorum through `select_valid_fileinfo`, the same rule heal uses and MinIO's `getObjectFileInfo`: the most common mod_time wins and a tie goes to the newer one, so a write that failed after reaching exactly `data_blocks` copies is served as listing shows it, instead of `ErasureReadQuorum` until heal runs. Two identities tied at the selected mod_time (a data = parity set split in half, such as a metadata update that failed after reaching half the drives) resolve to the one on the lowest-index disk, in reads and heal alike, so a retried metadata write can repair the split. A selected version with local shards also needs as many copies as its own `data_blocks`, since delete markers and empty versions can lower the common parity's read quorum. A metadata update whose target disks cannot reach write quorum is refused before any write. A version replaced by a committed write keeps fewer than `data_blocks` copies and can never be selected again; the newer side of a failed write can be served and then lose to the older side if one of its disks fails before heal converges the set. Reads in the system bucket (`is_meta_bucketname`) keep the commit-proof rule in `select_committed_fileinfo`: when enough copies answer to prove write quorum, the latest version must hold it, and a degraded read never serves a newer version below write quorum. Pool metadata's PREPARE/COMMIT recovery and identity epochs rely on an ordinary read implying a committed record, so a split stays unreadable until native recovery resolves it. When every drive answered and an S3 latest read still cannot select a version, `read_latest_past_dangling_versions` walks the version stack newest first through exact-version reads, as MinIO's `pickLatestQuorumFilesInfo` resolves stacks: it serves the readable version with the newest selected mod_time, skips versions whose exact read proves them dangling (the NoSuchVersion an explicit GET reports) and queues heal for them, and keeps the read-quorum error on any other outcome. A failed versioned PUT that left the new version on fewer drives than its data blocks therefore no longer hides the committed version below it; ListObjectVersions can still show the dangling version as latest until heal removes it.
 - **INVARIANT — decode needs ≥ `data_blocks` shards.** The stripe reader requires `available_shards ≥ data_shards`; below that the read fails closed with a read-quorum error (never silent truncation) ([set_disk/read.rs](../../crates/ecstore/src/set_disk/read.rs), [set_disk/shard_source.rs](../../crates/ecstore/src/set_disk/shard_source.rs)). Codec geometry is validated at construction: `Erasure::try_new` / `try_new_with_options` ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)) return `ErasureConstructionError` for `data_shards == 0`, `block_size == 0`, shard-count overflow, or an unsupported shard configuration, so a read never reaches a `block_size` / `data_shards` division with invalid geometry (§13).
 - If `available ≥ data_blocks` but some shards are missing, the read is served **and** a background read-repair heal is enqueued.
 - **INVARIANT — cross-stripe read verification.** When a data shard is missing and `available > data_blocks`, reconstruction regenerates parity and compares it to the surviving parity; a mismatch is `InvalidData "inconsistent read source shards"` (backlog#832), catching corruption that passed per-shard bitrot but disagrees across the stripe ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)).

docs/architecture/heal-terminal-reports.md

@@ -4,10 +4,6 @@ Admin heal tokens retain their terminal status for ten minutes from the original
 
 ## Publication and Recovery
 
-Authenticated token queries first inspect the receiving node's in-memory automatic tasks. An original Scanner, MRF, automatic-Heal, or read-repair token can return its path-bound snapshot on that node, including the retained outcome and incremental cursor. The task source and original ID remain attached to the completion snapshot; aliases cannot enter this local query path. The local lookup never reads administrator recovery records.
-
-Administrator tokens and unknown local identities continue through the cluster coordinator. Starting and cancelling tasks retain their coordinator route. Querying another node does not discover or aggregate a remote automatic task, and this path does not extend automatic-task retention across process restart.
-
 The original schema-1 `terminal-root-heal-<token>.json` remains the commit marker. Its separate `heal-terminal-report-<token>.json` report has its own schema version and embeds the exact terminal identity. Both files belong to the same coordinator disk as the pending intent.
 
 Publication writes the report through the storage owner's conditional file update, writes the terminal marker, and then conditionally removes the pending intent. Recovery never treats a report without its marker as committed. A committed terminal continues to suppress stale pending work even when its report is unreadable. An initial publication failure leaves the existing responsibility available for recovery.

docs/operations/scanner-runtime-controls.md

@@ -529,21 +529,17 @@ The OTLP exporter records these node-local histograms for completed manager admi
 
 The two new histograms aggregate all observed admission sources and results without per-request labels. An ordinary uninterrupted admission partitions the phase into wait and hold. Admin forceStart releases these guards while cancelling overlaps, then reacquires them: the cancellation gap is included only in the original total phase, and the second acquisition contributes to wait. Cancellation subroutine work is outside these outer admission guard intervals. Recording happens after all three guards are released. They measure manager admission ownership, not namespace/object-lock waiting during repair. The status field `maxLockPhaseMicros` remains a manager-lifetime maximum, not a p95 sample. Keep same-window histogram counts and bucket resolution when deriving percentiles; missing observations do not mean zero latency. These metrics do not establish performance acceptance by themselves.
 
-## Experimental shared node admission
+## Experimental heal node admission
 
-`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback shared by the heal manager’s four object-heal entry points and local Scanner namespace reads and object accounting. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. This does not yet control the abandoned-child streaming list coordinator, bucket/format repair or physical devices, and it is not a replacement for existing worker, disk or byte budgets.
+`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback for the four object-heal entry points owned by the heal manager. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. This does not yet control scanner walks, bucket/format repair or physical devices, and it is not a replacement for existing worker, disk or byte budgets.
 
 The node samples its effective CPU quota/cpuset, ancestor memory headroom, CPU usage/throttling and CPU/memory/I/O pressure once per second. Sampling uses one blocking worker at a time. Resource reads retain their real start/end times: reads spanning over 20 ms or older than one second after all inputs return cannot renew admission. Counter rates use both read brackets conservatively. The concurrency cap is at most the rounded-up effective CPU count and 256 scheduling units; existing operator limits still apply independently. At most 1% memory headroom blocks new work. This is a pressure safeguard, not a memory reservation or a claim that concurrent repair buffers fit in the remaining bytes.
 
 Each complete sample leases admission for three seconds. Missing resource caps block new work; missing or low-confidence feedback retains at most one maintenance unit. CPU use above 90%, throttling or pressure above 1% of elapsed time, memory headroom at most 10%, foreground errors/cancellations, a foreground queue or a comparable foreground p99 regression reduce concurrency immediately. Recovery requires CPU use at most 70%, throttling/stalls at most 0.1% of elapsed time, at least 20% memory headroom, fresh foreground evidence and actual successful work. The existing controller probes one additional unit only after eight consecutive healthy seconds and cooldown. Limits affect new admission and do not revoke active work.
 
-The Scanner captures the same node owner once per bucket walk. Directory open/refill, entry type and size lookup, and object accounting try admission before their first storage poll; each unit ends when its operation future finishes. On contention, local and remote coordinators suspend the same traversal, release the bucket cache lease, and join the shared FIFO queue. They reacquire the cache lease and verify both cache revisions before resuming. A changed cache rejects the suspended result. Cancellation also drops the queued request; the existing cycle budget still bounds the walk. This retains directory and recursion state while allowing Heal to advance at capacity one, without repeatedly replaying a partially classified directory.
-
-Calls without a coordinating cache lease retain immediate resource deferral and useful partial-cache handling. Existing remote single-session ownership and local-disk admission shares remain in force. Disabled admission creates no request channel. Enabled walks use one bounded channel, with a response allocation and cache revision reads only on contention; the uncontended per-entry path adds no global lookup or task. The streaming abandoned-child coordinator requires a separate ownership boundary: retaining a node unit while it waits for a bounded-channel consumer that needs the same unit would deadlock at capacity one.
-
 Foreground comparison uses the existing complete response-body windows and shared admission snapshots. Each active class needs at least 100 completions and a finite p99 interval; this permits the percentile rank but is not a statistical confidence guarantee. Comparisons require the same active classes and rates within 80–125% of a retained reference. A changed workload resets recovery. The reference is an observed workload window, not an independent background-off baseline. A p99 interval can justify stability only when its upper bound is within 5% of the reference lower bound; a lower bound above 105% of the reference upper bound is pressure. Overlapping/uncertain comparisons cannot justify growth. Missing queues, stale or repeated cuts, uncertain cut boundaries and low counts are not idle evidence.
 
-Heal admission waits at most 250 ms before returning the existing typed transient deferral, before polling storage or taking namespace locks. No-lock callers never wait. Queued read repair uses the same node gate; direct foreground ECStore calls remain outside this heal-manager adapter. Cancellation releases the non-cloneable node scheduling unit. Native work retains separate physical-I/O units, and unsuccessful/cancelled/dry-run work cannot increment the node's completed-work counter. This scheduling counter is not an authoritative heal receipt and cannot discharge repair responsibility.
+Admission waits at most 250 ms before returning the existing typed transient deferral, before polling storage or taking namespace locks. No-lock callers never wait. Queued read repair uses the same node gate; direct foreground ECStore calls remain outside this heal-manager adapter. Cancellation releases the non-cloneable node scheduling unit. Native work retains separate physical-I/O units, and unsuccessful/cancelled/dry-run work cannot increment the node's completed-work counter. This scheduling counter is not an authoritative heal receipt and cannot discharge repair responsibility.
 
 The `rustfs_background_node_admission_limit` and `rustfs_background_node_admission_active` gauges describe the last published sample, not a live reservation proof. A blocked sampler can leave these exports stale even though the admission lease has expired. Full disk-owner feedback, shared scanner scheduling, escaping-buffer byte ownership and the matched-release SSD matrix/hot-shrink/72-hour soak remain required before enabling this by default or claiming optimal performance.
 

docs/testing/README.md

@@ -34,8 +34,6 @@ The [Pool layout compatibility reference](pool-layout-compatibility.md) defines
 
 The [Scanner/Heal topology protection gates](scanner-heal-topology-gates.md) map G-EXP, G-REB and G-DEC and their fault windows to the `scanner_heal_topology_` cases, their recorded inventory, and the basis of their timeouts.
 
-The [immutable validation candidate reference](validation-candidates.md) describes isolated package builds and exact candidate resolution for the current auto-testing fleet.
-
 The [distributed multipart RPC delay gates](multipart-rpc-delay.md) define authenticated method-entry barriers, latency and peer-loss oracles, and the boundary between hook evidence and production incident attribution.
 
 ## Naming conventions

docs/testing/validation-candidates.md

@@ -1,52 +0,0 @@
-# Immutable validation candidates
-
-Canonical validation on the current auto-testing fleet consumes one exact
-package, source commit and candidate-bound tool build. A successful package
-build is provenance evidence, not a canonical test PASS.
-
-The scheduled Functional Chain continues to resolve successful Nightly GNU
-Build attempts from `main`. Optional manual validation can also resolve a
-validation package built without publishing nightly aliases or release assets.
-
-## Build a validation package
-
-Dispatch **RustFS Functional Chain** from the desired driver revision with
-only `validation_package_sha` set to the full 40-character RustFS source SHA.
-Keep candidate, case, helper, tools and cleanup inputs empty or false. The
-request validator rejects mixed package/fleet execution before building.
-
-The package job calls `rustfs-validation-package.yml`, checks out the requested
-source, confirms its Git SHA and builds the ordinary release binary and DEB.
-This job uses build runners and does not acquire or mutate the Heal fleet.
-The same workflow can be dispatched directly with its `source_sha` input when
-it is available on the repository's default branch.
-
-Publication requires complete R2 configuration, conditional `PutObject` with
-`If-None-Match: *`, and a public download whose SHA-256 matches the local DEB.
-Failure emits no candidate manifest. An existing object is never overwritten.
-The only object path is:
-
-```text
-artifacts/rustfs/packages/validation/runs/<run>/<attempt>/<package-sha256>/rustfs.deb
-```
-
-The schema-3 artifact `validation-candidate-<run>-<attempt>` records the
-workflow SHA independently of the exact source SHA and source ref, the
-`validation` channel, build run/attempt, immutable URL and package checksum.
-The resolver verifies the successful manual run, repository/workflow identity,
-exact artifact name and digest, archive members, schema and immutable URL.
-The ordinary scheduled chain does not enable validation package resolution.
-
-## Validate the candidate
-
-After the package run succeeds, dispatch Functional Chain with its exact
-`build_run_id` and `build_run_attempt`, a canonical `validation_case`, and an
-exact `auto_testing_sha`. Heal also requires a new tool build verified against
-that source and helper revision. Reassess source-bound contracts and native
-prerequisites when the source changes; an older tool artifact does not certify
-a new candidate.
-
-The fleet lane retains the shared environment lock, per-case evidence and
-cleanup gate. Record new results separately. A package build, native test,
-diagnostic replay, clean recovery or partial topology result cannot replace
-complete original source assertions or rewrite a retained canonical FAIL.

fuzz/Cargo.toml

@@ -19,18 +19,6 @@ rustfs-security-governance = { path = "../crates/security-governance" }
 rustfs-utils = { path = "../crates/utils", features = ["compress", "path"] }
 serde_json = "1"
 
-[patch.crates-io]
-# Standalone workspaces must repeat the root workspace's fixed SigV4a backend.
-aws-sigv4 = { git = "https://github.com/smithy-lang/smithy-rs", rev = "5c1c4bca2ea7af97496e077c9f1a718eaac64bf8" }
-
-[patch."https://github.com/smithy-lang/smithy-rs"]
-# Keep the signer's path dependencies on the SDK's existing registry types.
-aws-credential-types = { version = "=1.3.1" }
-aws-smithy-eventstream = { version = "=0.61.6" }
-aws-smithy-http = { version = "=0.64.3" }
-aws-smithy-runtime-api = { version = "=1.19.2" }
-aws-smithy-types = { version = "=1.9.1" }
-
 [[bin]]
 name = "bucket_validation"
 path = "fuzz_targets/bucket_validation.rs"

rustfs/src/admin/handlers/heal.rs

@@ -1051,21 +1051,6 @@ async fn route_cluster_heal_control(
         })
 }
 
-async fn query_heal_status_on_owner<L, C, CF>(
-    local: L,
-    coordinator: C,
-) -> S3Result<rustfs_heal_contracts::heal_channel::HealChannelResponse>
-where
-    L: Future<Output = S3Result<Option<rustfs_heal_contracts::heal_channel::HealChannelResponse>>>,
-    C: FnOnce() -> CF,
-    CF: Future<Output = S3Result<rustfs_heal_contracts::heal_channel::HealChannelResponse>>,
-{
-    match local.await? {
-        Some(response) => Ok(response),
-        None => coordinator().await,
-    }
-}
-
 async fn execute_after_heal_start_preflight<P, PF, E, EF, T>(
     endpoints: &EndpointServerPools,
     options: &HealOpts,
@@ -1529,41 +1514,20 @@ impl Operation for HealHandler {
         let heal_path = encode_heal_control_path(&hip.bucket, &hip.obj_prefix);
         if !hip.client_token.is_empty() && !hip.force_start && !hip.force_stop {
             let client_token = hip.client_token.clone();
-            // Automatic workers run on their owning node. Only an exact
-            // path/token/source snapshot can answer locally; Admin tokens and
-            // unknown identities retain the coordinator authority boundary.
-            let response = query_heal_status_on_owner(
-                async {
-                    let Some(manager) = rustfs_heal::get_heal_manager() else {
-                        return Ok(None);
-                    };
-                    let processor = rustfs_heal::HealChannelProcessor::new(manager.clone());
-                    timeout(
-                        PEER_HEAL_STATUS_TIMEOUT,
-                        processor.execute_local_automatic_query(&heal_path, &client_token, hip.since_seq),
-                    )
-                    .await
-                    .map_err(|_| cluster_heal_control_unavailable("local_task_query_expired"))?
-                    .map_err(|_| cluster_heal_control_unavailable("local_task_query_failed"))
-                },
-                || async {
-                    let request_id = uuid::Uuid::new_v4().to_string();
-                    let context = app_context
-                        .clone()
-                        .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?;
-                    let route = prepare_heal_control_route(&context)?;
-                    let envelope = rustfs_protos::heal_control::Envelope::query(
-                        request_id.clone(),
-                        new_heal_control_metadata(&route)?,
-                        heal_path.clone(),
-                        client_token.clone(),
-                        hip.since_seq,
-                    )
-                    .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?;
-                    submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token.clone()).await
-                },
+            let request_id = uuid::Uuid::new_v4().to_string();
+            let context = app_context
+                .clone()
+                .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?;
+            let route = prepare_heal_control_route(&context)?;
+            let envelope = rustfs_protos::heal_control::Envelope::query(
+                request_id.clone(),
+                new_heal_control_metadata(&route)?,
+                heal_path,
+                client_token.clone(),
+                hip.since_seq,
             )
-            .await?;
+            .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?;
+            let response = submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token).await?;
             if !response.success {
                 return Err(s3_error!(
                     InternalError,
@@ -2031,54 +1995,6 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use uuid::Uuid;
 
-    #[tokio::test]
-    async fn local_automatic_heal_query_preserves_owner_and_coordinator_errors() {
-        let response = || rustfs_heal_contracts::heal_channel::HealChannelResponse {
-            request_id: "original-token".to_string(),
-            success: true,
-            data: Some(b"retained-owner-report".to_vec()),
-            error: None,
-        };
-        let called = AtomicBool::new(false);
-        let local = super::query_heal_status_on_owner(async { Ok(Some(response())) }, || async {
-            called.store(true, Ordering::SeqCst);
-            Ok(response())
-        })
-        .await
-        .expect("local automatic owner response");
-        assert!(!called.load(Ordering::SeqCst));
-        assert_eq!(local.request_id, "original-token");
-        assert_eq!(local.data.as_deref(), Some(b"retained-owner-report".as_slice()));
-
-        let fallback = super::query_heal_status_on_owner(async { Ok(None) }, || async {
-            called.store(true, Ordering::SeqCst);
-            Ok(response())
-        })
-        .await
-        .expect("unknown local identity uses coordinator");
-        assert!(called.swap(false, Ordering::SeqCst));
-        assert_eq!(fallback.data, local.data);
-
-        let error = super::query_heal_status_on_owner(
-            async { Err(super::admin_error(S3ErrorCode::InternalError, "local lookup failed")) },
-            || async {
-                called.store(true, Ordering::SeqCst);
-                Ok(response())
-            },
-        )
-        .await
-        .expect_err("local failure must not become coordinator success");
-        assert_eq!(error.code(), &S3ErrorCode::InternalError);
-        assert!(!called.load(Ordering::SeqCst));
-
-        let error = super::query_heal_status_on_owner(async { Ok(None) }, || async {
-            Err(super::admin_error(S3ErrorCode::ServiceUnavailable, "coordinator unavailable"))
-        })
-        .await
-        .expect_err("coordinator error must remain visible");
-        assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
-    }
-
     #[test]
     fn mrf_legacy_only_refresh_accepts_an_unobserved_incarnation() {
         for action in ["refresh", "recheck", "acceptUnverifiedRisk"] {

rustfs/src/app/object/get.rs

@@ -3141,10 +3141,7 @@ impl DefaultObjectUsecase {
             // resolver; only the response headers, authorization and audit
             // summary are derived here, without a second KMS unwrap.
             Some(headers) => (
-                headers
-                    .sse_customer_algorithm
-                    .is_none()
-                    .then_some(headers.server_side_encryption),
+                Some(headers.server_side_encryption),
                 headers.sse_customer_algorithm,
                 headers.sse_customer_key_md5,
                 headers.ssekms_key_id,

rustfs/src/app/object/head.rs

@@ -462,13 +462,12 @@ impl DefaultObjectUsecase {
         })?;
 
         let metadata_map = info.user_defined.clone();
+        let server_side_encryption = metadata_map
+            .get("x-amz-server-side-encryption")
+            .map(|v| ServerSideEncryption::from(v.clone()));
         let sse_customer_algorithm = metadata_map
             .get("x-amz-server-side-encryption-customer-algorithm")
             .map(|v| SSECustomerAlgorithm::from(v.clone()));
-        let server_side_encryption = metadata_map
-            .get("x-amz-server-side-encryption")
-            .filter(|_| sse_customer_algorithm.is_none())
-            .map(|v| ServerSideEncryption::from(v.clone()));
         let sse_customer_key_md5 = metadata_map.get("x-amz-server-side-encryption-customer-key-md5").cloned();
         let sse_kms_key_id = metadata_map.get("x-amz-server-side-encryption-aws-kms-key-id").cloned();
         let storage_class = response_storage_class(&info, &metadata_map);

rustfs/src/app/object/put.rs

@@ -1435,10 +1435,7 @@ impl DefaultObjectUsecase {
 
         let output = PutObjectOutput {
             e_tag,
-            server_side_encryption: committed
-                .effective_sse
-                .clone()
-                .filter(|_| committed.sse_customer_algorithm.is_none()),
+            server_side_encryption: committed.effective_sse.clone(),
             sse_customer_algorithm: committed.sse_customer_algorithm.clone(),
             sse_customer_key_md5: committed.sse_customer_key_md5.clone(),
             ssekms_key_id: committed.effective_kms_key_id.clone(),
@@ -3967,12 +3964,6 @@ mod tests {
     async fn sse_c_put_is_accepted_on_a_bucket_with_default_encryption() {
         use md5::{Digest as _, Md5};
 
-        let context = crate::storage::storage_api::bootstrap_instance_ctx();
-        assert!(
-            context.object_encryption_resolver().is_some(),
-            "real SSE-C reads need the production encryption resolver"
-        );
-
         for (algorithm, kms_key_id, prefix) in [
             (ServerSideEncryption::AES256, None, "ssec-over-aes256-default"),
             (ServerSideEncryption::AWS_KMS, Some("bucket-key"), "ssec-over-kms-default"),
@@ -3994,69 +3985,14 @@ mod tests {
                 .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
                 .sse_customer_algorithm(Some("AES256".to_string()))
                 .sse_customer_key(Some(base64_simd::STANDARD.encode_to_string(customer_key)))
-                .sse_customer_key_md5(Some(key_md5.clone()))
+                .sse_customer_key_md5(Some(key_md5))
                 .build()
                 .expect("SSE-C PUT input must build");
 
-            let usecase = DefaultObjectUsecase::from_global();
-            let response = usecase
+            DefaultObjectUsecase::from_global()
                 .execute_put_object(&FS::new(), build_request(input, Method::PUT))
                 .await
                 .unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}"));
-            assert_eq!(response.output.server_side_encryption, None);
-            assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256"));
-            assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
-
-            let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key);
-            let mut read_headers = HeaderMap::new();
-            read_headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
-            read_headers.insert(
-                "x-amz-server-side-encryption-customer-key",
-                HeaderValue::from_str(&encoded_key).expect("encoded customer key is a valid header"),
-            );
-            read_headers.insert(
-                "x-amz-server-side-encryption-customer-key-md5",
-                HeaderValue::from_str(&key_md5).expect("customer key MD5 is a valid header"),
-            );
-            let head_input = HeadObjectInput::builder()
-                .bucket(bucket.clone())
-                .key("ledger.csv".to_string())
-                .sse_customer_algorithm(Some("AES256".to_string()))
-                .sse_customer_key(Some(encoded_key.clone()))
-                .sse_customer_key_md5(Some(key_md5.clone()))
-                .build()
-                .expect("SSE-C HEAD input must build");
-            let mut head_request = build_request(head_input, Method::HEAD);
-            head_request.headers = read_headers.clone();
-            let head = Box::pin(usecase.execute_head_object(head_request))
-                .await
-                .expect("matching SSE-C key must authorize HEAD");
-            assert_eq!(head.output.server_side_encryption, None);
-            assert_eq!(head.output.sse_customer_algorithm.as_deref(), Some("AES256"));
-            assert_eq!(head.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
-
-            let get_input = GetObjectInput::builder()
-                .bucket(bucket.clone())
-                .key("ledger.csv".to_string())
-                .sse_customer_algorithm(Some("AES256".to_string()))
-                .sse_customer_key(Some(encoded_key))
-                .sse_customer_key_md5(Some(key_md5.clone()))
-                .build()
-                .expect("SSE-C GET input must build");
-            let mut get_request = build_request(get_input, Method::GET);
-            get_request.headers = read_headers;
-            let get = Box::pin(usecase.execute_get_object(get_request))
-                .await
-                .expect("matching SSE-C key must authorize GET");
-            assert_eq!(get.output.server_side_encryption, None);
-            assert_eq!(get.output.sse_customer_algorithm.as_deref(), Some("AES256"));
-            assert_eq!(get.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
-            let mut body = get.output.body.expect("SSE-C GET must include the decrypted body");
-            let mut plaintext = Vec::new();
-            while let Some(chunk) = body.next().await {
-                plaintext.extend_from_slice(&chunk.expect("SSE-C body must decrypt completely"));
-            }
-            assert_eq!(plaintext, payload, "header classification must preserve the complete decrypted payload");
 
             let stored = store
                 .get_object_info(&bucket, "ledger.csv", &ObjectOptions::default())

rustfs/src/error.rs

@@ -310,7 +310,6 @@ impl std::error::Error for ApiError {
 pub(crate) struct ApiErrorDiagnostic {
     pub storage_code: Option<StorageErrorCode>,
     pub io_kind: Option<std::io::ErrorKind>,
-    pub io_os_error: Option<i32>,
     pub rpc_code: Option<tonic::Code>,
     pub pool_metadata: Option<PoolMetadataDiagnostic>,
     pub truncated: bool,
@@ -334,16 +333,10 @@ impl From<&PoolMetadataError> for PoolMetadataDiagnostic {
     }
 }
 
-impl ApiErrorDiagnostic {
-    pub(crate) fn from_error(error: &(dyn std::error::Error + 'static)) -> Self {
+impl ApiError {
+    pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic {
         let mut diagnostic = ApiErrorDiagnostic::default();
-        // S3Error retains its payload through an inherent accessor; its
-        // std::error::Error implementation does not expose that source.
-        let mut current = if let Some(error) = error.downcast_ref::<S3Error>() {
-            error.source().map(|source| source as &(dyn std::error::Error + 'static))
-        } else {
-            std::error::Error::source(error)
-        };
+        let mut current = std::error::Error::source(self);
         for _ in 0..16 {
             let Some(error) = current else {
                 return diagnostic;
@@ -361,11 +354,8 @@ impl ApiErrorDiagnostic {
             }
             current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
                 diagnostic.io_kind = Some(io.kind());
-                diagnostic.io_os_error = io.raw_os_error();
                 // io::Error::source can skip the wrapped error itself.
                 io.get_ref().map(|source| source as &(dyn std::error::Error + 'static))
-            } else if let Some(error) = error.downcast_ref::<S3Error>() {
-                error.source().map(|source| source as &(dyn std::error::Error + 'static))
             } else {
                 error.source()
             };
@@ -373,12 +363,6 @@ impl ApiErrorDiagnostic {
         diagnostic.truncated = current.is_some();
         diagnostic
     }
-}
-
-impl ApiError {
-    pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic {
-        ApiErrorDiagnostic::from_error(self)
-    }
 
     /// Access-denied error with the exact message emitted by the authorization
     /// paths in `storage::access`; callers there match on the code only.
@@ -571,7 +555,6 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
     let mut body_size_exceeded = false;
     let mut aws_chunked_error = None;
     let mut incomplete_body = false;
-    let mut rpc_unavailable = false;
     let digest = error_chain_find(err, |error| {
         let upload_error = error.downcast_ref::<UploadStreamError>();
         if matches!(upload_error, Some(UploadStreamError::Sha256Mismatch)) || error.is::<rustfs_rio::Sha256Mismatch>() {
@@ -581,9 +564,6 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
             return Some(S3ErrorCode::BadDigest);
         }
 
-        rpc_unavailable |= error
-            .downcast_ref::<tonic::Status>()
-            .is_some_and(|status| status.code() == tonic::Code::Unavailable);
         upload_limit_exceeded |= error.is::<UploadLimitExceeded>();
         server_source_failed |= error.is::<ServerSideSourceReadError>();
         client_body_timed_out |= error.is::<ClientBodyReadTimeout>();
@@ -611,8 +591,6 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
             Some(code)
         } else if incomplete_body {
             Some(S3ErrorCode::IncompleteBody)
-        } else if rpc_unavailable {
-            Some(S3ErrorCode::ServiceUnavailable)
         } else {
             None
         }
@@ -1263,35 +1241,6 @@ mod tests {
         assert!(!format!("{:?}", error.diagnostic()).contains("secret"));
     }
 
-    #[test]
-    fn api_error_diagnostic_survives_s3_conversion_without_changing_response() {
-        let api = ApiError::from(StorageError::Io(IoError::from_raw_os_error(30)));
-        let before = api.diagnostic();
-        assert_eq!(before.io_os_error, Some(30));
-        let mut error: S3Error = api.into();
-        error.set_request_id("server-owned-request");
-        let code = error.code().clone();
-        let status = error.status_code();
-        let message = error.message().map(str::to_owned);
-        assert_eq!(super::ApiErrorDiagnostic::from_error(&error), before);
-        assert_eq!(error.code(), &code);
-        assert_eq!(error.status_code(), status);
-        assert_eq!(error.message(), message.as_deref());
-        assert_eq!(error.request_id(), Some("server-owned-request"));
-        assert!(error.source().is_some());
-        assert!(std::error::Error::source(&error).is_none());
-
-        let error: S3Error = ApiError::from(StorageError::Io(IoError::other(StorageError::ErasureReadQuorum))).into();
-        assert_eq!(
-            super::ApiErrorDiagnostic::from_error(&error).storage_code,
-            Some(StorageErrorCode::ErasureReadQuorum)
-        );
-
-        let nested: S3Error = ApiError::from(StorageError::ErasureReadQuorum).into();
-        let api = ApiError::from(StorageError::Io(IoError::other(nested)));
-        assert_eq!(api.diagnostic().storage_code, Some(StorageErrorCode::ErasureReadQuorum));
-    }
-
     #[test]
     fn api_error_diagnostic_bounds_cyclic_error_chains() {
         #[derive(Debug)]
@@ -1313,11 +1262,6 @@ mod tests {
         };
         assert!(error.diagnostic().truncated);
         assert!(!format!("{:?}", error.diagnostic()).contains("secret"));
-        let error: S3Error = error.into();
-        let diagnostic = super::ApiErrorDiagnostic::from_error(&error);
-        assert!(diagnostic.truncated);
-        assert!(!format!("{diagnostic:?}").contains("secret"));
-
         let error = S3Error::from(ApiError {
             code: S3ErrorCode::ServiceUnavailable,
             message: "safe".into(),
@@ -1628,85 +1572,6 @@ mod tests {
         assert!(error_chain_has_type::<AwsChunkedStreamError>(&error));
     }
 
-    #[test]
-    fn rpc_unavailable_preserves_service_refusal_across_io_boundaries() {
-        let status = || tonic::Status::unavailable("private transport detail must not reach the S3 response");
-        let storage = || StorageError::from(status());
-        let original = storage();
-        let cloned = original.clone();
-        for error in [
-            ApiError::from(original),
-            ApiError::from(cloned),
-            ApiError::from(StorageError::Io(IoError::other(IoError::other(status())))),
-            ApiError::from(IoError::other(status())),
-            ApiError::from(IoError::other(storage())),
-        ] {
-            assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
-            assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable));
-            assert_eq!(error.diagnostic().rpc_code, Some(tonic::Code::Unavailable));
-            let s3_error = S3Error::from(error);
-            assert_eq!(s3_error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
-        }
-    }
-
-    #[test]
-    fn rpc_unavailable_mapping_does_not_match_other_codes_or_untyped_messages() {
-        for code in [
-            tonic::Code::Cancelled,
-            tonic::Code::Unknown,
-            tonic::Code::InvalidArgument,
-            tonic::Code::DeadlineExceeded,
-            tonic::Code::NotFound,
-            tonic::Code::AlreadyExists,
-            tonic::Code::PermissionDenied,
-            tonic::Code::ResourceExhausted,
-            tonic::Code::FailedPrecondition,
-            tonic::Code::Aborted,
-            tonic::Code::OutOfRange,
-            tonic::Code::Unimplemented,
-            tonic::Code::Internal,
-            tonic::Code::DataLoss,
-            tonic::Code::Unauthenticated,
-        ] {
-            for error in [
-                ApiError::from(StorageError::from(tonic::Status::new(code, "rpc unavailable"))),
-                ApiError::from(IoError::other(tonic::Status::new(code, "rpc unavailable"))),
-            ] {
-                assert_eq!(error.code, S3ErrorCode::InternalError, "{code:?}");
-            }
-        }
-        for error in [
-            ApiError::from(StorageError::Io(IoError::other("status: Unavailable"))),
-            ApiError::from(IoError::other("status: Unavailable")),
-        ] {
-            assert_eq!(error.code, S3ErrorCode::InternalError);
-        }
-    }
-
-    #[test]
-    fn rpc_unavailable_mapping_keeps_client_digest_error_precedence() {
-        for sha256 in [false, true] {
-            let source = || {
-                let mut status = tonic::Status::unavailable("private transport failure");
-                if sha256 {
-                    status.set_source(std::sync::Arc::new(UploadStreamError::Sha256Mismatch));
-                } else {
-                    status.set_source(std::sync::Arc::new(rustfs_rio::BadDigest {
-                        expected_md5: "expected".into(),
-                        calculated_md5: "calculated".into(),
-                    }));
-                }
-                status
-            };
-            for error in [
-                ApiError::from(StorageError::Io(IoError::other(source()))),
-                ApiError::from(IoError::other(source())),
-            ] {
-                assert_eq!(error.code.as_str(), if sha256 { CONTENT_SHA256_MISMATCH_CODE } else { "BadDigest" });
-            }
-        }
-    }
-
     #[test]
     fn body_size_limit_exceeded_maps_to_entity_too_large_across_io_boundaries() {
         // Shape observed in production (issue #7596):

rustfs/src/startup_background.rs

@@ -81,14 +81,6 @@ pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Resu
         let admission = None;
         let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()).with_admission(admission.clone()));
         init_heal_manager_with_workload_provider(heal_storage, None, Some(workload_provider.clone())).await?;
-        if let Some(admission) = &admission
-            && !rustfs_scanner::set_scanner_node_admission(admission.clone())
-        {
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::AlreadyExists,
-                "scanner node admission is already initialized",
-            ));
-        }
         #[cfg(any(target_os = "linux", all(test, unix)))]
         if let Some(admission) = admission {
             crate::background_admission::start(admission, background_cancel.clone(), workload_provider);

rustfs/src/storage/ecfs.rs

@@ -22,7 +22,7 @@ use super::{
     update_bucket_metadata_config_if_incarnation,
 };
 use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
-use crate::error::{ApiError, ApiErrorDiagnostic};
+use crate::error::ApiError;
 use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header};
 use crate::storage::helper::OperationHelper;
 use crate::storage::options::get_opts;
@@ -60,32 +60,6 @@ const LOG_COMPONENT_STORAGE: &str = "storage";
 const LOG_SUBSYSTEM_OBJECT: &str = "object";
 const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
 const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
-const EVENT_OBJECT_INTERNAL_ERROR: &str = "object_internal_error";
-
-fn log_object_internal_error(operation: &'static str, request_id: Option<&str>, error: &S3Error) {
-    if error.code() != &S3ErrorCode::InternalError {
-        return;
-    }
-    // Generated S3 adapters serialize handler errors without retaining their
-    // typed sources in logs. Keep only bounded classifications at this boundary.
-    let diagnostic = ApiErrorDiagnostic::from_error(error);
-    error!(
-        event = EVENT_OBJECT_INTERNAL_ERROR,
-        component = LOG_COMPONENT_STORAGE,
-        subsystem = LOG_SUBSYSTEM_OBJECT,
-        result = "error",
-        operation,
-        request_id = request_id.unwrap_or("unavailable"),
-        storage_error_code = ?diagnostic.storage_code,
-        io_error_kind = ?diagnostic.io_kind,
-        io_os_error = diagnostic.io_os_error,
-        rpc_error_code = ?diagnostic.rpc_code,
-        pool_metadata_reason = diagnostic.pool_metadata.map(|context| context.reason),
-        pool_metadata_phase = diagnostic.pool_metadata.map(|context| context.phase),
-        source_chain_truncated = diagnostic.truncated,
-        "Object request returned an internal error"
-    );
-}
 
 use crate::app::storage_api::object_usecase::bucket::replication::{
     OperatorRuleContract, ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication,
@@ -671,16 +645,8 @@ impl S3 for FS {
     #[instrument(level = "debug", skip(self, req))]
     async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
         crate::hp_guard!("S3::delete_object");
-        let request_id = req
-            .extensions
-            .get::<super::request_context::RequestContext>()
-            .map(|ctx| ctx.request_id.clone());
         let usecase = s3_api::object_usecase_for(self);
-        let result = Box::pin(usecase.execute_delete_object(req)).await;
-        if let Err(error) = &result {
-            log_object_internal_error("DeleteObject", request_id.as_deref(), error);
-        }
-        result
+        Box::pin(usecase.execute_delete_object(req)).await
     }
 
     #[instrument(level = "debug", skip(self))]
@@ -998,16 +964,8 @@ impl S3 for FS {
     )]
     async fn get_object(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
         crate::hp_guard!("S3::get_object");
-        let request_id = req
-            .extensions
-            .get::<super::request_context::RequestContext>()
-            .map(|ctx| ctx.request_id.clone());
         let usecase = s3_api::object_usecase_for(self);
-        let result = usecase.execute_get_object(req).await;
-        if let Err(error) = &result {
-            log_object_internal_error("GetObject", request_id.as_deref(), error);
-        }
-        result
+        usecase.execute_get_object(req).await
     }
 
     async fn get_object_acl(&self, req: S3Request<GetObjectAclInput>) -> S3Result<S3Response<GetObjectAclOutput>> {
@@ -2115,73 +2073,6 @@ mod tests {
     use s3s::{S3, S3Error, S3ErrorCode, S3Request};
     use std::sync::atomic::Ordering;
 
-    #[test]
-    fn object_internal_error_logs_only_correlated_bounded_classifications() {
-        #[derive(Clone, Default)]
-        struct Writer(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
-        impl std::io::Write for Writer {
-            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
-                self.0.lock().expect("log buffer").extend_from_slice(bytes);
-                Ok(bytes.len())
-            }
-            fn flush(&mut self) -> std::io::Result<()> {
-                Ok(())
-            }
-        }
-        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Writer {
-            type Writer = Self;
-            fn make_writer(&'a self) -> Self::Writer {
-                self.clone()
-            }
-        }
-        let writer = Writer::default();
-        let subscriber = tracing_subscriber::fmt()
-            .json()
-            .without_time()
-            .with_writer(writer.clone())
-            .finish();
-        let mut status = tonic::Status::internal("secret source message");
-        status
-            .metadata_mut()
-            .insert("authorization", "secret-token".parse().expect("metadata"));
-        let error: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(status)).into();
-        let original_code = error.code().clone();
-        let original_status = error.status_code();
-        let original_message = error.message().map(str::to_owned);
-        tracing::subscriber::with_default(subscriber, || {
-            super::log_object_internal_error("GetObject", Some("server-owned-request"), &error);
-            let mut opaque = S3Error::with_message(S3ErrorCode::InternalError, "secret outer message");
-            opaque.set_source(Box::new(std::io::Error::other("secret inner message")));
-            super::log_object_internal_error("DeleteObject", None, &opaque);
-            let unavailable: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(
-                tonic::Status::unavailable("secret transport message"),
-            ))
-            .into();
-            assert_eq!(unavailable.code(), &S3ErrorCode::ServiceUnavailable);
-            assert_eq!(unavailable.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
-            super::log_object_internal_error("GetObject", None, &unavailable);
-        });
-        let output = String::from_utf8(writer.0.lock().expect("log buffer").clone()).expect("UTF8 log");
-        assert!(!output.contains("secret"));
-        assert!(!output.contains("authorization"));
-        let rows = output
-            .lines()
-            .map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON log"))
-            .collect::<Vec<_>>();
-        assert_eq!(rows.len(), 2);
-        for row in &rows {
-            assert_eq!(row["fields"]["event"], "object_internal_error");
-            assert_eq!(row["fields"]["source_chain_truncated"], false);
-        }
-        assert_eq!(rows[0]["fields"]["request_id"], "server-owned-request");
-        assert_eq!(rows[0]["fields"]["rpc_error_code"], "Some(Internal)");
-        assert_eq!(rows[1]["fields"]["request_id"], "unavailable");
-        assert_eq!(rows[1]["fields"]["io_error_kind"], "Some(Other)");
-        assert_eq!(error.code(), &original_code);
-        assert_eq!(error.status_code(), original_status);
-        assert_eq!(error.message(), original_message.as_deref());
-    }
-
     #[test]
     fn website_config_rejects_missing_index_and_invalid_redirect_rules() {
         assert!(validate_website_configuration(&WebsiteConfiguration::default()).is_err());

scripts/error-other-format-baseline.txt

@@ -62,7 +62,7 @@
 1|crates/ecstore/src/set_disk/ops/heal.rs
 8|crates/ecstore/src/set_disk/ops/multipart.rs
 2|crates/ecstore/src/set_disk/ops/object.rs
-1|crates/ecstore/src/set_disk/read.rs
+3|crates/ecstore/src/set_disk/read.rs
 5|crates/ecstore/src/store/bucket.rs
 1|crates/ecstore/src/store/heal_walk.rs
 10|crates/ecstore/src/store/init.rs

scripts/initialize_validation_evidence.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python3
-"""Retain self-hosted validation evidence outside Actions' temporary cleanup."""
-import json
-import os
-from pathlib import Path
-import re
-import stat
-import sys
-
-MINIMUM_FREE_BYTES = 10 * 1024**3
-MINIMUM_FREE_INODES = 50000
-
-
-def initialize(environ=None):
-    env = os.environ if environ is None else environ
-    run, attempt = env['GITHUB_RUN_ID'], env['GITHUB_RUN_ATTEMPT']
-    if not all(re.fullmatch(r'[1-9][0-9]*', value) for value in (run, attempt)):
-        raise ValueError('Evidence requires numeric workflow run and attempt identities')
-    tools_only = env['VALIDATION_TOOLS_ONLY']
-    if tools_only not in {'true', 'false'}:
-        raise ValueError('Evidence requires an explicit tools-only selection')
-    temporary = Path(env['RUNNER_TEMP'])
-    if not temporary.is_absolute() or temporary.resolve(strict=True) != temporary or not temporary.is_dir():
-        raise ValueError('Runner temporary directory must be an existing canonical absolute directory')
-    parent = temporary if tools_only == 'true' else temporary.parent / 'rustfs-validation-evidence'
-    if tools_only == 'false':
-        parent.mkdir(mode=0o700, exist_ok=True)
-        info = parent.lstat()
-        if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
-            raise ValueError('Persistent evidence parent must be a private runner-owned directory')
-    root = parent / f'rustfs-validation-{run}-{attempt}'
-    root.mkdir(mode=0o700)
-    observations = []
-    for path in dict.fromkeys((temporary, parent)):
-        usage = os.statvfs(path)
-        observations.append({'path': str(path), 'availableBytes': usage.f_bavail * usage.f_frsize,
-            'availableInodes': usage.f_favail, 'minimumFreeBytes': MINIMUM_FREE_BYTES,
-            'minimumFreeInodes': MINIMUM_FREE_INODES})
-    ready = all(item['availableBytes'] >= MINIMUM_FREE_BYTES and item['availableInodes'] >= MINIMUM_FREE_INODES
-                for item in observations)
-    report = {'schemaVersion': 1, 'runId': run, 'runAttempt': attempt, 'evidenceRoot': str(root),
-              'persistentOutsideRunnerTemp': tools_only == 'false', 'storage': observations, 'ready': ready,
-              'automaticEvidenceDeletion': False, 'canonicalPassClaimed': False}
-    with (root / 'runner-storage-preflight.json').open('x') as stream:
-        json.dump(report, stream, indent=2)
-        stream.write(chr(10))
-    with Path(env['GITHUB_ENV']).open('a') as stream:
-        if any(character in str(root) for character in (chr(10), chr(13))):
-            raise ValueError('Evidence path cannot contain environment-file delimiters')
-        stream.write('VALIDATION_ROOT=' + str(root) + chr(10))
-    print(json.dumps(report, sort_keys=True))
-    if not ready:
-        raise ValueError('Validation requires at least 10 GiB and 50000 free inodes on runner storage')
-    return root
-
-
-if __name__ == '__main__':
-    try:
-        initialize()
-    except (OSError, ValueError, KeyError) as error:
-        print(f'Validation evidence initialization failed: {error}', file=sys.stderr)
-        sys.exit(1)

scripts/inspect_validation_runner.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python3
-"""Inspect GitHub runner registration and queued jobs without accessing the fleet."""
-
-import argparse
-from datetime import datetime, timezone
-import json
-import os
-from pathlib import Path
-import sys
-
-from resolve_functional_candidate import REPOSITORY, api, require
-
-
-LABEL = "smoke-testing"
-ACTIVE_STATUSES = ("queued", "in_progress", "pending", "waiting", "requested")
-
-
-def validate_request(inputs):
-    require(isinstance(inputs, dict) and inputs.get("validation_runner_inspect_only") is True,
-            "runner inspection must be explicitly selected")
-    defaults = {"validation_runner_inspect_only": True,
-                "validation_retain_evidence_run_attempt": "1",
-                "i05_diagnostic_branch": "timeout", "validation_tools_run_attempt": "1"}
-    require(all(value == defaults[key] if key in defaults else value in ("", False, None)
-                for key, value in inputs.items()),
-            "runner inspection cannot select a candidate, tools, fleet case or recovery")
-
-
-def pages(path, field):
-    rows = []
-    for page in range(1, 11):
-        separator = "&" if "?" in path else "?"
-        batch = api(f"{path}{separator}per_page=100&page={page}").get(field)
-        require(isinstance(batch, list), "GitHub inventory response is incomplete")
-        rows.extend(batch)
-        if len(batch) < 100:
-            return rows
-    raise ValueError("GitHub inventory exceeds the bounded inspection limit")
-
-
-def collect(report):
-    prefix = f"repos/{REPOSITORY}/actions"
-    report["stage"] = "runner-inventory"
-    runners = pages(prefix + "/runners", "runners")
-    report["runners"] = [
-        {"id": row["id"], "name": row["name"], "status": row["status"],
-         "busy": row["busy"], "labels": [label["name"] for label in row["labels"]]}
-        for row in runners if LABEL in {label["name"] for label in row["labels"]}]
-    online = [row for row in report["runners"] if row["status"] == "online"]
-    report["runnerAvailability"] = (
-        "online-idle" if any(row["busy"] is False for row in online) else
-        "online-busy" if online else "offline" if report["runners"] else "unregistered")
-    report["stage"] = "active-workflows"
-    runs = {}
-    for status in ACTIVE_STATUSES:
-        for row in pages(prefix + f"/runs?status={status}", "workflow_runs"):
-            runs[row["id"]] = row
-    report["fleetJobs"] = []
-    report["runsWithoutJobs"] = []
-    for run_id, row in sorted(runs.items()):
-        report["stage"] = "workflow-jobs"
-        jobs = pages(prefix + f"/runs/{run_id}/jobs", "jobs")
-        identity = {"runId": run_id, "workflow": row["name"], "status": row["status"],
-                    "createdAt": row["created_at"], "url": row["html_url"]}
-        if not jobs:
-            report["runsWithoutJobs"].append(identity)
-        for job in jobs:
-            if LABEL in job.get("labels", []) and job["status"] != "completed":
-                report["fleetJobs"].append({**identity, "jobId": job["id"], "jobName": job["name"],
-                    "jobStatus": job["status"], "runnerName": job.get("runner_name"),
-                    "labels": job["labels"], "jobUrl": job["html_url"]})
-    report.update(stage="complete", complete=True, activeWorkflowCount=len(runs))
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--output", type=Path, required=True)
-    args = parser.parse_args()
-    require(not args.output.exists(), "retain the original runner inspection report")
-    report = {"schemaVersion": 1, "repository": REPOSITORY, "label": LABEL,
-              "observedAt": datetime.now(timezone.utc).isoformat(), "complete": False,
-              "runId": os.environ.get("GITHUB_RUN_ID"), "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
-              "workflowCommit": os.environ.get("GITHUB_SHA"),
-              "runnerAvailability": "unknown", "stage": "request-validation",
-              "fleetAccessed": False, "mutated": False,
-              "cleanStateCertified": False, "sharedWorkflowIdleCertified": False,
-              "canonicalCasePassClaimed": False}
-    try:
-        validate_request(json.loads(os.environ["RUNNER_INSPECTION_INPUTS"]))
-        collect(report)
-    except Exception as error:
-        # Subprocess diagnostics can include configuration or credentials.
-        report["errorType"] = type(error).__name__
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    with args.output.open("x") as stream:
-        json.dump(report, stream, indent=2)
-        stream.write("\n")
-    print(json.dumps({key: report[key] for key in ("complete", "stage", "runnerAvailability")}))
-    return 0 if report["complete"] else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/prepare_retained_validation_recovery.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-"""Copy verified local evidence into a recovery input without claiming an old artifact."""
-
-import argparse
-import json
-import os
-from pathlib import Path
-import re
-import sys
-
-from resolve_functional_candidate import REPOSITORY, api, require
-from retain_validation_evidence import inventory, prepare
-
-
-def stage(env, report):
-    source = env["FAILED_RUN_ID"]
-    require(re.fullmatch(r"[1-9][0-9]{0,19}", source), "exact failed run required")
-    run = api(f"repos/{REPOSITORY}/actions/runs/{source}")
-    require(run.get("head_branch") == env["GITHUB_REF_NAME"]
-            and type(run.get("run_attempt")) is int and run["run_attempt"] > 0,
-            "local recovery source branch or attempt differs")
-    root = prepare({**env, "EVIDENCE_RETAIN_INPUTS": json.dumps({
-        "validation_retain_evidence_run_id": source,
-        "validation_retain_evidence_run_attempt": str(run["run_attempt"])})}, report)
-    report.update(complete=False, stage="stable-recovery-copy")
-    destination = Path(env["VALIDATION_ROOT"]) / "failed-artifact"
-    require(destination.parent.is_absolute() and destination.parent.resolve(strict=True) == destination.parent,
-            "recovery evidence root is not canonical")
-    destination.mkdir(mode=0o700, exist_ok=False)
-    copied = destination / "retained-source-evidence"
-    files, size = inventory(root, copy_to=copied)
-    require(files == report["files"] and size == report["bytes"]
-            and inventory(copied) == (files, size), "retained evidence changed during recovery copy")
-    report.update(complete=True, stage="complete", recoveryInput=str(copied))
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--report", type=Path, required=True)
-    args = parser.parse_args()
-    report = {"schemaVersion": 1, "collectionRunId": os.environ["GITHUB_RUN_ID"],
-              "collectionAttempt": os.environ["GITHUB_RUN_ATTEMPT"], "complete": False,
-              "stage": "source-workflow",
-              "collectionKind": "runner-local-recovery-input-copy", "sourceArtifactClaimed": False,
-              "fleetAccessed": False, "originalEvidenceModified": False,
-              "sourceVerdictRewritten": False, "canonicalCasePassClaimed": False}
-    try:
-        stage(os.environ, report)
-    except Exception as error:
-        report.update(complete=False, errorType=type(error).__name__)
-    with args.report.open("x") as stream:
-        json.dump(report, stream, indent=2)
-        stream.write("\n")
-    print(json.dumps({key: report[key] for key in ("complete", "stage")}))
-    return 0 if report["complete"] else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/resolve_functional_candidate.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Resolve a candidate manifest from its exact GitHub build attempt and artifact."""
+"""Resolve a nightly manifest from its exact GitHub build attempt and artifact."""
 from __future__ import annotations
 
 import hashlib
@@ -15,7 +15,6 @@ import zipfile
 REPOSITORY = "rustfs/rustfs"
 ROOT = Path(__file__).resolve().parents[1]
 MAX_ARCHIVE = 1024 * 1024
-VALIDATION_BUILD_WORKFLOWS = {".github/workflows/rustfs-validation-package.yml", ".github/workflows/rustfs-functional-chain.yml"}
 
 
 def require(condition, message):
@@ -40,11 +39,8 @@ def validate_manifest(manifest, run):
     require(isinstance(manifest, dict), "manifest must be an object")
     common = {"schema", "source_sha", "build_run_id", "build_run_attempt", "package_url", "package_sha256"}
     version = manifest.get("schema")
-    require(type(version) is int and version in (1, 2, 3), "unsupported candidate schema")
-    expected_fields = common if version == 1 else common | {"workflow_sha", "source_ref"}
-    if version == 3:
-        expected_fields |= {"channel"}
-    require(set(manifest) == expected_fields, "unexpected candidate fields")
+    require(type(version) is int and version in (1, 2), "unsupported candidate schema")
+    require(set(manifest) == (common if version == 1 else common | {"workflow_sha", "source_ref"}), "unexpected candidate fields")
     require(positive(manifest["build_run_id"]) and positive(manifest["build_run_attempt"]), "invalid build identity")
     require((manifest["build_run_id"], manifest["build_run_attempt"]) == (run["id"], run["run_attempt"]), "candidate belongs to another build attempt")
     require(sha(manifest["source_sha"]) and sha(manifest["package_sha256"], 64), "invalid candidate hash")
@@ -53,11 +49,7 @@ def validate_manifest(manifest, run):
     else:
         require(manifest["workflow_sha"] == run["head_sha"] and sha(manifest["workflow_sha"]), "candidate workflow SHA differs from artifact provenance")
         require(isinstance(manifest["source_ref"], str) and bool(re.fullmatch(r"[A-Za-z0-9_./-]{1,200}", manifest["source_ref"])), "invalid build source ref")
-        if version == 3:
-            require(manifest["channel"] == "validation", "validation candidate has the wrong publication channel")
-            require(manifest["source_ref"] == manifest["source_sha"], "validation candidate must bind an exact source SHA")
-    channel = "validation" if version == 3 else "nightly"
-    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/{channel}/runs/{run['id']}/"
+    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/{run['id']}/"
                 f"{run['run_attempt']}/{manifest['package_sha256']}/rustfs.deb")
     require(manifest["package_url"] == expected, "package URL does not bind the run, attempt and checksum")
     return manifest
@@ -80,31 +72,19 @@ def read_json_artifact(archive, artifact, run, expected_name, member, max_json=1
 
 
 def read_manifest(archive, artifact, run):
-    prefix = "validation-candidate" if run.get("path") in VALIDATION_BUILD_WORKFLOWS else "nightly-candidate"
-    name = f"{prefix}-{run['id']}-{run['run_attempt']}"
-    manifest = validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
-    if prefix == "validation-candidate":
-        require(manifest["schema"] == 3 and manifest["channel"] == "validation",
-                "validation workflow requires a schema-3 validation candidate")
-    else:
-        require(manifest["schema"] != 3, "nightly workflow cannot publish a validation candidate")
-    return manifest
+    name = f"nightly-candidate-{run['id']}-{run['run_attempt']}"
+    return validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
 
 
-def resolve(run_id, attempt, *, allow_validation=False):
+def resolve(run_id, attempt):
     require(positive(run_id) and positive(attempt), "build run and attempt are required positive integers")
     endpoint = f"repos/{REPOSITORY}/actions/runs/{run_id}"
     run = api(f"{endpoint}/attempts/{attempt}")
     require(run.get("id") == run_id and run.get("run_attempt") == attempt, "GitHub returned a different build attempt")
-    path = run.get("path")
-    validation = allow_validation and path in VALIDATION_BUILD_WORKFLOWS
-    require(path == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main" or validation,
-            "candidate must come from nightly-gnu on main or an explicitly allowed validation build")
+    require(run.get("path") == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main", "candidate must come from nightly-gnu on main")
     require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "candidate came from another repository")
-    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "candidate attempt has not completed successfully")
-    if validation:
-        require(run.get("event") == "workflow_dispatch", "validation candidate must be manually dispatched")
-    name = f"{'validation' if validation else 'nightly'}-candidate-{run_id}-{attempt}"
+    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "nightly attempt has not completed successfully")
+    name = f"nightly-candidate-{run_id}-{attempt}"
     artifacts = []
     for page in range(1, 11):
         batch = api(f"{endpoint}/artifacts?per_page=100&page={page}")["artifacts"]

scripts/retain_validation_evidence.py

@@ -1,178 +0,0 @@
-#!/usr/bin/env python3
-"""Admit an interrupted run's existing runner-local evidence for read-only upload."""
-
-import argparse
-from datetime import datetime, timezone
-import hashlib
-import json
-import os
-from pathlib import Path
-import re
-import stat
-import sys
-
-from resolve_functional_candidate import REPOSITORY, api, require
-
-
-MAX_FILES = 200000
-MAX_BYTES = 4 * 1024**3
-
-
-def stable(info):
-    return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid,
-            info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
-
-
-def inventory(root, *, copy_to=None):
-    files, total, entries = [], 0, 0
-    device = root.lstat().st_dev
-    if copy_to is not None:
-        require(copy_to.is_absolute() and copy_to.parent.resolve(strict=True) == copy_to.parent,
-                "evidence copy parent is not canonical")
-        copy_to.mkdir(mode=0o700, exist_ok=False)
-
-    def visit(fd, prefix):
-        nonlocal total, entries
-        before = os.fstat(fd)
-        require(stat.S_ISDIR(before.st_mode) and before.st_uid == os.getuid() and before.st_dev == device,
-                "evidence directory has a different owner or type")
-        names = sorted(os.listdir(fd))
-        require(len(names) <= MAX_FILES, "evidence directory exceeds the entry budget")
-        for name in names:
-            entries += 1
-            require(entries <= MAX_FILES, "evidence exceeds the entry budget")
-            info = os.stat(name, dir_fd=fd, follow_symlinks=False)
-            path = prefix + name
-            require(info.st_uid == os.getuid() and info.st_dev == device, "evidence entry has a different owner or device")
-            if stat.S_ISDIR(info.st_mode):
-                if copy_to is not None:
-                    (copy_to / path).mkdir(mode=0o700, exist_ok=False)
-                child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
-                try:
-                    require(stable(os.fstat(child)) == stable(info), "evidence directory changed before inspection")
-                    visit(child, path + "/")
-                finally:
-                    os.close(child)
-            else:
-                require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1
-                        and len(files) < MAX_FILES and total + info.st_size <= MAX_BYTES,
-                        "evidence contains links, special files or exceeds its byte/file budget")
-                child = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
-                target = None
-                try:
-                    require(stable(os.fstat(child)) == stable(info), "evidence file changed before inspection")
-                    if copy_to is not None:
-                        target = os.open(copy_to / path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
-                    digest, size = hashlib.sha256(), 0
-                    while chunk := os.read(child, 1024**2):
-                        size += len(chunk)
-                        require(size <= info.st_size, "evidence file grew during inspection")
-                        digest.update(chunk)
-                        if target is not None:
-                            remaining = memoryview(chunk)
-                            while remaining:
-                                written = os.write(target, remaining)
-                                require(written > 0, "evidence copy made no progress")
-                                remaining = remaining[written:]
-                    require(size == info.st_size and stable(os.fstat(child)) == stable(info),
-                            "evidence file changed during inspection")
-                finally:
-                    os.close(child)
-                    if target is not None:
-                        os.close(target)
-                total += size
-                files.append({"path": path, "size": size, "sha256": digest.hexdigest()})
-            require(stable(os.stat(name, dir_fd=fd, follow_symlinks=False)) == stable(info),
-                    "evidence entry changed during inspection")
-        require(sorted(os.listdir(fd)) == names
-                and stable(os.fstat(fd)) == stable(before), "evidence population changed during inspection")
-
-    fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
-    try:
-        visit(fd, "")
-    finally:
-        os.close(fd)
-    return files, total
-
-
-def prepare(env, report):
-    inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"])
-    source = inputs.get("validation_retain_evidence_run_id")
-    attempt = inputs.get("validation_retain_evidence_run_attempt", "1")
-    require(isinstance(source, str) and re.fullmatch(r"[1-9][0-9]{0,19}", source), "exact source run ID required")
-    require(isinstance(attempt, str) and re.fullmatch(r"[1-9][0-9]{0,9}", attempt), "exact source run attempt required")
-    defaults = {"validation_retain_evidence_run_id": source,
-                "validation_retain_evidence_run_attempt": attempt,
-                "i05_diagnostic_branch": "timeout", "validation_tools_run_attempt": "1"}
-    require(all(value == defaults[key] if key in defaults else value in ("", False, None)
-                for key, value in inputs.items()), "evidence retention is exclusive with all execution modes")
-    require(source != env["GITHUB_RUN_ID"], "cannot retain the running collection job")
-    report["stage"] = "source-workflow"
-    run = api(f"repos/{REPOSITORY}/actions/runs/{source}/attempts/{attempt}")
-    require(run.get("id") == int(source) and type(run.get("run_attempt")) is int and run["run_attempt"] == int(attempt)
-            and run.get("repository", {}).get("full_name") == REPOSITORY
-            and run.get("status") == "completed" and run.get("conclusion") in {"failure", "cancelled", "timed_out"}
-            and run.get("event") == "workflow_dispatch"
-            and run.get("path") == ".github/workflows/rustfs-functional-chain.yml",
-            "source is not one completed interrupted validation workflow")
-    jobs = api(f"repos/{REPOSITORY}/actions/runs/{source}/attempts/{run['run_attempt']}/jobs?per_page=100")
-    require(jobs.get("total_count", 101) <= 100
-            and len([job for job in jobs.get("jobs", []) if job.get("name") == "validation"
-                     and job.get("runner_name") == env["RUNNER_NAME"]]) == 1,
-            "source validation did not execute on this runner")
-    report["sourceRun"] = {key: run[key] for key in ("id", "run_attempt", "head_sha", "status", "conclusion", "html_url")}
-    report["stage"] = "owned-persistent-root"
-    temporary = Path(env["RUNNER_TEMP"])
-    require(temporary.is_absolute() and temporary.resolve(strict=True) == temporary
-            and temporary.is_dir(), "runner temporary path is not canonical")
-    parent = temporary.parent / "rustfs-validation-evidence"
-    root = parent / f"rustfs-validation-{source}-{run['run_attempt']}"
-    for path in (parent, root):
-        info = path.lstat()
-        require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid()
-                and stat.S_IMODE(info.st_mode) == 0o700 and path.resolve(strict=True) == path,
-                "persistent evidence must be an existing private runner-owned directory")
-    report["stage"] = "stable-file-inventory"
-    files, size = inventory(root)
-    marker = root / "runner-storage-preflight.json"
-    require(any(row["path"] == marker.name for row in files) and marker.stat().st_size < 1024**2,
-            "source storage provenance is absent or oversized")
-    storage = json.loads(marker.read_text())
-    require(storage.get("runId") == source and storage.get("runAttempt") == str(run["run_attempt"])
-            and storage.get("evidenceRoot") == str(root)
-            and storage.get("persistentOutsideRunnerTemp") is True,
-            "evidence storage provenance belongs to another run or root")
-    require(not any(character in str(root) for character in ("\n", "\r")), "unsafe evidence export path")
-    report.update(complete=True, stage="complete", evidenceRoot=str(root), files=files, bytes=size)
-    return root
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--report", type=Path, required=True)
-    args = parser.parse_args()
-    report = {"schemaVersion": 1, "collectionRunId": os.environ["GITHUB_RUN_ID"],
-              "collectionAttempt": os.environ["GITHUB_RUN_ATTEMPT"], "complete": False,
-              "observedAt": datetime.now(timezone.utc).isoformat(),
-              "stage": "request-validation", "fleetAccessed": False, "originalEvidenceModified": False,
-              "sourceVerdictRewritten": False, "canonicalCasePassClaimed": False}
-    root = None
-    try:
-        root = prepare(os.environ, report)
-    except Exception as error:
-        report["errorType"] = type(error).__name__
-    args.report.parent.mkdir(parents=True, exist_ok=True)
-    with args.report.open("x") as stream:
-        json.dump(report, stream, indent=2)
-        stream.write("\n")
-    if root is not None:
-        with Path(os.environ["GITHUB_ENV"]).open("a") as stream:
-            stream.write("VALIDATION_RETAIN_ROOT=" + str(root) + "\n")
-            stream.write("VALIDATION_RETAIN_SOURCE_RUN=" + str(report["sourceRun"]["id"]) + "\n")
-            stream.write("VALIDATION_RETAIN_SOURCE_ATTEMPT=" + str(report["sourceRun"]["run_attempt"]) + "\n")
-    print(json.dumps({key: report[key] for key in ("complete", "stage")}))
-    return 0 if root is not None else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/scanner_process_receipt.py

@@ -18,7 +18,6 @@ SAFE_ENVIRONMENT = frozenset({
     "RUSTFS_STORAGE_CLASS_STANDARD", "RUSTFS_DURABILITY_MODE", "RUSTFS_NEW_BUCKET_DURABILITY_MODE",
     "RUSTFS_SCANNER_ENABLED", "RUSTFS_HEAL_ENABLED", "RUSTFS_SCANNER_SPEED", "RUSTFS_SCANNER_IDLE_MODE",
     "RUSTFS_SCANNER_START_DELAY_SECS", "RUSTFS_SCANNER_CYCLE",
-    "RUSTFS_HEAL_ADAPTIVE_ADMISSION",
     "RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "RUSTFS_OBS_LOGS_EXPORT_ENABLED", "RUSTFS_OBS_TRACES_EXPORT_ENABLED",
     "RUSTFS_OBS_METRICS_EXPORT_ENABLED", "RUST_LOG", "NO_PROXY", "no_proxy",
 })

scripts/test_functional_chain.py

@@ -4,7 +4,6 @@ import copy
 import hashlib
 import io
 import json
-import os
 from pathlib import Path
 import subprocess
 import tempfile
@@ -14,369 +13,6 @@ import zipfile
 
 import functional_chain_evidence as evidence
 import resolve_functional_candidate as candidate
-import initialize_validation_evidence as validation_storage
-
-
-class ValidationStorageTests(unittest.TestCase):
-    def fixture(self, root, *, tools_only=False):
-        temporary = root / '_work/_temp'
-        temporary.mkdir(parents=True)
-        return dict(RUNNER_TEMP=str(temporary.resolve()), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
-                    VALIDATION_TOOLS_ONLY=str(tools_only).lower(), GITHUB_ENV=str(root / 'env'))
-
-    def usage(self, *, available=None, inodes=None):
-        return mock.Mock(f_bavail=validation_storage.MINIMUM_FREE_BYTES if available is None else available,
-                         f_frsize=1, f_favail=validation_storage.MINIMUM_FREE_INODES if inodes is None else inodes)
-
-    def test_self_hosted_evidence_survives_temp_cleanup_and_refuses_reuse(self):
-        with tempfile.TemporaryDirectory() as temp:
-            root = Path(temp)
-            env = self.fixture(root)
-            with mock.patch.object(validation_storage.os, 'statvfs', return_value=self.usage()):
-                evidence_root = validation_storage.initialize(env)
-                with self.assertRaises(FileExistsError):
-                    validation_storage.initialize(env)
-            self.assertEqual(evidence_root, Path(env['RUNNER_TEMP']).parent /
-                             'rustfs-validation-evidence/rustfs-validation-77-1')
-            self.assertEqual(evidence_root.parent.stat().st_mode & 0o777, 0o700)
-            self.assertEqual(evidence_root.stat().st_mode & 0o777, 0o700)
-            Path(env['RUNNER_TEMP']).rmdir()
-            self.assertTrue((evidence_root / 'runner-storage-preflight.json').is_file())
-            self.assertEqual(Path(env['GITHUB_ENV']).read_text(), 'VALIDATION_ROOT=' + str(evidence_root) + chr(10))
-
-    def test_each_filesystem_byte_and_inode_budget_fails_before_candidate_execution(self):
-        for position in (0, 1):
-            for kind in ('bytes', 'inodes'):
-                with self.subTest(position=position, kind=kind), tempfile.TemporaryDirectory() as temp:
-                    root = Path(temp)
-                    env = self.fixture(root)
-                    usage = [self.usage(), self.usage()]
-                    usage[position] = self.usage(available=validation_storage.MINIMUM_FREE_BYTES - 1) if kind == 'bytes' \
-                        else self.usage(inodes=validation_storage.MINIMUM_FREE_INODES - 1)
-                    with mock.patch.object(validation_storage.os, 'statvfs', side_effect=usage):
-                        with self.assertRaisesRegex(ValueError, 'at least 10 GiB'):
-                            validation_storage.initialize(env)
-                    evidence_root = Path(Path(env['GITHUB_ENV']).read_text().strip().split('=', 1)[1])
-                    report = json.loads((evidence_root / 'runner-storage-preflight.json').read_text())
-                    self.assertFalse(report['ready'])
-                    self.assertFalse(report['automaticEvidenceDeletion'])
-                    self.assertEqual(len(report['storage']), 2)
-
-    def test_tools_resolver_keeps_its_disposable_hosted_location(self):
-        with tempfile.TemporaryDirectory() as temp:
-            env = self.fixture(Path(temp), tools_only=True)
-            with mock.patch.object(validation_storage.os, 'statvfs', return_value=self.usage()):
-                root = validation_storage.initialize(env)
-            self.assertEqual(root.parent, Path(env['RUNNER_TEMP']))
-            self.assertFalse(json.loads((root / 'runner-storage-preflight.json').read_text())
-                             ['persistentOutsideRunnerTemp'])
-
-    def test_rejects_unsafe_parent_and_untrusted_identities(self):
-        for kind in ('symlink', 'public', 'run', 'attempt', 'selection'):
-            with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp:
-                root = Path(temp)
-                env = self.fixture(root)
-                parent = Path(env['RUNNER_TEMP']).parent / 'rustfs-validation-evidence'
-                if kind == 'symlink':
-                    other = root / 'other'
-                    other.mkdir(mode=0o700)
-                    parent.symlink_to(other, target_is_directory=True)
-                elif kind == 'public':
-                    parent.mkdir(mode=0o755)
-                    parent.chmod(0o755)
-                else:
-                    env[{'run': 'GITHUB_RUN_ID', 'attempt': 'GITHUB_RUN_ATTEMPT',
-                         'selection': 'VALIDATION_TOOLS_ONLY'}[kind]] = '../foreign'
-                with self.assertRaises(ValueError):
-                    validation_storage.initialize(env)
-                self.assertFalse(Path(env['GITHUB_ENV']).exists())
-                self.assertFalse((parent / 'rustfs-validation-77-1').exists())
-
-    def test_workflow_uploads_persistent_evidence_and_preflights_before_resolution(self):
-        source = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
-        initialize = source.index('      - name: Initialize validation evidence')
-        self.assertLess(initialize, source.index('      - name: Resolve immutable candidate'))
-        self.assertIn('python3 -B scripts/initialize_validation_evidence.py', source)
-        self.assertIn('VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}', source)
-        upload = source.split('      - name: Upload validation evidence', 1)[1].split('  validation-tools:', 1)[0]
-        self.assertIn("if: ${{ always() && env.VALIDATION_ROOT != '' }}", upload)
-        self.assertIn('path: ${{ env.VALIDATION_ROOT }}', upload)
-
-
-class RaceNativeToolsWorkflowTests(unittest.TestCase):
-    def chain_shell_step(self, name):
-        source = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
-        step = source.split('      - name: ' + name + '\n', 1)[1].split('\n      - name:', 1)[0]
-        shell = step.split('        run: |\n', 1)[1]
-        return '\n'.join(line[10:] for line in shell.splitlines()
-                         if not line.strip() or line.startswith('          '))
-
-    def test_race_actual_workflow_selection_preserves_legacy_and_propagates_failures(self):
-        shell = self.chain_shell_step('Select exact-source race native evidence')
-        g01 = 'heal.race-unversioned-overwrite'
-        cases = [(False, 2, 0, 'false', 'a' * 40, g01), (True, 2, 0, 'false', 'a' * 40, g01),
-                 (True, 0, 0, 'true', 'a' * 40, g01), (True, 1, 1, None, 'a' * 40, g01),
-                 (True, 42, 42, None, 'a' * 40, g01)]
-        reviewed = [('6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca', g01)]
-        reviewed += [('bd0671ba781d8ebf509f4f03533228dfb44496f1', case) for case in
-                     (g01, 'heal.race-versioned-put', 'heal.race-version-delete')]
-        reviewed += [('8380466f3a27c3c3ef4115084178a0ab99a78f32', case) for case in
-                     (g01, 'heal.race-versioned-put', 'heal.race-version-delete', 'heal.race-markers-and-batch-delete')]
-        reviewed += [('757b751482bb970bc321e92059690a7698554358', case) for case in
-                     ('heal.admin-nolock-safety', 'heal.race-copy-and-metadata-update', 'heal.race-multipart-complete-abort')]
-        for source, case in reviewed:
-            cases += [(False, 2, 1, None, source, case), (True, 2, 1, None, source, case),
-                      (True, 0, 0, 'true', source, case)]
-        for helper, support, code, required, source, case in cases:
-            with self.subTest(helper=helper, support=support, source=source, case=case), tempfile.TemporaryDirectory() as name:
-                root = Path(name)
-                env = self.fixture(root, new_helper=helper)
-                output = root / 'output'
-                env.update(SUPPORT_STATUS=str(support), GITHUB_OUTPUT=str(output), CANDIDATE_SHA=source,
-                           VALIDATION_CASE_ID=case)
-                result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
-                self.assertEqual(result.returncode, code, result.stderr)
-                self.assertEqual(output.read_text() if output.exists() else None,
-                                 'required=' + required + '\n' if required else None)
-                if helper:
-                    calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()]
-                    self.assertEqual(calls, [{'kind': 'race_native', 'args': ['supports', '--commit', source,
-                                      '--case-id', case]}])
-
-    def test_race_actual_admission_requires_native_and_runtime_before_exporting_directory(self):
-        shell = self.chain_shell_step('Verify exact-source race native and runtime evidence')
-        for case in ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete',
-                     'heal.race-markers-and-batch-delete', 'heal.race-multipart-complete-abort',
-                     'heal.admin-nolock-safety', 'heal.race-copy-and-metadata-update'):
-            for native_exit, runtime in [(0, 0), (42, 0), (0, 42), (0, 'missing'), (0, 'collector-only')]:
-                with self.subTest(case=case, native_exit=native_exit, runtime=runtime), tempfile.TemporaryDirectory() as name:
-                    root = Path(name)
-                    env = self.fixture(root, new_helper=True)
-                    directory = root / 'auto-testing/scripts/validation_heal'
-                    runtime_driver = 'nolock.py' if case == 'heal.admin-nolock-safety' else 'race.py'
-                    (directory / 'race_native_evidence.py').write_text(
-                        'import os,sys\nprint("{}")\nsys.exit(int(os.environ["VERIFY_EXIT"]))\n')
-                    if runtime == 'collector-only':
-                        (directory / runtime_driver).write_text(
-                            'import argparse\np=argparse.ArgumentParser()\np.add_argument("--rc-bin", required=True)\np.parse_args()\n')
-                    elif runtime != 'missing':
-                        (directory / runtime_driver).write_text(
-                            'import json,os,pathlib,sys\n'
-                            'assert not pathlib.Path(os.environ["GITHUB_ENV"]).exists()\n'
-                            'assert sys.argv[1] == "preflight"\n'
-                            'assert sys.argv[-2:] == ["--case-id",os.environ["VALIDATION_CASE_ID"]]\n'
-                            'assert pathlib.Path(os.environ["VALIDATION_HEAL_NATIVE_DIRECTORY"],"tool-run.json").is_file()\n'
-                            'pathlib.Path("runtime-call.json").write_text(json.dumps(sys.argv[1:]))\n'
-                            'sys.exit(int(os.environ["RUNTIME_EXIT"]))\n')
-                    validation = root / 'validation'
-                    validation.mkdir()
-                    (validation / 'tool-run.json').write_text('{"id":123}')
-                    native = root / 'rustfs-validation-native-77-1'
-                    native.mkdir()
-                    output = root / 'env'
-                    env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
-                               VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(native_exit),
-                               RUNTIME_EXIT=str(runtime), VALIDATION_CASE_ID=case)
-                    result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
-                    expected = native_exit or (runtime if isinstance(runtime, int) else 2)
-                    self.assertEqual(result.returncode, expected, result.stderr)
-                    self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
-                    self.assertEqual(output.exists(), expected == 0)
-                    self.assertEqual((root / 'runtime-call.json').exists(), native_exit == 0 and isinstance(runtime, int))
-                    if expected == 0:
-                        self.assertEqual(output.read_text(), 'VALIDATION_HEAL_NATIVE_DIRECTORY=' + str(native) + '\n')
-        workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
-        self.assertLess(workflow.index('Verify exact-source race native and runtime'),
-                        workflow.index('Run selected canonical validation case'))
-        self.assertIn("steps.race_native.outputs.required == 'true'", workflow)
-        self.assertIn("inputs.validation_case == 'heal.race-markers-and-batch-delete'", workflow)
-        self.assertIn("inputs.validation_case == 'heal.admin-nolock-safety'", workflow)
-        self.assertIn("inputs.validation_case == 'heal.race-copy-and-metadata-update'", workflow)
-        selection = workflow.split('      - name: Select exact-source race native evidence\n', 1)[1].split('\n        env:', 1)[0]
-        self.assertIn("inputs.validation_case == 'heal.race-multipart-complete-abort'", selection)
-
-    def test_i05_native_prerequisite_propagates_failure_and_exports_only_after_verification(self):
-        shell = self.chain_shell_step('Verify I05 queued-successor native prerequisite')
-        for exit_code in (0, 42):
-            with self.subTest(exit_code=exit_code), tempfile.TemporaryDirectory() as name:
-                root = Path(name)
-                env = self.fixture(root, new_helper=True)
-                script = root / 'auto-testing/scripts/validation_heal/race_native_evidence.py'
-                script.write_text('import os,sys\n'
-                    'assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n'
-                    'assert sys.argv[-3] == os.environ["AUTO_TESTING_SHA"]\n'
-                    'sys.exit(int(os.environ["VERIFY_EXIT"]))\n')
-                validation = root / 'validation'; validation.mkdir()
-                (validation / 'tool-run.json').write_text('{"id":123}')
-                native = root / 'rustfs-validation-native-77-1'; native.mkdir()
-                output = root / 'env'
-                env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
-                           VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code))
-                result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
-                self.assertEqual(result.returncode, exit_code, result.stderr)
-                self.assertEqual(output.exists(), exit_code == 0)
-                self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
-        workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
-        self.assertLess(workflow.index('Verify I05 queued-successor native prerequisite'),
-                        workflow.index('Run selected canonical validation case'))
-        download = workflow.split('      - name: Download pinned native storage evidence\n', 1)[1].split('        uses:', 1)[0]
-        self.assertIn("inputs.i05_diagnostic_branch == 'queued-successor'", download)
-        execution = workflow.split('      - name: Run selected canonical validation case\n', 1)[1].split('        run:', 1)[0]
-        self.assertIn('I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}', execution)
-
-    def shell_step(self, name):
-        from check_test_wiring import yaml_block
-        source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
-        job = yaml_block(source.splitlines(), 'native-heal-regressions', 2)
-        start = job.index('      - name: ' + name)
-        end = next((index for index in range(start + 1, len(job))
-                    if job[index].startswith('      - name: ')), len(job))
-        step = job[start:end]
-        return '\n'.join(line[10:] for line in step[step.index('        run: |') + 1:]
-                         if not line.strip() or line.startswith('          '))
-
-    def fixture(self, root, *, new_helper):
-        directory = root / 'auto-testing/scripts/validation_heal'
-        directory.mkdir(parents=True)
-        stub = '''import json, os, pathlib, sys
-kind = pathlib.Path(__file__).stem
-with pathlib.Path('calls.jsonl').open('a') as out:
-    out.write(json.dumps({'kind': kind, 'args': sys.argv[1:]}) + '\\n')
-if sys.argv[1] == 'supports': sys.exit(int(os.environ['SUPPORT_STATUS']))
-sys.exit(int(os.environ.get(kind.upper() + '_FAIL', '0')))
-'''
-        (directory / 'native_regressions.py').write_text(stub)
-        if new_helper: (directory / 'race_native.py').write_text(stub)
-        bin_dir = root / 'bin'
-        bin_dir.mkdir()
-        git = bin_dir / 'git'
-        git.write_text('#!/bin/sh\nprintf "%s\\n" "$FAKE_HEAD"\n')
-        git.chmod(0o755)
-        env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, CANDIDATE_SHA='a' * 40,
-                   FAKE_HEAD='b' * 40, SUPPORT_STATUS='2', PATH=str(bin_dir) + os.pathsep + os.environ['PATH'])
-        env.pop('BASH_ENV', None)
-        return env
-
-    def test_actual_native_build_routes_old_new_unknown_and_failed_collectors(self):
-        shell = self.shell_step('Build and enumerate the unmodified candidate test executable')
-        for mode, helper, support, failure, head, expected_code, expected_calls in [
-            ('old-harness', False, 2, None, 'b', 0, [('native_regressions', 'build')]),
-            ('legacy', True, 2, None, 'b', 0, [('race_native', 'supports'), ('native_regressions', 'build')]),
-            ('race', True, 0, None, 'b', 0, [('race_native', 'supports'), ('race_native', 'build')]),
-            ('support-error', True, 42, None, 'b', 42, [('race_native', 'supports')]),
-            ('invalid-support', True, 1, None, 'b', 1, [('race_native', 'supports')]),
-            ('race-build-error', True, 0, 'RACE_NATIVE_FAIL', 'b', 42, [('race_native', 'supports'), ('race_native', 'build')]),
-            ('legacy-build-error', True, 2, 'NATIVE_REGRESSIONS_FAIL', 'b', 42, [('race_native', 'supports'), ('native_regressions', 'build')]),
-            ('wrong-helper', True, 0, None, 'c', 1, []),
-        ]:
-            with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode):
-                root = Path(temp)
-                env = self.fixture(root, new_helper=helper)
-                env.update(SUPPORT_STATUS=str(support), FAKE_HEAD=head * 40)
-                if failure: env[failure] = '42'
-                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell],
-                                        cwd=root, env=env, capture_output=True, text=True)
-                self.assertEqual(result.returncode, expected_code, result.stderr)
-                calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else []
-                self.assertEqual([(row['kind'], row['args'][0]) for row in calls], expected_calls)
-                for row in calls:
-                    if row['args'][0] == 'supports':
-                        self.assertEqual(row['args'], ['supports', '--commit', 'a' * 40])
-                    else:
-                        expected = ['build', '--source', 'candidate', '--commit', 'a' * 40]
-                        if row['kind'] == 'race_native': expected += ['--testing-commit', 'b' * 40]
-                        self.assertEqual(row['args'], expected + ['--out', 'native-tools'])
-
-    def test_actual_native_run_rejects_ambiguous_formats_and_propagates_failures(self):
-        shell = self.shell_step('Run exact native regressions separately from fleet validation')
-        for mode, helper, manifests, failure, expected_code, expected_kind in [
-            ('legacy', False, ['native-regressions.json'], None, 0, 'native_regressions'),
-            ('race', True, ['race-native.json'], None, 0, 'race_native'),
-            ('ambiguous', True, ['native-regressions.json', 'race-native.json'], None, 1, None),
-            ('legacy-error', True, ['native-regressions.json'], 'NATIVE_REGRESSIONS_FAIL', 42, 'native_regressions'),
-            ('race-error', True, ['race-native.json'], 'RACE_NATIVE_FAIL', 42, 'race_native'),
-            ('missing-race-helper', False, ['race-native.json'], None, 2, None),
-        ]:
-            with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode):
-                root = Path(temp)
-                env = self.fixture(root, new_helper=helper)
-                if failure: env[failure] = '42'
-                (root / 'native-tools').mkdir()
-                for manifest in manifests: (root / 'native-tools' / manifest).write_text('{}')
-                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell],
-                                        cwd=root, env=env, capture_output=True, text=True)
-                self.assertEqual(result.returncode, expected_code, result.stderr)
-                calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else []
-                self.assertEqual(len(calls), int(expected_kind is not None))
-                if expected_kind:
-                    self.assertEqual(calls, [{'kind': expected_kind, 'args': ['run', '--directory', 'native-tools', '--out', 'native-evidence']}])
-
-    def test_native_upload_retains_both_fixed_executable_kinds(self):
-        source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
-        self.assertIn('            native-tools/ecstore-native-tests\n', source)
-        self.assertIn('            native-tools/ecstore-race-native-tests\n', source)
-
-
-class PersistenceToolsWorkflowTests(unittest.TestCase):
-    def test_supplement_preserves_old_pins_and_propagates_actual_collector_failure(self):
-        from check_test_wiring import yaml_block
-        source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
-        job = yaml_block(source.splitlines(), 'native-heal-regressions', 2)
-        start = job.index('      - name: Run existing F07 directory fsync failure regressions')
-        end = job.index('      - name: Upload native executable identities and original diagnostics')
-        step = job[start:end]
-        shell = '\n'.join(line[10:] for line in step[step.index('        run: |') + 1:]
-                          if not line.strip() or line.startswith('          '))
-        for mode in ('old-harness', 'success', 'run-failure', 'verify-failure'):
-            with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode):
-                root = Path(temp)
-                if mode != 'old-harness':
-                    collector = root / 'auto-testing/scripts/validation_heal/persistence_native.py'
-                    collector.parent.mkdir(parents=True)
-                    collector.write_text('''import json, os, pathlib, sys
-with pathlib.Path('calls.jsonl').open('a') as out: out.write(json.dumps(sys.argv[1:]) + '\\n')
-sys.exit(42 if os.environ['COLLECTOR_MODE'] == sys.argv[1] + '-failure' else 0)
-''')
-                env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, COLLECTOR_MODE=mode)
-                env.pop('BASH_ENV', None)
-                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell],
-                                        cwd=root, env=env, capture_output=True, text=True)
-                self.assertEqual(result.returncode == 0, mode in ('old-harness', 'success'), result.stderr)
-                if mode == 'old-harness':
-                    report = json.loads((root / 'native-persistence-evidence/not-available.json').read_text())
-                    self.assertIs(report['complete'], False)
-                    self.assertIs(report['requiredNativeCoverageComplete'], False)
-                    self.assertIs(report['canonicalCasePassClaimed'], False)
-                    self.assertFalse((root / 'calls.jsonl').exists())
-                else:
-                    calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()]
-                    self.assertEqual([call[0] for call in calls], ['run'] if mode == 'run-failure' else ['run', 'verify'])
-                    for call in calls: self.assertEqual(call[-2:], ['--testing-commit', 'b' * 40])
-
-
-class InterruptedCleanupGuardTests(unittest.TestCase):
-    def test_actual_workflow_guard_rejects_live_success_and_unrelated_runs(self):
-        source = (candidate.ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
-        step = source[source.index('      - name: Audit and finalize retained validation environment'):]
-        guard = step.split("          python3 - <<'PY'", 1)[1].split("          PY", 1)[0]
-        guard = "\n".join(line[10:] for line in guard.splitlines())
-        for state, conclusion, branch, workflow, accepted in (
-            ("completed", "failure", "topic", "RustFS Functional Chain", True),
-            ("completed", "cancelled", "topic", "RustFS Functional Chain", True),
-            ("in_progress", "cancelled", "topic", "RustFS Functional Chain", False),
-            ("completed", "success", "topic", "RustFS Functional Chain", False),
-            ("completed", "cancelled", "other", "RustFS Functional Chain", False),
-            ("completed", "cancelled", "topic", "Other Workflow", False),
-        ):
-            with self.subTest(state=state, conclusion=conclusion, branch=branch, workflow=workflow), tempfile.TemporaryDirectory() as temp:
-                Path(temp, "failed-run.json").write_text(json.dumps({"status": state, "conclusion": conclusion,
-                    "headBranch": branch, "workflowName": workflow}))
-                with mock.patch.dict(os.environ, VALIDATION_ROOT=temp, GITHUB_REF_NAME="topic"):
-                    if accepted:
-                        exec(compile(guard, "workflow-cleanup-guard", "exec"), {})
-                    else:
-                        with self.assertRaises(ValueError):
-                            exec(compile(guard, "workflow-cleanup-guard", "exec"), {})
 
 
 class CandidateTests(unittest.TestCase):
@@ -440,98 +76,6 @@ class CandidateTests(unittest.TestCase):
         self.assertEqual(result["artifact_id"], 789)
         self.assertEqual(result["manifest"]["source_sha"], "b" * 40)
 
-    def test_validation_manifest_binds_requested_source_and_private_channel(self):
-        manifest = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"],
-                    "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")}
-        run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"}
-        payload = io.BytesIO()
-        with zipfile.ZipFile(payload, "w") as archive:
-            archive.writestr("validation-candidate-123-2.json", json.dumps(manifest))
-        raw = payload.getvalue()
-        artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False,
-                    "size_in_bytes": len(raw), "digest": "sha256:" + hashlib.sha256(raw).hexdigest(),
-                    "workflow_run": {"id": 123, "head_sha": "a" * 40}}
-        result = candidate.read_manifest(raw, artifact, run)
-        self.assertEqual(result["channel"], "validation")
-        self.assertIn("/packages/validation/runs/123/2/", result["package_url"])
-
-    def test_validation_resolver_requires_manual_workflow_dispatch(self):
-        payload, artifact = self.archive()
-        run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40,
-               "path": ".github/workflows/rustfs-validation-package.yml", "head_branch": "main",
-               "head_repository": {"full_name": "rustfs/rustfs"}, "event": "schedule",
-               "status": "completed", "conclusion": "success", "run_started_at": "2026-10-12T00:00:00Z"}
-        with mock.patch.object(candidate, "api", side_effect=[run]):
-            with self.assertRaisesRegex(ValueError, "manually dispatched"):
-                candidate.resolve(123, 2, allow_validation=True)
-
-    def test_validation_workflow_rejects_legacy_manifest_schema(self):
-        manifest = {**self.manifest}
-        payload_buffer = io.BytesIO()
-        with zipfile.ZipFile(payload_buffer, "w") as archive:
-            archive.writestr("validation-candidate-123-2.json", json.dumps(manifest))
-        payload = payload_buffer.getvalue()
-        artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False,
-                    "size_in_bytes": len(payload), "digest": "sha256:" + hashlib.sha256(payload).hexdigest(),
-                    "workflow_run": {"id": 123, "head_sha": "a" * 40}}
-        run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"}
-        with self.assertRaisesRegex(ValueError, "schema-3"):
-            candidate.read_manifest(payload, artifact, run)
-
-    def test_private_candidate_resolution_is_explicit_and_attempt_bound(self):
-        self.manifest.update(schema=3, channel="validation", source_ref=self.manifest["source_sha"],
-                             package_url=self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/"))
-        payload, artifact = self.archive(["validation-candidate-123-2.json"])
-        artifact["name"] = "validation-candidate-123-2"
-        base = {**self.run, "head_branch": "cxymds/validation-runner", "head_repository": {"full_name": "rustfs/rustfs"},
-                "event": "workflow_dispatch", "status": "completed", "conclusion": "success",
-                "run_started_at": "2026-10-12T00:00:00Z"}
-        for path in candidate.VALIDATION_BUILD_WORKFLOWS:
-            run = {**base, "path": path}
-            with self.subTest(path=path), mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]):
-                resolved = candidate.resolve(123, 2, allow_validation=True)
-                self.assertEqual(resolved["manifest"]["source_sha"], "b" * 40)
-                self.assertEqual(resolved["workflow_sha"], "a" * 40)
-            with mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError):
-                candidate.resolve(123, 2)
-        for field, value in (("id", 124), ("run_attempt", 1), ("event", "push"), ("status", "in_progress"),
-                             ("conclusion", "failure"), ("path", ".github/workflows/other.yml"),
-                             ("head_repository", {"full_name": "untrusted/rustfs"})):
-            run = {**base, "path": ".github/workflows/rustfs-functional-chain.yml", field: value}
-            with self.subTest(field=field), mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError):
-                candidate.resolve(123, 2, allow_validation=True)
-
-    def test_private_manifest_rejects_alias_channel_source_and_schema_substitution(self):
-        valid = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"],
-                 "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")}
-        for key, value in (("channel", "nightly"), ("source_ref", "main"), ("source_sha", "e" * 40),
-                           ("package_url", self.manifest["package_url"]), ("workflow_sha", "e" * 40),
-                           ("build_run_attempt", 3), ("schema", True)):
-            with self.subTest(key=key), self.assertRaises(ValueError):
-                candidate.validate_manifest({**valid, key: value}, self.run)
-
-    def test_package_only_mode_rejects_conflicting_fleet_and_tool_requests(self):
-        from check_test_wiring import yaml_block
-        workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml"
-        job = yaml_block(workflow.read_text().splitlines(), "validation-package-request", 2)
-        begin = job.index("        run: |") + 1
-        script = "\n".join(line[10:] for line in job[begin:] if not line.strip() or line.startswith("          "))
-        import os
-        inputs = {"validation_package_sha": "a" * 40}
-        env = {**os.environ, "PACKAGE_INPUTS": json.dumps(inputs)}
-        result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True)
-        self.assertEqual(result.returncode, 0, result.stderr)
-        for field in ("build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha", "validation_tools_only",
-                      "validation_native_tools", "validation_shared_tools", "validation_manager_tools",
-                      "validation_cleanup_run_id", "validation_tools_run_id"):
-            with self.subTest(field=field):
-                env["PACKAGE_INPUTS"] = json.dumps({**inputs, field: "not-empty"})
-                result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True)
-                self.assertNotEqual(result.returncode, 0)
-        for value in ("main", "a" * 39, "a" * 40 + "\nFORGED=value"):
-            env["PACKAGE_INPUTS"] = json.dumps({"validation_package_sha": value})
-            self.assertNotEqual(subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True).returncode, 0)
-
 
 class EvidenceTests(unittest.TestCase):
     def setUp(self):

scripts/test_inspect_validation_runner.py

@@ -1,115 +0,0 @@
-"""Runner diagnostics must remain read-only when the shared fleet is unavailable."""
-
-import json
-import os
-from pathlib import Path
-import subprocess
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import inspect_validation_runner as inspection
-
-
-class RunnerInspectionTests(unittest.TestCase):
-    def request(self):
-        return {"validation_runner_inspect_only": True, "validation_case": "",
-                "validation_tools_only": False, "i05_diagnostic_branch": "timeout",
-                "validation_tools_run_attempt": "1"}
-
-    def test_exclusive_request_rejects_every_execution_mode_before_api_access(self):
-        inspection.validate_request(self.request())
-        for key, value in [("validation_runner_inspect_only", False),
-                           ("validation_case", "environment.inspect"), ("build_run_id", "123"),
-                           ("validation_package_sha", "a" * 40), ("validation_tools_only", True),
-                           ("validation_native_tools", True), ("validation_shared_tools", True),
-                           ("validation_manager_tools", True), ("validation_cleanup_run_id", "123"),
-                           ("auto_testing_sha", "a" * 40), ("validation_tools_run_id", "123"),
-                           ("i05_diagnostic_branch", "queued-successor"),
-                           ("validation_tools_run_attempt", "2")]:
-            with self.subTest(key=key), self.assertRaises(ValueError):
-                inspection.validate_request({**self.request(), key: value})
-
-    def fixture(self, runners, *, unknown_jobs=False):
-        def api(path):
-            self.assertTrue(path.startswith("repos/rustfs/rustfs/actions/"))
-            if "/runners?" in path:
-                return {"runners": runners}
-            if "/runs?status=queued&" in path:
-                return {"workflow_runs": [{"id": 123, "name": "RustFS Functional Chain",
-                    "status": "queued", "created_at": "2026-10-26T00:00:00Z", "html_url": "run-url"}]}
-            if "/runs?status=" in path:
-                return {"workflow_runs": []}
-            self.assertIn("/runs/123/jobs?", path)
-            return {"jobs": [] if unknown_jobs else [
-                {"id": 456, "name": "validation", "status": "queued", "runner_name": "",
-                 "labels": ["smoke-testing"], "html_url": "job-url", "steps": [{"secret": "excluded"}]},
-                {"id": 457, "name": "native", "status": "in_progress", "labels": ["ubuntu-latest"]},
-                {"id": 458, "name": "prior", "status": "completed", "labels": ["smoke-testing"]}]}
-        return api
-
-    def test_offline_idle_busy_and_unregistered_are_distinct_with_only_fleet_jobs(self):
-        runner = {"id": 7, "name": "rustfs-smoke-testing", "status": "offline", "busy": False,
-                  "labels": [{"name": "smoke-testing"}], "extra": {"secret": "excluded"}}
-        for state, busy, expected in [("offline", False, "offline"), ("online", True, "online-busy"),
-                                       ("online", False, "online-idle"), (None, False, "unregistered")]:
-            with self.subTest(expected=expected):
-                rows = [{**runner, "status": state, "busy": busy}] if state else []
-                report = {}
-                with patch.object(inspection, "api", side_effect=self.fixture(rows)):
-                    inspection.collect(report)
-                self.assertEqual(report["runnerAvailability"], expected)
-                self.assertTrue(report["complete"])
-                self.assertEqual([row["jobId"] for row in report["fleetJobs"]], [456])
-                self.assertNotIn("secret", json.dumps(report))
-
-    def test_jobless_workflow_is_retained_without_inventing_a_runner_assignment(self):
-        report = {}
-        with patch.object(inspection, "api", side_effect=self.fixture([], unknown_jobs=True)):
-            inspection.collect(report)
-        self.assertEqual(report["fleetJobs"], [])
-        self.assertEqual(report["runsWithoutJobs"][0]["runId"], 123)
-
-    def test_inventory_pagination_is_bounded_and_cannot_claim_complete_when_truncated(self):
-        row = {"id": 7}
-        with patch.object(inspection, "api", return_value={"runners": [row] * 100}) as api:
-            with self.assertRaisesRegex(ValueError, "bounded inspection"):
-                inspection.pages("repos/rustfs/rustfs/actions/runners", "runners")
-        self.assertEqual(api.call_count, 10)
-
-    def test_api_failure_keeps_partial_report_without_subprocess_diagnostics_or_fleet_claims(self):
-        for inputs, expected_calls in [(self.request(), 1), ({**self.request(), "validation_tools_only": True}, 0)]:
-            with self.subTest(inputs=inputs), tempfile.TemporaryDirectory() as directory:
-                output = Path(directory) / "report.json"
-                with patch.dict(os.environ, RUNNER_INSPECTION_INPUTS=json.dumps(inputs)), \
-                     patch.object(inspection.sys, "argv", ["inspect", "--output", str(output)]), \
-                     patch.object(inspection, "api", side_effect=subprocess.CalledProcessError(
-                         1, ["gh", "api"], stderr=b"secret-token")) as api:
-                    self.assertEqual(inspection.main(), 1)
-                report = json.loads(output.read_text())
-                self.assertFalse(report["complete"])
-                for field in ("mutated", "fleetAccessed", "cleanStateCertified",
-                              "sharedWorkflowIdleCertified", "canonicalCasePassClaimed"):
-                    self.assertIs(report[field], False)
-                self.assertNotIn("secret-token", output.read_text())
-                self.assertEqual(api.call_count, expected_calls)
-
-    def test_workflow_is_hosted_and_uses_a_separate_concurrency_group(self):
-        workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml"
-        source = workflow.read_text()
-        job = source.split("  validation-runner-inspection:\n", 1)[1].split("  validation-package-request:", 1)[0]
-        self.assertIn("runs-on: ubuntu-latest", job)
-        self.assertIn("timeout-minutes: 10", job)
-        self.assertIn("RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}", job)
-        self.assertIn("if: ${{ always() }}", job)
-        self.assertNotIn("RUSTFS_SECRET_KEY", job)
-        concurrency = source.split("concurrency:\n", 1)[1].split("jobs:\n", 1)[0]
-        self.assertLess(concurrency.index("validation_runner_inspect_only"), concurrency.index("validation_package_sha"))
-        self.assertIn("rustfs-functional-validation-runner-inspection", concurrency)
-        for name in ("validation-package-request", "validation-package", "prepare", "validation"):
-            job = source.split("  " + name + ":\n", 1)[1].split("\n    runs-on:", 1)[0].split("\n    uses:", 1)[0]
-            self.assertIn("!inputs.validation_runner_inspect_only", job)
-
-
-if __name__ == "__main__":
-    unittest.main()

scripts/test_nightly_candidate.py

@@ -221,171 +221,5 @@ SH
         self.assertNotIn("          overwrite: true", upload)
 
 
-class ValidationPackageTests(unittest.TestCase):
-    def setUp(self):
-        # Exercise the real upload shell with the same conditional object-store
-        # fixture used by the nightly publication tests.
-        self.fixture = NightlyCandidateTests("test_success_binds_actual_package_checkout_and_attempt")
-        self.fixture.setUp()
-        self.addCleanup(self.fixture.temp.cleanup)
-        self.workflow = ROOT / ".github/workflows/rustfs-validation-package.yml"
-        job = yaml_block(self.workflow.read_text().splitlines(), "publish", 2)
-        start = job.index("      - name: Upload validation package to immutable R2 path")
-        end = job.index("      - name: Upload validation candidate manifest")
-        upload = job[start:end]
-        begin = upload.index("        run: |") + 1
-        self.fixture.shell = "\n".join(line[10:] for line in upload[begin:] if not line.strip() or line.startswith("          "))
-        self.fixture.shims.write_text(self.fixture.shims.read_text().replace("/packages/nightly/runs/*", "/packages/validation/runs/*"))
-        self.fixture.env.update(PACKAGE_FILE=self.fixture.package.name, VALIDATION_SOURCE_SHA=self.fixture.sha,
-                                VALIDATION_SOURCE_REF=self.fixture.sha, GITHUB_SHA="f" * 40, AWS_CLI="aws")
-        def shell(name, end_name):
-            step = job[job.index("      - name: " + name):job.index("      - name: " + end_name)]
-            return "\n".join(line[10:] for line in step[step.index("        run: |") + 1:]
-                             if not line.strip() or line.startswith("          "))
-        self.stage_shell = shell("Validate and stage opaque package bytes", "Prepare trusted publication tools")
-        self.tools_shell = shell("Prepare trusted publication tools", "Upload validation package to immutable R2 path")
-
-    def stage(self):
-        env = dict(os.environ, RUNNER_TEMP=str(self.fixture.root))
-        env.pop("BASH_ENV", None)
-        return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.stage_shell],
-                              cwd=self.fixture.root, env=env, capture_output=True, text=True)
-
-    def test_success_uploads_only_one_immutable_object_and_exact_source(self):
-        result = self.fixture.run_publish()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        manifest = self.fixture.manifest()
-        self.assertEqual(manifest["schema"], 3)
-        self.assertEqual(manifest["channel"], "validation")
-        self.assertEqual(manifest["source_sha"], self.fixture.sha)
-        self.assertEqual(manifest["source_ref"], self.fixture.sha)
-        self.assertEqual(manifest["workflow_sha"], "f" * 40)
-        validate_manifest(manifest, {"id": 12345, "run_attempt": 1, "head_sha": "f" * 40})
-        objects = [path for path in self.fixture.store.rglob("*") if path.is_file()]
-        self.assertEqual(len(objects), 1)
-        self.assertEqual(objects[0].read_bytes(), self.fixture.package.read_bytes())
-        self.assertEqual(str(objects[0].relative_to(self.fixture.store)),
-                         f"artifacts/rustfs/packages/validation/runs/12345/1/{self.fixture.digest}/rustfs.deb")
-        self.assertNotIn("s3 cp", (self.fixture.root / "aws.log").read_text())
-
-    def test_failure_or_missing_credentials_cannot_certify_a_candidate(self):
-        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download")):
-            with self.subTest(mode=mode):
-                result = self.fixture.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
-                self.assertNotEqual(result.returncode, 0)
-                self.assertFalse(self.fixture.output.exists())
-                self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), [])
-        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
-            with self.subTest(key=key):
-                self.assertNotEqual(self.fixture.run_publish(**{key: ""}).returncode, 0)
-                self.assertFalse(self.fixture.output.exists())
-
-    def test_stale_source_and_duplicate_objects_are_rejected(self):
-        for source in ("not-an-exact-sha", "e" * 40):
-            with self.subTest(source=source):
-                result = self.fixture.run_publish(VALIDATION_SOURCE_SHA=source)
-                self.assertNotEqual(result.returncode, 0)
-                self.assertFalse(self.fixture.store.exists())
-        result = self.fixture.run_publish()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        manifest = self.fixture.manifest()
-        key = manifest["package_url"].removeprefix("https://dl.rustfs.com/")
-        stored = self.fixture.store / key
-        stored.write_bytes(b"preexisting conflicting object")
-        (self.fixture.root / "validation-candidate-12345-1.json").unlink()
-        result = self.fixture.run_publish()
-        self.assertNotEqual(result.returncode, 0)
-        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
-        self.assertFalse(self.fixture.output.exists())
-
-    def test_exact_source_and_workflow_identity_are_preserved_separately(self):
-        lines = self.workflow.read_text().splitlines()
-        build = "\n".join(yaml_block(lines, "build", 2))
-        publish = "\n".join(yaml_block(lines, "publish", 2))
-        source = "\n".join(yaml_block(lines, "resolve-source", 2))
-        self.assertIn('[[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]', source)
-        self.assertIn('test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"', source)
-        self.assertIn("ref: ${{ needs.resolve-source.outputs.source_sha }}", build)
-        self.assertIn('--arg source_sha "${VALIDATION_SOURCE_SHA}"', publish)
-        self.assertIn('--arg workflow_sha "${GITHUB_SHA}"', publish)
-        self.assertNotIn("PF_TESTING_GH_TOKEN", build)
-        self.assertNotIn("publish_nightly_assets", build)
-
-    def test_candidate_execution_and_publication_credentials_are_in_separate_jobs(self):
-        lines = self.workflow.read_text().splitlines()
-        build = "\n".join(yaml_block(lines, "build", 2))
-        publish = "\n".join(yaml_block(lines, "publish", 2))
-        self.assertIn("uses: ./.github/actions/setup", build)
-        self.assertIn("cargo build", build)
-        self.assertNotIn("secrets.", build)
-        self.assertNotIn("id-token: write", build)
-        self.assertIn("needs: [resolve-source, build]", publish)
-        self.assertIn("runs-on: ubuntu-latest", publish)
-        for forbidden in ("actions/checkout@", "actions/cache@", "uses: ./", "needs.build.outputs", "cargo ", "dpkg", "source "):
-            with self.subTest(forbidden=forbidden): self.assertNotIn(forbidden, publish)
-        name = "name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}"
-        self.assertIn(name, build)
-        self.assertIn(name, publish)
-        self.assertIn("VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}", publish)
-        self.assertIn("VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}", publish)
-        prepare = publish.split("      - name: Upload validation package to immutable R2 path")[0]
-        self.assertNotIn("secrets.", prepare)
-        self.assertIn("secrets.R2_SECRET_ACCESS_KEY", publish)
-
-    def test_staging_rejects_nonregular_empty_or_extra_artifact_entries(self):
-        incoming = self.fixture.root / "validation-package-input"
-        incoming.mkdir()
-        package = incoming / "rustfs.deb"
-        for kind in ("missing", "empty", "symlink", "directory", "fifo", "extra", "directory-symlink"):
-            with self.subTest(kind=kind):
-                if kind == "empty": package.touch()
-                elif kind == "symlink": package.symlink_to(self.fixture.package)
-                elif kind == "directory": package.mkdir()
-                elif kind == "fifo": os.mkfifo(package)
-                elif kind == "extra":
-                    package.write_bytes(b"opaque")
-                    (incoming / "BASH_ENV").write_text("exit 1")
-                elif kind == "directory-symlink":
-                    incoming.rmdir()
-                    incoming.symlink_to(self.fixture.root, target_is_directory=True)
-                self.assertNotEqual(self.stage().returncode, 0)
-                self.assertFalse((self.fixture.root / "validation-publication").exists())
-                if incoming.is_symlink():
-                    incoming.unlink()
-                    incoming.mkdir()
-                elif package.is_dir(): package.rmdir()
-                else: package.unlink(missing_ok=True)
-                (incoming / "BASH_ENV").unlink(missing_ok=True)
-
-    def test_staging_and_publication_never_execute_candidate_package_or_python_hooks(self):
-        incoming = self.fixture.root / "validation-package-input"
-        incoming.mkdir()
-        marker = self.fixture.root / "candidate-executed"
-        payload = f'#!/bin/bash\nprintf "%s" "$R2_SECRET_ACCESS_KEY" > "{marker}"\n'.encode()
-        (incoming / "rustfs.deb").write_bytes(payload)
-        (self.fixture.root / "sitecustomize.py").write_text(f"open({str(marker)!r}, 'w').write('python hook')")
-        result = self.stage()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        staged = self.fixture.root / "validation-publication/rustfs.deb"
-        self.assertEqual(staged.read_bytes(), payload)
-        result = self.fixture.run_publish(PACKAGE_FILE=str(staged))
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertFalse(marker.exists())
-        self.assertEqual(self.fixture.manifest()["package_sha256"], hashlib.sha256(payload).hexdigest())
-
-    def test_cli_bootstrap_finishes_without_publication_secrets(self):
-        for mode in ("success", "old-cli", "bootstrap-failure", "install-failure", "broken-install"):
-            with self.subTest(mode=mode):
-                (self.fixture.root / "installed").unlink(missing_ok=True)
-                env = dict(self.fixture.env, FAKE_MODE=mode)
-                for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
-                    env.pop(key)
-                result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", self.tools_shell],
-                                        cwd=self.fixture.root, env=env, capture_output=True, text=True)
-                self.assertEqual(result.returncode == 0, mode in ("success", "old-cli"), result.stderr)
-                self.assertFalse(self.fixture.store.exists())
-                self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), [])
-
-
 if __name__ == "__main__":
     unittest.main()

scripts/test_retain_validation_evidence.py

@@ -1,189 +0,0 @@
-"""Interrupted evidence is read-only and never an implicit recovery or PASS."""
-
-import hashlib
-import json
-import os
-from pathlib import Path
-import tempfile
-import unittest
-from unittest.mock import patch
-
-import retain_validation_evidence as retain
-import prepare_retained_validation_recovery as recovery_input
-
-
-class RetainEvidenceTests(unittest.TestCase):
-    def fixture(self, directory):
-        temporary = Path(directory).resolve() / "_work/_temp"
-        temporary.mkdir(parents=True)
-        parent = temporary.parent / "rustfs-validation-evidence"
-        parent.mkdir(mode=0o700)
-        root = parent / "rustfs-validation-77-1"
-        root.mkdir(mode=0o700)
-        (root / "runner-storage-preflight.json").write_text(json.dumps({
-            "runId": "77", "runAttempt": "1", "evidenceRoot": str(root),
-            "persistentOutsideRunnerTemp": True}))
-        (root / "original.json").write_bytes(b'{"verdict":"RUNNING"}')
-        env = {"RUNNER_TEMP": str(temporary), "GITHUB_RUN_ID": "88", "GITHUB_RUN_ATTEMPT": "1",
-               "RUNNER_NAME": "owned-runner", "GITHUB_ENV": str(temporary / "env"),
-               "EVIDENCE_RETAIN_INPUTS": json.dumps({"validation_retain_evidence_run_id": "77",
-                    "i05_diagnostic_branch": "timeout", "validation_tools_run_attempt": "1"})}
-        run = {"id": 77, "run_attempt": 1, "status": "completed", "conclusion": "failure",
-               "event": "workflow_dispatch", "repository": {"full_name": "rustfs/rustfs"},
-               "path": ".github/workflows/rustfs-functional-chain.yml", "head_sha": "a" * 40,
-               "html_url": "source-url"}
-        jobs = {"total_count": 1, "jobs": [{"name": "validation", "runner_name": "owned-runner"}]}
-        return env, root, run, jobs
-
-    def test_existing_bytes_and_original_running_verdict_remain_unchanged(self):
-        with tempfile.TemporaryDirectory() as directory:
-            env, root, run, jobs = self.fixture(directory)
-            before = {p.name: p.read_bytes() for p in root.iterdir()}
-            report = {}
-            with patch.object(retain, "api", side_effect=[run, jobs]) as api:
-                self.assertEqual(retain.prepare(env, report), root)
-            self.assertTrue(report["complete"])
-            self.assertEqual({p.name: p.read_bytes() for p in root.iterdir()}, before)
-            self.assertEqual({row["path"]: row["sha256"] for row in report["files"]},
-                             {name: hashlib.sha256(raw).hexdigest() for name, raw in before.items()})
-            self.assertEqual([call.args[0] for call in api.call_args_list], [
-                "repos/rustfs/rustfs/actions/runs/77/attempts/1",
-                "repos/rustfs/rustfs/actions/runs/77/attempts/1/jobs?per_page=100"])
-
-    def test_mixed_execution_requests_and_unsafe_run_ids_refuse_before_api(self):
-        with tempfile.TemporaryDirectory() as directory:
-            env, root, run, jobs = self.fixture(directory)
-            inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"])
-            for field, value in [("validation_case", "heal.admin-healthy-noop"),
-                                 ("validation_runner_inspect_only", True), ("validation_tools_only", True),
-                                 ("build_run_id", "1"), ("validation_retain_evidence_run_id", "../77"),
-                                 ("validation_retain_evidence_run_id", "88")]:
-                with self.subTest(field=field, value=value), patch.object(retain, "api") as api:
-                    altered = {**env, "EVIDENCE_RETAIN_INPUTS": json.dumps({**inputs, field: value})}
-                    with self.assertRaises(ValueError): retain.prepare(altered, {})
-                    api.assert_not_called()
-
-    def test_wrong_repository_running_source_and_different_runner_refuse(self):
-        for mutation in ("repo", "running", "success", "runner", "attempt"):
-            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
-                env, root, run, jobs = self.fixture(directory)
-                if mutation == "repo": run["repository"]["full_name"] = "foreign/repo"
-                elif mutation == "running": run["status"] = "in_progress"
-                elif mutation == "success": run["conclusion"] = "success"
-                elif mutation == "attempt": run["run_attempt"] = 2
-                else: jobs["jobs"][0]["runner_name"] = "other-runner"
-                with patch.object(retain, "api", side_effect=[run, jobs]), self.assertRaises(ValueError):
-                    retain.prepare(env, {})
-
-    def test_links_special_files_and_byte_budget_do_not_escape_the_original_root(self):
-        for mutation in ("symlink", "hardlink", "fifo", "budget", "public-root"):
-            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
-                env, root, run, jobs = self.fixture(directory)
-                if mutation == "symlink": (root / "foreign").symlink_to(Path(directory) / "secret")
-                elif mutation == "hardlink": os.link(root / "original.json", root / "alias")
-                elif mutation == "fifo": os.mkfifo(root / "fifo")
-                elif mutation == "public-root": root.chmod(0o755)
-                with patch.object(retain, "api", side_effect=[run, jobs]), \
-                     patch.object(retain, "MAX_BYTES", 1 if mutation == "budget" else retain.MAX_BYTES), \
-                     self.assertRaises(ValueError):
-                    retain.prepare(env, {})
-
-    def test_file_changes_and_foreign_storage_provenance_are_rejected(self):
-        for mutation in ("write", "marker"):
-            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
-                env, root, run, jobs = self.fixture(directory)
-                if mutation == "marker":
-                    (root / "runner-storage-preflight.json").write_text('{"runId":"foreign"}')
-                real = os.read
-                changed = []
-                def read(fd, size):
-                    raw = real(fd, size)
-                    if mutation == "write" and not changed:
-                        changed.append(True)
-                        with (root / "original.json").open("ab") as stream: stream.write(b"changed")
-                    return raw
-                with patch.object(retain, "api", side_effect=[run, jobs]), patch.object(retain.os, "read", side_effect=read), \
-                     self.assertRaises(ValueError):
-                    retain.prepare(env, {})
-
-    def test_cli_exports_only_admitted_root_and_emits_no_case_or_recovery_claim(self):
-        with tempfile.TemporaryDirectory() as directory:
-            env, root, run, jobs = self.fixture(directory)
-            output = Path(directory) / "receipt.json"
-            with patch.dict(os.environ, env), patch.object(retain, "api", side_effect=[run, jobs]), \
-                 patch.object(retain.sys, "argv", ["retain", "--report", str(output)]):
-                self.assertEqual(retain.main(), 0)
-            value = json.loads(output.read_text())
-            for field in ("fleetAccessed", "originalEvidenceModified", "sourceVerdictRewritten", "canonicalCasePassClaimed"):
-                self.assertIs(value[field], False)
-            self.assertIn("VALIDATION_RETAIN_ROOT=" + str(root), Path(env["GITHUB_ENV"]).read_text())
-
-    def test_cli_keeps_incomplete_receipt_without_exporting_a_missing_original_root(self):
-        with tempfile.TemporaryDirectory() as directory:
-            env, root, run, jobs = self.fixture(directory)
-            env["EVIDENCE_RETAIN_INPUTS"] = '{"validation_retain_evidence_run_id":"99"}'
-            run["id"] = 99
-            output = Path(directory) / "receipt.json"
-            with patch.dict(os.environ, env), patch.object(retain, "api", side_effect=[run, jobs]), \
-                 patch.object(retain.sys, "argv", ["retain", "--report", str(output)]):
-                self.assertEqual(retain.main(), 1)
-            value = json.loads(output.read_text())
-            self.assertFalse(value["complete"])
-            self.assertEqual(value["errorType"], "FileNotFoundError")
-            self.assertFalse(Path(env["GITHUB_ENV"]).exists())
-
-    def test_recovery_copy_keeps_original_bytes_and_separate_collection_identity(self):
-        with tempfile.TemporaryDirectory() as directory:
-            env, root, run, jobs = self.fixture(directory)
-            destination = Path(directory).resolve() / 'new-recovery'; destination.mkdir(mode=0o700)
-            env.update(FAILED_RUN_ID='77', GITHUB_REF_NAME='topic', VALIDATION_ROOT=str(destination))
-            run['head_branch'] = 'topic'
-            nested = root / 'nested'; nested.mkdir(); (nested / 'receipt').write_bytes(b'ACK')
-            before = retain.inventory(root)
-            report = {'collectionRunId': '88', 'sourceArtifactClaimed': False}
-            with patch.object(recovery_input, 'api', return_value=run), patch.object(retain, 'api', side_effect=[run, jobs]):
-                recovery_input.stage(env, report)
-            self.assertTrue(report['complete'])
-            self.assertEqual(report['collectionRunId'], '88')
-            self.assertEqual(report['sourceRun']['id'], 77)
-            self.assertIs(report['sourceArtifactClaimed'], False)
-            copied = Path(report['recoveryInput'])
-            self.assertEqual(retain.inventory(copied), before)
-            self.assertEqual(retain.inventory(root), before)
-            self.assertNotEqual((copied / 'original.json').stat().st_ino, (root / 'original.json').stat().st_ino)
-
-    def test_recovery_copy_refuses_changed_source_and_foreign_branch(self):
-        for mutation in ('drift', 'branch'):
-            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
-                env, root, run, jobs = self.fixture(directory)
-                destination = Path(directory).resolve() / 'recovery'; destination.mkdir(mode=0o700)
-                env.update(FAILED_RUN_ID='77', GITHUB_REF_NAME='topic', VALIDATION_ROOT=str(destination))
-                run['head_branch'] = 'foreign' if mutation == 'branch' else 'topic'
-                original = retain.prepare
-                def changed(environment, report):
-                    source = original(environment, report)
-                    (source / 'original.json').write_bytes(b'new bytes after admission')
-                    return source
-                with patch.object(recovery_input, 'api', return_value=run), \
-                     patch.object(retain, 'api', side_effect=[run, jobs]), \
-                     patch.object(recovery_input, 'prepare', side_effect=changed), self.assertRaises(ValueError):
-                    recovery_input.stage(env, {})
-
-    def test_copy_does_not_follow_a_symlink_swapped_after_directory_listing(self):
-        with tempfile.TemporaryDirectory() as directory:
-            env, root, run, jobs = self.fixture(directory)
-            secret = Path(directory) / 'private'; secret.write_bytes(b'outside-root-secret')
-            destination = Path(directory).resolve() / 'copy'
-            original_stat = os.stat
-            def swapped(path, *args, **kwargs):
-                info = original_stat(path, *args, **kwargs)
-                if path == 'original.json' and kwargs.get('follow_symlinks') is False:
-                    (root / path).unlink(); (root / path).symlink_to(secret)
-                return info
-            with patch.object(retain.os, 'stat', side_effect=swapped), self.assertRaises(OSError):
-                retain.inventory(root, copy_to=destination)
-            self.assertFalse((destination / 'original.json').exists())
-
-
-if __name__ == "__main__":
-    unittest.main()

scripts/test_scanner_process_receipt.py

@@ -108,21 +108,6 @@ class ProcessReceiptTest(unittest.TestCase):
         with self.assertRaisesRegex(ValueError, "not active"):
             self.attest()
 
-    def test_adaptive_admission_is_bound_to_actual_process_environment(self):
-        key = "RUSTFS_HEAL_ADAPTIVE_ADMISSION"
-        self.public[key] = "true"
-        raw = self.environment + b"RUSTFS_HEAL_ADAPTIVE_ADMISSION=true\0"
-        (self.pid / "environ").write_bytes(raw)
-        result = self.attest()
-        self.assertEqual(result["environment_sha256"], receipt.fingerprint(self.public))
-        self.assertIn(key, result["environment_keys"])
-        for observed in (self.environment, raw.replace(b"ADMISSION=true", b"ADMISSION=false"),
-                         raw + b"RUSTFS_HEAL_ADAPTIVE_ADMISSION=true\0"):
-            with self.subTest(raw_hash=hashlib.sha256(observed).hexdigest()), self.assertRaises(ValueError):
-                receipt.environment_digest(observed, self.public)
-        with self.assertRaisesRegex(ValueError, "mismatch"):
-            receipt.environment_digest(raw, {**self.public, key: "false"})
-
     def test_wrong_binary_content_is_rejected(self):
         self.binary.write_bytes(b"replaced executable")
         with self.assertRaisesRegex(ValueError, "binary digest"):