ci(validation): bind recovery to retained runner evidence

.config/make/tests.mak

@@ -63,6 +63,8 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py

.github/workflows/rustfs-functional-chain.yml

@@ -20,13 +20,91 @@ on:
   workflow_dispatch:
     inputs:
       build_run_id:
-        description: Successful nightly build run on main
+        description: Successful nightly or validation package run; empty for package-only build
         type: string
-        required: true
+        required: false
+        default: ''
       build_run_attempt:
         description: Exact successful build attempt
         type: string
-        required: true
+        required: false
+        default: ''
+      validation_package_sha:
+        description: Exact source SHA for an isolated package-only build
+        type: string
+        required: false
+        default: ''
+      validation_case:
+        description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain
+        type: string
+        required: false
+        default: ''
+      i05_diagnostic_branch:
+        description: I05 diagnostic-only branch; queued-successor requires the exact native evidence bundle
+        type: choice
+        options: [timeout, queued-successor]
+        required: false
+        default: timeout
+      auto_testing_sha:
+        description: Exact auto-testing commit for a validation case
+        type: string
+        required: false
+        default: ''
+      validation_retain_evidence_run_id:
+        description: Upload existing persistent evidence from one interrupted validation run without fleet access
+        type: string
+        required: false
+        default: ''
+      validation_retain_evidence_run_attempt:
+        description: Exact interrupted run attempt to retain
+        type: string
+        required: false
+        default: '1'
+      validation_runner_inspect_only:
+        description: Inspect GitHub runner registration and queued jobs on a hosted runner without fleet access
+        type: boolean
+        required: false
+        default: false
+      validation_tools_only:
+        description: Build candidate-bound validation tools without running a fleet case
+        type: boolean
+        required: false
+        default: false
+      validation_native_tools:
+        description: Include exact-source native regressions in a tools-only build
+        type: boolean
+        required: false
+        default: false
+      validation_shared_tools:
+        description: Include the controlled C10 shared-reference encoder in a tools-only build
+        type: boolean
+        required: false
+        default: false
+      validation_manager_tools:
+        description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build
+        type: boolean
+        required: false
+        default: false
+      validation_cleanup_run_id:
+        description: Failed validation run whose supported retained environment needs evidence-based finalization
+        type: string
+        required: false
+        default: ''
+      validation_cleanup_retained_evidence:
+        description: Recover from verified persistent runner-local evidence when the failed run uploaded no artifact
+        type: boolean
+        required: false
+        default: false
+      validation_tools_run_id:
+        description: Successful tool build run; required by Heal physical metadata checks
+        type: string
+        required: false
+        default: ''
+      validation_tools_run_attempt:
+        description: Exact tool build attempt
+        type: string
+        required: false
+        default: '1'
   workflow_run:
     workflows: ["Nightly GNU Build"]
     types: [completed]
@@ -36,12 +114,103 @@ permissions:
   actions: read
 
 concurrency:
-  group: rustfs-functional-chain-runs
+  group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only && 'rustfs-functional-validation-runner-inspection' || github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }}
   cancel-in-progress: false
 
 jobs:
+  validation-runner-inspection:
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - name: Inspect shared runner registration and queue without fleet access
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}
+        run: >-
+          python3 -B scripts/inspect_validation_runner.py
+          --output "${RUNNER_TEMP}/validation-runner-inspection/report.json"
+      - name: Upload read-only runner inspection including incomplete reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: validation-runner-inspection-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/validation-runner-inspection/report.json
+          if-no-files-found: error
+  validation-evidence-retention:
+    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id != '' }}
+    # The original interrupted evidence exists only on this long-lived runner.
+    runs-on: smoke-testing
+    timeout-minutes: 15
+    concurrency:
+      group: rustfs-shared-functional-tests-v2
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - name: Verify existing interrupted evidence without changing it
+        env:
+          GH_TOKEN: ${{ github.token }}
+          EVIDENCE_RETAIN_INPUTS: ${{ toJSON(inputs) }}
+        run: >-
+          python3 -B scripts/retain_validation_evidence.py
+          --report "${RUNNER_TEMP}/validation-evidence-retention/report.json"
+      - name: Upload admitted original evidence under collection provenance
+        if: ${{ env.VALIDATION_RETAIN_ROOT != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-retained-${{ env.VALIDATION_RETAIN_SOURCE_RUN }}-${{ env.VALIDATION_RETAIN_SOURCE_ATTEMPT }}-by-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.VALIDATION_RETAIN_ROOT }}
+          if-no-files-found: error
+          include-hidden-files: true
+      - name: Upload retention receipt including incomplete reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: validation-evidence-retention-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/validation-evidence-retention/report.json
+          if-no-files-found: error
+  validation-package-request:
+    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Validate exclusive package-only request
+        env:
+          PACKAGE_INPUTS: ${{ toJSON(inputs) }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          import re
+          value = json.loads(os.environ["PACKAGE_INPUTS"])
+          if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None:
+              raise ValueError("package build requires an exact source SHA")
+          exclusive = ("validation_retain_evidence_run_id", "validation_runner_inspect_only", "build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha",
+                       "validation_tools_only", "validation_native_tools", "validation_shared_tools",
+                       "validation_manager_tools", "validation_cleanup_run_id", "validation_cleanup_retained_evidence", "validation_tools_run_id")
+          if any(value.get(name) for name in exclusive):
+              raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution")
+          PY
+
+  validation-package:
+    needs: validation-package-request
+    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
+    uses: ./.github/workflows/rustfs-validation-package.yml
+    with:
+      source_sha: ${{ inputs.validation_package_sha }}
+    secrets:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+
   prepare:
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
+    if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && !inputs.validation_cleanup_retained_evidence && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }}
     runs-on: smoke-testing
     timeout-minutes: 10
     outputs:
@@ -89,6 +258,395 @@ jobs:
             echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
           fi
 
+  validation:
+    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
+    runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }}
+    timeout-minutes: 90
+    outputs:
+      source_sha: ${{ steps.candidate.outputs.source_sha }}
+    concurrency:
+      group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }}
+      cancel-in-progress: false
+    env:
+      RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }}
+      RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }}
+      RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }}
+      RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }}
+      RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }}
+    steps:
+      - name: Checkout candidate resolver
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize validation evidence
+        env:
+          VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}
+        run: |
+          set -euo pipefail
+          umask 077
+          python3 -B scripts/initialize_validation_evidence.py
+
+      - name: Resolve immutable candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+          CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
+          CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
+          SELECTED_CASE: ${{ inputs.validation_case }}
+          TOOLS_ONLY: ${{ inputs.validation_tools_only }}
+          NATIVE_TOOLS: ${{ inputs.validation_native_tools }}
+          SHARED_TOOLS: ${{ inputs.validation_shared_tools }}
+          MANAGER_TOOLS: ${{ inputs.validation_manager_tools }}
+          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          import re
+          from scripts.resolve_functional_candidate import resolve
+
+          revision = os.environ["AUTO_TESTING_SHA"]
+          cleanup = os.environ["CLEANUP_RUN_ID"]
+          if os.environ["CLEANUP_RETAINED_EVIDENCE"] == "true" and not cleanup:
+              raise ValueError("persistent recovery input requires one exact cleanup run")
+          if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("tools-only execution cannot select a fleet case or cleanup")
+          if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup")
+          if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup")
+          if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
+          if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
+              raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
+          diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "timeout"
+          if diagnostic_branch not in {"timeout", "queued-successor"}:
+              raise ValueError("I05 diagnostic branch is not registered")
+          if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort":
+              raise ValueError("queued-successor is available only for the I05 diagnostic case")
+          if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
+              raise ValueError("validation requires an exact auto-testing commit SHA")
+          candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True)
+          manifest = candidate["manifest"]
+          root = Path(os.environ["VALIDATION_ROOT"])
+          (root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n")
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout exact auto-testing revision
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Prepare validation Python runtime
+        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+
+      - name: Install validation Python client
+        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
+        run: |
+          set -euo pipefail
+          validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          test ! -e "${validation_python}"
+          python3 -m venv "${validation_python}"
+          "${validation_python}/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
+
+      - name: Validate Heal tool build identity
+        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }}
+          TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }}
+        run: |
+          set -euo pipefail
+          [[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+          [[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+          gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \
+            > "${VALIDATION_ROOT}/tool-run.json"
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text())
+          if (value.get("conclusion") != "success" or value.get("status") != "completed"
+                  or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"])
+                  or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}):
+              raise ValueError("Heal tool run is not an exact successful validation build attempt")
+          PY
+
+      - name: Download pinned Heal metadata inspector
+        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify Heal metadata inspector
+        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
+        run: |
+          set -euo pipefail
+          TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          python3 -B auto-testing/scripts/validation_heal/inspector.py \
+            --directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            > "${VALIDATION_ROOT}/tool-manifest.json"
+          printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Select exact-source race native evidence
+        id: race_native
+        if: ${{ inputs.validation_case == 'heal.race-unversioned-overwrite' || inputs.validation_case == 'heal.race-versioned-put' || inputs.validation_case == 'heal.race-version-delete' || inputs.validation_case == 'heal.race-markers-and-batch-delete' || inputs.validation_case == 'heal.race-multipart-complete-abort' || inputs.validation_case == 'heal.admin-nolock-safety' || inputs.validation_case == 'heal.race-copy-and-metadata-update' }}
+        env:
+          CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }}
+          VALIDATION_CASE_ID: ${{ inputs.validation_case }}
+        run: |
+          set -euo pipefail
+          required=false
+          if [ -f auto-testing/scripts/validation_heal/race_native.py ]; then
+            status=0
+            python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
+              --commit "${CANDIDATE_SHA}" --case-id "${VALIDATION_CASE_ID}" || status=$?
+            if [ "${status}" -eq 0 ]; then
+              required=true
+            elif [ "${status}" -ne 2 ]; then
+              exit "${status}"
+            fi
+          fi
+          reviewed=false
+          case "${CANDIDATE_SHA}:${VALIDATION_CASE_ID}" in
+            6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca:heal.race-unversioned-overwrite | \
+            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-unversioned-overwrite | \
+            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-versioned-put | \
+            bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-version-delete | \
+            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-unversioned-overwrite | \
+            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-versioned-put | \
+            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-version-delete | \
+            8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-markers-and-batch-delete | \
+            757b751482bb970bc321e92059690a7698554358:heal.admin-nolock-safety | \
+            757b751482bb970bc321e92059690a7698554358:heal.race-multipart-complete-abort | \
+            757b751482bb970bc321e92059690a7698554358:heal.race-copy-and-metadata-update) reviewed=true ;;
+          esac
+          if [ "${reviewed}" = true ] && [ "${required}" != true ]; then
+            printf 'Race candidate requires its reviewed native collector before fleet execution\n' >&2
+            exit 1
+          fi
+          printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
+
+      - name: Download pinned native storage evidence
+        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor') }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify C07 native source and exact test evidence
+        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
+        run: |
+          set -euo pipefail
+          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/authority.py \
+            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            > "${VALIDATION_ROOT}/native-authority-manifest.json"
+          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Verify C11 physical storage and exact final-rename barrier
+        if: ${{ inputs.validation_case == 'heal.bucket-incarnation-recreation' }}
+        run: |
+          set -euo pipefail
+          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/incarnation_native.py \
+            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --testing-commit "${{ inputs.auto_testing_sha }}" \
+            > "${VALIDATION_ROOT}/native-incarnation-manifest.json"
+          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Verify exact-source race native and runtime evidence
+        if: ${{ steps.race_native.outputs.required == 'true' }}
+        env:
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+          VALIDATION_CASE_ID: ${{ inputs.validation_case }}
+        run: |
+          set -euo pipefail
+          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
+            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --testing-commit "${AUTO_TESTING_SHA}" --case-id "${VALIDATION_CASE_ID}" \
+            > "${VALIDATION_ROOT}/native-race-manifest.json"
+          runtime_driver=race.py
+          if [ "${VALIDATION_CASE_ID}" = 'heal.admin-nolock-safety' ]; then
+            runtime_driver=nolock.py
+          fi
+          VALIDATION_HEAL_NATIVE_DIRECTORY="${NATIVE_DIRECTORY}" \
+            python3 -B "auto-testing/scripts/validation_heal/${runtime_driver}" preflight \
+              --candidate "${VALIDATION_ROOT}/candidate.json" --testing-commit "${AUTO_TESTING_SHA}" \
+              --case-id "${VALIDATION_CASE_ID}" \
+              > "${VALIDATION_ROOT}/native-race-runtime-preflight.json"
+          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Verify I05 queued-successor native prerequisite
+        if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor' }}
+        env:
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+        run: |
+          set -euo pipefail
+          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
+            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
+            > "${VALIDATION_ROOT}/native-i05-prerequisite.json"
+          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Download pinned manager persistence and incarnation evidence
+        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify exact manager native prerequisites before fleet mutation
+        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
+        env:
+          VALIDATION_CASE: ${{ inputs.validation_case }}
+        run: |
+          set -euo pipefail
+          MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \
+            --directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json"
+          printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Download pinned C10 shared-reference encoder
+        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify C10 exact-source codec and original tests
+        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
+        run: |
+          set -euo pipefail
+          SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \
+            --directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            > "${VALIDATION_ROOT}/shared-fixture-manifest.json"
+          printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Run selected canonical validation case
+        if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
+        env:
+          VALIDATION_CASE: ${{ inputs.validation_case }}
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+          I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          ./auto-testing/scripts/run_validation_case.sh \
+            "${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \
+            "${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run"
+
+      - name: Inspect current Heal fleet resources without mutation
+        if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }}
+        env:
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          python3 -B auto-testing/scripts/validation_heal/environment.py \
+            --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment"
+
+      - name: Audit and finalize retained validation environment
+        if: ${{ inputs.validation_cleanup_run_id != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+          CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
+            --json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json"
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text())
+          if (run["status"] != "completed" or run["conclusion"] not in {"failure", "cancelled"}
+              or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]):
+              raise ValueError("cleanup source must be a completed failed or cancelled Functional Chain run")
+          PY
+          if [ "${CLEANUP_RETAINED_EVIDENCE}" = true ]; then
+            python3 -B scripts/prepare_retained_validation_recovery.py \
+              --report "${VALIDATION_ROOT}/retained-recovery-input.json"
+          else
+            gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
+              --pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact"
+          fi
+          python3 -B auto-testing/scripts/validation_heal/recover_environment.py \
+            --failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \
+            --candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery"
+
+      - name: Show validation report
+        if: ${{ always() }}
+        run: |
+          if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then
+            cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}"
+          fi
+
+      - name: Upload validation evidence
+        if: ${{ always() && env.VALIDATION_ROOT != '' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.VALIDATION_ROOT }}
+          if-no-files-found: error
+
+  validation-tools:
+    needs: [validation]
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }}
+    uses: ./.github/workflows/rustfs-validation-tools.yml
+    with:
+      candidate_sha: ${{ needs.validation.outputs.source_sha }}
+      auto_testing_sha: ${{ inputs.auto_testing_sha }}
+      build_native_heal: ${{ inputs.validation_native_tools }}
+      build_shared_heal: ${{ inputs.validation_shared_tools }}
+      build_manager_native: ${{ inputs.validation_manager_tools }}
+    secrets: inherit
+
   upgrade:
     needs: [prepare]
     if: ${{ always() && needs.prepare.result == 'success' }}

.github/workflows/rustfs-validation-package.yml

@@ -0,0 +1,244 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+name: RustFS Validation Package
+
+on:
+  workflow_call:
+    inputs:
+      source_sha:
+        description: Exact RustFS commit SHA to build
+        required: true
+        type: string
+    secrets:
+      R2_ACCESS_KEY_ID:
+        required: true
+      R2_SECRET_ACCESS_KEY:
+        required: true
+      R2_ENDPOINT:
+        required: true
+      R2_BUCKET:
+        required: true
+  workflow_dispatch:
+    inputs:
+      source_sha:
+        description: Exact RustFS commit SHA to build
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-package-${{ inputs.source_sha }}
+  cancel-in-progress: false
+
+jobs:
+  resolve-source:
+    if: github.repository == 'rustfs/rustfs'
+    name: Resolve exact validation source
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      source_sha: ${{ steps.source.outputs.sha }}
+      source_ref: ${{ steps.source.outputs.ref }}
+    steps:
+      - name: Validate requested source
+        id: source
+        env:
+          REQUESTED_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          [[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")"
+          trap 'rm -rf "${source_directory}"' EXIT
+          git init -q "${source_directory}"
+          git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}"
+          test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"
+          echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
+          echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
+
+  build:
+    needs: resolve-source
+    name: Build immutable validation DEB
+    runs-on: ubuntu-latest
+    timeout-minutes: 150
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
+    steps:
+      - name: Checkout exact source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
+
+      - name: Confirm checked out source
+        env:
+          EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }}
+        run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          cache-shared-key: validation-package-hosted-x86_64-unknown-linux-gnu
+          cache-save-if: "false"
+          install-build-packaging-tools: "false"
+          install-test-tools: "false"
+
+      - name: Build RustFS
+        run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
+
+      - name: Build validation DEB
+        shell: bash
+        run: |
+          set -euo pipefail
+          command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot
+          package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}"
+          package_file="${RUNNER_TEMP}/validation-package/rustfs.deb"
+          mkdir -p "${RUNNER_TEMP}/validation-package"
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          test -x "${binary}"
+          mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \
+            "${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs"
+          cp "${binary}" "${package_dir}/usr/bin/rustfs"
+          chmod 755 "${package_dir}/usr/bin/rustfs"
+          cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/"
+          cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF'
+          # RustFS Environment Configuration
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
+          # RUSTFS_VOLUMES=""
+          # RUSTFS_ADDRESS=":9000"
+          ENVEOF
+          cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF'
+          Package: rustfs
+          Version: 0.0.0~validation
+          Section: utils
+          Priority: optional
+          Architecture: amd64
+          Depends: libc6 (>= 2.31)
+          Maintainer: RustFS Team <support@rustfs.com>
+          Description: RustFS immutable validation package
+           Candidate package for RustFS validation only.
+          CONTROLEOF
+          printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles"
+          ./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst"
+          ./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm"
+          ./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm"
+          chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm"
+          cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/"
+          fakeroot dpkg-deb --build "${package_dir}" "${package_file}"
+          test -s "${package_file}"
+
+      - name: Upload opaque validation package
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/validation-package/rustfs.deb
+          if-no-files-found: error
+
+  publish:
+    needs: [resolve-source, build]
+    name: Publish immutable validation DEB
+    # Candidate-controlled build scripts never run on this fresh hosted runner.
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
+      VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}
+    steps:
+      - name: Download opaque validation package
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/validation-package-input
+
+      - name: Validate and stage opaque package bytes
+        shell: bash
+        run: |
+          set -euo pipefail
+          python3 -I - <<'PYTHON'
+          import os
+          from pathlib import Path
+          import shutil
+          import stat
+
+          root = Path(os.environ["RUNNER_TEMP"])
+          incoming = root / "validation-package-input"
+          if incoming.is_symlink() or not incoming.is_dir() or sorted(p.name for p in incoming.iterdir()) != ["rustfs.deb"]:
+              raise SystemExit("Validation artifact must contain exactly one rustfs.deb file")
+          fd = os.open(incoming / "rustfs.deb", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+          with os.fdopen(fd, "rb") as source:
+              info = os.fstat(source.fileno())
+              if not stat.S_ISREG(info.st_mode) or info.st_size == 0:
+                  raise SystemExit("Validation package must be a nonempty regular file")
+              staged = root / "validation-publication"
+              staged.mkdir(mode=0o700)
+              with (staged / "rustfs.deb").open("xb") as target:
+                  shutil.copyfileobj(source, target)
+          PYTHON
+
+      - name: Prepare trusted publication tools
+        id: tools
+        env:
+          AWS_EC2_METADATA_DISABLED: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          AWS_CLI=aws
+          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install -y -qq python3-venv
+            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")"
+            python3 -m venv "${AWS_CLI_DIR}"
+            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+          fi
+          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+          echo "aws_cli=${AWS_CLI}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload validation package to immutable R2 path
+        id: publish
+        env:
+          PACKAGE_FILE: ${{ runner.temp }}/validation-publication/rustfs.deb
+          AWS_CLI: ${{ steps.tools.outputs.aws_cli }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+          R2_BUCKET: ${{ secrets.R2_BUCKET }}
+          AWS_EC2_METADATA_DISABLED: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then
+            echo "Validation package publication requires complete R2 configuration" >&2
+            exit 1
+          fi
+          [[ "${VALIDATION_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          test "${VALIDATION_SOURCE_REF}" = "${VALIDATION_SOURCE_SHA}"
+          test -s "${PACKAGE_FILE}" && test ! -L "${PACKAGE_FILE}"
+          export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto
+          package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)"
+          key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb"
+          url="https://dl.rustfs.com/${key}"
+          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \
+            --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+          downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)"
+          test "${downloaded_sha256}" = "${package_sha256}"
+          manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \
+            --arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \
+            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+            --arg package_url "${url}" --arg package_sha256 "${package_sha256}" \
+            '{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}"
+          echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload validation candidate manifest
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.publish.outputs.candidate_file }}
+          if-no-files-found: error

.github/workflows/rustfs-validation-tools.yml

@@ -0,0 +1,343 @@
+# Build source-bound helpers separately so live validation needs only artifacts.
+name: RustFS Validation Tools
+
+on:
+  workflow_call:
+    inputs:
+      candidate_sha:
+        type: string
+        required: true
+      auto_testing_sha:
+        type: string
+        required: true
+      build_native_heal:
+        type: boolean
+        required: false
+        default: false
+      build_shared_heal:
+        type: boolean
+        required: false
+        default: false
+      build_manager_native:
+        type: boolean
+        required: false
+        default: false
+    secrets:
+      PF_TESTING_GH_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      candidate_sha:
+        description: Exact RustFS source SHA from the validation candidate manifest
+        type: string
+        required: true
+      auto_testing_sha:
+        description: Exact auto-testing SHA containing the Heal metadata inspector
+        type: string
+        required: true
+      build_native_heal:
+        description: Also compile and run separate exact-source native Heal regressions
+        type: boolean
+        required: false
+        default: false
+      build_shared_heal:
+        description: Build the controlled C10 shared-reference fixture encoder
+        type: boolean
+        required: false
+        default: false
+      build_manager_native:
+        description: Build and run exact candidate checkpoint/MRF/incarnation regressions
+        type: boolean
+        required: false
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  manager-native-regressions:
+    if: ${{ inputs.build_manager_native }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 180
+    env:
+      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+    steps:
+      - name: Validate immutable manager native inputs
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
+
+      - name: Checkout exact manager candidate source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ inputs.candidate_sha }}
+          path: candidate
+          persist-credentials: false
+
+      - name: Checkout exact manager collector source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Install manager native build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
+          df -h .
+
+      - name: Build exact unmodified candidate manager test executable
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          python3 -B auto-testing/scripts/validation_heal/manager_native.py build \
+            --source candidate --commit "${CANDIDATE_SHA}" --out manager-tools
+
+      - name: Run exact manager regressions independently
+        run: |
+          set -euo pipefail
+          python3 -B auto-testing/scripts/validation_heal/manager_native.py run \
+            --directory manager-tools --out manager-evidence
+
+      - name: Upload manager executable identities and original diagnostics
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            manager-tools/heal-manager-native-tests
+            manager-tools/*.json
+            manager-tools/*.jsonl
+            manager-tools/*.stderr
+            manager-tools/*.log
+            manager-evidence/
+          if-no-files-found: warn
+
+  native-heal-regressions:
+    if: ${{ inputs.build_native_heal }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 180
+    env:
+      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+    steps:
+      - name: Validate immutable native source inputs
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
+
+      - name: Checkout exact native candidate source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ inputs.candidate_sha }}
+          path: candidate
+          persist-credentials: false
+
+      - name: Checkout exact native collector source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Install native build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
+          df -h .
+
+      - name: Build and enumerate the unmodified candidate test executable
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          use_race_native=false
+          if [[ -f auto-testing/scripts/validation_heal/race_native.py ]]; then
+            support_rc=0
+            python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
+              --commit "${CANDIDATE_SHA}" || support_rc=$?
+            case "${support_rc}" in
+              0) use_race_native=true ;;
+              2) ;; # The original exact-source collector handles legacy candidates.
+              *) exit "${support_rc}" ;;
+            esac
+          fi
+          if [[ "${use_race_native}" == true ]]; then
+            python3 -B auto-testing/scripts/validation_heal/race_native.py build \
+              --source candidate --commit "${CANDIDATE_SHA}" --testing-commit "${AUTO_TESTING_SHA}" --out native-tools
+          else
+            python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \
+              --source candidate --commit "${CANDIDATE_SHA}" --out native-tools
+          fi
+
+      - name: Run exact native regressions separately from fleet validation
+        run: |
+          set -euo pipefail
+          if [[ -f native-tools/race-native.json ]]; then
+            test ! -e native-tools/native-regressions.json
+            python3 -B auto-testing/scripts/validation_heal/race_native.py run \
+              --directory native-tools --out native-evidence
+          else
+            python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \
+              --directory native-tools --out native-evidence
+          fi
+
+      - name: Run existing F07 directory fsync failure regressions
+        if: ${{ inputs.candidate_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
+        run: |
+          set -euo pipefail
+          # Older immutable harness pins retain their original native suite;
+          # absence of this additive collector must never fabricate its proof.
+          if [[ ! -f auto-testing/scripts/validation_heal/persistence_native.py ]]; then
+            mkdir native-persistence-evidence
+            echo '{"complete":false,"reason":"pinned harness predates F07 directory-fsync collector","requiredNativeCoverageComplete":false,"canonicalCasePassClaimed":false}' \
+              > native-persistence-evidence/not-available.json
+            exit 0
+          fi
+          python3 -B auto-testing/scripts/validation_heal/persistence_native.py run \
+            --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
+            --testing-commit "${AUTO_TESTING_SHA}"
+          python3 -B auto-testing/scripts/validation_heal/persistence_native.py verify \
+            --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
+            --testing-commit "${AUTO_TESTING_SHA}"
+
+      - name: Upload native executable identities and original diagnostics
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            native-tools/ecstore-native-tests
+            native-tools/ecstore-race-native-tests
+            native-tools/*.json
+            native-tools/*.jsonl
+            native-tools/*.stderr
+            native-tools/*.log
+            native-evidence/
+            native-persistence-evidence/
+          if-no-files-found: warn
+
+  heal-metadata-inspector:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+    steps:
+      - name: Validate immutable source inputs
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
+
+      - name: Checkout exact candidate source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ inputs.candidate_sha }}
+          path: candidate
+          persist-credentials: false
+
+      - name: Checkout exact inspector source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Build the read-only candidate metadata decoder
+        run: |
+          set -euo pipefail
+          test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}"
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          df -h .
+          python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \
+            --source candidate --commit "${CANDIDATE_SHA}" --out inspector-build
+          # Seed the helper's separate graph from the candidate's frozen graph.
+          # Cargo adds the helper package; both lock identities are retained.
+          cp candidate/Cargo.lock inspector-build/Cargo.lock
+          cargo build --release --manifest-path inspector-build/Cargo.toml
+          python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \
+            --binary inspector-build/target/release/heal-meta-inspect --source candidate
+          mkdir validation-tools
+          cp inspector-build/target/release/heal-meta-inspect validation-tools/
+          cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock
+          python3 - <<'PY'
+          import hashlib
+          import json
+          import os
+          from pathlib import Path
+          import subprocess
+
+          source = json.loads(Path("inspector-build/source.json").read_text())
+          binary = Path("validation-tools/heal-meta-inspect")
+          source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"],
+                        binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
+                        inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(),
+                        toolchain=subprocess.check_output(["rustc", "-vV"], text=True),
+                        target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"],
+                        runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
+          Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n")
+          PY
+          git -C candidate diff --exit-code
+
+      - name: Build and test the controlled C10 metadata encoder
+        if: ${{ inputs.build_shared_heal }}
+        run: |
+          set -euo pipefail
+          python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \
+            --source candidate --commit "${CANDIDATE_SHA}" --out shared-build
+          cp candidate/Cargo.lock shared-build/Cargo.lock
+          export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target"
+          cargo build --release --manifest-path shared-build/Cargo.toml
+          mkdir shared-tools
+          if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \
+            > shared-tools/shared-fixture-tests.log 2>&1; then
+            cat shared-tools/shared-fixture-tests.log
+            exit 1
+          fi
+          cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/
+          cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock
+          python3 - <<'PY'
+          import hashlib
+          import json
+          import os
+          from pathlib import Path
+          import subprocess
+
+          source = json.loads(Path("shared-build/source.json").read_text())
+          source.update(schemaVersion=1, built=True, sourceUnmodified=True,
+              candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(),
+              testingCommit=os.environ["AUTO_TESTING_SHA"],
+              binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(),
+              sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(),
+              toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu",
+              runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
+          Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n")
+          PY
+          git -C candidate diff --exit-code
+
+      - name: Upload controlled C10 fixture encoder evidence
+        if: ${{ always() && inputs.build_shared_heal }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
+          path: shared-tools/
+          if-no-files-found: warn
+
+      - name: Upload immutable validation tools
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
+          path: validation-tools/
+          if-no-files-found: error

crates/ecstore/src/bucket/metadata_sys.rs

@@ -3186,9 +3186,17 @@ pub mod test_support {
     use crate::store::init_local_disks_with_instance_ctx;
 
     pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
-        let mut dirs = Vec::with_capacity(4);
-        let mut endpoints = Vec::with_capacity(4);
-        for disk_idx in 0..4 {
+        isolated_store_over_temp_disks_with_count(4).await
+    }
+
+    pub async fn isolated_store_over_temp_disks_with_count(disk_count: usize) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+        assert!(
+            matches!(disk_count, 4 | 16),
+            "isolated store fixture requires a registered logical set geometry"
+        );
+        let mut dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        for disk_idx in 0..disk_count {
             let dir = tempfile::tempdir().expect("tempdir should be created");
             let mut endpoint =
                 Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
@@ -3201,7 +3209,7 @@ pub mod test_support {
         let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
             legacy: false,
             set_count: 1,
-            drives_per_set: 4,
+            drives_per_set: disk_count,
             endpoints: Endpoints::from(endpoints),
             cmd_line: "metadata-sys-cache-test".to_string(),
             platform: "test".to_string(),

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -603,6 +603,10 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
     request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string);
     // A degraded read's surviving minority is not a recreate instruction; the
     // locked storage read decides whether the target is absent (backlog#2900).
+    // Reader failures include checksum damage that metadata-only healing cannot detect.
+    if matches!(reason, "missing_shards" | "decode_error") {
+        request.scan_mode = Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
+    }
 
     let request_id = request.id.clone();
     let bucket = bucket.to_string();
@@ -13580,6 +13584,61 @@ mod tests {
         release_read_repair_heal_reservation(&key).await;
     }
 
+    #[tokio::test]
+    async fn read_repair_reader_failure_requests_deep_scan_for_the_exact_version() {
+        fn captured() -> &'static std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>> {
+            static REQUEST: OnceLock<std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>>> =
+                OnceLock::new();
+            REQUEST.get_or_init(|| std::sync::Mutex::new(None))
+        }
+        fn capture(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
+            *captured().lock().expect("read-repair request capture must be available") = Some(request);
+            Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
+        }
+
+        for (reason, expected_scan) in [
+            ("missing_shards", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
+            ("decode_error", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
+            ("metadata_read_error", None),
+        ] {
+            let object = format!("reader-failure-{}", Uuid::new_v4());
+            let version = Uuid::new_v4().to_string();
+            submit_read_repair_heal_with_submitter(
+                ReadRepairHealSubmission {
+                    bucket: "reader-failure",
+                    object: &object,
+                    version_id: Some(&version),
+                    pool_index: 1,
+                    set_index: 2,
+                    part_number: Some(1),
+                    reason,
+                    mrf_intent: None,
+                },
+                capture,
+            )
+            .await;
+            let request = tokio::time::timeout(Duration::from_secs(5), async {
+                loop {
+                    if let Some(request) = captured().lock().expect("captured request must be readable").take() {
+                        break request;
+                    }
+                    tokio::task::yield_now().await;
+                }
+            })
+            .await
+            .expect("the actual asynchronous admission must be captured");
+            assert_eq!(request.scan_mode, expected_scan, "reader damage needs payload verification");
+            assert_eq!(request.source, HealRequestSource::ReadRepair);
+            assert_eq!(request.bucket, "reader-failure");
+            assert_eq!(request.object_prefix.as_deref(), Some(object.as_str()));
+            assert_eq!(request.object_version_id.as_deref(), Some(version.as_str()));
+            assert_eq!((request.pool_index, request.set_index), (Some(1), Some(2)));
+            assert_eq!(request.recreate_missing, None, "storage evidence decides absence");
+            let key = ReadRepairHealCacheKey::new("reader-failure", &object, Some(&version), 1, 2);
+            release_read_repair_heal_reservation(&key).await;
+        }
+    }
+
     // ========================================================================
     // backlog#898 — groups A/C: pure old-data-dir cleanup classification.
     // ========================================================================

crates/ecstore/src/set_disk/mod.rs

@@ -889,6 +889,8 @@ pub(crate) use ops::heal::AbsenceProofRequest;
 #[cfg(test)]
 pub(crate) use ops::heal::DanglingDeleteFailure;
 pub(crate) use ops::heal::HealedObjectAbsence;
+#[cfg(test)]
+pub(crate) use ops::heal::ReadRepairPauseScope;
 
 #[cfg(test)]
 pub(crate) use ops::hermetic_set_disks_isolated;

crates/ecstore/src/set_disk/ops/object.rs

@@ -8175,13 +8175,23 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
     }
 }
 
+#[cfg(test)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum DeleteObjectCommitPause {
+    BeforeNamespace,
+    BeforeCommit,
+    AfterPublish,
+}
+
 #[cfg(test)]
 struct DeleteObjectCommitBarrierState {
     bucket: String,
     object: String,
-    pause_after_publish: bool,
+    pause: DeleteObjectCommitPause,
     arrived: tokio::sync::Notify,
     release: tokio::sync::Notify,
+    namespace_pending: tokio::sync::Notify,
+    namespace_acquired: std::sync::atomic::AtomicBool,
 }
 
 #[cfg(test)]
@@ -8196,21 +8206,27 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
 #[cfg(test)]
 impl DeleteObjectCommitBarrier {
     pub(crate) fn install(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, false)
+        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeCommit)
+    }
+
+    pub(crate) fn install_before_namespace(bucket: &str, object: &str) -> Self {
+        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeNamespace)
     }
 
     #[cfg(feature = "test-util")]
     pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self {
-        Self::install_with_mode(bucket, object, true)
+        Self::install_with_mode(bucket, object, DeleteObjectCommitPause::AfterPublish)
     }
 
-    fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self {
+    fn install_with_mode(bucket: &str, object: &str, pause: DeleteObjectCommitPause) -> Self {
         let state = Arc::new(DeleteObjectCommitBarrierState {
             bucket: bucket.to_string(),
             object: object.to_string(),
-            pause_after_publish,
+            pause,
             arrived: tokio::sync::Notify::new(),
             release: tokio::sync::Notify::new(),
+            namespace_pending: tokio::sync::Notify::new(),
+            namespace_acquired: std::sync::atomic::AtomicBool::new(false),
         });
         let mut slot = DELETE_OBJECT_COMMIT_BARRIER
             .get_or_init(|| std::sync::Mutex::new(None))
@@ -8230,6 +8246,19 @@ impl DeleteObjectCommitBarrier {
     pub(crate) fn release(&self) {
         self.state.release.notify_one();
     }
+
+    pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
+        assert_eq!(self.state.pause, DeleteObjectCommitPause::BeforeNamespace);
+        let pending = self.state.namespace_pending.notified();
+        self.release();
+        tokio::time::timeout(Duration::from_secs(5), pending)
+            .await
+            .expect("delete object must poll its actual namespace acquisition to Pending");
+    }
+
+    pub(crate) fn namespace_acquired(&self) -> bool {
+        self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
+    }
 }
 
 #[cfg(test)]
@@ -8246,6 +8275,43 @@ impl Drop for DeleteObjectCommitBarrier {
     }
 }
 
+#[cfg(test)]
+async fn pause_delete_object_before_namespace(bucket: &str, object: &str) {
+    let barrier = DELETE_OBJECT_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("delete object commit barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| {
+            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
+        })
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
+#[cfg(test)]
+fn notify_delete_object_namespace(bucket: &str, object: &str, acquired: bool) {
+    let barrier = DELETE_OBJECT_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("delete object commit barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| {
+            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
+        })
+        .cloned();
+    if let Some(barrier) = barrier {
+        if acquired {
+            barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
+        } else {
+            barrier.namespace_pending.notify_one();
+        }
+    }
+}
+
 #[cfg(test)]
 async fn pause_delete_object_commit(bucket: &str, object: &str) {
     let barrier = DELETE_OBJECT_COMMIT_BARRIER
@@ -8253,7 +8319,9 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) {
         .lock()
         .expect("delete object commit barrier mutex should not poison")
         .as_ref()
-        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish)
+        .filter(|barrier| {
+            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeCommit
+        })
         .cloned();
     if let Some(barrier) = barrier {
         barrier.arrived.notify_one();
@@ -8268,7 +8336,9 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
         .lock()
         .expect("delete object commit barrier mutex should not poison")
         .as_ref()
-        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish)
+        .filter(|barrier| {
+            barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::AfterPublish
+        })
         .cloned();
     if let Some(barrier) = barrier {
         barrier.arrived.notify_one();
@@ -10527,7 +10597,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         // Guard lock for single object delete
         let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock {
-            Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?))
+            #[cfg(test)]
+            let guard = {
+                pause_delete_object_before_namespace(bucket, object).await;
+                let guard = self
+                    .acquire_write_lock_diag_with_pending_hook("delete_object", bucket, object, || {
+                        notify_delete_object_namespace(bucket, object, false);
+                    })
+                    .await?;
+                notify_delete_object_namespace(bucket, object, true);
+                guard
+            };
+            #[cfg(not(test))]
+            let guard = self.acquire_write_lock_diag("delete_object", bucket, object).await?;
+            Some(Arc::new(guard))
         } else {
             None
         };
@@ -21432,6 +21515,206 @@ mod put_object_tmp_cleanup_tests {
         .await;
     }
 
+    #[tokio::test]
+    async fn deep_heal_preserves_committed_generation_during_old_data_cleanup() {
+        use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
+
+        for (disk_count, parity, read_repair) in [(4, 2, false), (4, 2, true), (16, 4, false), (16, 4, true)] {
+            let (dirs, disks, set) = super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated(
+                disk_count, 0, parity,
+            )
+            .await;
+            let bucket = "native-heal-commit-cleanup";
+            let object = format!("generation-{disk_count}-{parity}-{read_repair}.bin");
+            for disk in &disks {
+                disk.make_volume(bucket).await.expect("create actual fixture volume");
+            }
+            let options = ObjectOptions {
+                write_completion: crate::object_api::WriteCompletion::TailDrained,
+                shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
+                ..Default::default()
+            };
+            set.put_object(bucket, &object, &mut PutObjReader::from_vec(vec![0x17; TEST_OBJECT_SIZE]), &options)
+                .await
+                .expect("acknowledge original generation on every disk");
+            let old = disks[0]
+                .read_version("", bucket, &object, "", &ReadOptions::default())
+                .await
+                .expect("read actual original generation");
+            let old_dir = old.data_dir.expect("original external data directory");
+            let old_part = dirs[0]
+                .path()
+                .join(bucket)
+                .join(&object)
+                .join(old_dir.to_string())
+                .join("part.1");
+            assert!(tokio::fs::try_exists(&old_part).await.expect("original part exists"));
+
+            // Cleanup slots follow the writer's shuffled erasure distribution,
+            // not the physical fixture order. Pause every real cleanup slot so
+            // no physical old shard can disappear before the proof below.
+            let barriers: Vec<_> = (0..disk_count)
+                .map(|slot| rename_fanout_barrier::arm(&object, slot, rename_fanout_barrier::PHASE_CLEANUP))
+                .collect();
+            let writer_set = Arc::clone(&set);
+            let writer_object = object.clone();
+            let writer_options = options.clone();
+            let payload = vec![0xa3; TEST_OBJECT_SIZE];
+            let writer_payload = payload.clone();
+            let writer = tokio::spawn(async move {
+                writer_set
+                    .put_object(bucket, &writer_object, &mut PutObjReader::from_vec(writer_payload), &writer_options)
+                    .await
+            });
+            tokio::time::timeout(Duration::from_secs(30), async {
+                for barrier in &barriers {
+                    barrier.wait_until_paused().await;
+                }
+            })
+            .await
+            .expect("overwrite commits before its real old-data cleanup pause");
+            assert!(!writer.is_finished(), "tail-drained ACK must remain pending at cleanup");
+            assert!(
+                tokio::fs::try_exists(&old_part)
+                    .await
+                    .expect("old source still present at pause")
+            );
+
+            let mut committed = Vec::new();
+            let mut healthy_metadata = Vec::new();
+            for disk in &disks {
+                let info = disk
+                    .read_version("", bucket, &object, "", &ReadOptions::default())
+                    .await
+                    .expect("read independently committed physical generation without GET");
+                assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (disk_count - parity, parity));
+                assert!(info.data.is_none(), "use actual external parts");
+                assert_ne!(info.data_dir, Some(old_dir));
+                committed.push(info);
+                healthy_metadata.push(
+                    disk.read_all(bucket, &format!("{object}/xl.meta"))
+                        .await
+                        .expect("committed raw metadata"),
+                );
+            }
+            let new_dir = committed[0].data_dir.expect("new committed data directory");
+            assert!(committed.iter().all(|info| info.data_dir == Some(new_dir)));
+            let new_part = dirs[0]
+                .path()
+                .join(bucket)
+                .join(&object)
+                .join(new_dir.to_string())
+                .join("part.1");
+            let expected_part = tokio::fs::read(&new_part).await.expect("exact new source bytes");
+            let mut committed_parts = Vec::new();
+            for dir in &dirs {
+                let original = dir.path().join(bucket).join(&object).join(old_dir.to_string()).join("part.1");
+                assert!(
+                    tokio::fs::try_exists(original)
+                        .await
+                        .expect("every old cleanup slot is paused")
+                );
+                committed_parts.push(
+                    tokio::fs::read(dir.path().join(bucket).join(&object).join(new_dir.to_string()).join("part.1"))
+                        .await
+                        .expect("every new encoded shard before fault"),
+                );
+            }
+            tokio::fs::remove_file(&new_part)
+                .await
+                .expect("remove only the newly committed target shard");
+            println!(
+                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} old_cleanup_paused new_generation_committed target_part_missing"
+            );
+
+            let (result, error) = tokio::time::timeout(
+                Duration::from_secs(30),
+                set.heal_object(
+                    bucket,
+                    &object,
+                    "",
+                    &HealOpts {
+                        scan_mode: HealScanMode::Deep,
+                        read_repair,
+                        ..Default::default()
+                    },
+                ),
+            )
+            .await
+            .expect("real locked Heal must progress during old cleanup")
+            .expect("complete actual deep Heal");
+            assert!(error.is_none(), "new generation must remain repairable: {error:?}");
+            assert_eq!(result.drives_healed(), Some(1));
+            // Shared data-phase repair deliberately does not certify full integrity.
+            assert_eq!(result.integrity_verified, !read_repair);
+            assert_eq!(tokio::fs::read(&new_part).await.expect("rebuilt part before any GET"), expected_part);
+            let repaired = disks[0]
+                .read_version("", bucket, &object, "", &ReadOptions::default())
+                .await
+                .expect("read actual repaired generation");
+            assert_eq!(repaired.data_dir, Some(new_dir));
+            assert_eq!(repaired.mod_time, committed[0].mod_time);
+            for (disk, expected) in disks.iter().zip(&healthy_metadata).skip(1) {
+                assert_eq!(
+                    disk.read_all(bucket, &format!("{object}/xl.meta"))
+                        .await
+                        .expect("healthy raw metadata after Heal"),
+                    *expected
+                );
+            }
+            println!(
+                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} locked_heal_completed new_generation_part_rebuilt_before_get"
+            );
+
+            for barrier in &barriers {
+                barrier.release();
+            }
+            let acknowledged = tokio::time::timeout(Duration::from_secs(30), writer)
+                .await
+                .expect("cleanup drains")
+                .expect("writer joins")
+                .expect("overwrite ACK remains successful");
+            assert_eq!(acknowledged.size, i64::try_from(payload.len()).expect("fixture size fits i64"));
+            drop(barriers);
+            for (slot, dir) in dirs.iter().enumerate() {
+                let root = dir.path().join(bucket).join(&object);
+                assert!(
+                    !tokio::fs::try_exists(root.join(old_dir.to_string()))
+                        .await
+                        .expect("old cleanup completed")
+                );
+                assert_eq!(
+                    tokio::fs::read(root.join(new_dir.to_string()).join("part.1"))
+                        .await
+                        .expect("new shard survives cleanup before GET"),
+                    committed_parts[slot],
+                    "old cleanup must preserve every exact committed encoded shard"
+                );
+            }
+            let mut reader = set
+                .get_object_reader(bucket, &object, None, HeaderMap::new(), &options)
+                .await
+                .expect("read final acknowledged generation");
+            let mut body = Vec::new();
+            reader
+                .stream
+                .read_to_end(&mut body)
+                .await
+                .expect("read complete final payload");
+            assert_eq!(body, payload);
+            for disk in &disks {
+                let final_info = disk
+                    .read_version("", bucket, &object, "", &ReadOptions::default())
+                    .await
+                    .expect("final physical generation");
+                assert_eq!(final_info.data_dir, Some(new_dir), "old cleanup must not revert the healed generation");
+            }
+            println!(
+                "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} overwrite_ack_preserved final_physical_generation_and_full_body_verified"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;

crates/ecstore/src/set_disk/read.rs

@@ -1432,7 +1432,10 @@ impl SetDisks {
                         "Create bitrot reader failed read quorum"
                     );
                     record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
-                    return Err(to_object_err(read_err.into(), vec![bucket, object]));
+                    // The metadata selected this object, but the part cannot
+                    // be reconstructed. A dominant disk I/O/not-found error
+                    // must not erase the object's read-quorum failure.
+                    return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
                 }
                 let reason = GetObjectFailureReason::ReadQuorum;
                 error!(
@@ -1457,7 +1460,7 @@ impl SetDisks {
                     "Create bitrot reader did not have enough disks"
                 );
                 record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
-                return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
+                return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
             }
 
             // Check if we have missing shards even though we can read successfully
@@ -1992,10 +1995,10 @@ impl SetDisks {
             if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
                 let reason = classify_disk_error(&read_err);
                 record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
-                return Err(to_object_err(read_err.into(), vec![bucket, object]));
+                return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
             }
             record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
-            return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
+            return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
         }
 
         let missing_shards = reader_setup.completed_failed_shards();
@@ -7801,6 +7804,105 @@ mod tests {
         (data_dir, files)
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn external_missing_shards_keep_typed_read_quorum() {
+        use tokio::io::AsyncReadExt;
+
+        let payload = vec![0x37; 256 * 1024];
+        for readable_shards in [1, 2, 4] {
+            let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
+            let mut fi = codec_streaming_test_fileinfo(i64::try_from(payload.len()).expect("fixture size fits i64"), 1);
+            fi.erasure.data_blocks = erasure.data_shards;
+            fi.erasure.parity_blocks = erasure.parity_shards;
+            fi.erasure.block_size = erasure.block_size;
+            fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
+            let (dirs, disks) = local_test_disks(erasure.total_shard_count(), CODEC_STREAMING_TEST_BUCKET).await;
+            let (data_dir, files) = codec_streaming_external_files(&erasure, &payload, &disks).await;
+            fi.data_dir = Some(data_dir);
+            for directory in dirs.iter().skip(readable_shards) {
+                tokio::fs::remove_file(
+                    directory
+                        .path()
+                        .join(CODEC_STREAMING_TEST_BUCKET)
+                        .join(CODEC_STREAMING_TEST_OBJECT)
+                        .join(data_dir.to_string())
+                        .join("part.1"),
+                )
+                .await
+                .expect("remove real part while retaining its selected metadata");
+            }
+
+            let mut output = Vec::new();
+            let legacy = SetDisks::get_object_with_fileinfo(
+                CODEC_STREAMING_TEST_BUCKET,
+                CODEC_STREAMING_TEST_OBJECT,
+                Arc::new(ErasureCache::new()),
+                0,
+                fi.size,
+                &mut output,
+                fi.clone(),
+                files.clone(),
+                &disks,
+                0,
+                0,
+                false,
+                true,
+                false,
+                false,
+                GET_OBJECT_PATH_LEGACY_DUPLEX,
+                "plain_single_part",
+                "le_1mib",
+            )
+            .await;
+            let streaming = SetDisks::get_object_mid_size_reader_with_fileinfo(
+                CODEC_STREAMING_TEST_BUCKET,
+                CODEC_STREAMING_TEST_OBJECT,
+                Arc::new(ErasureCache::new()),
+                &fi,
+                &files,
+                &disks,
+                0,
+                0,
+                false,
+                true,
+                "plain_single_part",
+                "le_1mib",
+                false,
+            )
+            .await;
+            if readable_shards < erasure.data_shards {
+                for error in [
+                    legacy.expect_err("legacy read must reject quorum minus one"),
+                    streaming.err().expect("streaming setup must reject quorum minus one"),
+                ] {
+                    assert_eq!(
+                        error,
+                        Error::InsufficientReadQuorum(
+                            CODEC_STREAMING_TEST_BUCKET.to_owned(),
+                            CODEC_STREAMING_TEST_OBJECT.to_owned(),
+                        )
+                    );
+                }
+                assert!(output.is_empty(), "quorum rejection must precede all response bytes");
+            } else {
+                legacy.expect("legacy reader retains reconstructable quorum");
+                assert_eq!(output, payload, "legacy read returns every original byte at quorum");
+                let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) =
+                    streaming.expect("streaming quorum remains readable")
+                else {
+                    panic!("mid-size quorum must yield an actual reader");
+                };
+                let mut decoded = Vec::new();
+                reader
+                    .read_to_end(&mut decoded)
+                    .await
+                    .expect("streaming reader returns the complete body");
+                assert_eq!(decoded, payload, "streaming read returns every original byte at quorum");
+            }
+        }
+    }
+
     #[test]
     #[serial]
     fn payload_read_repair_is_suppressed_only_for_read_only_requests() {

crates/ecstore/src/store/heal.rs

@@ -2167,6 +2167,210 @@ mod tests {
         shutdown.cancel();
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit() {
+        use crate::set_disk::ReadRepairPauseScope;
+
+        let (_root, store, shutdown) = multi_pool_heal_store().await;
+        let bucket = format!("heal-commit-owner-{}", Uuid::new_v4().simple());
+        let object = "history.bin";
+        let version = Uuid::new_v4().to_string();
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("original bucket");
+        let old = store
+            .bucket_incarnation_id_from_disk(&bucket)
+            .await
+            .expect("original identity");
+        let set = store.pools[0].get_disks(0);
+        let mut reader = PutObjReader::from_vec(vec![0x17; 1024 * 1024]);
+        set.put_object(
+            &bucket,
+            object,
+            &mut reader,
+            &ObjectOptions {
+                versioned: true,
+                version_id: Some(version.clone()),
+                write_completion: crate::object_api::WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("original acknowledged version");
+        let missing = set.disks.read().await[0].clone().expect("repair target");
+        let healthy = set.disks.read().await[1].clone().expect("healthy target");
+        let baseline = healthy
+            .read_all(&bucket, &format!("{object}/xl.meta"))
+            .await
+            .expect("healthy metadata baseline");
+        let original = healthy
+            .read_version("", &bucket, object, &version, &Default::default())
+            .await
+            .expect("original version metadata");
+        assert!(original.data.is_none(), "the commit barrier fixture must use actual external data shards");
+        missing
+            .delete(
+                &bucket,
+                object,
+                DeleteOptions {
+                    recursive: true,
+                    immediate: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("remove one physical shard");
+
+        // The scope pauses the real repair after reconstruction, immediately
+        // before its generation check and final shard rename.
+        let (_pause, arrived, release) = ReadRepairPauseScope::install_incarnation_commit(&bucket, object);
+        let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
+        let worker_store = store.clone();
+        let worker_bucket = bucket.clone();
+        let worker_version = version.clone();
+        let opts = HealOpts {
+            pool: Some(0),
+            set: Some(0),
+            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
+            ..Default::default()
+        };
+        let mut waiter = tokio::spawn(async move {
+            worker_store
+                .run_bucket_heal_at_incarnation(&worker_bucket, old, &opts, move |store, bucket, opts| async move {
+                    let result = store.heal_object_with_proof(&bucket, object, &worker_version, &opts).await?;
+                    drained_tx
+                        .send(result)
+                        .expect("retain the actual storage result after caller cancellation");
+                    Ok(())
+                })
+                .await
+        });
+        tokio::select! {
+            paused = tokio::time::timeout(std::time::Duration::from_secs(10), arrived.notified()) => {
+                paused.expect("real missing-shard repair reaches the final commit pause");
+            }
+            completed = &mut waiter => panic!("repair ended before the actual final commit barrier: {completed:?}"),
+        }
+        assert!(
+            missing.read_xl(&bucket, object, false).await.is_err(),
+            "final shard is not published at the pause"
+        );
+        println!("incarnation_commit_barrier: final_rename_paused");
+        waiter.abort();
+        assert!(waiter.await.expect_err("caller aborted").is_cancelled());
+        let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
+        assert!(
+            futures::poll!(&mut writer).is_pending(),
+            "bucket deletion/recreation cannot acquire its generation writer while the cancelled repair still owns commit"
+        );
+        println!("incarnation_commit_barrier: caller_aborted_writer_pending");
+        release.notify_one();
+        let result = tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
+            .await
+            .expect("actual repair drains after release")
+            .expect("retained repair result");
+        assert!(
+            result.error.is_none(),
+            "physical repair must succeed before lifecycle handoff: {:?}",
+            result.error
+        );
+        let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
+            .await
+            .expect("generation writer acquires only after physical repair drains")
+            .expect("generation writer");
+        assert!(
+            missing.read_xl(&bucket, object, false).await.is_ok(),
+            "the physical shard is published before writer admission"
+        );
+        let repaired = missing
+            .read_version("", &bucket, object, &version, &Default::default())
+            .await
+            .expect("repaired exact version metadata");
+        assert_eq!(repaired.version_id, original.version_id);
+        assert_eq!(repaired.data_dir, original.data_dir);
+        let checked = missing
+            .check_parts(&bucket, object, &repaired)
+            .await
+            .expect("verify the published physical shard");
+        assert_eq!(
+            checked.results,
+            vec![crate::disk::CHECK_PART_SUCCESS],
+            "the actual published part passes bitrot verification"
+        );
+        assert_eq!(
+            healthy
+                .read_all(&bucket, &format!("{object}/xl.meta"))
+                .await
+                .expect("healthy after repair"),
+            baseline,
+            "repair does not overwrite the surviving version"
+        );
+        println!("incarnation_commit_barrier: repair_drained_writer_acquired");
+        drop(guard);
+
+        set.delete_object(
+            &bucket,
+            object,
+            ObjectOptions {
+                versioned: true,
+                version_id: Some(version.clone()),
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("normal explicit deletion of the old version");
+        store
+            .delete_bucket(&bucket, &DeleteBucketOptions::default())
+            .await
+            .expect("normal empty bucket deletion");
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("same-name successor bucket");
+        let new = store
+            .bucket_incarnation_id_from_disk(&bucket)
+            .await
+            .expect("successor identity");
+        assert_ne!(old, new);
+        let successor_version = Uuid::new_v4().to_string();
+        let mut successor_reader = PutObjReader::from_vec(vec![0x29; 512 * 1024]);
+        set.put_object(
+            &bucket,
+            object,
+            &mut successor_reader,
+            &ObjectOptions {
+                versioned: true,
+                version_id: Some(successor_version),
+                write_completion: crate::object_api::WriteCompletion::TailDrained,
+                ..Default::default()
+            },
+        )
+        .await
+        .expect("acknowledged successor version");
+        let successor = healthy
+            .read_all(&bucket, &format!("{object}/xl.meta"))
+            .await
+            .expect("successor metadata");
+        assert!(
+            store
+                .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
+                .await
+                .is_err()
+        );
+        assert_eq!(
+            healthy
+                .read_all(&bucket, &format!("{object}/xl.meta"))
+                .await
+                .expect("successor after obsolete admission"),
+            successor,
+            "obsolete admission cannot change the same-name successor"
+        );
+        println!("incarnation_commit_barrier: successor_rejects_obsolete_admission");
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {

crates/filemeta/src/filemeta.rs

@@ -2427,6 +2427,52 @@ mod test {
         }
     }
 
+    #[test]
+    fn truncated_inline_xlmeta_is_file_corrupt_only_for_supported_formats() {
+        let mut meta = FileMeta::new();
+        meta.data
+            .replace("fixture", vec![0x5a; 4095])
+            .expect("seed real inline framing");
+        let inline_len = meta.data.as_slice().len();
+        let original = meta.marshal_msg().expect("serialize valid inline metadata");
+        let inline_start = original.len() - inline_len;
+        let parsed = FileMeta::load(&original).expect("complete inline metadata must decode");
+        assert_eq!(parsed.data, meta.data);
+        for cut in inline_start + 1..original.len() {
+            assert_eq!(
+                FileMeta::load(&original[..cut]).expect_err("incomplete inline framing must fail"),
+                Error::FileCorrupt,
+                "inline truncation at byte {cut} must remain repairable"
+            );
+        }
+
+        for inline_version in [0, 2] {
+            let mut unsupported = original[..original.len() / 2].to_vec();
+            unsupported[inline_start] = inline_version;
+            assert_ne!(
+                FileMeta::load(&unsupported).expect_err("malformed unsupported inline format must fail"),
+                Error::FileCorrupt,
+                "unknown inline version must not authorize repair"
+            );
+        }
+        for (header_ver, meta_ver) in [
+            (XL_HEADER_VERSION + 1, XL_META_VERSION),
+            (XL_HEADER_VERSION, XL_META_VERSION + 1),
+        ] {
+            let mut headers = Vec::new();
+            rmp::encode::write_uint(&mut headers, u64::from(header_ver)).expect("write future header");
+            rmp::encode::write_uint(&mut headers, u64::from(meta_ver)).expect("write future metadata version");
+            rmp::encode::write_uint(&mut headers, 0).expect("write empty version count");
+            let mut unsupported = build_xl_buffer(&headers);
+            unsupported.extend_from_slice(&meta.data.as_slice()[..inline_len / 2]);
+            assert_ne!(
+                FileMeta::load(&unsupported).expect_err("unsupported metadata with truncated inline data must fail"),
+                Error::FileCorrupt,
+                "future metadata schema must not authorize repair"
+            );
+        }
+    }
+
     #[test]
     fn truncated_xlmeta_index_and_format_reads_are_file_corrupt() {
         let buf = FileMeta::default()

crates/filemeta/src/filemeta/codec.rs

@@ -188,7 +188,13 @@ impl FileMeta {
         if !buf.is_empty() {
             InlineData::validate_slice(buf).map_err(|e| {
                 error!("data validation failed: {}", e);
-                e
+                // Validation only examines bytes already read. Classify damage
+                // as repairable corruption only for formats we understand.
+                if InlineData::version_ok_slice(buf) && Self::decode_xl_headers(meta).is_ok() {
+                    Error::FileCorrupt
+                } else {
+                    e
+                }
             })?;
         }
 

crates/filemeta/src/filemeta_inline.rs

@@ -168,11 +168,11 @@ impl InlineData {
         self.0.as_slice()
     }
     pub fn version_ok(&self) -> bool {
-        if self.0.is_empty() {
-            return true;
-        }
+        Self::version_ok_slice(&self.0)
+    }
 
-        self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
+    pub(crate) fn version_ok_slice(raw: &[u8]) -> bool {
+        raw.first().is_none_or(|version| *version > 0 && *version <= INLINE_DATA_VER)
     }
 
     pub fn after_version(&self) -> &[u8] {

crates/heal/src/heal/channel.rs

@@ -196,6 +196,20 @@ impl HealChannelProcessor {
             .map_err(Error::other)
     }
 
+    /// Query a local automatic task; other tokens require coordinator routing.
+    pub async fn execute_local_automatic_query(
+        &self,
+        heal_path: &str,
+        client_token: &str,
+        since_seq: Option<u64>,
+    ) -> Result<Option<HealChannelResponse>> {
+        self.heal_manager
+            .get_local_automatic_task_report(heal_path, client_token, since_seq)
+            .await?
+            .map(|report| Self::query_report_response(client_token.to_owned(), Ok(report)))
+            .transpose()
+    }
+
     /// Execute cancellation directly against the manager.
     pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
         let (response_tx, response_rx) = oneshot::channel();
@@ -463,6 +477,13 @@ impl HealChannelProcessor {
                 .await
         };
 
+        let response = Self::query_report_response(client_token, report)?;
+        let _ = response_tx.send(Ok(response.clone()));
+        self.publish_response(response);
+        Ok(())
+    }
+
+    fn query_report_response(client_token: String, report: Result<HealTaskReport>) -> Result<HealChannelResponse> {
         let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
         let settings = report
             .as_ref()
@@ -581,27 +602,21 @@ impl HealChannelProcessor {
                 0,
             ),
             Err(crate::Error::InvalidClientToken) => {
-                let response = HealChannelResponse {
+                return Ok(HealChannelResponse {
                     request_id: client_token,
                     success: false,
                     data: None,
                     error: Some("invalid heal client token".to_string()),
-                };
-                let _ = response_tx.send(Ok(response.clone()));
-                self.publish_response(response);
-                return Ok(());
+                });
             }
             Err(err) => {
                 let error_text = err.to_string();
-                let response = HealChannelResponse {
+                return Ok(HealChannelResponse {
                     request_id: client_token,
                     success: false,
                     data: None,
                     error: Some(error_text),
-                };
-                let _ = response_tx.send(Ok(response.clone()));
-                self.publish_response(response);
-                return Ok(());
+                });
             }
         };
 
@@ -618,17 +633,12 @@ impl HealChannelProcessor {
             },
         )?;
 
-        let response = HealChannelResponse {
+        Ok(HealChannelResponse {
             request_id: client_token,
             success: true,
             data: Some(data),
             error: detail,
-        };
-
-        let _ = response_tx.send(Ok(response.clone()));
-        self.publish_response(response);
-
-        Ok(())
+        })
     }
 
     /// Process cancel request

crates/heal/src/heal/manager.rs

@@ -234,6 +234,8 @@ fn record_displaced_terminal(
     request: &HealRequest,
 ) -> Arc<CompletedHealStatus> {
     let terminal = Arc::new(CompletedHealStatus {
+        task_id: request.id.clone(),
+        source: request.source,
         outcome: None,
         progress: None,
         retained_bytes: std::sync::OnceLock::new(),
@@ -669,7 +671,10 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
     if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
         return None;
     }
-    if task.has_batch_failure() && !retryable_batch_failure {
+    // Admin traversals already exhaust their per-object retries. Restarting
+    // the whole traversal under the same token discards those receipts and
+    // reports previously repaired objects as unchanged on the next scan.
+    if task.has_batch_failure() && (task.source == HealRequestSource::Admin || !retryable_batch_failure) {
         return None;
     }
 
@@ -1056,9 +1061,9 @@ pub struct HealManager {
 /// cascade without re-locking.
 enum TaskStateLookup {
     Active(Arc<HealTask>),
-    Retrying(HealTaskStatus, HealOptions),
+    Retrying(HealTaskStatus, HealOptions, HealRequestSource),
     Completed(Arc<CompletedHealStatus>),
-    Queued(HealOptions),
+    Queued(HealOptions, HealRequestSource),
     /// A durable root heal record that cannot be admitted or quarantined;
     /// it answers status queries with the reason until an operator repairs it.
     Unrunnable(String),
@@ -2505,6 +2510,23 @@ impl HealManager {
     /// completed entry. `heal_path` additionally constrains the map matches
     /// the way the `*_for_path` variants always have.
     async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
+        let state = self.lookup_memory_task_state(canonical_task_id, heal_path).await;
+        if !matches!(state, TaskStateLookup::NotFound) {
+            return Ok(state);
+        }
+        if let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
+            && heal_path.is_none_or(|path| heal_type_matches_path(&completed.heal_type, path))
+        {
+            return Ok(TaskStateLookup::Completed(Arc::new(completed)));
+        }
+        // A durable record that cannot run still answers with its reason.
+        if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
+            return Ok(TaskStateLookup::Unrunnable(reason));
+        }
+        Ok(TaskStateLookup::NotFound)
+    }
+
+    async fn lookup_memory_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
         let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
 
         {
@@ -2513,7 +2535,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|task| matches_path(&task.heal_type))
             {
-                return Ok(TaskStateLookup::Active(Arc::clone(task)));
+                return TaskStateLookup::Active(Arc::clone(task));
             }
         }
 
@@ -2523,7 +2545,7 @@ impl HealManager {
                 .get(canonical_task_id)
                 .filter(|retrying| matches_path(&retrying.request.heal_type))
             {
-                return Ok(TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone()));
+                return TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone(), retrying.request.source);
             }
         }
 
@@ -2536,7 +2558,7 @@ impl HealManager {
             prune_completed_heal_statuses(&mut completed_heals);
             if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
                 if completed_status_is_retrying(&completed.status) {
-                    return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
+                    return TaskStateLookup::Completed(Arc::clone(completed));
                 }
                 terminal_completed = Some(Arc::clone(completed));
             }
@@ -2545,7 +2567,7 @@ impl HealManager {
         {
             let queue = self.heal_queue.lock().await;
             if let Some(request) = queue.request_matching_id_and_path(canonical_task_id, heal_path) {
-                return Ok(TaskStateLookup::Queued(request.options.clone()));
+                return TaskStateLookup::Queued(request.options.clone(), request.source);
             }
         }
 
@@ -2558,22 +2580,10 @@ impl HealManager {
                 .cloned();
         }
 
-        if terminal_completed.is_none()
-            && let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
-            && matches_path(&completed.heal_type)
-        {
-            terminal_completed = Some(Arc::new(completed));
-        }
-
         if let Some(completed) = terminal_completed {
-            return Ok(TaskStateLookup::Completed(completed));
+            return TaskStateLookup::Completed(completed);
         }
-        // A durable root record that can neither run nor be quarantined still
-        // answers status queries with its reason.
-        if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
-            return Ok(TaskStateLookup::Unrunnable(reason));
-        }
-        Ok(TaskStateLookup::NotFound)
+        TaskStateLookup::NotFound
     }
 
     async fn publish_admin_terminal(
@@ -2597,6 +2607,8 @@ impl HealManager {
     ) -> Result<bool> {
         let previous = self.completed_heals.lock().await.get(task_id).cloned();
         let mut completed = CompletedHealStatus {
+            task_id: task_id.to_owned(),
+            source,
             outcome: None,
             progress: None,
             retained_bytes: std::sync::OnceLock::new(),
@@ -2632,9 +2644,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
-            TaskStateLookup::Retrying(status, _) => Ok(status),
+            TaskStateLookup::Retrying(status, _, _) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
-            TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
+            TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
             TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
             TaskStateLookup::NotFound => Err(Error::TaskNotFound {
                 task_id: task_id.to_string(),
@@ -2653,9 +2665,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, None).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
-            TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
+            TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
-            TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
+            TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
             TaskStateLookup::Unrunnable(error) => {
                 Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
             }
@@ -2669,6 +2681,42 @@ impl HealManager {
         self.get_task_report_for_path_since(heal_path, task_id, None).await
     }
 
+    /// Resolve a node-owned automatic task without granting local control over
+    /// Admin tasks. The source and report come from the same state snapshot.
+    pub async fn get_local_automatic_task_report(
+        &self,
+        heal_path: &str,
+        task_id: &str,
+        since: Option<u64>,
+    ) -> Result<Option<HealTaskReport>> {
+        let canonical_task_id = self.canonical_task_id(task_id).await;
+        if canonical_task_id != task_id {
+            return Ok(None);
+        }
+        let state = self.lookup_memory_task_state(&canonical_task_id, Some(heal_path)).await;
+        let source = match &state {
+            TaskStateLookup::Active(task) => task.source,
+            TaskStateLookup::Retrying(_, _, source) | TaskStateLookup::Queued(_, source) => *source,
+            TaskStateLookup::Completed(completed) if completed.task_id == task_id => completed.source,
+            TaskStateLookup::Completed(_) => return Ok(None),
+            TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
+        };
+        if !matches!(
+            source,
+            HealRequestSource::Scanner | HealRequestSource::Mrf | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
+        ) {
+            return Ok(None);
+        }
+        let report = match state {
+            TaskStateLookup::Active(task) => active_task_report(&task, since).await,
+            TaskStateLookup::Retrying(status, options, _) => empty_task_report_with_options(status, options),
+            TaskStateLookup::Completed(completed) => completed_task_report(&completed, since),
+            TaskStateLookup::Queued(options, _) => empty_task_report_with_options(HealTaskStatus::Pending, options),
+            TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
+        };
+        Ok(Some(report))
+    }
+
     /// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
     pub async fn get_task_report_for_path_since(
         &self,
@@ -2679,9 +2727,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
-            TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
+            TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
             TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
-            TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
+            TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
             TaskStateLookup::Unrunnable(error) => {
                 Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
             }
@@ -2705,9 +2753,9 @@ impl HealManager {
         let canonical_task_id = self.canonical_task_id(task_id).await;
         match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
             TaskStateLookup::Active(task) => Ok(task.get_status().await),
-            TaskStateLookup::Retrying(status, _) => Ok(status),
+            TaskStateLookup::Retrying(status, _, _) => Ok(status),
             TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
-            TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
+            TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
             TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
             TaskStateLookup::NotFound => {
                 if self.path_has_task(heal_path).await {

crates/heal/src/heal/manager/queue.rs

@@ -107,7 +107,9 @@ pub(super) enum QueuePushOutcome {
 
 #[derive(Debug, Clone)]
 pub(super) struct CompletedHealStatus {
+    pub(super) task_id: String,
     pub(super) heal_type: HealType,
+    pub(super) source: HealRequestSource,
     /// Options used to execute the task, retained for token-scoped status.
     pub(super) options: HealOptions,
     pub(super) status: HealTaskStatus,
@@ -135,6 +137,7 @@ impl CompletedHealStatus {
     fn measure_retained_bytes(&self) -> usize {
         let mut bytes = size_of::<Self>();
         let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
+        add(self.task_id.capacity());
         add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
         match &self.heal_type {
             HealType::Cluster => {}
@@ -256,7 +259,9 @@ impl CompletedHealStatus {
             outcome.finish(Some(crate::heal::outcome::HealAbortReason::Cancelled));
         }
         let mut snapshot = Self {
+            task_id: task.id.clone(),
             heal_type: task.heal_type.clone(),
+            source: task.source,
             options: task.options.clone(),
             status,
             progress: Some(task.get_progress().await),

crates/heal/src/heal/manager/root_recovery.rs

@@ -460,6 +460,8 @@ impl RootHealTerminal {
 
     fn into_completed(self) -> CompletedHealStatus {
         CompletedHealStatus {
+            task_id: self.task_id,
+            source: HealRequestSource::Admin,
             outcome: None,
             progress: self.progress,
             retained_bytes: std::sync::OnceLock::new(),

crates/heal/src/heal/manager/tests.rs

@@ -118,6 +118,8 @@ struct MockStorage;
 
 fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
+        task_id: "fixture-token".to_string(),
+        source: HealRequestSource::Internal,
         outcome: None,
         heal_type: HealType::Cluster,
         options: HealOptions::default(),
@@ -2776,6 +2778,181 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
     );
 }
 
+#[tokio::test]
+async fn local_automatic_query_preserves_source_path_token_and_state() {
+    use crate::heal::channel::HealChannelProcessor;
+    use crate::heal::outcome::{
+        HealExecutionOutcome, HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome,
+        HealTraversalCoverage,
+    };
+
+    for source in [
+        HealRequestSource::Scanner,
+        HealRequestSource::Mrf,
+        HealRequestSource::AutoHeal,
+        HealRequestSource::ReadRepair,
+        HealRequestSource::Admin,
+        HealRequestSource::Internal,
+    ] {
+        for state in ["queued", "active", "retrying", "completed", "displaced"] {
+            let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+            let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
+            let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+            request.source = source;
+            let token = request.id.clone();
+            let task = Arc::new(HealTask::from_request(request.clone(), storage));
+            let mut snapshot = CompletedHealStatus::snapshot(&task, HealTaskStatus::Completed).await;
+            assert_eq!(snapshot.source, source, "completion must retain actual task source");
+            let mut outcome = HealTaskOutcome::default();
+            outcome.execution = HealExecutionOutcome::Completed;
+            outcome.coverage = HealTraversalCoverage::Complete;
+            outcome.record(HealObjectOutcome {
+                identity: HealObjectIdentity {
+                    kind: HealObjectKind::Object,
+                    bucket: "bucket".to_string(),
+                    object: "object".to_string(),
+                    version_id: None,
+                    bucket_incarnation_id: Some(uuid::Uuid::nil()),
+                    pool_index: Some(0),
+                    set_index: Some(0),
+                },
+                disposition: HealObjectDisposition::VerifiedHealthy,
+                detail: None,
+            });
+            snapshot.outcome = Some(Arc::new(outcome));
+            snapshot.seqed_items = vec![(
+                3,
+                HealResultItem {
+                    bucket: "bucket".to_string(),
+                    object: "object".to_string(),
+                    ..Default::default()
+                },
+            )];
+            snapshot.min_seq = 3;
+            snapshot.next_seq = 4;
+            let expected_status = match state {
+                "queued" => {
+                    manager.heal_queue.lock().await.push(request.clone());
+                    HealTaskStatus::Pending
+                }
+                "active" => {
+                    manager.active_heals.lock().await.insert(token.clone(), task);
+                    HealTaskStatus::Pending
+                }
+                "retrying" => {
+                    manager.retrying_heals.lock().await.insert(
+                        token.clone(),
+                        RetryingHeal {
+                            request: request.clone(),
+                            error: "transient".to_string(),
+                            cancel_token: CancellationToken::new(),
+                        },
+                    );
+                    HealTaskStatus::Retrying {
+                        error: "transient".to_string(),
+                        retry_attempt: request.retry_attempts,
+                    }
+                }
+                "completed" => {
+                    manager.completed_heals.lock().await.insert(token.clone(), Arc::new(snapshot));
+                    HealTaskStatus::Completed
+                }
+                "displaced" => {
+                    let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
+                    terminal.status.clone()
+                }
+                _ => unreachable!("fixed test states"),
+            };
+            let automatic = !matches!(source, HealRequestSource::Admin | HealRequestSource::Internal);
+            let report = manager
+                .get_local_automatic_task_report("bucket/object", &token, Some(2))
+                .await
+                .expect("local source lookup");
+            assert_eq!(report.is_some(), automatic, "source={source:?} state={state}");
+            if let Some(report) = report {
+                assert_eq!(report.status, expected_status);
+                if state == "completed" {
+                    let outcome = report.outcome.as_ref().expect("retained structured outcome");
+                    assert_eq!(outcome.objects.len(), 1);
+                    assert_eq!(outcome.objects[0].identity.object, "object");
+                    assert_eq!(outcome.objects[0].identity.pool_index, Some(0));
+                    assert_eq!(outcome.objects[0].identity.set_index, Some(0));
+                    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::VerifiedHealthy);
+                    assert_eq!(report.min_seq, 3);
+                    assert_eq!(report.next_seq, 4);
+                    assert_eq!(report.result_items.len(), 1);
+                    assert_eq!(report.result_items[0].object, "object");
+                }
+                let processor = HealChannelProcessor::new(manager.clone());
+                let local = processor
+                    .execute_local_automatic_query("bucket/object", &token, Some(2))
+                    .await
+                    .expect("local response")
+                    .expect("automatic task response");
+                let direct = processor
+                    .execute_query_request_since("bucket/object".to_string(), token.clone(), Some(2))
+                    .await
+                    .expect("original query response");
+                assert!(local.success);
+                assert_eq!(local.request_id, direct.request_id);
+                assert_eq!(local.data, direct.data, "same report encoding and cursor contract");
+                assert_eq!(local.error, direct.error);
+                let coordinator = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
+                let missing = HealChannelProcessor::new(coordinator)
+                    .execute_query_request("bucket/object".to_string(), token.clone())
+                    .await
+                    .expect("coordinator query");
+                let body: serde_json::Value =
+                    serde_json::from_slice(missing.data.as_deref().expect("coordinator response")).expect("coordinator JSON");
+                assert_eq!(body["summary"], "notFound", "a remote manager does not own the automatic task");
+                if state == "completed" {
+                    let completed = manager
+                        .completed_heals
+                        .lock()
+                        .await
+                        .get(&token)
+                        .cloned()
+                        .expect("original completion");
+                    manager
+                        .completed_heals
+                        .lock()
+                        .await
+                        .insert("retired-alias".to_string(), completed);
+                    assert!(
+                        manager
+                            .get_local_automatic_task_report("bucket/object", "retired-alias", None)
+                            .await
+                            .expect("retired alias lookup")
+                            .is_none()
+                    );
+                }
+            }
+            assert!(
+                manager
+                    .get_local_automatic_task_report("bucket/other", &token, None)
+                    .await
+                    .expect("wrong path lookup")
+                    .is_none()
+            );
+            assert!(
+                manager
+                    .get_local_automatic_task_report("bucket/object", "unknown", None)
+                    .await
+                    .expect("unknown token lookup")
+                    .is_none()
+            );
+            manager.insert_task_alias("admin-alias", &token).await;
+            assert!(
+                manager
+                    .get_local_automatic_task_report("bucket/object", "admin-alias", None)
+                    .await
+                    .expect("alias lookup")
+                    .is_none()
+            );
+        }
+    }
+}
+
 #[tokio::test]
 async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -3591,6 +3768,38 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab
     }
 }
 
+#[tokio::test]
+async fn admin_exhausted_batches_do_not_restart_the_traversal() {
+    for heal_type in [
+        HealType::Bucket {
+            bucket: "bucket".to_owned(),
+        },
+        HealType::Prefix {
+            bucket: "bucket".to_owned(),
+            prefix: "prefix/".to_owned(),
+        },
+        HealType::Cluster,
+    ] {
+        let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::Normal);
+        request.source = HealRequestSource::Admin;
+        let task = HealTask::from_request(request, Arc::new(MockStorage));
+        let result = Err(task
+            .record_batch_failure(BatchHealFailure {
+                scope: "completed-admin-traversal".to_owned(),
+                failed: 1,
+                retryable: 1,
+                permanent: 0,
+                first_object: "exhausted-object".to_owned(),
+                first_error: Error::Storage(EcstoreError::SlowDown).to_string(),
+            })
+            .await);
+        assert!(
+            retry_request_for_result_with_budget(&task, &result).await.is_none(),
+            "the exhausted admin batch must retain its original object receipts"
+        );
+    }
+}
+
 #[tokio::test]
 async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
     for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
@@ -3673,6 +3882,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
+            source: request.source,
+            task_id: request.id.clone(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type,
             options: request.options.clone(),
@@ -4521,6 +4732,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
+            source: request.source,
+            task_id: request.id.clone(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: request.heal_type.clone(),
             options: request.options.clone(),
@@ -4560,6 +4773,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
+            source: HealRequestSource::Internal,
+            task_id: "completed-token".to_string(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Bucket {
                 bucket: "bucket".to_string(),
@@ -4593,6 +4808,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
         Arc::new(CompletedHealStatus {
             progress: None,
             outcome: None,
+            source: HealRequestSource::Internal,
+            task_id: "completed-token".to_string(),
             retained_bytes: std::sync::OnceLock::new(),
             heal_type: HealType::Object {
                 bucket: "bucket".to_string(),

crates/heal/src/heal/manager/tests/root_recovery.rs

@@ -117,6 +117,8 @@ fn admin_request(heal_type: HealType) -> HealRequest {
 
 fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus {
     CompletedHealStatus {
+        task_id: "admin-fixture".to_string(),
+        source: HealRequestSource::Admin,
         outcome: None,
         heal_type: heal_type.clone(),
         options: HealOptions::default(),
@@ -1239,6 +1241,8 @@ async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() {
         .expect("durable bucket responsibility");
     let completed = CompletedHealStatus {
         outcome: None,
+        source: request.source,
+        task_id: request.id.clone(),
         heal_type: request.heal_type.clone(),
         options: request.options.clone(),
         status: HealTaskStatus::Completed,

crates/heal/src/heal/task.rs

@@ -37,7 +37,7 @@ use std::{
     collections::VecDeque,
     future::Future,
     sync::{
-        Arc,
+        Arc, OnceLock,
         atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
     },
     time::{Duration, Instant, SystemTime},
@@ -59,6 +59,19 @@ pub(crate) struct OutcomeFinishTestHook {
 pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
     std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
 
+#[cfg(test)]
+static STORAGE_RECEIPT_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
+    std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
+
+#[cfg(test)]
+async fn pause_storage_receipt(task_id: &str) {
+    let hook = STORAGE_RECEIPT_TEST_HOOK.lock().await.clone();
+    if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
+        hook.reached.notify_one();
+        hook.release.notified().await;
+    }
+}
+
 #[cfg(test)]
 async fn pause_outcome_finish(task_id: &str) {
     let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
@@ -436,6 +449,9 @@ pub struct HealTask {
     replacement_start_retry_count: Arc<AtomicU32>,
     /// Set when the locked storage read classified the single target as absent.
     object_absent: Arc<AtomicBool>,
+    /// The first resolved incarnation belongs to this task even if storage
+    /// fails or the bucket is recreated before its terminal outcome is recorded.
+    single_object_incarnation: Arc<OnceLock<Option<Uuid>>>,
     /// Task status
     pub status: Arc<RwLock<HealTaskStatus>>,
     /// Progress tracking
@@ -508,6 +524,7 @@ impl HealTask {
             replacement_running: Arc::new(AtomicBool::new(false)),
             replacement_start_retry_count: Arc::new(AtomicU32::new(0)),
             object_absent: Arc::new(AtomicBool::new(false)),
+            single_object_incarnation: Arc::new(OnceLock::new()),
             status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
             progress: Arc::new(RwLock::new(HealProgress::new())),
             result_items: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_RETAINED_HEAL_RESULT_ITEMS))),
@@ -669,7 +686,9 @@ impl HealTask {
             } => (bucket, object, Some(version_id.as_str())),
             _ => return None,
         };
-        Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
+        let mut identity = self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index);
+        identity.bucket_incarnation_id = self.single_object_incarnation.get().copied().flatten();
+        Some(identity)
     }
 
     pub(super) async fn record_verified_storage_receipt(

crates/heal/src/heal/task/heal_bucket.rs

@@ -306,9 +306,9 @@ impl HealTask {
                     let mut progress = self.progress.write().await;
                     progress.update_stage(3, 3);
                 }
-                Err(Error::TaskExecutionFailed {
-                    message: format!("Failed to heal bucket {bucket}: {e}"),
-                })
+                // Keep storage/quorum identity for the cluster retry classifier.
+                // The structured log above already carries the bucket context.
+                Err(e)
             }
         }
     }
@@ -796,21 +796,27 @@ impl HealTask {
                                 match unavailable_recreate_error(&storage_result.item, &heal_opts) {
                                     Some(error) => Some(error),
                                     None => {
-                                        telemetry_unknown |= !increment_counter(&mut healed);
-                                        telemetry_unknown |= !add_bytes(
-                                            &mut bytes,
-                                            u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
-                                        );
+                                        #[cfg(test)]
+                                        super::pause_storage_receipt(&self.id).await;
                                         recorded_authoritative_outcome =
                                             self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
-                                        if !recorded_authoritative_outcome
-                                            && !heal_opts.dry_run
-                                            && !storage_result.item.detail.is_empty()
-                                        {
-                                            detail = Some(storage_result.item.detail.clone());
+                                        if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
+                                            Some(Error::TaskCancelled)
+                                        } else {
+                                            telemetry_unknown |= !increment_counter(&mut healed);
+                                            telemetry_unknown |= !add_bytes(
+                                                &mut bytes,
+                                                u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
+                                            );
+                                            if !recorded_authoritative_outcome
+                                                && !heal_opts.dry_run
+                                                && !storage_result.item.detail.is_empty()
+                                            {
+                                                detail = Some(storage_result.item.detail.clone());
+                                            }
+                                            self.record_result_item(storage_result.item).await;
+                                            None
                                         }
-                                        self.record_result_item(storage_result.item).await;
-                                        None
                                     }
                                 }
                             }

crates/heal/src/heal/task/heal_object.rs

@@ -220,6 +220,11 @@ impl HealTask {
         let mut expected_identity =
             self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
         expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
+        if matches!(self.heal_type, HealType::Object { .. }) {
+            expected_identity.bucket_incarnation_id = *self
+                .single_object_incarnation
+                .get_or_init(|| expected_identity.bucket_incarnation_id);
+        }
 
         let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
         let heal_result = if self.source == HealRequestSource::ReadRepair {

crates/heal/src/heal/task/tests.rs

@@ -305,6 +305,53 @@ mod canonical_outcome {
         assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
     }
 
+    #[tokio::test]
+    async fn bucket_cancellation_after_storage_success_keeps_typed_cancelled_responsibility() {
+        let incarnation = Uuid::new_v4();
+        let storage = Arc::new(MockStorage {
+            heal_object_receipts: Mutex::new(HashMap::from([(
+                "object-a".to_string(),
+                VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
+            )])),
+            bucket_incarnation_id: Mutex::new(Some(incarnation)),
+            ..Default::default()
+        });
+        let task = Arc::new(bucket_task(storage.clone()));
+        let hook = Arc::new(OutcomeFinishTestHook {
+            task_id: task.id.clone(),
+            reached: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        *STORAGE_RECEIPT_TEST_HOOK.lock().await = Some(hook.clone());
+        let runner = {
+            let task = task.clone();
+            tokio::spawn(async move { task.execute().await })
+        };
+        tokio::time::timeout(Duration::from_secs(10), hook.reached.notified())
+            .await
+            .expect("storage success must reach the receipt boundary");
+        assert_eq!(storage.healed_objects.lock().expect("storage mutations").as_slice(), ["object-a"]);
+        task.cancel().await.expect("cancel before receipt recording");
+        hook.release.notify_one();
+        let result = tokio::time::timeout(Duration::from_secs(10), runner)
+            .await
+            .expect("cancelled worker must stop")
+            .expect("worker must not panic");
+        STORAGE_RECEIPT_TEST_HOOK.lock().await.take();
+        assert!(matches!(result, Err(Error::TaskCancelled)));
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Cancelled));
+        assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+        assert_eq!(outcome.counters.processed, 1);
+        assert_eq!(outcome.counters.healed, 0);
+        assert_eq!(outcome.counters.unchanged, 0);
+        assert_eq!(outcome.counters.unknown, 0);
+        assert_eq!(outcome.counters.skipped, 1);
+        assert_eq!(outcome.objects.len(), 1);
+        assert_eq!(outcome.objects[0].identity.object, "object-a");
+        assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
+    }
+
     #[tokio::test]
     async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
         let expected_incarnation = Uuid::new_v4();
@@ -809,6 +856,85 @@ mod canonical_outcome {
         );
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() {
+        for (failures, succeeds) in [(1, true), (4, false)] {
+            let storage = Arc::new(MockStorage {
+                listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
+                typed_bucket_heal_errors: Mutex::new(HashMap::from([(
+                    "bucket-a".to_string(),
+                    (0..failures)
+                        .map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum))
+                        .collect(),
+                )])),
+                ..Default::default()
+            });
+            let task = HealTask::from_request(
+                HealRequest::new(
+                    HealType::Cluster,
+                    HealOptions {
+                        recursive: true,
+                        timeout: None,
+                        ..Default::default()
+                    },
+                    HealPriority::Normal,
+                ),
+                storage.clone(),
+            );
+            let result = task.execute().await;
+            assert_eq!(
+                result.is_ok(),
+                succeeds,
+                "original cluster task must honor its bounded bucket retry budget"
+            );
+            let outcome = task.get_outcome().await;
+            if succeeds {
+                assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
+                assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+            } else {
+                assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
+                assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+                let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure");
+                assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
+            }
+            let calls = storage.bucket_heal_calls.lock().expect("original bucket calls");
+            assert_eq!(
+                calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(),
+                if succeeds { 2 } else { 4 }
+            );
+            assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited");
+        }
+    }
+
+    #[tokio::test]
+    async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() {
+        for error in [
+            Error::Storage(EcstoreError::ErasureWriteQuorum),
+            Error::Disk(DiskError::FileAccessDenied),
+        ] {
+            let retryable = error.is_recoverable_heal();
+            let storage = Arc::new(MockStorage {
+                typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])),
+                ..Default::default()
+            });
+            let task = bucket_task(storage.clone());
+            let observed = task
+                .heal_bucket("bucket-a")
+                .await
+                .expect_err("bucket prepass must retain the storage error");
+            assert!(matches!(
+                observed,
+                Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied)
+            ));
+            assert_eq!(observed.is_recoverable_heal(), retryable);
+            assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
+            assert!(
+                storage.heal_object_calls.lock().expect("object calls").is_empty(),
+                "failed prepass must not heal objects"
+            );
+        }
+    }
+
     #[tokio::test(start_paused = true)]
     async fn cancellation_and_deadline_leave_partial_coverage() {
         for cancel in [false, true] {
@@ -1567,6 +1693,7 @@ struct MockStorage {
     listed_buckets: Mutex<Option<Vec<String>>>,
     bucket_discovery_incomplete: bool,
     bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
+    typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
     bucket_heal_calls: Mutex<Vec<String>>,
     delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
     delete_recovery_calls: Mutex<Vec<String>>,
@@ -1815,6 +1942,127 @@ async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
                 | HealObjectDisposition::AuthoritativelyAbsent
         )
     }));
+    assert_eq!(outcome.objects.len(), 1);
+    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
+}
+
+#[tokio::test]
+async fn object_failure_retains_incarnation_for_inner_and_outer_storage_errors() {
+    for failure in [
+        MockHealObjectOutcome::PermissionDenied,
+        MockHealObjectOutcome::RetryableReadQuorum,
+        MockHealObjectOutcome::RetryableLockTimeout,
+        MockHealObjectOutcome::ErrOther("backend failed"),
+    ] {
+        let incarnation = Uuid::new_v4();
+        let storage = Arc::new(MockStorage {
+            bucket_incarnation_id: Mutex::new(Some(incarnation)),
+            heal_object_outcome: Mutex::new(Some(failure)),
+            ..Default::default()
+        });
+        let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), Some("version-a".to_owned()));
+        request.options.pool_index = Some(2);
+        request.options.set_index = Some(3);
+        let task = HealTask::from_request(request, storage);
+        task.execute()
+            .await
+            .expect_err("the failed storage operation must remain failed");
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.objects.len(), 1);
+        assert_eq!(
+            (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
+            (1, 1, 1)
+        );
+        assert_eq!((outcome.counters.healed, outcome.counters.unchanged, outcome.counters.unknown), (0, 0, 0));
+        let object = &outcome.objects[0];
+        assert_eq!(object.identity.bucket_incarnation_id, Some(incarnation));
+        assert_eq!(object.identity.bucket, "bucket-a");
+        assert_eq!(object.identity.object, "object-a");
+        assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
+        assert_eq!((object.identity.pool_index, object.identity.set_index), (Some(2), Some(3)));
+        assert!(matches!(object.disposition, HealObjectDisposition::Failed(_)));
+    }
+}
+
+#[tokio::test(start_paused = true)]
+async fn object_abort_keeps_pre_storage_incarnation_after_bucket_recreation() {
+    for cancel in [false, true] {
+        let original = Uuid::new_v4();
+        let storage = Arc::new(MockStorage {
+            bucket_incarnation_id: Mutex::new(Some(original)),
+            block_heal_object: Mutex::new(true),
+            ..Default::default()
+        });
+        let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
+        request.options.timeout = Some(Duration::from_secs(10));
+        let task = HealTask::from_request(request, storage.clone());
+        let execution = task.execute();
+        tokio::pin!(execution);
+        assert!(futures::poll!(&mut execution).is_pending());
+        assert_eq!(storage.heal_object_calls.lock().expect("storage boundary").as_slice(), ["object-a"]);
+        *storage.bucket_incarnation_id.lock().expect("recreated bucket") = Some(Uuid::new_v4());
+        if cancel {
+            task.cancel().await.expect("cancel blocked storage");
+        } else {
+            tokio::time::advance(Duration::from_secs(11)).await;
+        }
+        let result = execution.await;
+        assert!(matches!(
+            (&result, cancel),
+            (Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
+        ));
+        let outcome = task.get_outcome().await;
+        assert_eq!(outcome.objects.len(), 1);
+        assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
+        assert_eq!((outcome.counters.processed, outcome.counters.skipped), (1, 1));
+        assert_eq!((outcome.counters.failed, outcome.counters.healed, outcome.counters.unknown), (0, 0, 0));
+        assert_eq!(
+            outcome.objects[0].disposition,
+            if cancel {
+                HealObjectDisposition::Cancelled
+            } else {
+                HealObjectDisposition::Deferred {
+                    reason: HealDeferredReason::Deadline,
+                    retry_not_before: None,
+                }
+            }
+        );
+    }
+}
+
+#[tokio::test]
+async fn object_unverified_outcome_keeps_original_incarnation_without_claiming_repair() {
+    let original = Uuid::new_v4();
+    let successor = Uuid::new_v4();
+    let storage = Arc::new(MockStorage {
+        bucket_incarnation_id: Mutex::new(Some(original)),
+        bucket_incarnation_after_object_heal: Mutex::new(Some(successor)),
+        ..Default::default()
+    });
+    let task = HealTask::from_request(HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None), storage.clone());
+    task.execute().await.expect("legacy storage keeps its unverified result");
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
+    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
+    assert_eq!((outcome.counters.healed, outcome.counters.unknown), (0, 1));
+    assert_eq!(*storage.bucket_incarnation_id.lock().expect("successor"), Some(successor));
+}
+
+#[tokio::test]
+async fn object_dry_run_does_not_adopt_bucket_incarnation() {
+    let storage = Arc::new(MockStorage {
+        bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
+        ..Default::default()
+    });
+    let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
+    request.options.dry_run = true;
+    let task = HealTask::from_request(request, storage);
+    task.execute().await.expect("dry-run observation");
+    let outcome = task.get_outcome().await;
+    assert_eq!(outcome.objects.len(), 1);
+    assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, None);
+    assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
+    assert_eq!(outcome.counters.healed, 0);
 }
 
 #[tokio::test]
@@ -2387,6 +2635,15 @@ impl HealStorageAPI for MockStorage {
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
         self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
         self.bucket_heal_opts.lock().unwrap().push(*opts);
+        if let Some(error) = self
+            .typed_bucket_heal_errors
+            .lock()
+            .unwrap()
+            .get_mut(bucket)
+            .and_then(VecDeque::pop_front)
+        {
+            return Err(error);
+        }
         if let Some(message) = self
             .bucket_heal_errors
             .lock()

crates/heal/tests/shard_identity_receipt_test.rs

@@ -550,3 +550,159 @@ fn admin_normal_protected_repairs_report_repaired_for_data_and_parity() {
         .join()
         .expect("heal fixture result");
 }
+
+#[test]
+#[serial]
+fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
+    std::thread::Builder::new()
+        .stack_size(32 * 1024 * 1024)
+        .spawn(|| {
+            tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("mixed batch runtime")
+                .block_on(async {
+                    use rustfs_heal::heal::{
+                        manager::{HealConfig, HealManager},
+                        outcome::{HealExecutionOutcome, HealFailureClass, HealTraversalCoverage},
+                        task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
+                    };
+                    use rustfs_heal_contracts::heal_channel::HealRequestSource;
+                    use std::sync::Arc;
+
+                    let root = tempfile::tempdir().expect("mixed batch fixture");
+                    let env = TestECStoreEnv::builder().base_dir(root.path()).build().await;
+                    let bucket = "admin-mixed-batch";
+                    env.make_bucket(bucket, false).await;
+                    let set = env.ecstore.pools[0].get_disks(0);
+                    let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
+                    let mut originals = Vec::new();
+                    for (object, missing, byte) in [("healthy", 0, 0x11), ("repaired", 1, 0x52), ("failed", 3, 0xa3)] {
+                        env.ecstore
+                            .put_object(
+                                bucket,
+                                object,
+                                &mut PutObjReader::from_vec(vec![byte; 5 * 1024 * 1024 + 123]),
+                                &ObjectOptions {
+                                    shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
+                                    write_completion: WriteCompletion::TailDrained,
+                                    ..Default::default()
+                                },
+                            )
+                            .await
+                            .expect("commit complete fixture");
+                        for (slot, disk) in disks.iter().enumerate() {
+                            let meta = disk
+                                .read_version("", bucket, object, "", &ReadOptions::default())
+                                .await
+                                .expect("fixture metadata");
+                            assert_eq!(meta.erasure.data_blocks, 2, "failed fixture must retain R-1 shards");
+                            let path = env.disk_paths[slot]
+                                .join(bucket)
+                                .join(object)
+                                .join(meta.data_dir.expect("external shard").to_string())
+                                .join("part.1");
+                            let bytes = tokio::fs::read(&path).await.expect("original part");
+                            if slot < missing {
+                                tokio::fs::remove_file(&path).await.expect("inject missing part");
+                            }
+                            originals.push((object, path, bytes, slot < missing));
+                        }
+                    }
+                    let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
+                    let manager = HealManager::new(
+                        storage.clone(),
+                        Some(HealConfig {
+                            enable_auto_heal: false,
+                            ..Default::default()
+                        }),
+                    );
+                    manager.start().await.expect("start mixed batch manager");
+                    let mut request = HealRequest::new(
+                        HealType::Bucket {
+                            bucket: bucket.to_owned(),
+                        },
+                        HealOptions {
+                            recursive: true,
+                            scan_mode: HealScanMode::Deep,
+                            pool_index: Some(0),
+                            set_index: Some(0),
+                            timeout: Some(Duration::from_secs(150)),
+                            ..Default::default()
+                        },
+                        HealPriority::High,
+                    );
+                    request.source = HealRequestSource::Admin;
+                    let token = request.id.clone();
+                    manager.submit_heal_request(request).await.expect("admit original mixed task");
+                    let report = tokio::time::timeout(Duration::from_secs(150), async {
+                        loop {
+                            let report = manager.get_task_report(&token).await.expect("same-token report");
+                            if matches!(
+                                report.status,
+                                HealTaskStatus::Completed | HealTaskStatus::Failed { .. } | HealTaskStatus::Timeout
+                            ) {
+                                break report;
+                            }
+                            tokio::time::sleep(Duration::from_millis(20)).await;
+                        }
+                    })
+                    .await
+                    .expect("mixed task terminal deadline");
+                    manager.stop().await.expect("stop mixed task manager");
+                    for (object, path, original, missing) in originals {
+                        if object == "failed" && missing {
+                            assert!(!path.exists(), "below-R failure must not invent a shard");
+                        } else {
+                            assert_eq!(tokio::fs::read(&path).await.expect("preserved or repaired shard"), original);
+                        }
+                    }
+                    assert!(matches!(report.status, HealTaskStatus::Failed { .. }), "{report:?}");
+                    let outcome = report.outcome.as_ref().expect("canonical mixed outcome");
+                    assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
+                    assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+                    assert_eq!(
+                        (
+                            outcome.counters.processed,
+                            outcome.counters.healed,
+                            outcome.counters.unchanged,
+                            outcome.counters.failed,
+                            outcome.counters.skipped,
+                            outcome.counters.unknown,
+                            outcome.counters.attempt_failures
+                        ),
+                        (3, 1, 1, 1, 0, 0, 4),
+                        "{outcome:?}"
+                    );
+                    for (object, disposition) in [
+                        ("healthy", HealObjectDisposition::VerifiedHealthy),
+                        ("repaired", HealObjectDisposition::Repaired),
+                        ("failed", HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)),
+                    ] {
+                        let receipt = outcome
+                            .objects
+                            .iter()
+                            .find(|item| item.identity.object == object)
+                            .expect("exact object receipt");
+                        assert_eq!(receipt.disposition, disposition);
+                    }
+                    drop(manager);
+                    let restarted = HealManager::new(
+                        storage,
+                        Some(HealConfig {
+                            enable_auto_heal: false,
+                            ..Default::default()
+                        }),
+                    );
+                    restarted.start().await.expect("restart with retained failed batch");
+                    let restored = restarted.get_task_report(&token).await.expect("original token after restart");
+                    assert_eq!(restored.status, report.status);
+                    assert_eq!(restored.outcome, report.outcome);
+                    assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
+                    restarted.stop().await.expect("stop restarted mixed batch manager");
+                });
+        })
+        .expect("mixed batch thread")
+        .join()
+        .expect("mixed batch result");
+}

crates/scanner/src/scanner/tests.rs

@@ -5440,8 +5440,8 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
         .await
         .expect_err("truncated primary must fail in the real object reader");
     assert!(
-        error.to_string().contains("InlineData value out of range"),
-        "unexpected truncated-primary error: {error}"
+        matches!(error, EcstoreError::FileCorrupt),
+        "unexpected truncated-primary error: {error:?}"
     );
     assert_eq!(
         read_config_with_revision(store.clone(), &backup_path)

crates/scanner/src/scanner_folder.rs

@@ -20,10 +20,10 @@ use std::time::{Duration, Instant, SystemTime};
 
 use crate::ReplTargetSizeSummary;
 use crate::data_usage_define::{
-    DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
-    DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH,
-    PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
-    scan_frontier_covers_path,
+    DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageHash,
+    DataUsageHashMap, DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason,
+    MAX_DATA_USAGE_CACHE_DEPTH, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry,
+    SizeSummary, hash_path, scan_frontier_covers_path,
 };
 use crate::error::ScannerError;
 use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter};
@@ -1586,7 +1586,7 @@ impl FolderScanner {
         let candidate_type = pending_scanner_heal_candidate_type(kind);
         let priority = request.priority;
         let scan_mode = request.scan_mode.unwrap_or(self.scan_mode);
-        let result = match send_scanner_heal_request(candidate_type, request).await {
+        let result = match send_scanner_heal_request(candidate_type, request, scanner_disk_heal_scope(&self.local_disk)).await {
             Ok(result) => result,
             Err(err) => {
                 self.update_pending_scanner_heal_after_admission(
@@ -2178,6 +2178,7 @@ impl FolderScanner {
                     lifecycle: active_life_cycle.clone(),
                     object_lock: active_object_lock.clone(),
                     replication: active_replication.clone(),
+                    heal_scope: None,
                     heal_enabled,
                     heal_bitrot: self.scan_mode == HealScanMode::Deep,
                     debug: self.data_usage_scanner_debug,
@@ -3405,6 +3406,7 @@ pub(crate) async fn scan_data_folder_scoped(
 mod item_actions;
 mod ledger;
 
+pub(crate) use item_actions::scanner_disk_heal_scope;
 use item_actions::*;
 pub use item_actions::{GetSizeFn, ScannerItem};
 use ledger::*;

crates/scanner/src/scanner_folder/item_actions.rs

@@ -474,10 +474,23 @@ pub(super) fn record_heal_candidate_admission(
     }
 }
 
+pub(crate) fn scanner_disk_heal_scope(disk: &Disk) -> Option<DataUsageCacheSource> {
+    let location = disk.get_disk_location();
+    location.valid().then_some(())?;
+    Some(DataUsageCacheSource::new(location.pool_idx?, location.set_idx?))
+}
+
 pub(super) async fn send_scanner_heal_request(
     candidate_type: &'static str,
-    request: HealChannelRequest,
+    mut request: HealChannelRequest,
+    scope: Option<DataUsageCacheSource>,
 ) -> Result<HealAdmissionResult, ScannerError> {
+    if request.object_prefix.as_deref().is_some_and(|object| !object.is_empty()) {
+        let scope = scope
+            .ok_or_else(|| ScannerError::Other("scanner object heal requires the scanning disk's pool and set".to_string()))?;
+        request.pool_index = Some(scope.pool_index);
+        request.set_index = Some(scope.set_index);
+    }
     let priority = request.priority;
     let trace_context = scanner_heal_candidate_trace_context(&request);
     let candidate = crate::heal_completion::prepare(&request);
@@ -537,6 +550,8 @@ pub struct ScannerItem {
     pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
     pub object_lock: Option<Arc<ObjectLockConfiguration>>,
     pub replication: Option<Arc<ReplicationConfig>>,
+    /// Actual scanning-disk location; refreshed by the disk reader before actions.
+    pub heal_scope: Option<DataUsageCacheSource>,
     pub heal_enabled: bool,
     pub heal_bitrot: bool,
     pub debug: bool,
@@ -1161,6 +1176,7 @@ impl ScannerItem {
                 scan_mode,
                 HealChannelPriority::Low,
             ),
+            self.heal_scope,
         )
         .await;
 
@@ -1322,6 +1338,7 @@ mod tests {
             lifecycle: None,
             object_lock: None,
             replication: None,
+            heal_scope: None,
             heal_enabled: false,
             heal_bitrot: false,
             debug: false,

crates/scanner/src/scanner_folder/tests.rs

@@ -336,7 +336,10 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         .await
         .expect("failed to create test directory");
 
-    let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
+    let mut endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(0);
     let disk = new_disk(
         &endpoint,
         &DiskOption {
@@ -768,6 +771,7 @@ fn test_classify_get_size_failure_marks_metadata_heal_object_path() {
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -799,6 +803,7 @@ fn test_classify_get_size_failure_records_transient_metadata_error_without_heal(
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -855,6 +860,7 @@ async fn test_heal_replication_skips_only_unversioned_data_versions() {
         lifecycle: None,
         object_lock: None,
         replication: Some(Arc::new(ReplicationConfig::new(None, None))),
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -2265,6 +2271,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() {
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: true,
         heal_bitrot: true,
         debug: false,
@@ -2509,7 +2516,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
     let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
     let heal_starts = Arc::new(AtomicUsize::new(0));
     let heal_starts_clone = heal_starts.clone();
-    let healed_versions = Arc::new(Mutex::new(Vec::<Option<String>>::new()));
+    let healed_versions = Arc::new(Mutex::new(Vec::new()));
     let healed_versions_clone = healed_versions.clone();
     let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels()
         .expect("heal channels should initialize once for scanner tests");
@@ -2521,7 +2528,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
                 healed_versions_clone
                     .lock()
                     .expect("heal version capture lock should not be poisoned")
-                    .push(request.object_version_id);
+                    .push((request.object_prefix, request.object_version_id, request.pool_index, request.set_index));
                 let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt {
                     result: HealAdmissionResult::Accepted,
                     task_id: request.id,
@@ -2602,9 +2609,90 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
             .lock()
             .expect("heal version capture lock should not be poisoned")
             .iter()
-            .any(|version| version.as_deref() == Some(orphan_version_text.as_str())),
+            .any(|(_, version, _, _)| version.as_deref() == Some(orphan_version_text.as_str())),
         "sub-quorum orphan version must be submitted as an exact heal candidate"
     );
+    assert!(
+        healed_versions
+            .lock()
+            .expect("heal scope capture lock should not be poisoned")
+            .iter()
+            .filter(|(object, _, _, _)| object.is_some())
+            .all(|(_, _, pool, set)| (*pool, *set) == (Some(0), Some(0))),
+        "high-priority discovered objects must retain the scanning disk's scope"
+    );
+    let bucket_admission = scanner
+        .send_required_scanner_heal_request(
+            PendingScannerHealKind::Bucket,
+            bucket.to_string(),
+            None,
+            None,
+            build_bucket_heal_request(bucket.to_string(), HealChannelPriority::High),
+        )
+        .await
+        .expect("submit bucket request");
+    assert_eq!(bucket_admission, HealAdmissionResult::Accepted);
+    assert_eq!(
+        healed_versions.lock().expect("read captured bucket scope").last(),
+        Some(&(None, None, None, None)),
+        "bucket healing must retain its cross-pool scope"
+    );
+    scanner.new_cache.info.name = bucket.to_string();
+    scanner.new_cache.info.pending_heals = vec![pending_heal(
+        PendingScannerHealKind::Object,
+        bucket,
+        Some("pending-object"),
+        Some(&orphan_version_text),
+        0,
+        1,
+    )];
+    scanner
+        .retry_pending_scanner_heals()
+        .await
+        .expect("retry persisted object heal");
+    assert_eq!(
+        healed_versions.lock().expect("read retried object scope").last(),
+        Some(&(Some("pending-object".to_string()), Some(orphan_version_text.clone()), Some(0), Some(0))),
+        "persisted retries must bind the current scanning disk"
+    );
+    let unscoped_endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("parse uninitialized endpoint");
+    scanner.local_disk = new_disk(
+        &unscoped_endpoint,
+        &DiskOption {
+            cleanup: false,
+            health_check: false,
+        },
+    )
+    .await
+    .expect("open disk without initialized location");
+    let request_count = healed_versions.lock().expect("count admitted requests").len();
+    let deferred = scanner
+        .send_required_scanner_heal_request(
+            PendingScannerHealKind::Object,
+            bucket.to_string(),
+            Some("unscoped-object".to_string()),
+            Some(orphan_version_text.clone()),
+            build_object_heal_request(
+                bucket.to_string(),
+                "unscoped-object".to_string(),
+                Some(orphan_version_text),
+                HealScanMode::Deep,
+                HealChannelPriority::High,
+            ),
+        )
+        .await
+        .expect("defer object with unknown disk location");
+    assert_eq!(deferred, HealAdmissionResult::Full);
+    assert_eq!(healed_versions.lock().expect("count unchanged admissions").len(), request_count);
+    assert!(
+        scanner
+            .new_cache
+            .info
+            .pending_heals
+            .iter()
+            .any(|entry| entry.object.as_deref() == Some("unscoped-object")),
+        "an unscoped high-priority object must remain pending"
+    );
     for (path, expected) in expected_metadata {
         assert_eq!(
             tokio::fs::read(&path)

crates/scanner/src/scanner_io/io_disk.rs

@@ -130,6 +130,8 @@ impl ScannerIODisk for Disk {
         // `object_infos`.
         global_metrics().record_scanner_versions_scanned(object_infos.len() as u64);
 
+        // A caller-provided item or persisted usage cache cannot choose another set.
+        item.heal_scope = crate::scanner_folder::scanner_disk_heal_scope(self);
         item.apply_actions(object_infos, lock_config, versioning_config, tier_names, &mut size_summary)
             .await;
 

crates/scanner/src/scanner_io/tests.rs

@@ -1770,6 +1770,7 @@ async fn scanner_item_object_lock_uses_cached_config() {
         lifecycle: None,
         object_lock: Some(cached.clone()),
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4378,6 +4379,7 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4433,6 +4435,7 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -4515,6 +4518,7 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,

crates/scanner/tests/lifecycle/scanner_heal_scope.rs

@@ -0,0 +1,144 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_filemeta::{FileInfo, FileMeta};
+use rustfs_heal_contracts::heal_channel::{
+    HealAdmissionReceipt, HealAdmissionResult, HealReceiptCommand, HealRequestSource, init_heal_channels,
+};
+use rustfs_scanner::DataUsageCacheSource;
+use rustfs_scanner::scanner_folder::ScannerItem;
+use rustfs_scanner::scanner_io::ScannerIODisk;
+use tokio::sync::mpsc;
+
+use super::storage_api::lifecycle::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
+
+#[test]
+fn scanner_object_requests_bind_the_actual_disk_scope() {
+    const CHILD: &str = "RUSTFS_TEST_SCANNER_SCOPE_CHILD";
+    if std::env::var_os(CHILD).is_none() {
+        let status = std::process::Command::new(std::env::current_exe().expect("locate integration test executable"))
+            .args([
+                "--exact",
+                "scanner_heal_scope::scanner_object_requests_bind_the_actual_disk_scope",
+                "--nocapture",
+            ])
+            .env(CHILD, "1")
+            .status()
+            .expect("run scanner scope regression in an isolated process");
+        assert!(status.success(), "isolated scanner scope regression failed: {status}");
+        return;
+    }
+    tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .expect("build isolated scanner runtime")
+        .block_on(check_scanner_object_scope());
+}
+
+async fn check_scanner_object_scope() {
+    let (_commands, mut receipts) = init_heal_channels().expect("initialize isolated scanner request channels");
+    let (captured, mut requests) = mpsc::unbounded_channel();
+    let responder = tokio::spawn(async move {
+        while let Some(command) = receipts.recv().await {
+            let HealReceiptCommand { request, response_tx } = command;
+            let receipt = HealAdmissionReceipt {
+                result: HealAdmissionResult::Accepted,
+                task_id: request.id.clone(),
+            };
+            captured.send(request).expect("retain scanner request");
+            let _ = response_tx.send(Ok(receipt));
+        }
+    });
+
+    for (pool, set, disk_index) in [
+        (Some(0), Some(0), Some(1)),
+        (Some(2), Some(3), Some(4)),
+        (None, Some(3), Some(4)),
+        (Some(2), None, Some(4)),
+        (Some(2), Some(3), None),
+        (None, None, None),
+    ] {
+        for version_id in [None, Some(uuid::Uuid::from_u128(42))] {
+            let root = tempfile::tempdir().expect("create scanner disk");
+            let bucket = "scanner-scope-bucket";
+            let object = "object";
+            let path = root.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE);
+            tokio::fs::create_dir_all(path.parent().expect("metadata parent"))
+                .await
+                .expect("create object directory");
+            let mut info = FileInfo::new(object, 4, 2);
+            info.volume = bucket.to_string();
+            info.name = object.to_string();
+            info.size = 1;
+            info.version_id = version_id;
+            info.data_dir = Some(uuid::Uuid::new_v4());
+            info.erasure.index = 1;
+            info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1).expect("valid object timestamp"));
+            info.add_object_part(1, "fixture-etag".to_string(), 1, info.mod_time, 1, None, None);
+            let mut meta = FileMeta::new();
+            meta.add_version(info).expect("encode original null version");
+            let original = meta.marshal_msg().expect("marshal original metadata");
+            tokio::fs::write(&path, &original).await.expect("write original metadata");
+            let mut endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("parse scanning disk endpoint");
+            if let Some(pool) = pool {
+                endpoint.set_pool_index(pool);
+            }
+            if let Some(set) = set {
+                endpoint.set_set_index(set);
+            }
+            if let Some(disk_index) = disk_index {
+                endpoint.set_disk_index(disk_index);
+            }
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("open actual scanning disk");
+            let item = ScannerItem {
+                path: path.to_string_lossy().to_string(),
+                bucket: bucket.to_string(),
+                prefix: object.to_string(),
+                object_name: STORAGE_FORMAT_FILE.to_string(),
+                file_type: tokio::fs::metadata(&path).await.expect("stat original metadata").file_type(),
+                lifecycle: None,
+                object_lock: None,
+                replication: None,
+                // This foreign caller-provided scope must never reach admission.
+                heal_scope: Some(DataUsageCacheSource::new(99, 98)),
+                heal_enabled: true,
+                heal_bitrot: true,
+                debug: false,
+            };
+            disk.get_size_with_tier_names(item, &[]).await.expect("scan original object");
+            if let (Some(pool), Some(set), Some(_)) = (pool, set, disk_index) {
+                let request = requests.try_recv().expect("scanner submitted one real object request");
+                assert_eq!(request.pool_index, Some(pool));
+                assert_eq!(request.set_index, Some(set));
+                assert_eq!(request.bucket, bucket);
+                assert_eq!(request.object_prefix.as_deref(), Some(object));
+                assert_eq!(request.object_version_id, version_id.map(|value| value.to_string()));
+                assert_eq!(request.source, HealRequestSource::Scanner);
+                assert_eq!(request.no_lock, None);
+                assert_eq!(request.recreate_missing, Some(false));
+            }
+            assert!(requests.try_recv().is_err(), "no duplicate or unscoped request may be admitted");
+            assert_eq!(tokio::fs::read(&path).await.expect("read original metadata"), original);
+        }
+    }
+    responder.abort();
+}

crates/scanner/tests/lifecycle_integration_test.rs

@@ -39,6 +39,9 @@ use uuid::Uuid;
 
 mod storage_api;
 
+#[path = "lifecycle/scanner_heal_scope.rs"]
+mod scanner_heal_scope;
+
 use storage_api::lifecycle::{
     BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart,
     DeleteAfterObjectLockSnapshotBarrier, DiskOption, ECStore, EcstoreError, Endpoint, EndpointServerPools, Endpoints,
@@ -467,6 +470,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
         lifecycle,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,
@@ -504,6 +508,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
         lifecycle: None,
         object_lock: None,
         replication: None,
+        heal_scope: None,
         heal_enabled: false,
         heal_bitrot: false,
         debug: false,

docs/architecture/heal-terminal-reports.md

@@ -4,6 +4,10 @@ Admin heal tokens retain their terminal status for ten minutes from the original
 
 ## Publication and Recovery
 
+Authenticated token queries first inspect the receiving node's in-memory automatic tasks. An original Scanner, MRF, automatic-Heal, or read-repair token can return its path-bound snapshot on that node, including the retained outcome and incremental cursor. The task source and original ID remain attached to the completion snapshot; aliases cannot enter this local query path. The local lookup never reads administrator recovery records.
+
+Administrator tokens and unknown local identities continue through the cluster coordinator. Starting and cancelling tasks retain their coordinator route. Querying another node does not discover or aggregate a remote automatic task, and this path does not extend automatic-task retention across process restart.
+
 The original schema-1 `terminal-root-heal-<token>.json` remains the commit marker. Its separate `heal-terminal-report-<token>.json` report has its own schema version and embeds the exact terminal identity. Both files belong to the same coordinator disk as the pending intent.
 
 Publication writes the report through the storage owner's conditional file update, writes the terminal marker, and then conditionally removes the pending intent. Recovery never treats a report without its marker as committed. A committed terminal continues to suppress stale pending work even when its report is unreadable. An initial publication failure leaves the existing responsibility available for recovery.

docs/testing/README.md

@@ -32,6 +32,8 @@ The [Pool layout compatibility reference](pool-layout-compatibility.md) defines
 
 The [Scanner/Heal topology protection gates](scanner-heal-topology-gates.md) map G-EXP, G-REB and G-DEC and their fault windows to the `scanner_heal_topology_` cases, their recorded inventory, and the basis of their timeouts.
 
+The [immutable validation candidate reference](validation-candidates.md) describes isolated package builds and exact candidate resolution for the current auto-testing fleet.
+
 The [distributed multipart RPC delay gates](multipart-rpc-delay.md) define authenticated method-entry barriers, latency and peer-loss oracles, and the boundary between hook evidence and production incident attribution.
 
 ## Naming conventions

docs/testing/validation-candidates.md

@@ -0,0 +1,52 @@
+# Immutable validation candidates
+
+Canonical validation on the current auto-testing fleet consumes one exact
+package, source commit and candidate-bound tool build. A successful package
+build is provenance evidence, not a canonical test PASS.
+
+The scheduled Functional Chain continues to resolve successful Nightly GNU
+Build attempts from `main`. Optional manual validation can also resolve a
+validation package built without publishing nightly aliases or release assets.
+
+## Build a validation package
+
+Dispatch **RustFS Functional Chain** from the desired driver revision with
+only `validation_package_sha` set to the full 40-character RustFS source SHA.
+Keep candidate, case, helper, tools and cleanup inputs empty or false. The
+request validator rejects mixed package/fleet execution before building.
+
+The package job calls `rustfs-validation-package.yml`, checks out the requested
+source, confirms its Git SHA and builds the ordinary release binary and DEB.
+This job uses build runners and does not acquire or mutate the Heal fleet.
+The same workflow can be dispatched directly with its `source_sha` input when
+it is available on the repository's default branch.
+
+Publication requires complete R2 configuration, conditional `PutObject` with
+`If-None-Match: *`, and a public download whose SHA-256 matches the local DEB.
+Failure emits no candidate manifest. An existing object is never overwritten.
+The only object path is:
+
+```text
+artifacts/rustfs/packages/validation/runs/<run>/<attempt>/<package-sha256>/rustfs.deb
+```
+
+The schema-3 artifact `validation-candidate-<run>-<attempt>` records the
+workflow SHA independently of the exact source SHA and source ref, the
+`validation` channel, build run/attempt, immutable URL and package checksum.
+The resolver verifies the successful manual run, repository/workflow identity,
+exact artifact name and digest, archive members, schema and immutable URL.
+The ordinary scheduled chain does not enable validation package resolution.
+
+## Validate the candidate
+
+After the package run succeeds, dispatch Functional Chain with its exact
+`build_run_id` and `build_run_attempt`, a canonical `validation_case`, and an
+exact `auto_testing_sha`. Heal also requires a new tool build verified against
+that source and helper revision. Reassess source-bound contracts and native
+prerequisites when the source changes; an older tool artifact does not certify
+a new candidate.
+
+The fleet lane retains the shared environment lock, per-case evidence and
+cleanup gate. Record new results separately. A package build, native test,
+diagnostic replay, clean recovery or partial topology result cannot replace
+complete original source assertions or rewrite a retained canonical FAIL.

rustfs/src/admin/handlers/heal.rs

@@ -1051,6 +1051,21 @@ async fn route_cluster_heal_control(
         })
 }
 
+async fn query_heal_status_on_owner<L, C, CF>(
+    local: L,
+    coordinator: C,
+) -> S3Result<rustfs_heal_contracts::heal_channel::HealChannelResponse>
+where
+    L: Future<Output = S3Result<Option<rustfs_heal_contracts::heal_channel::HealChannelResponse>>>,
+    C: FnOnce() -> CF,
+    CF: Future<Output = S3Result<rustfs_heal_contracts::heal_channel::HealChannelResponse>>,
+{
+    match local.await? {
+        Some(response) => Ok(response),
+        None => coordinator().await,
+    }
+}
+
 async fn execute_after_heal_start_preflight<P, PF, E, EF, T>(
     endpoints: &EndpointServerPools,
     options: &HealOpts,
@@ -1514,20 +1529,41 @@ impl Operation for HealHandler {
         let heal_path = encode_heal_control_path(&hip.bucket, &hip.obj_prefix);
         if !hip.client_token.is_empty() && !hip.force_start && !hip.force_stop {
             let client_token = hip.client_token.clone();
-            let request_id = uuid::Uuid::new_v4().to_string();
-            let context = app_context
-                .clone()
-                .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?;
-            let route = prepare_heal_control_route(&context)?;
-            let envelope = rustfs_protos::heal_control::Envelope::query(
-                request_id.clone(),
-                new_heal_control_metadata(&route)?,
-                heal_path,
-                client_token.clone(),
-                hip.since_seq,
+            // Automatic workers run on their owning node. Only an exact
+            // path/token/source snapshot can answer locally; Admin tokens and
+            // unknown identities retain the coordinator authority boundary.
+            let response = query_heal_status_on_owner(
+                async {
+                    let Some(manager) = rustfs_heal::get_heal_manager() else {
+                        return Ok(None);
+                    };
+                    let processor = rustfs_heal::HealChannelProcessor::new(manager.clone());
+                    timeout(
+                        PEER_HEAL_STATUS_TIMEOUT,
+                        processor.execute_local_automatic_query(&heal_path, &client_token, hip.since_seq),
+                    )
+                    .await
+                    .map_err(|_| cluster_heal_control_unavailable("local_task_query_expired"))?
+                    .map_err(|_| cluster_heal_control_unavailable("local_task_query_failed"))
+                },
+                || async {
+                    let request_id = uuid::Uuid::new_v4().to_string();
+                    let context = app_context
+                        .clone()
+                        .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?;
+                    let route = prepare_heal_control_route(&context)?;
+                    let envelope = rustfs_protos::heal_control::Envelope::query(
+                        request_id.clone(),
+                        new_heal_control_metadata(&route)?,
+                        heal_path.clone(),
+                        client_token.clone(),
+                        hip.since_seq,
+                    )
+                    .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?;
+                    submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token.clone()).await
+                },
             )
-            .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?;
-            let response = submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token).await?;
+            .await?;
             if !response.success {
                 return Err(s3_error!(
                     InternalError,
@@ -1995,6 +2031,54 @@ mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
     use uuid::Uuid;
 
+    #[tokio::test]
+    async fn local_automatic_heal_query_preserves_owner_and_coordinator_errors() {
+        let response = || rustfs_heal_contracts::heal_channel::HealChannelResponse {
+            request_id: "original-token".to_string(),
+            success: true,
+            data: Some(b"retained-owner-report".to_vec()),
+            error: None,
+        };
+        let called = AtomicBool::new(false);
+        let local = super::query_heal_status_on_owner(async { Ok(Some(response())) }, || async {
+            called.store(true, Ordering::SeqCst);
+            Ok(response())
+        })
+        .await
+        .expect("local automatic owner response");
+        assert!(!called.load(Ordering::SeqCst));
+        assert_eq!(local.request_id, "original-token");
+        assert_eq!(local.data.as_deref(), Some(b"retained-owner-report".as_slice()));
+
+        let fallback = super::query_heal_status_on_owner(async { Ok(None) }, || async {
+            called.store(true, Ordering::SeqCst);
+            Ok(response())
+        })
+        .await
+        .expect("unknown local identity uses coordinator");
+        assert!(called.swap(false, Ordering::SeqCst));
+        assert_eq!(fallback.data, local.data);
+
+        let error = super::query_heal_status_on_owner(
+            async { Err(super::admin_error(S3ErrorCode::InternalError, "local lookup failed")) },
+            || async {
+                called.store(true, Ordering::SeqCst);
+                Ok(response())
+            },
+        )
+        .await
+        .expect_err("local failure must not become coordinator success");
+        assert_eq!(error.code(), &S3ErrorCode::InternalError);
+        assert!(!called.load(Ordering::SeqCst));
+
+        let error = super::query_heal_status_on_owner(async { Ok(None) }, || async {
+            Err(super::admin_error(S3ErrorCode::ServiceUnavailable, "coordinator unavailable"))
+        })
+        .await
+        .expect_err("coordinator error must remain visible");
+        assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
+    }
+
     #[test]
     fn mrf_legacy_only_refresh_accepts_an_unobserved_incarnation() {
         for action in ["refresh", "recheck", "acceptUnverifiedRisk"] {

rustfs/src/app/object/get.rs

@@ -3141,7 +3141,10 @@ impl DefaultObjectUsecase {
             // resolver; only the response headers, authorization and audit
             // summary are derived here, without a second KMS unwrap.
             Some(headers) => (
-                Some(headers.server_side_encryption),
+                headers
+                    .sse_customer_algorithm
+                    .is_none()
+                    .then_some(headers.server_side_encryption),
                 headers.sse_customer_algorithm,
                 headers.sse_customer_key_md5,
                 headers.ssekms_key_id,

rustfs/src/app/object/head.rs

@@ -462,12 +462,13 @@ impl DefaultObjectUsecase {
         })?;
 
         let metadata_map = info.user_defined.clone();
-        let server_side_encryption = metadata_map
-            .get("x-amz-server-side-encryption")
-            .map(|v| ServerSideEncryption::from(v.clone()));
         let sse_customer_algorithm = metadata_map
             .get("x-amz-server-side-encryption-customer-algorithm")
             .map(|v| SSECustomerAlgorithm::from(v.clone()));
+        let server_side_encryption = metadata_map
+            .get("x-amz-server-side-encryption")
+            .filter(|_| sse_customer_algorithm.is_none())
+            .map(|v| ServerSideEncryption::from(v.clone()));
         let sse_customer_key_md5 = metadata_map.get("x-amz-server-side-encryption-customer-key-md5").cloned();
         let sse_kms_key_id = metadata_map.get("x-amz-server-side-encryption-aws-kms-key-id").cloned();
         let storage_class = response_storage_class(&info, &metadata_map);

rustfs/src/app/object/put.rs

@@ -1435,7 +1435,10 @@ impl DefaultObjectUsecase {
 
         let output = PutObjectOutput {
             e_tag,
-            server_side_encryption: committed.effective_sse.clone(),
+            server_side_encryption: committed
+                .effective_sse
+                .clone()
+                .filter(|_| committed.sse_customer_algorithm.is_none()),
             sse_customer_algorithm: committed.sse_customer_algorithm.clone(),
             sse_customer_key_md5: committed.sse_customer_key_md5.clone(),
             ssekms_key_id: committed.effective_kms_key_id.clone(),
@@ -3964,6 +3967,12 @@ mod tests {
     async fn sse_c_put_is_accepted_on_a_bucket_with_default_encryption() {
         use md5::{Digest as _, Md5};
 
+        let context = crate::storage::storage_api::bootstrap_instance_ctx();
+        assert!(
+            context.object_encryption_resolver().is_some(),
+            "real SSE-C reads need the production encryption resolver"
+        );
+
         for (algorithm, kms_key_id, prefix) in [
             (ServerSideEncryption::AES256, None, "ssec-over-aes256-default"),
             (ServerSideEncryption::AWS_KMS, Some("bucket-key"), "ssec-over-kms-default"),
@@ -3985,14 +3994,69 @@ mod tests {
                 .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
                 .sse_customer_algorithm(Some("AES256".to_string()))
                 .sse_customer_key(Some(base64_simd::STANDARD.encode_to_string(customer_key)))
-                .sse_customer_key_md5(Some(key_md5))
+                .sse_customer_key_md5(Some(key_md5.clone()))
                 .build()
                 .expect("SSE-C PUT input must build");
 
-            DefaultObjectUsecase::from_global()
+            let usecase = DefaultObjectUsecase::from_global();
+            let response = usecase
                 .execute_put_object(&FS::new(), build_request(input, Method::PUT))
                 .await
                 .unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}"));
+            assert_eq!(response.output.server_side_encryption, None);
+            assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256"));
+            assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
+
+            let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key);
+            let mut read_headers = HeaderMap::new();
+            read_headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
+            read_headers.insert(
+                "x-amz-server-side-encryption-customer-key",
+                HeaderValue::from_str(&encoded_key).expect("encoded customer key is a valid header"),
+            );
+            read_headers.insert(
+                "x-amz-server-side-encryption-customer-key-md5",
+                HeaderValue::from_str(&key_md5).expect("customer key MD5 is a valid header"),
+            );
+            let head_input = HeadObjectInput::builder()
+                .bucket(bucket.clone())
+                .key("ledger.csv".to_string())
+                .sse_customer_algorithm(Some("AES256".to_string()))
+                .sse_customer_key(Some(encoded_key.clone()))
+                .sse_customer_key_md5(Some(key_md5.clone()))
+                .build()
+                .expect("SSE-C HEAD input must build");
+            let mut head_request = build_request(head_input, Method::HEAD);
+            head_request.headers = read_headers.clone();
+            let head = Box::pin(usecase.execute_head_object(head_request))
+                .await
+                .expect("matching SSE-C key must authorize HEAD");
+            assert_eq!(head.output.server_side_encryption, None);
+            assert_eq!(head.output.sse_customer_algorithm.as_deref(), Some("AES256"));
+            assert_eq!(head.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
+
+            let get_input = GetObjectInput::builder()
+                .bucket(bucket.clone())
+                .key("ledger.csv".to_string())
+                .sse_customer_algorithm(Some("AES256".to_string()))
+                .sse_customer_key(Some(encoded_key))
+                .sse_customer_key_md5(Some(key_md5.clone()))
+                .build()
+                .expect("SSE-C GET input must build");
+            let mut get_request = build_request(get_input, Method::GET);
+            get_request.headers = read_headers;
+            let get = Box::pin(usecase.execute_get_object(get_request))
+                .await
+                .expect("matching SSE-C key must authorize GET");
+            assert_eq!(get.output.server_side_encryption, None);
+            assert_eq!(get.output.sse_customer_algorithm.as_deref(), Some("AES256"));
+            assert_eq!(get.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
+            let mut body = get.output.body.expect("SSE-C GET must include the decrypted body");
+            let mut plaintext = Vec::new();
+            while let Some(chunk) = body.next().await {
+                plaintext.extend_from_slice(&chunk.expect("SSE-C body must decrypt completely"));
+            }
+            assert_eq!(plaintext, payload, "header classification must preserve the complete decrypted payload");
 
             let stored = store
                 .get_object_info(&bucket, "ledger.csv", &ObjectOptions::default())

rustfs/src/error.rs

@@ -310,6 +310,7 @@ impl std::error::Error for ApiError {
 pub(crate) struct ApiErrorDiagnostic {
     pub storage_code: Option<StorageErrorCode>,
     pub io_kind: Option<std::io::ErrorKind>,
+    pub io_os_error: Option<i32>,
     pub rpc_code: Option<tonic::Code>,
     pub pool_metadata: Option<PoolMetadataDiagnostic>,
     pub truncated: bool,
@@ -333,10 +334,16 @@ impl From<&PoolMetadataError> for PoolMetadataDiagnostic {
     }
 }
 
-impl ApiError {
-    pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic {
+impl ApiErrorDiagnostic {
+    pub(crate) fn from_error(error: &(dyn std::error::Error + 'static)) -> Self {
         let mut diagnostic = ApiErrorDiagnostic::default();
-        let mut current = std::error::Error::source(self);
+        // S3Error retains its payload through an inherent accessor; its
+        // std::error::Error implementation does not expose that source.
+        let mut current = if let Some(error) = error.downcast_ref::<S3Error>() {
+            error.source().map(|source| source as &(dyn std::error::Error + 'static))
+        } else {
+            std::error::Error::source(error)
+        };
         for _ in 0..16 {
             let Some(error) = current else {
                 return diagnostic;
@@ -354,8 +361,11 @@ impl ApiError {
             }
             current = if let Some(io) = error.downcast_ref::<std::io::Error>() {
                 diagnostic.io_kind = Some(io.kind());
+                diagnostic.io_os_error = io.raw_os_error();
                 // io::Error::source can skip the wrapped error itself.
                 io.get_ref().map(|source| source as &(dyn std::error::Error + 'static))
+            } else if let Some(error) = error.downcast_ref::<S3Error>() {
+                error.source().map(|source| source as &(dyn std::error::Error + 'static))
             } else {
                 error.source()
             };
@@ -363,6 +373,12 @@ impl ApiError {
         diagnostic.truncated = current.is_some();
         diagnostic
     }
+}
+
+impl ApiError {
+    pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic {
+        ApiErrorDiagnostic::from_error(self)
+    }
 
     /// Access-denied error with the exact message emitted by the authorization
     /// paths in `storage::access`; callers there match on the code only.
@@ -555,6 +571,7 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
     let mut body_size_exceeded = false;
     let mut aws_chunked_error = None;
     let mut incomplete_body = false;
+    let mut rpc_unavailable = false;
     let digest = error_chain_find(err, |error| {
         let upload_error = error.downcast_ref::<UploadStreamError>();
         if matches!(upload_error, Some(UploadStreamError::Sha256Mismatch)) || error.is::<rustfs_rio::Sha256Mismatch>() {
@@ -564,6 +581,9 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
             return Some(S3ErrorCode::BadDigest);
         }
 
+        rpc_unavailable |= error
+            .downcast_ref::<tonic::Status>()
+            .is_some_and(|status| status.code() == tonic::Code::Unavailable);
         upload_limit_exceeded |= error.is::<UploadLimitExceeded>();
         server_source_failed |= error.is::<ServerSideSourceReadError>();
         client_body_timed_out |= error.is::<ClientBodyReadTimeout>();
@@ -591,6 +611,8 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option<S3Err
             Some(code)
         } else if incomplete_body {
             Some(S3ErrorCode::IncompleteBody)
+        } else if rpc_unavailable {
+            Some(S3ErrorCode::ServiceUnavailable)
         } else {
             None
         }
@@ -1241,6 +1263,35 @@ mod tests {
         assert!(!format!("{:?}", error.diagnostic()).contains("secret"));
     }
 
+    #[test]
+    fn api_error_diagnostic_survives_s3_conversion_without_changing_response() {
+        let api = ApiError::from(StorageError::Io(IoError::from_raw_os_error(30)));
+        let before = api.diagnostic();
+        assert_eq!(before.io_os_error, Some(30));
+        let mut error: S3Error = api.into();
+        error.set_request_id("server-owned-request");
+        let code = error.code().clone();
+        let status = error.status_code();
+        let message = error.message().map(str::to_owned);
+        assert_eq!(super::ApiErrorDiagnostic::from_error(&error), before);
+        assert_eq!(error.code(), &code);
+        assert_eq!(error.status_code(), status);
+        assert_eq!(error.message(), message.as_deref());
+        assert_eq!(error.request_id(), Some("server-owned-request"));
+        assert!(error.source().is_some());
+        assert!(std::error::Error::source(&error).is_none());
+
+        let error: S3Error = ApiError::from(StorageError::Io(IoError::other(StorageError::ErasureReadQuorum))).into();
+        assert_eq!(
+            super::ApiErrorDiagnostic::from_error(&error).storage_code,
+            Some(StorageErrorCode::ErasureReadQuorum)
+        );
+
+        let nested: S3Error = ApiError::from(StorageError::ErasureReadQuorum).into();
+        let api = ApiError::from(StorageError::Io(IoError::other(nested)));
+        assert_eq!(api.diagnostic().storage_code, Some(StorageErrorCode::ErasureReadQuorum));
+    }
+
     #[test]
     fn api_error_diagnostic_bounds_cyclic_error_chains() {
         #[derive(Debug)]
@@ -1262,6 +1313,11 @@ mod tests {
         };
         assert!(error.diagnostic().truncated);
         assert!(!format!("{:?}", error.diagnostic()).contains("secret"));
+        let error: S3Error = error.into();
+        let diagnostic = super::ApiErrorDiagnostic::from_error(&error);
+        assert!(diagnostic.truncated);
+        assert!(!format!("{diagnostic:?}").contains("secret"));
+
         let error = S3Error::from(ApiError {
             code: S3ErrorCode::ServiceUnavailable,
             message: "safe".into(),
@@ -1572,6 +1628,85 @@ mod tests {
         assert!(error_chain_has_type::<AwsChunkedStreamError>(&error));
     }
 
+    #[test]
+    fn rpc_unavailable_preserves_service_refusal_across_io_boundaries() {
+        let status = || tonic::Status::unavailable("private transport detail must not reach the S3 response");
+        let storage = || StorageError::from(status());
+        let original = storage();
+        let cloned = original.clone();
+        for error in [
+            ApiError::from(original),
+            ApiError::from(cloned),
+            ApiError::from(StorageError::Io(IoError::other(IoError::other(status())))),
+            ApiError::from(IoError::other(status())),
+            ApiError::from(IoError::other(storage())),
+        ] {
+            assert_eq!(error.code, S3ErrorCode::ServiceUnavailable);
+            assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable));
+            assert_eq!(error.diagnostic().rpc_code, Some(tonic::Code::Unavailable));
+            let s3_error = S3Error::from(error);
+            assert_eq!(s3_error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
+        }
+    }
+
+    #[test]
+    fn rpc_unavailable_mapping_does_not_match_other_codes_or_untyped_messages() {
+        for code in [
+            tonic::Code::Cancelled,
+            tonic::Code::Unknown,
+            tonic::Code::InvalidArgument,
+            tonic::Code::DeadlineExceeded,
+            tonic::Code::NotFound,
+            tonic::Code::AlreadyExists,
+            tonic::Code::PermissionDenied,
+            tonic::Code::ResourceExhausted,
+            tonic::Code::FailedPrecondition,
+            tonic::Code::Aborted,
+            tonic::Code::OutOfRange,
+            tonic::Code::Unimplemented,
+            tonic::Code::Internal,
+            tonic::Code::DataLoss,
+            tonic::Code::Unauthenticated,
+        ] {
+            for error in [
+                ApiError::from(StorageError::from(tonic::Status::new(code, "rpc unavailable"))),
+                ApiError::from(IoError::other(tonic::Status::new(code, "rpc unavailable"))),
+            ] {
+                assert_eq!(error.code, S3ErrorCode::InternalError, "{code:?}");
+            }
+        }
+        for error in [
+            ApiError::from(StorageError::Io(IoError::other("status: Unavailable"))),
+            ApiError::from(IoError::other("status: Unavailable")),
+        ] {
+            assert_eq!(error.code, S3ErrorCode::InternalError);
+        }
+    }
+
+    #[test]
+    fn rpc_unavailable_mapping_keeps_client_digest_error_precedence() {
+        for sha256 in [false, true] {
+            let source = || {
+                let mut status = tonic::Status::unavailable("private transport failure");
+                if sha256 {
+                    status.set_source(std::sync::Arc::new(UploadStreamError::Sha256Mismatch));
+                } else {
+                    status.set_source(std::sync::Arc::new(rustfs_rio::BadDigest {
+                        expected_md5: "expected".into(),
+                        calculated_md5: "calculated".into(),
+                    }));
+                }
+                status
+            };
+            for error in [
+                ApiError::from(StorageError::Io(IoError::other(source()))),
+                ApiError::from(IoError::other(source())),
+            ] {
+                assert_eq!(error.code.as_str(), if sha256 { CONTENT_SHA256_MISMATCH_CODE } else { "BadDigest" });
+            }
+        }
+    }
+
     #[test]
     fn body_size_limit_exceeded_maps_to_entity_too_large_across_io_boundaries() {
         // Shape observed in production (issue #7596):

rustfs/src/storage/ecfs.rs

@@ -22,7 +22,7 @@ use super::{
     update_bucket_metadata_config_if_incarnation,
 };
 use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
-use crate::error::ApiError;
+use crate::error::{ApiError, ApiErrorDiagnostic};
 use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header};
 use crate::storage::helper::OperationHelper;
 use crate::storage::options::get_opts;
@@ -60,6 +60,32 @@ const LOG_COMPONENT_STORAGE: &str = "storage";
 const LOG_SUBSYSTEM_OBJECT: &str = "object";
 const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
 const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
+const EVENT_OBJECT_INTERNAL_ERROR: &str = "object_internal_error";
+
+fn log_object_internal_error(operation: &'static str, request_id: Option<&str>, error: &S3Error) {
+    if error.code() != &S3ErrorCode::InternalError {
+        return;
+    }
+    // Generated S3 adapters serialize handler errors without retaining their
+    // typed sources in logs. Keep only bounded classifications at this boundary.
+    let diagnostic = ApiErrorDiagnostic::from_error(error);
+    error!(
+        event = EVENT_OBJECT_INTERNAL_ERROR,
+        component = LOG_COMPONENT_STORAGE,
+        subsystem = LOG_SUBSYSTEM_OBJECT,
+        result = "error",
+        operation,
+        request_id = request_id.unwrap_or("unavailable"),
+        storage_error_code = ?diagnostic.storage_code,
+        io_error_kind = ?diagnostic.io_kind,
+        io_os_error = diagnostic.io_os_error,
+        rpc_error_code = ?diagnostic.rpc_code,
+        pool_metadata_reason = diagnostic.pool_metadata.map(|context| context.reason),
+        pool_metadata_phase = diagnostic.pool_metadata.map(|context| context.phase),
+        source_chain_truncated = diagnostic.truncated,
+        "Object request returned an internal error"
+    );
+}
 
 use crate::app::storage_api::object_usecase::bucket::replication::{
     OperatorRuleContract, ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication,
@@ -645,8 +671,16 @@ impl S3 for FS {
     #[instrument(level = "debug", skip(self, req))]
     async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
         crate::hp_guard!("S3::delete_object");
+        let request_id = req
+            .extensions
+            .get::<super::request_context::RequestContext>()
+            .map(|ctx| ctx.request_id.clone());
         let usecase = s3_api::object_usecase_for(self);
-        Box::pin(usecase.execute_delete_object(req)).await
+        let result = Box::pin(usecase.execute_delete_object(req)).await;
+        if let Err(error) = &result {
+            log_object_internal_error("DeleteObject", request_id.as_deref(), error);
+        }
+        result
     }
 
     #[instrument(level = "debug", skip(self))]
@@ -964,8 +998,16 @@ impl S3 for FS {
     )]
     async fn get_object(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
         crate::hp_guard!("S3::get_object");
+        let request_id = req
+            .extensions
+            .get::<super::request_context::RequestContext>()
+            .map(|ctx| ctx.request_id.clone());
         let usecase = s3_api::object_usecase_for(self);
-        usecase.execute_get_object(req).await
+        let result = usecase.execute_get_object(req).await;
+        if let Err(error) = &result {
+            log_object_internal_error("GetObject", request_id.as_deref(), error);
+        }
+        result
     }
 
     async fn get_object_acl(&self, req: S3Request<GetObjectAclInput>) -> S3Result<S3Response<GetObjectAclOutput>> {
@@ -2073,6 +2115,73 @@ mod tests {
     use s3s::{S3, S3Error, S3ErrorCode, S3Request};
     use std::sync::atomic::Ordering;
 
+    #[test]
+    fn object_internal_error_logs_only_correlated_bounded_classifications() {
+        #[derive(Clone, Default)]
+        struct Writer(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
+        impl std::io::Write for Writer {
+            fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+                self.0.lock().expect("log buffer").extend_from_slice(bytes);
+                Ok(bytes.len())
+            }
+            fn flush(&mut self) -> std::io::Result<()> {
+                Ok(())
+            }
+        }
+        impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Writer {
+            type Writer = Self;
+            fn make_writer(&'a self) -> Self::Writer {
+                self.clone()
+            }
+        }
+        let writer = Writer::default();
+        let subscriber = tracing_subscriber::fmt()
+            .json()
+            .without_time()
+            .with_writer(writer.clone())
+            .finish();
+        let mut status = tonic::Status::internal("secret source message");
+        status
+            .metadata_mut()
+            .insert("authorization", "secret-token".parse().expect("metadata"));
+        let error: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(status)).into();
+        let original_code = error.code().clone();
+        let original_status = error.status_code();
+        let original_message = error.message().map(str::to_owned);
+        tracing::subscriber::with_default(subscriber, || {
+            super::log_object_internal_error("GetObject", Some("server-owned-request"), &error);
+            let mut opaque = S3Error::with_message(S3ErrorCode::InternalError, "secret outer message");
+            opaque.set_source(Box::new(std::io::Error::other("secret inner message")));
+            super::log_object_internal_error("DeleteObject", None, &opaque);
+            let unavailable: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(
+                tonic::Status::unavailable("secret transport message"),
+            ))
+            .into();
+            assert_eq!(unavailable.code(), &S3ErrorCode::ServiceUnavailable);
+            assert_eq!(unavailable.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
+            super::log_object_internal_error("GetObject", None, &unavailable);
+        });
+        let output = String::from_utf8(writer.0.lock().expect("log buffer").clone()).expect("UTF8 log");
+        assert!(!output.contains("secret"));
+        assert!(!output.contains("authorization"));
+        let rows = output
+            .lines()
+            .map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON log"))
+            .collect::<Vec<_>>();
+        assert_eq!(rows.len(), 2);
+        for row in &rows {
+            assert_eq!(row["fields"]["event"], "object_internal_error");
+            assert_eq!(row["fields"]["source_chain_truncated"], false);
+        }
+        assert_eq!(rows[0]["fields"]["request_id"], "server-owned-request");
+        assert_eq!(rows[0]["fields"]["rpc_error_code"], "Some(Internal)");
+        assert_eq!(rows[1]["fields"]["request_id"], "unavailable");
+        assert_eq!(rows[1]["fields"]["io_error_kind"], "Some(Other)");
+        assert_eq!(error.code(), &original_code);
+        assert_eq!(error.status_code(), original_status);
+        assert_eq!(error.message(), original_message.as_deref());
+    }
+
     #[test]
     fn website_config_rejects_missing_index_and_invalid_redirect_rules() {
         assert!(validate_website_configuration(&WebsiteConfiguration::default()).is_err());

scripts/error-other-format-baseline.txt

@@ -62,7 +62,7 @@
 1|crates/ecstore/src/set_disk/ops/heal.rs
 8|crates/ecstore/src/set_disk/ops/multipart.rs
 2|crates/ecstore/src/set_disk/ops/object.rs
-3|crates/ecstore/src/set_disk/read.rs
+1|crates/ecstore/src/set_disk/read.rs
 5|crates/ecstore/src/store/bucket.rs
 1|crates/ecstore/src/store/heal_walk.rs
 10|crates/ecstore/src/store/init.rs

scripts/initialize_validation_evidence.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Retain self-hosted validation evidence outside Actions' temporary cleanup."""
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import sys
+
+MINIMUM_FREE_BYTES = 10 * 1024**3
+MINIMUM_FREE_INODES = 50000
+
+
+def initialize(environ=None):
+    env = os.environ if environ is None else environ
+    run, attempt = env['GITHUB_RUN_ID'], env['GITHUB_RUN_ATTEMPT']
+    if not all(re.fullmatch(r'[1-9][0-9]*', value) for value in (run, attempt)):
+        raise ValueError('Evidence requires numeric workflow run and attempt identities')
+    tools_only = env['VALIDATION_TOOLS_ONLY']
+    if tools_only not in {'true', 'false'}:
+        raise ValueError('Evidence requires an explicit tools-only selection')
+    temporary = Path(env['RUNNER_TEMP'])
+    if not temporary.is_absolute() or temporary.resolve(strict=True) != temporary or not temporary.is_dir():
+        raise ValueError('Runner temporary directory must be an existing canonical absolute directory')
+    parent = temporary if tools_only == 'true' else temporary.parent / 'rustfs-validation-evidence'
+    if tools_only == 'false':
+        parent.mkdir(mode=0o700, exist_ok=True)
+        info = parent.lstat()
+        if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
+            raise ValueError('Persistent evidence parent must be a private runner-owned directory')
+    root = parent / f'rustfs-validation-{run}-{attempt}'
+    root.mkdir(mode=0o700)
+    observations = []
+    for path in dict.fromkeys((temporary, parent)):
+        usage = os.statvfs(path)
+        observations.append({'path': str(path), 'availableBytes': usage.f_bavail * usage.f_frsize,
+            'availableInodes': usage.f_favail, 'minimumFreeBytes': MINIMUM_FREE_BYTES,
+            'minimumFreeInodes': MINIMUM_FREE_INODES})
+    ready = all(item['availableBytes'] >= MINIMUM_FREE_BYTES and item['availableInodes'] >= MINIMUM_FREE_INODES
+                for item in observations)
+    report = {'schemaVersion': 1, 'runId': run, 'runAttempt': attempt, 'evidenceRoot': str(root),
+              'persistentOutsideRunnerTemp': tools_only == 'false', 'storage': observations, 'ready': ready,
+              'automaticEvidenceDeletion': False, 'canonicalPassClaimed': False}
+    with (root / 'runner-storage-preflight.json').open('x') as stream:
+        json.dump(report, stream, indent=2)
+        stream.write(chr(10))
+    with Path(env['GITHUB_ENV']).open('a') as stream:
+        if any(character in str(root) for character in (chr(10), chr(13))):
+            raise ValueError('Evidence path cannot contain environment-file delimiters')
+        stream.write('VALIDATION_ROOT=' + str(root) + chr(10))
+    print(json.dumps(report, sort_keys=True))
+    if not ready:
+        raise ValueError('Validation requires at least 10 GiB and 50000 free inodes on runner storage')
+    return root
+
+
+if __name__ == '__main__':
+    try:
+        initialize()
+    except (OSError, ValueError, KeyError) as error:
+        print(f'Validation evidence initialization failed: {error}', file=sys.stderr)
+        sys.exit(1)

scripts/inspect_validation_runner.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Inspect GitHub runner registration and queued jobs without accessing the fleet."""
+
+import argparse
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import sys
+
+from resolve_functional_candidate import REPOSITORY, api, require
+
+
+LABEL = "smoke-testing"
+ACTIVE_STATUSES = ("queued", "in_progress", "pending", "waiting", "requested")
+
+
+def validate_request(inputs):
+    require(isinstance(inputs, dict) and inputs.get("validation_runner_inspect_only") is True,
+            "runner inspection must be explicitly selected")
+    defaults = {"validation_runner_inspect_only": True,
+                "validation_retain_evidence_run_attempt": "1",
+                "i05_diagnostic_branch": "timeout", "validation_tools_run_attempt": "1"}
+    require(all(value == defaults[key] if key in defaults else value in ("", False, None)
+                for key, value in inputs.items()),
+            "runner inspection cannot select a candidate, tools, fleet case or recovery")
+
+
+def pages(path, field):
+    rows = []
+    for page in range(1, 11):
+        separator = "&" if "?" in path else "?"
+        batch = api(f"{path}{separator}per_page=100&page={page}").get(field)
+        require(isinstance(batch, list), "GitHub inventory response is incomplete")
+        rows.extend(batch)
+        if len(batch) < 100:
+            return rows
+    raise ValueError("GitHub inventory exceeds the bounded inspection limit")
+
+
+def collect(report):
+    prefix = f"repos/{REPOSITORY}/actions"
+    report["stage"] = "runner-inventory"
+    runners = pages(prefix + "/runners", "runners")
+    report["runners"] = [
+        {"id": row["id"], "name": row["name"], "status": row["status"],
+         "busy": row["busy"], "labels": [label["name"] for label in row["labels"]]}
+        for row in runners if LABEL in {label["name"] for label in row["labels"]}]
+    online = [row for row in report["runners"] if row["status"] == "online"]
+    report["runnerAvailability"] = (
+        "online-idle" if any(row["busy"] is False for row in online) else
+        "online-busy" if online else "offline" if report["runners"] else "unregistered")
+    report["stage"] = "active-workflows"
+    runs = {}
+    for status in ACTIVE_STATUSES:
+        for row in pages(prefix + f"/runs?status={status}", "workflow_runs"):
+            runs[row["id"]] = row
+    report["fleetJobs"] = []
+    report["runsWithoutJobs"] = []
+    for run_id, row in sorted(runs.items()):
+        report["stage"] = "workflow-jobs"
+        jobs = pages(prefix + f"/runs/{run_id}/jobs", "jobs")
+        identity = {"runId": run_id, "workflow": row["name"], "status": row["status"],
+                    "createdAt": row["created_at"], "url": row["html_url"]}
+        if not jobs:
+            report["runsWithoutJobs"].append(identity)
+        for job in jobs:
+            if LABEL in job.get("labels", []) and job["status"] != "completed":
+                report["fleetJobs"].append({**identity, "jobId": job["id"], "jobName": job["name"],
+                    "jobStatus": job["status"], "runnerName": job.get("runner_name"),
+                    "labels": job["labels"], "jobUrl": job["html_url"]})
+    report.update(stage="complete", complete=True, activeWorkflowCount=len(runs))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    require(not args.output.exists(), "retain the original runner inspection report")
+    report = {"schemaVersion": 1, "repository": REPOSITORY, "label": LABEL,
+              "observedAt": datetime.now(timezone.utc).isoformat(), "complete": False,
+              "runId": os.environ.get("GITHUB_RUN_ID"), "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+              "workflowCommit": os.environ.get("GITHUB_SHA"),
+              "runnerAvailability": "unknown", "stage": "request-validation",
+              "fleetAccessed": False, "mutated": False,
+              "cleanStateCertified": False, "sharedWorkflowIdleCertified": False,
+              "canonicalCasePassClaimed": False}
+    try:
+        validate_request(json.loads(os.environ["RUNNER_INSPECTION_INPUTS"]))
+        collect(report)
+    except Exception as error:
+        # Subprocess diagnostics can include configuration or credentials.
+        report["errorType"] = type(error).__name__
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("x") as stream:
+        json.dump(report, stream, indent=2)
+        stream.write("\n")
+    print(json.dumps({key: report[key] for key in ("complete", "stage", "runnerAvailability")}))
+    return 0 if report["complete"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/prepare_retained_validation_recovery.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Copy verified local evidence into a recovery input without claiming an old artifact."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import sys
+
+from resolve_functional_candidate import REPOSITORY, api, require
+from retain_validation_evidence import inventory, prepare
+
+
+def stage(env, report):
+    source = env["FAILED_RUN_ID"]
+    require(re.fullmatch(r"[1-9][0-9]{0,19}", source), "exact failed run required")
+    run = api(f"repos/{REPOSITORY}/actions/runs/{source}")
+    require(run.get("head_branch") == env["GITHUB_REF_NAME"]
+            and type(run.get("run_attempt")) is int and run["run_attempt"] > 0,
+            "local recovery source branch or attempt differs")
+    root = prepare({**env, "EVIDENCE_RETAIN_INPUTS": json.dumps({
+        "validation_retain_evidence_run_id": source,
+        "validation_retain_evidence_run_attempt": str(run["run_attempt"])})}, report)
+    report.update(complete=False, stage="stable-recovery-copy")
+    destination = Path(env["VALIDATION_ROOT"]) / "failed-artifact"
+    require(destination.parent.is_absolute() and destination.parent.resolve(strict=True) == destination.parent,
+            "recovery evidence root is not canonical")
+    destination.mkdir(mode=0o700, exist_ok=False)
+    copied = destination / "retained-source-evidence"
+    files, size = inventory(root, copy_to=copied)
+    require(files == report["files"] and size == report["bytes"]
+            and inventory(copied) == (files, size), "retained evidence changed during recovery copy")
+    report.update(complete=True, stage="complete", recoveryInput=str(copied))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+    report = {"schemaVersion": 1, "collectionRunId": os.environ["GITHUB_RUN_ID"],
+              "collectionAttempt": os.environ["GITHUB_RUN_ATTEMPT"], "complete": False,
+              "stage": "source-workflow",
+              "collectionKind": "runner-local-recovery-input-copy", "sourceArtifactClaimed": False,
+              "fleetAccessed": False, "originalEvidenceModified": False,
+              "sourceVerdictRewritten": False, "canonicalCasePassClaimed": False}
+    try:
+        stage(os.environ, report)
+    except Exception as error:
+        report.update(complete=False, errorType=type(error).__name__)
+    with args.report.open("x") as stream:
+        json.dump(report, stream, indent=2)
+        stream.write("\n")
+    print(json.dumps({key: report[key] for key in ("complete", "stage")}))
+    return 0 if report["complete"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/resolve_functional_candidate.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Resolve a nightly manifest from its exact GitHub build attempt and artifact."""
+"""Resolve a candidate manifest from its exact GitHub build attempt and artifact."""
 from __future__ import annotations
 
 import hashlib
@@ -15,6 +15,7 @@ import zipfile
 REPOSITORY = "rustfs/rustfs"
 ROOT = Path(__file__).resolve().parents[1]
 MAX_ARCHIVE = 1024 * 1024
+VALIDATION_BUILD_WORKFLOWS = {".github/workflows/rustfs-validation-package.yml", ".github/workflows/rustfs-functional-chain.yml"}
 
 
 def require(condition, message):
@@ -39,8 +40,11 @@ def validate_manifest(manifest, run):
     require(isinstance(manifest, dict), "manifest must be an object")
     common = {"schema", "source_sha", "build_run_id", "build_run_attempt", "package_url", "package_sha256"}
     version = manifest.get("schema")
-    require(type(version) is int and version in (1, 2), "unsupported candidate schema")
-    require(set(manifest) == (common if version == 1 else common | {"workflow_sha", "source_ref"}), "unexpected candidate fields")
+    require(type(version) is int and version in (1, 2, 3), "unsupported candidate schema")
+    expected_fields = common if version == 1 else common | {"workflow_sha", "source_ref"}
+    if version == 3:
+        expected_fields |= {"channel"}
+    require(set(manifest) == expected_fields, "unexpected candidate fields")
     require(positive(manifest["build_run_id"]) and positive(manifest["build_run_attempt"]), "invalid build identity")
     require((manifest["build_run_id"], manifest["build_run_attempt"]) == (run["id"], run["run_attempt"]), "candidate belongs to another build attempt")
     require(sha(manifest["source_sha"]) and sha(manifest["package_sha256"], 64), "invalid candidate hash")
@@ -49,7 +53,11 @@ def validate_manifest(manifest, run):
     else:
         require(manifest["workflow_sha"] == run["head_sha"] and sha(manifest["workflow_sha"]), "candidate workflow SHA differs from artifact provenance")
         require(isinstance(manifest["source_ref"], str) and bool(re.fullmatch(r"[A-Za-z0-9_./-]{1,200}", manifest["source_ref"])), "invalid build source ref")
-    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/{run['id']}/"
+        if version == 3:
+            require(manifest["channel"] == "validation", "validation candidate has the wrong publication channel")
+            require(manifest["source_ref"] == manifest["source_sha"], "validation candidate must bind an exact source SHA")
+    channel = "validation" if version == 3 else "nightly"
+    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/{channel}/runs/{run['id']}/"
                 f"{run['run_attempt']}/{manifest['package_sha256']}/rustfs.deb")
     require(manifest["package_url"] == expected, "package URL does not bind the run, attempt and checksum")
     return manifest
@@ -72,19 +80,31 @@ def read_json_artifact(archive, artifact, run, expected_name, member, max_json=1
 
 
 def read_manifest(archive, artifact, run):
-    name = f"nightly-candidate-{run['id']}-{run['run_attempt']}"
-    return validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
+    prefix = "validation-candidate" if run.get("path") in VALIDATION_BUILD_WORKFLOWS else "nightly-candidate"
+    name = f"{prefix}-{run['id']}-{run['run_attempt']}"
+    manifest = validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
+    if prefix == "validation-candidate":
+        require(manifest["schema"] == 3 and manifest["channel"] == "validation",
+                "validation workflow requires a schema-3 validation candidate")
+    else:
+        require(manifest["schema"] != 3, "nightly workflow cannot publish a validation candidate")
+    return manifest
 
 
-def resolve(run_id, attempt):
+def resolve(run_id, attempt, *, allow_validation=False):
     require(positive(run_id) and positive(attempt), "build run and attempt are required positive integers")
     endpoint = f"repos/{REPOSITORY}/actions/runs/{run_id}"
     run = api(f"{endpoint}/attempts/{attempt}")
     require(run.get("id") == run_id and run.get("run_attempt") == attempt, "GitHub returned a different build attempt")
-    require(run.get("path") == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main", "candidate must come from nightly-gnu on main")
+    path = run.get("path")
+    validation = allow_validation and path in VALIDATION_BUILD_WORKFLOWS
+    require(path == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main" or validation,
+            "candidate must come from nightly-gnu on main or an explicitly allowed validation build")
     require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "candidate came from another repository")
-    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "nightly attempt has not completed successfully")
-    name = f"nightly-candidate-{run_id}-{attempt}"
+    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "candidate attempt has not completed successfully")
+    if validation:
+        require(run.get("event") == "workflow_dispatch", "validation candidate must be manually dispatched")
+    name = f"{'validation' if validation else 'nightly'}-candidate-{run_id}-{attempt}"
     artifacts = []
     for page in range(1, 11):
         batch = api(f"{endpoint}/artifacts?per_page=100&page={page}")["artifacts"]

scripts/retain_validation_evidence.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Admit an interrupted run's existing runner-local evidence for read-only upload."""
+
+import argparse
+from datetime import datetime, timezone
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import sys
+
+from resolve_functional_candidate import REPOSITORY, api, require
+
+
+MAX_FILES = 200000
+MAX_BYTES = 4 * 1024**3
+
+
+def stable(info):
+    return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid,
+            info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
+
+
+def inventory(root, *, copy_to=None):
+    files, total, entries = [], 0, 0
+    device = root.lstat().st_dev
+    if copy_to is not None:
+        require(copy_to.is_absolute() and copy_to.parent.resolve(strict=True) == copy_to.parent,
+                "evidence copy parent is not canonical")
+        copy_to.mkdir(mode=0o700, exist_ok=False)
+
+    def visit(fd, prefix):
+        nonlocal total, entries
+        before = os.fstat(fd)
+        require(stat.S_ISDIR(before.st_mode) and before.st_uid == os.getuid() and before.st_dev == device,
+                "evidence directory has a different owner or type")
+        names = sorted(os.listdir(fd))
+        require(len(names) <= MAX_FILES, "evidence directory exceeds the entry budget")
+        for name in names:
+            entries += 1
+            require(entries <= MAX_FILES, "evidence exceeds the entry budget")
+            info = os.stat(name, dir_fd=fd, follow_symlinks=False)
+            path = prefix + name
+            require(info.st_uid == os.getuid() and info.st_dev == device, "evidence entry has a different owner or device")
+            if stat.S_ISDIR(info.st_mode):
+                if copy_to is not None:
+                    (copy_to / path).mkdir(mode=0o700, exist_ok=False)
+                child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
+                try:
+                    require(stable(os.fstat(child)) == stable(info), "evidence directory changed before inspection")
+                    visit(child, path + "/")
+                finally:
+                    os.close(child)
+            else:
+                require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1
+                        and len(files) < MAX_FILES and total + info.st_size <= MAX_BYTES,
+                        "evidence contains links, special files or exceeds its byte/file budget")
+                child = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
+                target = None
+                try:
+                    require(stable(os.fstat(child)) == stable(info), "evidence file changed before inspection")
+                    if copy_to is not None:
+                        target = os.open(copy_to / path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+                    digest, size = hashlib.sha256(), 0
+                    while chunk := os.read(child, 1024**2):
+                        size += len(chunk)
+                        require(size <= info.st_size, "evidence file grew during inspection")
+                        digest.update(chunk)
+                        if target is not None:
+                            remaining = memoryview(chunk)
+                            while remaining:
+                                written = os.write(target, remaining)
+                                require(written > 0, "evidence copy made no progress")
+                                remaining = remaining[written:]
+                    require(size == info.st_size and stable(os.fstat(child)) == stable(info),
+                            "evidence file changed during inspection")
+                finally:
+                    os.close(child)
+                    if target is not None:
+                        os.close(target)
+                total += size
+                files.append({"path": path, "size": size, "sha256": digest.hexdigest()})
+            require(stable(os.stat(name, dir_fd=fd, follow_symlinks=False)) == stable(info),
+                    "evidence entry changed during inspection")
+        require(sorted(os.listdir(fd)) == names
+                and stable(os.fstat(fd)) == stable(before), "evidence population changed during inspection")
+
+    fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        visit(fd, "")
+    finally:
+        os.close(fd)
+    return files, total
+
+
+def prepare(env, report):
+    inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"])
+    source = inputs.get("validation_retain_evidence_run_id")
+    attempt = inputs.get("validation_retain_evidence_run_attempt", "1")
+    require(isinstance(source, str) and re.fullmatch(r"[1-9][0-9]{0,19}", source), "exact source run ID required")
+    require(isinstance(attempt, str) and re.fullmatch(r"[1-9][0-9]{0,9}", attempt), "exact source run attempt required")
+    defaults = {"validation_retain_evidence_run_id": source,
+                "validation_retain_evidence_run_attempt": attempt,
+                "i05_diagnostic_branch": "timeout", "validation_tools_run_attempt": "1"}
+    require(all(value == defaults[key] if key in defaults else value in ("", False, None)
+                for key, value in inputs.items()), "evidence retention is exclusive with all execution modes")
+    require(source != env["GITHUB_RUN_ID"], "cannot retain the running collection job")
+    report["stage"] = "source-workflow"
+    run = api(f"repos/{REPOSITORY}/actions/runs/{source}/attempts/{attempt}")
+    require(run.get("id") == int(source) and type(run.get("run_attempt")) is int and run["run_attempt"] == int(attempt)
+            and run.get("repository", {}).get("full_name") == REPOSITORY
+            and run.get("status") == "completed" and run.get("conclusion") in {"failure", "cancelled", "timed_out"}
+            and run.get("event") == "workflow_dispatch"
+            and run.get("path") == ".github/workflows/rustfs-functional-chain.yml",
+            "source is not one completed interrupted validation workflow")
+    jobs = api(f"repos/{REPOSITORY}/actions/runs/{source}/attempts/{run['run_attempt']}/jobs?per_page=100")
+    require(jobs.get("total_count", 101) <= 100
+            and len([job for job in jobs.get("jobs", []) if job.get("name") == "validation"
+                     and job.get("runner_name") == env["RUNNER_NAME"]]) == 1,
+            "source validation did not execute on this runner")
+    report["sourceRun"] = {key: run[key] for key in ("id", "run_attempt", "head_sha", "status", "conclusion", "html_url")}
+    report["stage"] = "owned-persistent-root"
+    temporary = Path(env["RUNNER_TEMP"])
+    require(temporary.is_absolute() and temporary.resolve(strict=True) == temporary
+            and temporary.is_dir(), "runner temporary path is not canonical")
+    parent = temporary.parent / "rustfs-validation-evidence"
+    root = parent / f"rustfs-validation-{source}-{run['run_attempt']}"
+    for path in (parent, root):
+        info = path.lstat()
+        require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid()
+                and stat.S_IMODE(info.st_mode) == 0o700 and path.resolve(strict=True) == path,
+                "persistent evidence must be an existing private runner-owned directory")
+    report["stage"] = "stable-file-inventory"
+    files, size = inventory(root)
+    marker = root / "runner-storage-preflight.json"
+    require(any(row["path"] == marker.name for row in files) and marker.stat().st_size < 1024**2,
+            "source storage provenance is absent or oversized")
+    storage = json.loads(marker.read_text())
+    require(storage.get("runId") == source and storage.get("runAttempt") == str(run["run_attempt"])
+            and storage.get("evidenceRoot") == str(root)
+            and storage.get("persistentOutsideRunnerTemp") is True,
+            "evidence storage provenance belongs to another run or root")
+    require(not any(character in str(root) for character in ("\n", "\r")), "unsafe evidence export path")
+    report.update(complete=True, stage="complete", evidenceRoot=str(root), files=files, bytes=size)
+    return root
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--report", type=Path, required=True)
+    args = parser.parse_args()
+    report = {"schemaVersion": 1, "collectionRunId": os.environ["GITHUB_RUN_ID"],
+              "collectionAttempt": os.environ["GITHUB_RUN_ATTEMPT"], "complete": False,
+              "observedAt": datetime.now(timezone.utc).isoformat(),
+              "stage": "request-validation", "fleetAccessed": False, "originalEvidenceModified": False,
+              "sourceVerdictRewritten": False, "canonicalCasePassClaimed": False}
+    root = None
+    try:
+        root = prepare(os.environ, report)
+    except Exception as error:
+        report["errorType"] = type(error).__name__
+    args.report.parent.mkdir(parents=True, exist_ok=True)
+    with args.report.open("x") as stream:
+        json.dump(report, stream, indent=2)
+        stream.write("\n")
+    if root is not None:
+        with Path(os.environ["GITHUB_ENV"]).open("a") as stream:
+            stream.write("VALIDATION_RETAIN_ROOT=" + str(root) + "\n")
+            stream.write("VALIDATION_RETAIN_SOURCE_RUN=" + str(report["sourceRun"]["id"]) + "\n")
+            stream.write("VALIDATION_RETAIN_SOURCE_ATTEMPT=" + str(report["sourceRun"]["run_attempt"]) + "\n")
+    print(json.dumps({key: report[key] for key in ("complete", "stage")}))
+    return 0 if root is not None else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/test_functional_chain.py

@@ -4,6 +4,7 @@ import copy
 import hashlib
 import io
 import json
+import os
 from pathlib import Path
 import subprocess
 import tempfile
@@ -13,6 +14,369 @@ import zipfile
 
 import functional_chain_evidence as evidence
 import resolve_functional_candidate as candidate
+import initialize_validation_evidence as validation_storage
+
+
+class ValidationStorageTests(unittest.TestCase):
+    def fixture(self, root, *, tools_only=False):
+        temporary = root / '_work/_temp'
+        temporary.mkdir(parents=True)
+        return dict(RUNNER_TEMP=str(temporary.resolve()), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
+                    VALIDATION_TOOLS_ONLY=str(tools_only).lower(), GITHUB_ENV=str(root / 'env'))
+
+    def usage(self, *, available=None, inodes=None):
+        return mock.Mock(f_bavail=validation_storage.MINIMUM_FREE_BYTES if available is None else available,
+                         f_frsize=1, f_favail=validation_storage.MINIMUM_FREE_INODES if inodes is None else inodes)
+
+    def test_self_hosted_evidence_survives_temp_cleanup_and_refuses_reuse(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            env = self.fixture(root)
+            with mock.patch.object(validation_storage.os, 'statvfs', return_value=self.usage()):
+                evidence_root = validation_storage.initialize(env)
+                with self.assertRaises(FileExistsError):
+                    validation_storage.initialize(env)
+            self.assertEqual(evidence_root, Path(env['RUNNER_TEMP']).parent /
+                             'rustfs-validation-evidence/rustfs-validation-77-1')
+            self.assertEqual(evidence_root.parent.stat().st_mode & 0o777, 0o700)
+            self.assertEqual(evidence_root.stat().st_mode & 0o777, 0o700)
+            Path(env['RUNNER_TEMP']).rmdir()
+            self.assertTrue((evidence_root / 'runner-storage-preflight.json').is_file())
+            self.assertEqual(Path(env['GITHUB_ENV']).read_text(), 'VALIDATION_ROOT=' + str(evidence_root) + chr(10))
+
+    def test_each_filesystem_byte_and_inode_budget_fails_before_candidate_execution(self):
+        for position in (0, 1):
+            for kind in ('bytes', 'inodes'):
+                with self.subTest(position=position, kind=kind), tempfile.TemporaryDirectory() as temp:
+                    root = Path(temp)
+                    env = self.fixture(root)
+                    usage = [self.usage(), self.usage()]
+                    usage[position] = self.usage(available=validation_storage.MINIMUM_FREE_BYTES - 1) if kind == 'bytes' \
+                        else self.usage(inodes=validation_storage.MINIMUM_FREE_INODES - 1)
+                    with mock.patch.object(validation_storage.os, 'statvfs', side_effect=usage):
+                        with self.assertRaisesRegex(ValueError, 'at least 10 GiB'):
+                            validation_storage.initialize(env)
+                    evidence_root = Path(Path(env['GITHUB_ENV']).read_text().strip().split('=', 1)[1])
+                    report = json.loads((evidence_root / 'runner-storage-preflight.json').read_text())
+                    self.assertFalse(report['ready'])
+                    self.assertFalse(report['automaticEvidenceDeletion'])
+                    self.assertEqual(len(report['storage']), 2)
+
+    def test_tools_resolver_keeps_its_disposable_hosted_location(self):
+        with tempfile.TemporaryDirectory() as temp:
+            env = self.fixture(Path(temp), tools_only=True)
+            with mock.patch.object(validation_storage.os, 'statvfs', return_value=self.usage()):
+                root = validation_storage.initialize(env)
+            self.assertEqual(root.parent, Path(env['RUNNER_TEMP']))
+            self.assertFalse(json.loads((root / 'runner-storage-preflight.json').read_text())
+                             ['persistentOutsideRunnerTemp'])
+
+    def test_rejects_unsafe_parent_and_untrusted_identities(self):
+        for kind in ('symlink', 'public', 'run', 'attempt', 'selection'):
+            with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp:
+                root = Path(temp)
+                env = self.fixture(root)
+                parent = Path(env['RUNNER_TEMP']).parent / 'rustfs-validation-evidence'
+                if kind == 'symlink':
+                    other = root / 'other'
+                    other.mkdir(mode=0o700)
+                    parent.symlink_to(other, target_is_directory=True)
+                elif kind == 'public':
+                    parent.mkdir(mode=0o755)
+                    parent.chmod(0o755)
+                else:
+                    env[{'run': 'GITHUB_RUN_ID', 'attempt': 'GITHUB_RUN_ATTEMPT',
+                         'selection': 'VALIDATION_TOOLS_ONLY'}[kind]] = '../foreign'
+                with self.assertRaises(ValueError):
+                    validation_storage.initialize(env)
+                self.assertFalse(Path(env['GITHUB_ENV']).exists())
+                self.assertFalse((parent / 'rustfs-validation-77-1').exists())
+
+    def test_workflow_uploads_persistent_evidence_and_preflights_before_resolution(self):
+        source = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
+        initialize = source.index('      - name: Initialize validation evidence')
+        self.assertLess(initialize, source.index('      - name: Resolve immutable candidate'))
+        self.assertIn('python3 -B scripts/initialize_validation_evidence.py', source)
+        self.assertIn('VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}', source)
+        upload = source.split('      - name: Upload validation evidence', 1)[1].split('  validation-tools:', 1)[0]
+        self.assertIn("if: ${{ always() && env.VALIDATION_ROOT != '' }}", upload)
+        self.assertIn('path: ${{ env.VALIDATION_ROOT }}', upload)
+
+
+class RaceNativeToolsWorkflowTests(unittest.TestCase):
+    def chain_shell_step(self, name):
+        source = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
+        step = source.split('      - name: ' + name + '\n', 1)[1].split('\n      - name:', 1)[0]
+        shell = step.split('        run: |\n', 1)[1]
+        return '\n'.join(line[10:] for line in shell.splitlines()
+                         if not line.strip() or line.startswith('          '))
+
+    def test_race_actual_workflow_selection_preserves_legacy_and_propagates_failures(self):
+        shell = self.chain_shell_step('Select exact-source race native evidence')
+        g01 = 'heal.race-unversioned-overwrite'
+        cases = [(False, 2, 0, 'false', 'a' * 40, g01), (True, 2, 0, 'false', 'a' * 40, g01),
+                 (True, 0, 0, 'true', 'a' * 40, g01), (True, 1, 1, None, 'a' * 40, g01),
+                 (True, 42, 42, None, 'a' * 40, g01)]
+        reviewed = [('6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca', g01)]
+        reviewed += [('bd0671ba781d8ebf509f4f03533228dfb44496f1', case) for case in
+                     (g01, 'heal.race-versioned-put', 'heal.race-version-delete')]
+        reviewed += [('8380466f3a27c3c3ef4115084178a0ab99a78f32', case) for case in
+                     (g01, 'heal.race-versioned-put', 'heal.race-version-delete', 'heal.race-markers-and-batch-delete')]
+        reviewed += [('757b751482bb970bc321e92059690a7698554358', case) for case in
+                     ('heal.admin-nolock-safety', 'heal.race-copy-and-metadata-update', 'heal.race-multipart-complete-abort')]
+        for source, case in reviewed:
+            cases += [(False, 2, 1, None, source, case), (True, 2, 1, None, source, case),
+                      (True, 0, 0, 'true', source, case)]
+        for helper, support, code, required, source, case in cases:
+            with self.subTest(helper=helper, support=support, source=source, case=case), tempfile.TemporaryDirectory() as name:
+                root = Path(name)
+                env = self.fixture(root, new_helper=helper)
+                output = root / 'output'
+                env.update(SUPPORT_STATUS=str(support), GITHUB_OUTPUT=str(output), CANDIDATE_SHA=source,
+                           VALIDATION_CASE_ID=case)
+                result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode, code, result.stderr)
+                self.assertEqual(output.read_text() if output.exists() else None,
+                                 'required=' + required + '\n' if required else None)
+                if helper:
+                    calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()]
+                    self.assertEqual(calls, [{'kind': 'race_native', 'args': ['supports', '--commit', source,
+                                      '--case-id', case]}])
+
+    def test_race_actual_admission_requires_native_and_runtime_before_exporting_directory(self):
+        shell = self.chain_shell_step('Verify exact-source race native and runtime evidence')
+        for case in ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete',
+                     'heal.race-markers-and-batch-delete', 'heal.race-multipart-complete-abort',
+                     'heal.admin-nolock-safety', 'heal.race-copy-and-metadata-update'):
+            for native_exit, runtime in [(0, 0), (42, 0), (0, 42), (0, 'missing'), (0, 'collector-only')]:
+                with self.subTest(case=case, native_exit=native_exit, runtime=runtime), tempfile.TemporaryDirectory() as name:
+                    root = Path(name)
+                    env = self.fixture(root, new_helper=True)
+                    directory = root / 'auto-testing/scripts/validation_heal'
+                    runtime_driver = 'nolock.py' if case == 'heal.admin-nolock-safety' else 'race.py'
+                    (directory / 'race_native_evidence.py').write_text(
+                        'import os,sys\nprint("{}")\nsys.exit(int(os.environ["VERIFY_EXIT"]))\n')
+                    if runtime == 'collector-only':
+                        (directory / runtime_driver).write_text(
+                            'import argparse\np=argparse.ArgumentParser()\np.add_argument("--rc-bin", required=True)\np.parse_args()\n')
+                    elif runtime != 'missing':
+                        (directory / runtime_driver).write_text(
+                            'import json,os,pathlib,sys\n'
+                            'assert not pathlib.Path(os.environ["GITHUB_ENV"]).exists()\n'
+                            'assert sys.argv[1] == "preflight"\n'
+                            'assert sys.argv[-2:] == ["--case-id",os.environ["VALIDATION_CASE_ID"]]\n'
+                            'assert pathlib.Path(os.environ["VALIDATION_HEAL_NATIVE_DIRECTORY"],"tool-run.json").is_file()\n'
+                            'pathlib.Path("runtime-call.json").write_text(json.dumps(sys.argv[1:]))\n'
+                            'sys.exit(int(os.environ["RUNTIME_EXIT"]))\n')
+                    validation = root / 'validation'
+                    validation.mkdir()
+                    (validation / 'tool-run.json').write_text('{"id":123}')
+                    native = root / 'rustfs-validation-native-77-1'
+                    native.mkdir()
+                    output = root / 'env'
+                    env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
+                               VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(native_exit),
+                               RUNTIME_EXIT=str(runtime), VALIDATION_CASE_ID=case)
+                    result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
+                    expected = native_exit or (runtime if isinstance(runtime, int) else 2)
+                    self.assertEqual(result.returncode, expected, result.stderr)
+                    self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
+                    self.assertEqual(output.exists(), expected == 0)
+                    self.assertEqual((root / 'runtime-call.json').exists(), native_exit == 0 and isinstance(runtime, int))
+                    if expected == 0:
+                        self.assertEqual(output.read_text(), 'VALIDATION_HEAL_NATIVE_DIRECTORY=' + str(native) + '\n')
+        workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
+        self.assertLess(workflow.index('Verify exact-source race native and runtime'),
+                        workflow.index('Run selected canonical validation case'))
+        self.assertIn("steps.race_native.outputs.required == 'true'", workflow)
+        self.assertIn("inputs.validation_case == 'heal.race-markers-and-batch-delete'", workflow)
+        self.assertIn("inputs.validation_case == 'heal.admin-nolock-safety'", workflow)
+        self.assertIn("inputs.validation_case == 'heal.race-copy-and-metadata-update'", workflow)
+        selection = workflow.split('      - name: Select exact-source race native evidence\n', 1)[1].split('\n        env:', 1)[0]
+        self.assertIn("inputs.validation_case == 'heal.race-multipart-complete-abort'", selection)
+
+    def test_i05_native_prerequisite_propagates_failure_and_exports_only_after_verification(self):
+        shell = self.chain_shell_step('Verify I05 queued-successor native prerequisite')
+        for exit_code in (0, 42):
+            with self.subTest(exit_code=exit_code), tempfile.TemporaryDirectory() as name:
+                root = Path(name)
+                env = self.fixture(root, new_helper=True)
+                script = root / 'auto-testing/scripts/validation_heal/race_native_evidence.py'
+                script.write_text('import os,sys\n'
+                    'assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n'
+                    'assert sys.argv[-3] == os.environ["AUTO_TESTING_SHA"]\n'
+                    'sys.exit(int(os.environ["VERIFY_EXIT"]))\n')
+                validation = root / 'validation'; validation.mkdir()
+                (validation / 'tool-run.json').write_text('{"id":123}')
+                native = root / 'rustfs-validation-native-77-1'; native.mkdir()
+                output = root / 'env'
+                env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
+                           VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code))
+                result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode, exit_code, result.stderr)
+                self.assertEqual(output.exists(), exit_code == 0)
+                self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
+        workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
+        self.assertLess(workflow.index('Verify I05 queued-successor native prerequisite'),
+                        workflow.index('Run selected canonical validation case'))
+        download = workflow.split('      - name: Download pinned native storage evidence\n', 1)[1].split('        uses:', 1)[0]
+        self.assertIn("inputs.i05_diagnostic_branch == 'queued-successor'", download)
+        execution = workflow.split('      - name: Run selected canonical validation case\n', 1)[1].split('        run:', 1)[0]
+        self.assertIn('I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}', execution)
+
+    def shell_step(self, name):
+        from check_test_wiring import yaml_block
+        source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
+        job = yaml_block(source.splitlines(), 'native-heal-regressions', 2)
+        start = job.index('      - name: ' + name)
+        end = next((index for index in range(start + 1, len(job))
+                    if job[index].startswith('      - name: ')), len(job))
+        step = job[start:end]
+        return '\n'.join(line[10:] for line in step[step.index('        run: |') + 1:]
+                         if not line.strip() or line.startswith('          '))
+
+    def fixture(self, root, *, new_helper):
+        directory = root / 'auto-testing/scripts/validation_heal'
+        directory.mkdir(parents=True)
+        stub = '''import json, os, pathlib, sys
+kind = pathlib.Path(__file__).stem
+with pathlib.Path('calls.jsonl').open('a') as out:
+    out.write(json.dumps({'kind': kind, 'args': sys.argv[1:]}) + '\\n')
+if sys.argv[1] == 'supports': sys.exit(int(os.environ['SUPPORT_STATUS']))
+sys.exit(int(os.environ.get(kind.upper() + '_FAIL', '0')))
+'''
+        (directory / 'native_regressions.py').write_text(stub)
+        if new_helper: (directory / 'race_native.py').write_text(stub)
+        bin_dir = root / 'bin'
+        bin_dir.mkdir()
+        git = bin_dir / 'git'
+        git.write_text('#!/bin/sh\nprintf "%s\\n" "$FAKE_HEAD"\n')
+        git.chmod(0o755)
+        env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, CANDIDATE_SHA='a' * 40,
+                   FAKE_HEAD='b' * 40, SUPPORT_STATUS='2', PATH=str(bin_dir) + os.pathsep + os.environ['PATH'])
+        env.pop('BASH_ENV', None)
+        return env
+
+    def test_actual_native_build_routes_old_new_unknown_and_failed_collectors(self):
+        shell = self.shell_step('Build and enumerate the unmodified candidate test executable')
+        for mode, helper, support, failure, head, expected_code, expected_calls in [
+            ('old-harness', False, 2, None, 'b', 0, [('native_regressions', 'build')]),
+            ('legacy', True, 2, None, 'b', 0, [('race_native', 'supports'), ('native_regressions', 'build')]),
+            ('race', True, 0, None, 'b', 0, [('race_native', 'supports'), ('race_native', 'build')]),
+            ('support-error', True, 42, None, 'b', 42, [('race_native', 'supports')]),
+            ('invalid-support', True, 1, None, 'b', 1, [('race_native', 'supports')]),
+            ('race-build-error', True, 0, 'RACE_NATIVE_FAIL', 'b', 42, [('race_native', 'supports'), ('race_native', 'build')]),
+            ('legacy-build-error', True, 2, 'NATIVE_REGRESSIONS_FAIL', 'b', 42, [('race_native', 'supports'), ('native_regressions', 'build')]),
+            ('wrong-helper', True, 0, None, 'c', 1, []),
+        ]:
+            with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode):
+                root = Path(temp)
+                env = self.fixture(root, new_helper=helper)
+                env.update(SUPPORT_STATUS=str(support), FAKE_HEAD=head * 40)
+                if failure: env[failure] = '42'
+                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell],
+                                        cwd=root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode, expected_code, result.stderr)
+                calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else []
+                self.assertEqual([(row['kind'], row['args'][0]) for row in calls], expected_calls)
+                for row in calls:
+                    if row['args'][0] == 'supports':
+                        self.assertEqual(row['args'], ['supports', '--commit', 'a' * 40])
+                    else:
+                        expected = ['build', '--source', 'candidate', '--commit', 'a' * 40]
+                        if row['kind'] == 'race_native': expected += ['--testing-commit', 'b' * 40]
+                        self.assertEqual(row['args'], expected + ['--out', 'native-tools'])
+
+    def test_actual_native_run_rejects_ambiguous_formats_and_propagates_failures(self):
+        shell = self.shell_step('Run exact native regressions separately from fleet validation')
+        for mode, helper, manifests, failure, expected_code, expected_kind in [
+            ('legacy', False, ['native-regressions.json'], None, 0, 'native_regressions'),
+            ('race', True, ['race-native.json'], None, 0, 'race_native'),
+            ('ambiguous', True, ['native-regressions.json', 'race-native.json'], None, 1, None),
+            ('legacy-error', True, ['native-regressions.json'], 'NATIVE_REGRESSIONS_FAIL', 42, 'native_regressions'),
+            ('race-error', True, ['race-native.json'], 'RACE_NATIVE_FAIL', 42, 'race_native'),
+            ('missing-race-helper', False, ['race-native.json'], None, 2, None),
+        ]:
+            with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode):
+                root = Path(temp)
+                env = self.fixture(root, new_helper=helper)
+                if failure: env[failure] = '42'
+                (root / 'native-tools').mkdir()
+                for manifest in manifests: (root / 'native-tools' / manifest).write_text('{}')
+                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell],
+                                        cwd=root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode, expected_code, result.stderr)
+                calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else []
+                self.assertEqual(len(calls), int(expected_kind is not None))
+                if expected_kind:
+                    self.assertEqual(calls, [{'kind': expected_kind, 'args': ['run', '--directory', 'native-tools', '--out', 'native-evidence']}])
+
+    def test_native_upload_retains_both_fixed_executable_kinds(self):
+        source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
+        self.assertIn('            native-tools/ecstore-native-tests\n', source)
+        self.assertIn('            native-tools/ecstore-race-native-tests\n', source)
+
+
+class PersistenceToolsWorkflowTests(unittest.TestCase):
+    def test_supplement_preserves_old_pins_and_propagates_actual_collector_failure(self):
+        from check_test_wiring import yaml_block
+        source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
+        job = yaml_block(source.splitlines(), 'native-heal-regressions', 2)
+        start = job.index('      - name: Run existing F07 directory fsync failure regressions')
+        end = job.index('      - name: Upload native executable identities and original diagnostics')
+        step = job[start:end]
+        shell = '\n'.join(line[10:] for line in step[step.index('        run: |') + 1:]
+                          if not line.strip() or line.startswith('          '))
+        for mode in ('old-harness', 'success', 'run-failure', 'verify-failure'):
+            with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode):
+                root = Path(temp)
+                if mode != 'old-harness':
+                    collector = root / 'auto-testing/scripts/validation_heal/persistence_native.py'
+                    collector.parent.mkdir(parents=True)
+                    collector.write_text('''import json, os, pathlib, sys
+with pathlib.Path('calls.jsonl').open('a') as out: out.write(json.dumps(sys.argv[1:]) + '\\n')
+sys.exit(42 if os.environ['COLLECTOR_MODE'] == sys.argv[1] + '-failure' else 0)
+''')
+                env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, COLLECTOR_MODE=mode)
+                env.pop('BASH_ENV', None)
+                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell],
+                                        cwd=root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode == 0, mode in ('old-harness', 'success'), result.stderr)
+                if mode == 'old-harness':
+                    report = json.loads((root / 'native-persistence-evidence/not-available.json').read_text())
+                    self.assertIs(report['complete'], False)
+                    self.assertIs(report['requiredNativeCoverageComplete'], False)
+                    self.assertIs(report['canonicalCasePassClaimed'], False)
+                    self.assertFalse((root / 'calls.jsonl').exists())
+                else:
+                    calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()]
+                    self.assertEqual([call[0] for call in calls], ['run'] if mode == 'run-failure' else ['run', 'verify'])
+                    for call in calls: self.assertEqual(call[-2:], ['--testing-commit', 'b' * 40])
+
+
+class InterruptedCleanupGuardTests(unittest.TestCase):
+    def test_actual_workflow_guard_rejects_live_success_and_unrelated_runs(self):
+        source = (candidate.ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
+        step = source[source.index('      - name: Audit and finalize retained validation environment'):]
+        guard = step.split("          python3 - <<'PY'", 1)[1].split("          PY", 1)[0]
+        guard = "\n".join(line[10:] for line in guard.splitlines())
+        for state, conclusion, branch, workflow, accepted in (
+            ("completed", "failure", "topic", "RustFS Functional Chain", True),
+            ("completed", "cancelled", "topic", "RustFS Functional Chain", True),
+            ("in_progress", "cancelled", "topic", "RustFS Functional Chain", False),
+            ("completed", "success", "topic", "RustFS Functional Chain", False),
+            ("completed", "cancelled", "other", "RustFS Functional Chain", False),
+            ("completed", "cancelled", "topic", "Other Workflow", False),
+        ):
+            with self.subTest(state=state, conclusion=conclusion, branch=branch, workflow=workflow), tempfile.TemporaryDirectory() as temp:
+                Path(temp, "failed-run.json").write_text(json.dumps({"status": state, "conclusion": conclusion,
+                    "headBranch": branch, "workflowName": workflow}))
+                with mock.patch.dict(os.environ, VALIDATION_ROOT=temp, GITHUB_REF_NAME="topic"):
+                    if accepted:
+                        exec(compile(guard, "workflow-cleanup-guard", "exec"), {})
+                    else:
+                        with self.assertRaises(ValueError):
+                            exec(compile(guard, "workflow-cleanup-guard", "exec"), {})
 
 
 class CandidateTests(unittest.TestCase):
@@ -76,6 +440,98 @@ class CandidateTests(unittest.TestCase):
         self.assertEqual(result["artifact_id"], 789)
         self.assertEqual(result["manifest"]["source_sha"], "b" * 40)
 
+    def test_validation_manifest_binds_requested_source_and_private_channel(self):
+        manifest = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"],
+                    "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")}
+        run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"}
+        payload = io.BytesIO()
+        with zipfile.ZipFile(payload, "w") as archive:
+            archive.writestr("validation-candidate-123-2.json", json.dumps(manifest))
+        raw = payload.getvalue()
+        artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False,
+                    "size_in_bytes": len(raw), "digest": "sha256:" + hashlib.sha256(raw).hexdigest(),
+                    "workflow_run": {"id": 123, "head_sha": "a" * 40}}
+        result = candidate.read_manifest(raw, artifact, run)
+        self.assertEqual(result["channel"], "validation")
+        self.assertIn("/packages/validation/runs/123/2/", result["package_url"])
+
+    def test_validation_resolver_requires_manual_workflow_dispatch(self):
+        payload, artifact = self.archive()
+        run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40,
+               "path": ".github/workflows/rustfs-validation-package.yml", "head_branch": "main",
+               "head_repository": {"full_name": "rustfs/rustfs"}, "event": "schedule",
+               "status": "completed", "conclusion": "success", "run_started_at": "2026-10-12T00:00:00Z"}
+        with mock.patch.object(candidate, "api", side_effect=[run]):
+            with self.assertRaisesRegex(ValueError, "manually dispatched"):
+                candidate.resolve(123, 2, allow_validation=True)
+
+    def test_validation_workflow_rejects_legacy_manifest_schema(self):
+        manifest = {**self.manifest}
+        payload_buffer = io.BytesIO()
+        with zipfile.ZipFile(payload_buffer, "w") as archive:
+            archive.writestr("validation-candidate-123-2.json", json.dumps(manifest))
+        payload = payload_buffer.getvalue()
+        artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False,
+                    "size_in_bytes": len(payload), "digest": "sha256:" + hashlib.sha256(payload).hexdigest(),
+                    "workflow_run": {"id": 123, "head_sha": "a" * 40}}
+        run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"}
+        with self.assertRaisesRegex(ValueError, "schema-3"):
+            candidate.read_manifest(payload, artifact, run)
+
+    def test_private_candidate_resolution_is_explicit_and_attempt_bound(self):
+        self.manifest.update(schema=3, channel="validation", source_ref=self.manifest["source_sha"],
+                             package_url=self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/"))
+        payload, artifact = self.archive(["validation-candidate-123-2.json"])
+        artifact["name"] = "validation-candidate-123-2"
+        base = {**self.run, "head_branch": "cxymds/validation-runner", "head_repository": {"full_name": "rustfs/rustfs"},
+                "event": "workflow_dispatch", "status": "completed", "conclusion": "success",
+                "run_started_at": "2026-10-12T00:00:00Z"}
+        for path in candidate.VALIDATION_BUILD_WORKFLOWS:
+            run = {**base, "path": path}
+            with self.subTest(path=path), mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]):
+                resolved = candidate.resolve(123, 2, allow_validation=True)
+                self.assertEqual(resolved["manifest"]["source_sha"], "b" * 40)
+                self.assertEqual(resolved["workflow_sha"], "a" * 40)
+            with mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError):
+                candidate.resolve(123, 2)
+        for field, value in (("id", 124), ("run_attempt", 1), ("event", "push"), ("status", "in_progress"),
+                             ("conclusion", "failure"), ("path", ".github/workflows/other.yml"),
+                             ("head_repository", {"full_name": "untrusted/rustfs"})):
+            run = {**base, "path": ".github/workflows/rustfs-functional-chain.yml", field: value}
+            with self.subTest(field=field), mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError):
+                candidate.resolve(123, 2, allow_validation=True)
+
+    def test_private_manifest_rejects_alias_channel_source_and_schema_substitution(self):
+        valid = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"],
+                 "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")}
+        for key, value in (("channel", "nightly"), ("source_ref", "main"), ("source_sha", "e" * 40),
+                           ("package_url", self.manifest["package_url"]), ("workflow_sha", "e" * 40),
+                           ("build_run_attempt", 3), ("schema", True)):
+            with self.subTest(key=key), self.assertRaises(ValueError):
+                candidate.validate_manifest({**valid, key: value}, self.run)
+
+    def test_package_only_mode_rejects_conflicting_fleet_and_tool_requests(self):
+        from check_test_wiring import yaml_block
+        workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml"
+        job = yaml_block(workflow.read_text().splitlines(), "validation-package-request", 2)
+        begin = job.index("        run: |") + 1
+        script = "\n".join(line[10:] for line in job[begin:] if not line.strip() or line.startswith("          "))
+        import os
+        inputs = {"validation_package_sha": "a" * 40}
+        env = {**os.environ, "PACKAGE_INPUTS": json.dumps(inputs)}
+        result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        for field in ("build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha", "validation_tools_only",
+                      "validation_native_tools", "validation_shared_tools", "validation_manager_tools",
+                      "validation_cleanup_run_id", "validation_tools_run_id"):
+            with self.subTest(field=field):
+                env["PACKAGE_INPUTS"] = json.dumps({**inputs, field: "not-empty"})
+                result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True)
+                self.assertNotEqual(result.returncode, 0)
+        for value in ("main", "a" * 39, "a" * 40 + "\nFORGED=value"):
+            env["PACKAGE_INPUTS"] = json.dumps({"validation_package_sha": value})
+            self.assertNotEqual(subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True).returncode, 0)
+
 
 class EvidenceTests(unittest.TestCase):
     def setUp(self):

scripts/test_inspect_validation_runner.py

@@ -0,0 +1,115 @@
+"""Runner diagnostics must remain read-only when the shared fleet is unavailable."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import inspect_validation_runner as inspection
+
+
+class RunnerInspectionTests(unittest.TestCase):
+    def request(self):
+        return {"validation_runner_inspect_only": True, "validation_case": "",
+                "validation_tools_only": False, "i05_diagnostic_branch": "timeout",
+                "validation_tools_run_attempt": "1"}
+
+    def test_exclusive_request_rejects_every_execution_mode_before_api_access(self):
+        inspection.validate_request(self.request())
+        for key, value in [("validation_runner_inspect_only", False),
+                           ("validation_case", "environment.inspect"), ("build_run_id", "123"),
+                           ("validation_package_sha", "a" * 40), ("validation_tools_only", True),
+                           ("validation_native_tools", True), ("validation_shared_tools", True),
+                           ("validation_manager_tools", True), ("validation_cleanup_run_id", "123"),
+                           ("auto_testing_sha", "a" * 40), ("validation_tools_run_id", "123"),
+                           ("i05_diagnostic_branch", "queued-successor"),
+                           ("validation_tools_run_attempt", "2")]:
+            with self.subTest(key=key), self.assertRaises(ValueError):
+                inspection.validate_request({**self.request(), key: value})
+
+    def fixture(self, runners, *, unknown_jobs=False):
+        def api(path):
+            self.assertTrue(path.startswith("repos/rustfs/rustfs/actions/"))
+            if "/runners?" in path:
+                return {"runners": runners}
+            if "/runs?status=queued&" in path:
+                return {"workflow_runs": [{"id": 123, "name": "RustFS Functional Chain",
+                    "status": "queued", "created_at": "2026-10-26T00:00:00Z", "html_url": "run-url"}]}
+            if "/runs?status=" in path:
+                return {"workflow_runs": []}
+            self.assertIn("/runs/123/jobs?", path)
+            return {"jobs": [] if unknown_jobs else [
+                {"id": 456, "name": "validation", "status": "queued", "runner_name": "",
+                 "labels": ["smoke-testing"], "html_url": "job-url", "steps": [{"secret": "excluded"}]},
+                {"id": 457, "name": "native", "status": "in_progress", "labels": ["ubuntu-latest"]},
+                {"id": 458, "name": "prior", "status": "completed", "labels": ["smoke-testing"]}]}
+        return api
+
+    def test_offline_idle_busy_and_unregistered_are_distinct_with_only_fleet_jobs(self):
+        runner = {"id": 7, "name": "rustfs-smoke-testing", "status": "offline", "busy": False,
+                  "labels": [{"name": "smoke-testing"}], "extra": {"secret": "excluded"}}
+        for state, busy, expected in [("offline", False, "offline"), ("online", True, "online-busy"),
+                                       ("online", False, "online-idle"), (None, False, "unregistered")]:
+            with self.subTest(expected=expected):
+                rows = [{**runner, "status": state, "busy": busy}] if state else []
+                report = {}
+                with patch.object(inspection, "api", side_effect=self.fixture(rows)):
+                    inspection.collect(report)
+                self.assertEqual(report["runnerAvailability"], expected)
+                self.assertTrue(report["complete"])
+                self.assertEqual([row["jobId"] for row in report["fleetJobs"]], [456])
+                self.assertNotIn("secret", json.dumps(report))
+
+    def test_jobless_workflow_is_retained_without_inventing_a_runner_assignment(self):
+        report = {}
+        with patch.object(inspection, "api", side_effect=self.fixture([], unknown_jobs=True)):
+            inspection.collect(report)
+        self.assertEqual(report["fleetJobs"], [])
+        self.assertEqual(report["runsWithoutJobs"][0]["runId"], 123)
+
+    def test_inventory_pagination_is_bounded_and_cannot_claim_complete_when_truncated(self):
+        row = {"id": 7}
+        with patch.object(inspection, "api", return_value={"runners": [row] * 100}) as api:
+            with self.assertRaisesRegex(ValueError, "bounded inspection"):
+                inspection.pages("repos/rustfs/rustfs/actions/runners", "runners")
+        self.assertEqual(api.call_count, 10)
+
+    def test_api_failure_keeps_partial_report_without_subprocess_diagnostics_or_fleet_claims(self):
+        for inputs, expected_calls in [(self.request(), 1), ({**self.request(), "validation_tools_only": True}, 0)]:
+            with self.subTest(inputs=inputs), tempfile.TemporaryDirectory() as directory:
+                output = Path(directory) / "report.json"
+                with patch.dict(os.environ, RUNNER_INSPECTION_INPUTS=json.dumps(inputs)), \
+                     patch.object(inspection.sys, "argv", ["inspect", "--output", str(output)]), \
+                     patch.object(inspection, "api", side_effect=subprocess.CalledProcessError(
+                         1, ["gh", "api"], stderr=b"secret-token")) as api:
+                    self.assertEqual(inspection.main(), 1)
+                report = json.loads(output.read_text())
+                self.assertFalse(report["complete"])
+                for field in ("mutated", "fleetAccessed", "cleanStateCertified",
+                              "sharedWorkflowIdleCertified", "canonicalCasePassClaimed"):
+                    self.assertIs(report[field], False)
+                self.assertNotIn("secret-token", output.read_text())
+                self.assertEqual(api.call_count, expected_calls)
+
+    def test_workflow_is_hosted_and_uses_a_separate_concurrency_group(self):
+        workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml"
+        source = workflow.read_text()
+        job = source.split("  validation-runner-inspection:\n", 1)[1].split("  validation-package-request:", 1)[0]
+        self.assertIn("runs-on: ubuntu-latest", job)
+        self.assertIn("timeout-minutes: 10", job)
+        self.assertIn("RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}", job)
+        self.assertIn("if: ${{ always() }}", job)
+        self.assertNotIn("RUSTFS_SECRET_KEY", job)
+        concurrency = source.split("concurrency:\n", 1)[1].split("jobs:\n", 1)[0]
+        self.assertLess(concurrency.index("validation_runner_inspect_only"), concurrency.index("validation_package_sha"))
+        self.assertIn("rustfs-functional-validation-runner-inspection", concurrency)
+        for name in ("validation-package-request", "validation-package", "prepare", "validation"):
+            job = source.split("  " + name + ":\n", 1)[1].split("\n    runs-on:", 1)[0].split("\n    uses:", 1)[0]
+            self.assertIn("!inputs.validation_runner_inspect_only", job)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_nightly_candidate.py

@@ -221,5 +221,171 @@ SH
         self.assertNotIn("          overwrite: true", upload)
 
 
+class ValidationPackageTests(unittest.TestCase):
+    def setUp(self):
+        # Exercise the real upload shell with the same conditional object-store
+        # fixture used by the nightly publication tests.
+        self.fixture = NightlyCandidateTests("test_success_binds_actual_package_checkout_and_attempt")
+        self.fixture.setUp()
+        self.addCleanup(self.fixture.temp.cleanup)
+        self.workflow = ROOT / ".github/workflows/rustfs-validation-package.yml"
+        job = yaml_block(self.workflow.read_text().splitlines(), "publish", 2)
+        start = job.index("      - name: Upload validation package to immutable R2 path")
+        end = job.index("      - name: Upload validation candidate manifest")
+        upload = job[start:end]
+        begin = upload.index("        run: |") + 1
+        self.fixture.shell = "\n".join(line[10:] for line in upload[begin:] if not line.strip() or line.startswith("          "))
+        self.fixture.shims.write_text(self.fixture.shims.read_text().replace("/packages/nightly/runs/*", "/packages/validation/runs/*"))
+        self.fixture.env.update(PACKAGE_FILE=self.fixture.package.name, VALIDATION_SOURCE_SHA=self.fixture.sha,
+                                VALIDATION_SOURCE_REF=self.fixture.sha, GITHUB_SHA="f" * 40, AWS_CLI="aws")
+        def shell(name, end_name):
+            step = job[job.index("      - name: " + name):job.index("      - name: " + end_name)]
+            return "\n".join(line[10:] for line in step[step.index("        run: |") + 1:]
+                             if not line.strip() or line.startswith("          "))
+        self.stage_shell = shell("Validate and stage opaque package bytes", "Prepare trusted publication tools")
+        self.tools_shell = shell("Prepare trusted publication tools", "Upload validation package to immutable R2 path")
+
+    def stage(self):
+        env = dict(os.environ, RUNNER_TEMP=str(self.fixture.root))
+        env.pop("BASH_ENV", None)
+        return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.stage_shell],
+                              cwd=self.fixture.root, env=env, capture_output=True, text=True)
+
+    def test_success_uploads_only_one_immutable_object_and_exact_source(self):
+        result = self.fixture.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        manifest = self.fixture.manifest()
+        self.assertEqual(manifest["schema"], 3)
+        self.assertEqual(manifest["channel"], "validation")
+        self.assertEqual(manifest["source_sha"], self.fixture.sha)
+        self.assertEqual(manifest["source_ref"], self.fixture.sha)
+        self.assertEqual(manifest["workflow_sha"], "f" * 40)
+        validate_manifest(manifest, {"id": 12345, "run_attempt": 1, "head_sha": "f" * 40})
+        objects = [path for path in self.fixture.store.rglob("*") if path.is_file()]
+        self.assertEqual(len(objects), 1)
+        self.assertEqual(objects[0].read_bytes(), self.fixture.package.read_bytes())
+        self.assertEqual(str(objects[0].relative_to(self.fixture.store)),
+                         f"artifacts/rustfs/packages/validation/runs/12345/1/{self.fixture.digest}/rustfs.deb")
+        self.assertNotIn("s3 cp", (self.fixture.root / "aws.log").read_text())
+
+    def test_failure_or_missing_credentials_cannot_certify_a_candidate(self):
+        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download")):
+            with self.subTest(mode=mode):
+                result = self.fixture.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
+                self.assertNotEqual(result.returncode, 0)
+                self.assertFalse(self.fixture.output.exists())
+                self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), [])
+        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
+            with self.subTest(key=key):
+                self.assertNotEqual(self.fixture.run_publish(**{key: ""}).returncode, 0)
+                self.assertFalse(self.fixture.output.exists())
+
+    def test_stale_source_and_duplicate_objects_are_rejected(self):
+        for source in ("not-an-exact-sha", "e" * 40):
+            with self.subTest(source=source):
+                result = self.fixture.run_publish(VALIDATION_SOURCE_SHA=source)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertFalse(self.fixture.store.exists())
+        result = self.fixture.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        manifest = self.fixture.manifest()
+        key = manifest["package_url"].removeprefix("https://dl.rustfs.com/")
+        stored = self.fixture.store / key
+        stored.write_bytes(b"preexisting conflicting object")
+        (self.fixture.root / "validation-candidate-12345-1.json").unlink()
+        result = self.fixture.run_publish()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
+        self.assertFalse(self.fixture.output.exists())
+
+    def test_exact_source_and_workflow_identity_are_preserved_separately(self):
+        lines = self.workflow.read_text().splitlines()
+        build = "\n".join(yaml_block(lines, "build", 2))
+        publish = "\n".join(yaml_block(lines, "publish", 2))
+        source = "\n".join(yaml_block(lines, "resolve-source", 2))
+        self.assertIn('[[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]', source)
+        self.assertIn('test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"', source)
+        self.assertIn("ref: ${{ needs.resolve-source.outputs.source_sha }}", build)
+        self.assertIn('--arg source_sha "${VALIDATION_SOURCE_SHA}"', publish)
+        self.assertIn('--arg workflow_sha "${GITHUB_SHA}"', publish)
+        self.assertNotIn("PF_TESTING_GH_TOKEN", build)
+        self.assertNotIn("publish_nightly_assets", build)
+
+    def test_candidate_execution_and_publication_credentials_are_in_separate_jobs(self):
+        lines = self.workflow.read_text().splitlines()
+        build = "\n".join(yaml_block(lines, "build", 2))
+        publish = "\n".join(yaml_block(lines, "publish", 2))
+        self.assertIn("uses: ./.github/actions/setup", build)
+        self.assertIn("cargo build", build)
+        self.assertNotIn("secrets.", build)
+        self.assertNotIn("id-token: write", build)
+        self.assertIn("needs: [resolve-source, build]", publish)
+        self.assertIn("runs-on: ubuntu-latest", publish)
+        for forbidden in ("actions/checkout@", "actions/cache@", "uses: ./", "needs.build.outputs", "cargo ", "dpkg", "source "):
+            with self.subTest(forbidden=forbidden): self.assertNotIn(forbidden, publish)
+        name = "name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}"
+        self.assertIn(name, build)
+        self.assertIn(name, publish)
+        self.assertIn("VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}", publish)
+        self.assertIn("VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}", publish)
+        prepare = publish.split("      - name: Upload validation package to immutable R2 path")[0]
+        self.assertNotIn("secrets.", prepare)
+        self.assertIn("secrets.R2_SECRET_ACCESS_KEY", publish)
+
+    def test_staging_rejects_nonregular_empty_or_extra_artifact_entries(self):
+        incoming = self.fixture.root / "validation-package-input"
+        incoming.mkdir()
+        package = incoming / "rustfs.deb"
+        for kind in ("missing", "empty", "symlink", "directory", "fifo", "extra", "directory-symlink"):
+            with self.subTest(kind=kind):
+                if kind == "empty": package.touch()
+                elif kind == "symlink": package.symlink_to(self.fixture.package)
+                elif kind == "directory": package.mkdir()
+                elif kind == "fifo": os.mkfifo(package)
+                elif kind == "extra":
+                    package.write_bytes(b"opaque")
+                    (incoming / "BASH_ENV").write_text("exit 1")
+                elif kind == "directory-symlink":
+                    incoming.rmdir()
+                    incoming.symlink_to(self.fixture.root, target_is_directory=True)
+                self.assertNotEqual(self.stage().returncode, 0)
+                self.assertFalse((self.fixture.root / "validation-publication").exists())
+                if incoming.is_symlink():
+                    incoming.unlink()
+                    incoming.mkdir()
+                elif package.is_dir(): package.rmdir()
+                else: package.unlink(missing_ok=True)
+                (incoming / "BASH_ENV").unlink(missing_ok=True)
+
+    def test_staging_and_publication_never_execute_candidate_package_or_python_hooks(self):
+        incoming = self.fixture.root / "validation-package-input"
+        incoming.mkdir()
+        marker = self.fixture.root / "candidate-executed"
+        payload = f'#!/bin/bash\nprintf "%s" "$R2_SECRET_ACCESS_KEY" > "{marker}"\n'.encode()
+        (incoming / "rustfs.deb").write_bytes(payload)
+        (self.fixture.root / "sitecustomize.py").write_text(f"open({str(marker)!r}, 'w').write('python hook')")
+        result = self.stage()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        staged = self.fixture.root / "validation-publication/rustfs.deb"
+        self.assertEqual(staged.read_bytes(), payload)
+        result = self.fixture.run_publish(PACKAGE_FILE=str(staged))
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertFalse(marker.exists())
+        self.assertEqual(self.fixture.manifest()["package_sha256"], hashlib.sha256(payload).hexdigest())
+
+    def test_cli_bootstrap_finishes_without_publication_secrets(self):
+        for mode in ("success", "old-cli", "bootstrap-failure", "install-failure", "broken-install"):
+            with self.subTest(mode=mode):
+                (self.fixture.root / "installed").unlink(missing_ok=True)
+                env = dict(self.fixture.env, FAKE_MODE=mode)
+                for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
+                    env.pop(key)
+                result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", self.tools_shell],
+                                        cwd=self.fixture.root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode == 0, mode in ("success", "old-cli"), result.stderr)
+                self.assertFalse(self.fixture.store.exists())
+                self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), [])
+
+
 if __name__ == "__main__":
     unittest.main()

scripts/test_retain_validation_evidence.py

@@ -0,0 +1,189 @@
+"""Interrupted evidence is read-only and never an implicit recovery or PASS."""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import retain_validation_evidence as retain
+import prepare_retained_validation_recovery as recovery_input
+
+
+class RetainEvidenceTests(unittest.TestCase):
+    def fixture(self, directory):
+        temporary = Path(directory).resolve() / "_work/_temp"
+        temporary.mkdir(parents=True)
+        parent = temporary.parent / "rustfs-validation-evidence"
+        parent.mkdir(mode=0o700)
+        root = parent / "rustfs-validation-77-1"
+        root.mkdir(mode=0o700)
+        (root / "runner-storage-preflight.json").write_text(json.dumps({
+            "runId": "77", "runAttempt": "1", "evidenceRoot": str(root),
+            "persistentOutsideRunnerTemp": True}))
+        (root / "original.json").write_bytes(b'{"verdict":"RUNNING"}')
+        env = {"RUNNER_TEMP": str(temporary), "GITHUB_RUN_ID": "88", "GITHUB_RUN_ATTEMPT": "1",
+               "RUNNER_NAME": "owned-runner", "GITHUB_ENV": str(temporary / "env"),
+               "EVIDENCE_RETAIN_INPUTS": json.dumps({"validation_retain_evidence_run_id": "77",
+                    "i05_diagnostic_branch": "timeout", "validation_tools_run_attempt": "1"})}
+        run = {"id": 77, "run_attempt": 1, "status": "completed", "conclusion": "failure",
+               "event": "workflow_dispatch", "repository": {"full_name": "rustfs/rustfs"},
+               "path": ".github/workflows/rustfs-functional-chain.yml", "head_sha": "a" * 40,
+               "html_url": "source-url"}
+        jobs = {"total_count": 1, "jobs": [{"name": "validation", "runner_name": "owned-runner"}]}
+        return env, root, run, jobs
+
+    def test_existing_bytes_and_original_running_verdict_remain_unchanged(self):
+        with tempfile.TemporaryDirectory() as directory:
+            env, root, run, jobs = self.fixture(directory)
+            before = {p.name: p.read_bytes() for p in root.iterdir()}
+            report = {}
+            with patch.object(retain, "api", side_effect=[run, jobs]) as api:
+                self.assertEqual(retain.prepare(env, report), root)
+            self.assertTrue(report["complete"])
+            self.assertEqual({p.name: p.read_bytes() for p in root.iterdir()}, before)
+            self.assertEqual({row["path"]: row["sha256"] for row in report["files"]},
+                             {name: hashlib.sha256(raw).hexdigest() for name, raw in before.items()})
+            self.assertEqual([call.args[0] for call in api.call_args_list], [
+                "repos/rustfs/rustfs/actions/runs/77/attempts/1",
+                "repos/rustfs/rustfs/actions/runs/77/attempts/1/jobs?per_page=100"])
+
+    def test_mixed_execution_requests_and_unsafe_run_ids_refuse_before_api(self):
+        with tempfile.TemporaryDirectory() as directory:
+            env, root, run, jobs = self.fixture(directory)
+            inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"])
+            for field, value in [("validation_case", "heal.admin-healthy-noop"),
+                                 ("validation_runner_inspect_only", True), ("validation_tools_only", True),
+                                 ("build_run_id", "1"), ("validation_retain_evidence_run_id", "../77"),
+                                 ("validation_retain_evidence_run_id", "88")]:
+                with self.subTest(field=field, value=value), patch.object(retain, "api") as api:
+                    altered = {**env, "EVIDENCE_RETAIN_INPUTS": json.dumps({**inputs, field: value})}
+                    with self.assertRaises(ValueError): retain.prepare(altered, {})
+                    api.assert_not_called()
+
+    def test_wrong_repository_running_source_and_different_runner_refuse(self):
+        for mutation in ("repo", "running", "success", "runner", "attempt"):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                env, root, run, jobs = self.fixture(directory)
+                if mutation == "repo": run["repository"]["full_name"] = "foreign/repo"
+                elif mutation == "running": run["status"] = "in_progress"
+                elif mutation == "success": run["conclusion"] = "success"
+                elif mutation == "attempt": run["run_attempt"] = 2
+                else: jobs["jobs"][0]["runner_name"] = "other-runner"
+                with patch.object(retain, "api", side_effect=[run, jobs]), self.assertRaises(ValueError):
+                    retain.prepare(env, {})
+
+    def test_links_special_files_and_byte_budget_do_not_escape_the_original_root(self):
+        for mutation in ("symlink", "hardlink", "fifo", "budget", "public-root"):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                env, root, run, jobs = self.fixture(directory)
+                if mutation == "symlink": (root / "foreign").symlink_to(Path(directory) / "secret")
+                elif mutation == "hardlink": os.link(root / "original.json", root / "alias")
+                elif mutation == "fifo": os.mkfifo(root / "fifo")
+                elif mutation == "public-root": root.chmod(0o755)
+                with patch.object(retain, "api", side_effect=[run, jobs]), \
+                     patch.object(retain, "MAX_BYTES", 1 if mutation == "budget" else retain.MAX_BYTES), \
+                     self.assertRaises(ValueError):
+                    retain.prepare(env, {})
+
+    def test_file_changes_and_foreign_storage_provenance_are_rejected(self):
+        for mutation in ("write", "marker"):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                env, root, run, jobs = self.fixture(directory)
+                if mutation == "marker":
+                    (root / "runner-storage-preflight.json").write_text('{"runId":"foreign"}')
+                real = os.read
+                changed = []
+                def read(fd, size):
+                    raw = real(fd, size)
+                    if mutation == "write" and not changed:
+                        changed.append(True)
+                        with (root / "original.json").open("ab") as stream: stream.write(b"changed")
+                    return raw
+                with patch.object(retain, "api", side_effect=[run, jobs]), patch.object(retain.os, "read", side_effect=read), \
+                     self.assertRaises(ValueError):
+                    retain.prepare(env, {})
+
+    def test_cli_exports_only_admitted_root_and_emits_no_case_or_recovery_claim(self):
+        with tempfile.TemporaryDirectory() as directory:
+            env, root, run, jobs = self.fixture(directory)
+            output = Path(directory) / "receipt.json"
+            with patch.dict(os.environ, env), patch.object(retain, "api", side_effect=[run, jobs]), \
+                 patch.object(retain.sys, "argv", ["retain", "--report", str(output)]):
+                self.assertEqual(retain.main(), 0)
+            value = json.loads(output.read_text())
+            for field in ("fleetAccessed", "originalEvidenceModified", "sourceVerdictRewritten", "canonicalCasePassClaimed"):
+                self.assertIs(value[field], False)
+            self.assertIn("VALIDATION_RETAIN_ROOT=" + str(root), Path(env["GITHUB_ENV"]).read_text())
+
+    def test_cli_keeps_incomplete_receipt_without_exporting_a_missing_original_root(self):
+        with tempfile.TemporaryDirectory() as directory:
+            env, root, run, jobs = self.fixture(directory)
+            env["EVIDENCE_RETAIN_INPUTS"] = '{"validation_retain_evidence_run_id":"99"}'
+            run["id"] = 99
+            output = Path(directory) / "receipt.json"
+            with patch.dict(os.environ, env), patch.object(retain, "api", side_effect=[run, jobs]), \
+                 patch.object(retain.sys, "argv", ["retain", "--report", str(output)]):
+                self.assertEqual(retain.main(), 1)
+            value = json.loads(output.read_text())
+            self.assertFalse(value["complete"])
+            self.assertEqual(value["errorType"], "FileNotFoundError")
+            self.assertFalse(Path(env["GITHUB_ENV"]).exists())
+
+    def test_recovery_copy_keeps_original_bytes_and_separate_collection_identity(self):
+        with tempfile.TemporaryDirectory() as directory:
+            env, root, run, jobs = self.fixture(directory)
+            destination = Path(directory).resolve() / 'new-recovery'; destination.mkdir(mode=0o700)
+            env.update(FAILED_RUN_ID='77', GITHUB_REF_NAME='topic', VALIDATION_ROOT=str(destination))
+            run['head_branch'] = 'topic'
+            nested = root / 'nested'; nested.mkdir(); (nested / 'receipt').write_bytes(b'ACK')
+            before = retain.inventory(root)
+            report = {'collectionRunId': '88', 'sourceArtifactClaimed': False}
+            with patch.object(recovery_input, 'api', return_value=run), patch.object(retain, 'api', side_effect=[run, jobs]):
+                recovery_input.stage(env, report)
+            self.assertTrue(report['complete'])
+            self.assertEqual(report['collectionRunId'], '88')
+            self.assertEqual(report['sourceRun']['id'], 77)
+            self.assertIs(report['sourceArtifactClaimed'], False)
+            copied = Path(report['recoveryInput'])
+            self.assertEqual(retain.inventory(copied), before)
+            self.assertEqual(retain.inventory(root), before)
+            self.assertNotEqual((copied / 'original.json').stat().st_ino, (root / 'original.json').stat().st_ino)
+
+    def test_recovery_copy_refuses_changed_source_and_foreign_branch(self):
+        for mutation in ('drift', 'branch'):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory:
+                env, root, run, jobs = self.fixture(directory)
+                destination = Path(directory).resolve() / 'recovery'; destination.mkdir(mode=0o700)
+                env.update(FAILED_RUN_ID='77', GITHUB_REF_NAME='topic', VALIDATION_ROOT=str(destination))
+                run['head_branch'] = 'foreign' if mutation == 'branch' else 'topic'
+                original = retain.prepare
+                def changed(environment, report):
+                    source = original(environment, report)
+                    (source / 'original.json').write_bytes(b'new bytes after admission')
+                    return source
+                with patch.object(recovery_input, 'api', return_value=run), \
+                     patch.object(retain, 'api', side_effect=[run, jobs]), \
+                     patch.object(recovery_input, 'prepare', side_effect=changed), self.assertRaises(ValueError):
+                    recovery_input.stage(env, {})
+
+    def test_copy_does_not_follow_a_symlink_swapped_after_directory_listing(self):
+        with tempfile.TemporaryDirectory() as directory:
+            env, root, run, jobs = self.fixture(directory)
+            secret = Path(directory) / 'private'; secret.write_bytes(b'outside-root-secret')
+            destination = Path(directory).resolve() / 'copy'
+            original_stat = os.stat
+            def swapped(path, *args, **kwargs):
+                info = original_stat(path, *args, **kwargs)
+                if path == 'original.json' and kwargs.get('follow_symlinks') is False:
+                    (root / path).unlink(); (root / path).symlink_to(secret)
+                return info
+            with patch.object(retain.os, 'stat', side_effect=swapped), self.assertRaises(OSError):
+                retain.inventory(root, copy_to=destination)
+            self.assertFalse((destination / 'original.json').exists())
+
+
+if __name__ == "__main__":
+    unittest.main()