diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 428ae33ec..60089f36c 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -64,6 +64,8 @@ script-tests: ## Run shell script tests $(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py $(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py $(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py + $(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py + $(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py $(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index 3e14e17b2..453a87956 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -20,13 +20,91 @@ on: workflow_dispatch: inputs: build_run_id: - description: Successful nightly build run on main + description: Successful nightly or validation package run; empty for package-only build type: string - required: true + required: false + default: '' build_run_attempt: description: Exact successful build attempt type: string - required: true + required: false + default: '' + validation_package_sha: + description: Exact source SHA for an isolated package-only build + type: string + required: false + default: '' + validation_case: + description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain + type: string + required: false + default: '' + i05_diagnostic_branch: + description: I05 source-plan runs both H1/H2 controllers with exact native evidence; subsets remain diagnostic + type: choice + options: [source-plan, timeout, queued-successor] + required: false + default: source-plan + auto_testing_sha: + description: Exact auto-testing commit for a validation case + type: string + required: false + default: '' + validation_retain_evidence_run_id: + description: Upload existing persistent evidence from one interrupted validation run without fleet access + type: string + required: false + default: '' + validation_retain_evidence_run_attempt: + description: Exact interrupted run attempt to retain + type: string + required: false + default: '1' + validation_runner_inspect_only: + description: Inspect GitHub runner registration and queued jobs on a hosted runner without fleet access + type: boolean + required: false + default: false + validation_tools_only: + description: Build candidate-bound validation tools without running a fleet case + type: boolean + required: false + default: false + validation_native_tools: + description: Include exact-source native regressions in a tools-only build + type: boolean + required: false + default: false + validation_shared_tools: + description: Include the controlled C10 shared-reference encoder in a tools-only build + type: boolean + required: false + default: false + validation_manager_tools: + description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build + type: boolean + required: false + default: false + validation_cleanup_run_id: + description: Failed validation run whose supported retained environment needs evidence-based finalization + type: string + required: false + default: '' + validation_cleanup_retained_evidence: + description: Recover from verified persistent runner-local evidence when the failed run uploaded no artifact + type: boolean + required: false + default: false + validation_tools_run_id: + description: Successful tool build run; required by Heal physical metadata checks + type: string + required: false + default: '' + validation_tools_run_attempt: + description: Exact tool build attempt + type: string + required: false + default: '1' workflow_run: workflows: ["Nightly GNU Build"] types: [completed] @@ -36,12 +114,103 @@ permissions: actions: read concurrency: - group: rustfs-functional-chain-runs + group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only && 'rustfs-functional-validation-runner-inspection' || github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }} cancel-in-progress: false jobs: + validation-runner-inspection: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + persist-credentials: false + - name: Inspect shared runner registration and queue without fleet access + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }} + run: >- + python3 -B scripts/inspect_validation_runner.py + --output "${RUNNER_TEMP}/validation-runner-inspection/report.json" + - name: Upload read-only runner inspection including incomplete reports + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: validation-runner-inspection-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/validation-runner-inspection/report.json + if-no-files-found: error + validation-evidence-retention: + if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id != '' }} + # The original interrupted evidence exists only on this long-lived runner. + runs-on: smoke-testing + timeout-minutes: 15 + concurrency: + group: rustfs-shared-functional-tests-v2 + cancel-in-progress: false + steps: + - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + persist-credentials: false + - name: Verify existing interrupted evidence without changing it + env: + GH_TOKEN: ${{ github.token }} + EVIDENCE_RETAIN_INPUTS: ${{ toJSON(inputs) }} + run: >- + python3 -B scripts/retain_validation_evidence.py + --report "${RUNNER_TEMP}/validation-evidence-retention/report.json" + - name: Upload admitted original evidence under collection provenance + if: ${{ env.VALIDATION_RETAIN_ROOT != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-retained-${{ env.VALIDATION_RETAIN_SOURCE_RUN }}-${{ env.VALIDATION_RETAIN_SOURCE_ATTEMPT }}-by-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.VALIDATION_RETAIN_ROOT }} + if-no-files-found: error + include-hidden-files: true + - name: Upload retention receipt including incomplete reports + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: validation-evidence-retention-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/validation-evidence-retention/report.json + if-no-files-found: error + validation-package-request: + if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Validate exclusive package-only request + env: + PACKAGE_INPUTS: ${{ toJSON(inputs) }} + run: | + python3 - <<'PY' + import json + import os + import re + value = json.loads(os.environ["PACKAGE_INPUTS"]) + if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None: + raise ValueError("package build requires an exact source SHA") + exclusive = ("validation_retain_evidence_run_id", "validation_runner_inspect_only", "build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha", + "validation_tools_only", "validation_native_tools", "validation_shared_tools", + "validation_manager_tools", "validation_cleanup_run_id", "validation_cleanup_retained_evidence", "validation_tools_run_id") + if any(value.get(name) for name in exclusive): + raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution") + PY + + validation-package: + needs: validation-package-request + if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }} + uses: ./.github/workflows/rustfs-validation-package.yml + with: + source_sha: ${{ inputs.validation_package_sha }} + secrets: + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + R2_BUCKET: ${{ secrets.R2_BUCKET }} + prepare: - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }} + if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && !inputs.validation_cleanup_retained_evidence && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }} runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -89,6 +258,411 @@ jobs: echo "performance_ready=offline" >> "$GITHUB_OUTPUT" fi + validation: + if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }} + runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }} + timeout-minutes: 90 + outputs: + source_sha: ${{ steps.candidate.outputs.source_sha }} + concurrency: + group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }} + cancel-in-progress: false + env: + RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }} + RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }} + RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }} + RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }} + RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }} + steps: + - name: Checkout candidate resolver + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + persist-credentials: false + + - name: Initialize validation evidence + env: + VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }} + run: | + set -euo pipefail + umask 077 + python3 -B scripts/initialize_validation_evidence.py + + - name: Resolve immutable candidate + id: candidate + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} + BUILD_RUN_ID: ${{ inputs.build_run_id }} + BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }} + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }} + CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }} + SELECTED_CASE: ${{ inputs.validation_case }} + TOOLS_ONLY: ${{ inputs.validation_tools_only }} + NATIVE_TOOLS: ${{ inputs.validation_native_tools }} + SHARED_TOOLS: ${{ inputs.validation_shared_tools }} + MANAGER_TOOLS: ${{ inputs.validation_manager_tools }} + I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }} + run: | + set -euo pipefail + python3 - <<'PY' + import json + import os + from pathlib import Path + import re + from scripts.resolve_functional_candidate import resolve + + revision = os.environ["AUTO_TESTING_SHA"] + cleanup = os.environ["CLEANUP_RUN_ID"] + if os.environ["CLEANUP_RETAINED_EVIDENCE"] == "true" and not cleanup: + raise ValueError("persistent recovery input requires one exact cleanup run") + if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]): + raise ValueError("tools-only execution cannot select a fleet case or cleanup") + if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]): + raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup") + if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]): + raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup") + if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]): + raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup") + if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"): + raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution") + diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "source-plan" + if diagnostic_branch not in {"source-plan", "timeout", "queued-successor"}: + raise ValueError("I05 diagnostic branch is not registered") + if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort": + raise ValueError("queued-successor is available only for the I05 diagnostic case") + if re.fullmatch(r"[0-9a-f]{40}", revision) is None: + raise ValueError("validation requires an exact auto-testing commit SHA") + candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True) + manifest = candidate["manifest"] + root = Path(os.environ["VALIDATION_ROOT"]) + (root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n") + (root / "candidate.json").write_text(json.dumps({ + "package_url": manifest["package_url"], + "package_sha256": manifest["package_sha256"], + "source_sha": manifest["source_sha"], + }, sort_keys=True) + "\n") + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write("source_sha=" + manifest["source_sha"] + "\n") + PY + + - name: Checkout exact auto-testing revision + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + repository: rustfs/auto-testing + ref: ${{ inputs.auto_testing_sha }} + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + path: auto-testing + persist-credentials: false + + - name: Prepare validation Python runtime + if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install validation Python client + if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }} + run: | + set -euo pipefail + validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + test ! -e "${validation_python}" + python3 -m venv "${validation_python}" + "${validation_python}/bin/python" -m pip install \ + -r auto-testing/scripts/validation_ilm/requirements.txt + printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}" + + - name: Validate Heal tool build identity + if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }} + env: + GH_TOKEN: ${{ github.token }} + TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }} + TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }} + run: | + set -euo pipefail + [[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]] + [[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]] + gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \ + > "${VALIDATION_ROOT}/tool-run.json" + python3 - <<'PY' + import json + import os + from pathlib import Path + value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text()) + if (value.get("conclusion") != "success" or value.get("status") != "completed" + or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"]) + or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}): + raise ValueError("Heal tool run is not an exact successful validation build attempt") + PY + + - name: Download pinned Heal metadata inspector + if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.validation_tools_run_id }} + name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }} + path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Verify Heal metadata inspector + if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }} + run: | + set -euo pipefail + TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + python3 -B auto-testing/scripts/validation_heal/inspector.py \ + --directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + > "${VALIDATION_ROOT}/tool-manifest.json" + printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Select exact-source race native evidence + id: race_native + if: ${{ inputs.validation_case == 'heal.race-unversioned-overwrite' || inputs.validation_case == 'heal.race-versioned-put' || inputs.validation_case == 'heal.race-version-delete' || inputs.validation_case == 'heal.race-markers-and-batch-delete' || inputs.validation_case == 'heal.race-multipart-complete-abort' || inputs.validation_case == 'heal.admin-nolock-safety' || inputs.validation_case == 'heal.race-copy-and-metadata-update' }} + env: + CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }} + VALIDATION_CASE_ID: ${{ inputs.validation_case }} + run: | + set -euo pipefail + required=false + if [ -f auto-testing/scripts/validation_heal/race_native.py ]; then + status=0 + python3 -B auto-testing/scripts/validation_heal/race_native.py supports \ + --commit "${CANDIDATE_SHA}" --case-id "${VALIDATION_CASE_ID}" || status=$? + if [ "${status}" -eq 0 ]; then + required=true + elif [ "${status}" -ne 2 ]; then + exit "${status}" + fi + fi + reviewed=false + case "${CANDIDATE_SHA}:${VALIDATION_CASE_ID}" in + 6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca:heal.race-unversioned-overwrite | \ + bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-unversioned-overwrite | \ + bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-versioned-put | \ + bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-version-delete | \ + 8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-unversioned-overwrite | \ + 8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-versioned-put | \ + 8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-version-delete | \ + 8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-markers-and-batch-delete | \ + 757b751482bb970bc321e92059690a7698554358:heal.admin-nolock-safety | \ + 757b751482bb970bc321e92059690a7698554358:heal.race-multipart-complete-abort | \ + 757b751482bb970bc321e92059690a7698554358:heal.race-copy-and-metadata-update) reviewed=true ;; + esac + if [ "${reviewed}" = true ] && [ "${required}" != true ]; then + printf 'Race candidate requires its reviewed native collector before fleet execution\n' >&2 + exit 1 + fi + printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}" + + - name: Download pinned native storage evidence + if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout') || (inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac') }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.validation_tools_run_id }} + name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }} + path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Verify C07 native source and exact test evidence + if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }} + run: | + set -euo pipefail + NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" + python3 -B auto-testing/scripts/validation_heal/authority.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + > "${VALIDATION_ROOT}/native-authority-manifest.json" + printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Verify C11 physical storage and exact final-rename barrier + if: ${{ inputs.validation_case == 'heal.bucket-incarnation-recreation' }} + run: | + set -euo pipefail + NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" + python3 -B auto-testing/scripts/validation_heal/incarnation_native.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${{ inputs.auto_testing_sha }}" \ + > "${VALIDATION_ROOT}/native-incarnation-manifest.json" + printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Verify exact-source race native and runtime evidence + if: ${{ steps.race_native.outputs.required == 'true' }} + env: + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + VALIDATION_CASE_ID: ${{ inputs.validation_case }} + run: | + set -euo pipefail + NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" + python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${AUTO_TESTING_SHA}" --case-id "${VALIDATION_CASE_ID}" \ + > "${VALIDATION_ROOT}/native-race-manifest.json" + runtime_driver=race.py + if [ "${VALIDATION_CASE_ID}" = 'heal.admin-nolock-safety' ]; then + runtime_driver=nolock.py + fi + VALIDATION_HEAL_NATIVE_DIRECTORY="${NATIVE_DIRECTORY}" \ + python3 -B "auto-testing/scripts/validation_heal/${runtime_driver}" preflight \ + --candidate "${VALIDATION_ROOT}/candidate.json" --testing-commit "${AUTO_TESTING_SHA}" \ + --case-id "${VALIDATION_CASE_ID}" \ + > "${VALIDATION_ROOT}/native-race-runtime-preflight.json" + printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Verify I05 source-plan or queued-successor native prerequisites + if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout' }} + env: + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }} + run: | + set -euo pipefail + NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" + if [ "${I05_DIAGNOSTIC_BRANCH}" = 'queued-successor' ]; then + python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \ + > "${VALIDATION_ROOT}/native-i05-prerequisite.json" + else + python3 -B auto-testing/scripts/validation_heal/lock_source.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${AUTO_TESTING_SHA}" \ + > "${VALIDATION_ROOT}/native-i05-source-prerequisites.json" + fi + printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Bind F07 exact-candidate directory fsync evidence + if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }} + run: | + set -euo pipefail + NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" + printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Download pinned manager persistence and incarnation evidence + if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.validation_tools_run_id }} + name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }} + path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Verify exact manager native prerequisites before fleet mutation + if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }} + env: + VALIDATION_CASE: ${{ inputs.validation_case }} + run: | + set -euo pipefail + MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json" + python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \ + --directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json" + printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Download pinned C10 shared-reference encoder + if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + run-id: ${{ inputs.validation_tools_run_id }} + name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }} + path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Verify C10 exact-source codec and original tests + if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }} + run: | + set -euo pipefail + SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json" + python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \ + --directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + > "${VALIDATION_ROOT}/shared-fixture-manifest.json" + printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Run selected canonical validation case + if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }} + env: + VALIDATION_CASE: ${{ inputs.validation_case }} + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }} + run: | + set -euo pipefail + test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}" + ./auto-testing/scripts/run_validation_case.sh \ + "${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \ + "${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run" + + - name: Inspect current Heal fleet resources without mutation + if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }} + env: + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + run: | + set -euo pipefail + test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}" + python3 -B auto-testing/scripts/validation_heal/environment.py \ + --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment" + + - name: Audit and finalize retained validation environment + if: ${{ inputs.validation_cleanup_run_id != '' }} + env: + GH_TOKEN: ${{ github.token }} + FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }} + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }} + run: | + set -euo pipefail + test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}" + gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \ + --json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json" + python3 - <<'PY' + import json + import os + from pathlib import Path + run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text()) + if (run["status"] != "completed" or run["conclusion"] not in {"failure", "cancelled"} + or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]): + raise ValueError("cleanup source must be a completed failed or cancelled Functional Chain run") + PY + if [ "${CLEANUP_RETAINED_EVIDENCE}" = true ]; then + python3 -B scripts/prepare_retained_validation_recovery.py \ + --report "${VALIDATION_ROOT}/retained-recovery-input.json" + else + gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \ + --pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact" + fi + python3 -B auto-testing/scripts/validation_heal/recover_environment.py \ + --failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \ + --candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery" + + - name: Show validation report + if: ${{ always() }} + run: | + if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then + cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Upload validation evidence + if: ${{ always() && env.VALIDATION_ROOT != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.VALIDATION_ROOT }} + if-no-files-found: error + + validation-tools: + needs: [validation] + if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }} + uses: ./.github/workflows/rustfs-validation-tools.yml + with: + candidate_sha: ${{ needs.validation.outputs.source_sha }} + auto_testing_sha: ${{ inputs.auto_testing_sha }} + build_native_heal: ${{ inputs.validation_native_tools }} + build_shared_heal: ${{ inputs.validation_shared_tools }} + build_manager_native: ${{ inputs.validation_manager_tools }} + secrets: inherit + upgrade: needs: [prepare] if: ${{ always() && needs.prepare.result == 'success' }} diff --git a/.github/workflows/rustfs-validation-package.yml b/.github/workflows/rustfs-validation-package.yml new file mode 100644 index 000000000..2e1e3f921 --- /dev/null +++ b/.github/workflows/rustfs-validation-package.yml @@ -0,0 +1,244 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +name: RustFS Validation Package + +on: + workflow_call: + inputs: + source_sha: + description: Exact RustFS commit SHA to build + required: true + type: string + secrets: + R2_ACCESS_KEY_ID: + required: true + R2_SECRET_ACCESS_KEY: + required: true + R2_ENDPOINT: + required: true + R2_BUCKET: + required: true + workflow_dispatch: + inputs: + source_sha: + description: Exact RustFS commit SHA to build + required: true + type: string + +permissions: + contents: read + +concurrency: + group: rustfs-validation-package-${{ inputs.source_sha }} + cancel-in-progress: false + +jobs: + resolve-source: + if: github.repository == 'rustfs/rustfs' + name: Resolve exact validation source + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + source_sha: ${{ steps.source.outputs.sha }} + source_ref: ${{ steps.source.outputs.ref }} + steps: + - name: Validate requested source + id: source + env: + REQUESTED_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + [[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]] + source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")" + trap 'rm -rf "${source_directory}"' EXIT + git init -q "${source_directory}" + git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}" + test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}" + echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}" + echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}" + + build: + needs: resolve-source + name: Build immutable validation DEB + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }} + steps: + - name: Checkout exact source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + persist-credentials: false + ref: ${{ needs.resolve-source.outputs.source_sha }} + + - name: Confirm checked out source + env: + EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }} + run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + cache-shared-key: validation-package-hosted-x86_64-unknown-linux-gnu + cache-save-if: "false" + install-build-packaging-tools: "false" + install-test-tools: "false" + + - name: Build RustFS + run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins + + - name: Build validation DEB + shell: bash + run: | + set -euo pipefail + command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot + package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}" + package_file="${RUNNER_TEMP}/validation-package/rustfs.deb" + mkdir -p "${RUNNER_TEMP}/validation-package" + binary="target/x86_64-unknown-linux-gnu/release/rustfs" + test -x "${binary}" + mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \ + "${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs" + cp "${binary}" "${package_dir}/usr/bin/rustfs" + chmod 755 "${package_dir}/usr/bin/rustfs" + cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/" + cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF' + # RustFS Environment Configuration + # RUSTFS_ACCESS_KEY= + # RUSTFS_SECRET_KEY= + # RUSTFS_VOLUMES="" + # RUSTFS_ADDRESS=":9000" + ENVEOF + cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF' + Package: rustfs + Version: 0.0.0~validation + Section: utils + Priority: optional + Architecture: amd64 + Depends: libc6 (>= 2.31) + Maintainer: RustFS Team + Description: RustFS immutable validation package + Candidate package for RustFS validation only. + CONTROLEOF + printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles" + ./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst" + ./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm" + ./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm" + chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm" + cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/" + fakeroot dpkg-deb --build "${package_dir}" "${package_file}" + test -s "${package_file}" + + - name: Upload opaque validation package + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/validation-package/rustfs.deb + if-no-files-found: error + + publish: + needs: [resolve-source, build] + name: Publish immutable validation DEB + # Candidate-controlled build scripts never run on this fresh hosted runner. + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }} + VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }} + steps: + - name: Download opaque validation package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/validation-package-input + + - name: Validate and stage opaque package bytes + shell: bash + run: | + set -euo pipefail + python3 -I - <<'PYTHON' + import os + from pathlib import Path + import shutil + import stat + + root = Path(os.environ["RUNNER_TEMP"]) + incoming = root / "validation-package-input" + if incoming.is_symlink() or not incoming.is_dir() or sorted(p.name for p in incoming.iterdir()) != ["rustfs.deb"]: + raise SystemExit("Validation artifact must contain exactly one rustfs.deb file") + fd = os.open(incoming / "rustfs.deb", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size == 0: + raise SystemExit("Validation package must be a nonempty regular file") + staged = root / "validation-publication" + staged.mkdir(mode=0o700) + with (staged / "rustfs.deb").open("xb") as target: + shutil.copyfileobj(source, target) + PYTHON + + - name: Prepare trusted publication tools + id: tools + env: + AWS_EC2_METADATA_DISABLED: true + shell: bash + run: | + set -euo pipefail + AWS_CLI=aws + if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then + sudo apt-get update + sudo apt-get install -y -qq python3-venv + AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")" + python3 -m venv "${AWS_CLI_DIR}" + "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79' + AWS_CLI="${AWS_CLI_DIR}/bin/aws" + fi + "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null + echo "aws_cli=${AWS_CLI}" >> "${GITHUB_OUTPUT}" + + - name: Upload validation package to immutable R2 path + id: publish + env: + PACKAGE_FILE: ${{ runner.temp }}/validation-publication/rustfs.deb + AWS_CLI: ${{ steps.tools.outputs.aws_cli }} + R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} + R2_BUCKET: ${{ secrets.R2_BUCKET }} + AWS_EC2_METADATA_DISABLED: true + shell: bash + run: | + set -euo pipefail + if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then + echo "Validation package publication requires complete R2 configuration" >&2 + exit 1 + fi + [[ "${VALIDATION_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] + test "${VALIDATION_SOURCE_REF}" = "${VALIDATION_SOURCE_SHA}" + test -s "${PACKAGE_FILE}" && test ! -L "${PACKAGE_FILE}" + export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto + package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)" + key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb" + url="https://dl.rustfs.com/${key}" + "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \ + --if-none-match '*' --endpoint-url "${R2_ENDPOINT}" + downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)" + test "${downloaded_sha256}" = "${package_sha256}" + manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json" + jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \ + --arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \ + --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg package_url "${url}" --arg package_sha256 "${package_sha256}" \ + '{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}" + echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}" + + - name: Upload validation candidate manifest + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.publish.outputs.candidate_file }} + if-no-files-found: error diff --git a/.github/workflows/rustfs-validation-tools.yml b/.github/workflows/rustfs-validation-tools.yml new file mode 100644 index 000000000..44dbf76e6 --- /dev/null +++ b/.github/workflows/rustfs-validation-tools.yml @@ -0,0 +1,343 @@ +# Build source-bound helpers separately so live validation needs only artifacts. +name: RustFS Validation Tools + +on: + workflow_call: + inputs: + candidate_sha: + type: string + required: true + auto_testing_sha: + type: string + required: true + build_native_heal: + type: boolean + required: false + default: false + build_shared_heal: + type: boolean + required: false + default: false + build_manager_native: + type: boolean + required: false + default: false + secrets: + PF_TESTING_GH_TOKEN: + required: true + workflow_dispatch: + inputs: + candidate_sha: + description: Exact RustFS source SHA from the validation candidate manifest + type: string + required: true + auto_testing_sha: + description: Exact auto-testing SHA containing the Heal metadata inspector + type: string + required: true + build_native_heal: + description: Also compile and run separate exact-source native Heal regressions + type: boolean + required: false + default: false + build_shared_heal: + description: Build the controlled C10 shared-reference fixture encoder + type: boolean + required: false + default: false + build_manager_native: + description: Build and run exact candidate checkpoint/MRF/incarnation regressions + type: boolean + required: false + default: false + +permissions: + contents: read + +jobs: + manager-native-regressions: + if: ${{ inputs.build_manager_native }} + runs-on: ubuntu-22.04 + timeout-minutes: 180 + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + steps: + - name: Validate immutable manager native inputs + run: | + set -euo pipefail + [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]] + + - name: Checkout exact manager candidate source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + ref: ${{ inputs.candidate_sha }} + path: candidate + persist-credentials: false + + - name: Checkout exact manager collector source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + repository: rustfs/auto-testing + ref: ${{ inputs.auto_testing_sha }} + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + path: auto-testing + persist-credentials: false + + - name: Install manager native build dependencies + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler + df -h . + + - name: Build exact unmodified candidate manager test executable + run: | + set -euo pipefail + test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}" + python3 -B auto-testing/scripts/validation_heal/manager_native.py build \ + --source candidate --commit "${CANDIDATE_SHA}" --out manager-tools + + - name: Run exact manager regressions independently + run: | + set -euo pipefail + python3 -B auto-testing/scripts/validation_heal/manager_native.py run \ + --directory manager-tools --out manager-evidence + + - name: Upload manager executable identities and original diagnostics + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }} + path: | + manager-tools/heal-manager-native-tests + manager-tools/*.json + manager-tools/*.jsonl + manager-tools/*.stderr + manager-tools/*.log + manager-evidence/ + if-no-files-found: warn + + native-heal-regressions: + if: ${{ inputs.build_native_heal }} + runs-on: ubuntu-22.04 + timeout-minutes: 180 + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + steps: + - name: Validate immutable native source inputs + run: | + set -euo pipefail + [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]] + + - name: Checkout exact native candidate source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + ref: ${{ inputs.candidate_sha }} + path: candidate + persist-credentials: false + + - name: Checkout exact native collector source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + repository: rustfs/auto-testing + ref: ${{ inputs.auto_testing_sha }} + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + path: auto-testing + persist-credentials: false + + - name: Install native build dependencies + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler + df -h . + + - name: Build and enumerate the unmodified candidate test executable + run: | + set -euo pipefail + test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}" + use_race_native=false + if [[ -f auto-testing/scripts/validation_heal/race_native.py ]]; then + support_rc=0 + python3 -B auto-testing/scripts/validation_heal/race_native.py supports \ + --commit "${CANDIDATE_SHA}" || support_rc=$? + case "${support_rc}" in + 0) use_race_native=true ;; + 2) ;; # The original exact-source collector handles legacy candidates. + *) exit "${support_rc}" ;; + esac + fi + if [[ "${use_race_native}" == true ]]; then + python3 -B auto-testing/scripts/validation_heal/race_native.py build \ + --source candidate --commit "${CANDIDATE_SHA}" --testing-commit "${AUTO_TESTING_SHA}" --out native-tools + else + python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \ + --source candidate --commit "${CANDIDATE_SHA}" --out native-tools + fi + + - name: Run exact native regressions separately from fleet validation + run: | + set -euo pipefail + if [[ -f native-tools/race-native.json ]]; then + test ! -e native-tools/native-regressions.json + python3 -B auto-testing/scripts/validation_heal/race_native.py run \ + --directory native-tools --out native-evidence + else + python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \ + --directory native-tools --out native-evidence + fi + + - name: Run existing F07 directory fsync failure regressions + if: ${{ inputs.candidate_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }} + run: | + set -euo pipefail + # Older immutable harness pins retain their original native suite; + # absence of this additive collector must never fabricate its proof. + if [[ ! -f auto-testing/scripts/validation_heal/persistence_native.py ]]; then + mkdir native-persistence-evidence + echo '{"complete":false,"reason":"pinned harness predates F07 directory-fsync collector","requiredNativeCoverageComplete":false,"canonicalCasePassClaimed":false}' \ + > native-persistence-evidence/not-available.json + exit 0 + fi + python3 -B auto-testing/scripts/validation_heal/persistence_native.py run \ + --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \ + --testing-commit "${AUTO_TESTING_SHA}" + python3 -B auto-testing/scripts/validation_heal/persistence_native.py verify \ + --directory native-tools --native-evidence native-evidence --out native-persistence-evidence \ + --testing-commit "${AUTO_TESTING_SHA}" + + - name: Upload native executable identities and original diagnostics + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }} + path: | + native-tools/ecstore-native-tests + native-tools/ecstore-race-native-tests + native-tools/*.json + native-tools/*.jsonl + native-tools/*.stderr + native-tools/*.log + native-evidence/ + native-persistence-evidence/ + if-no-files-found: warn + + heal-metadata-inspector: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + steps: + - name: Validate immutable source inputs + run: | + set -euo pipefail + [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]] + [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]] + + - name: Checkout exact candidate source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + ref: ${{ inputs.candidate_sha }} + path: candidate + persist-credentials: false + + - name: Checkout exact inspector source + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + repository: rustfs/auto-testing + ref: ${{ inputs.auto_testing_sha }} + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + path: auto-testing + persist-credentials: false + + - name: Build the read-only candidate metadata decoder + run: | + set -euo pipefail + test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}" + test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}" + df -h . + python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \ + --source candidate --commit "${CANDIDATE_SHA}" --out inspector-build + # Seed the helper's separate graph from the candidate's frozen graph. + # Cargo adds the helper package; both lock identities are retained. + cp candidate/Cargo.lock inspector-build/Cargo.lock + cargo build --release --manifest-path inspector-build/Cargo.toml + python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \ + --binary inspector-build/target/release/heal-meta-inspect --source candidate + mkdir validation-tools + cp inspector-build/target/release/heal-meta-inspect validation-tools/ + cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + import subprocess + + source = json.loads(Path("inspector-build/source.json").read_text()) + binary = Path("validation-tools/heal-meta-inspect") + source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"], + binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(), + inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(), + toolchain=subprocess.check_output(["rustc", "-vV"], text=True), + target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"], + runAttempt=os.environ["GITHUB_RUN_ATTEMPT"]) + Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n") + PY + git -C candidate diff --exit-code + + - name: Build and test the controlled C10 metadata encoder + if: ${{ inputs.build_shared_heal }} + run: | + set -euo pipefail + python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \ + --source candidate --commit "${CANDIDATE_SHA}" --out shared-build + cp candidate/Cargo.lock shared-build/Cargo.lock + export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target" + cargo build --release --manifest-path shared-build/Cargo.toml + mkdir shared-tools + if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \ + > shared-tools/shared-fixture-tests.log 2>&1; then + cat shared-tools/shared-fixture-tests.log + exit 1 + fi + cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/ + cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + import subprocess + + source = json.loads(Path("shared-build/source.json").read_text()) + source.update(schemaVersion=1, built=True, sourceUnmodified=True, + candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(), + testingCommit=os.environ["AUTO_TESTING_SHA"], + binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(), + sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(), + toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu", + runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"]) + Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n") + PY + git -C candidate diff --exit-code + + - name: Upload controlled C10 fixture encoder evidence + if: ${{ always() && inputs.build_shared_heal }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }} + path: shared-tools/ + if-no-files-found: warn + + - name: Upload immutable validation tools + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }} + path: validation-tools/ + if-no-files-found: error diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 99825a832..64425b155 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -3213,9 +3213,17 @@ pub mod test_support { use crate::store::init_local_disks_with_instance_ctx; pub async fn isolated_store_over_temp_disks() -> (Vec, Arc) { - let mut dirs = Vec::with_capacity(4); - let mut endpoints = Vec::with_capacity(4); - for disk_idx in 0..4 { + isolated_store_over_temp_disks_with_count(4).await + } + + pub async fn isolated_store_over_temp_disks_with_count(disk_count: usize) -> (Vec, Arc) { + assert!( + matches!(disk_count, 4 | 16), + "isolated store fixture requires a registered logical set geometry" + ); + let mut dirs = Vec::with_capacity(disk_count); + let mut endpoints = Vec::with_capacity(disk_count); + for disk_idx in 0..disk_count { let dir = tempfile::tempdir().expect("tempdir should be created"); let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse"); @@ -3228,7 +3236,7 @@ pub mod test_support { let endpoint_pools = EndpointServerPools(vec![PoolEndpoints { legacy: false, set_count: 1, - drives_per_set: 4, + drives_per_set: disk_count, endpoints: Endpoints::from(endpoints), cmd_line: "metadata-sys-cache-test".to_string(), platform: "test".to_string(), diff --git a/crates/ecstore/src/cluster/rpc/remote_locker.rs b/crates/ecstore/src/cluster/rpc/remote_locker.rs index df11fdb03..9e5b9a910 100644 --- a/crates/ecstore/src/cluster/rpc/remote_locker.rs +++ b/crates/ecstore/src/cluster/rpc/remote_locker.rs @@ -1074,7 +1074,6 @@ impl RemoteClient { #[async_trait] impl LockClient for RemoteClient { async fn acquire_lock(&self, request: &LockRequest) -> Result { - info!("remote acquire_exclusive for {}", request.resource); let admission = match self.admit_lock_request("lock") { Some(admission) => admission, None => return Ok(Self::rpc_backoff_failure_response(request)), diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 132ecc1e0..9b389f94c 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -603,6 +603,10 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter( request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string); // A degraded read's surviving minority is not a recreate instruction; the // locked storage read decides whether the target is absent (backlog#2900). + // Reader failures include checksum damage that metadata-only healing cannot detect. + if matches!(reason, "missing_shards" | "decode_error") { + request.scan_mode = Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep); + } let request_id = request.id.clone(); let bucket = bucket.to_string(); @@ -14220,6 +14224,61 @@ mod tests { release_read_repair_heal_reservation(&key).await; } + #[tokio::test] + async fn read_repair_reader_failure_requests_deep_scan_for_the_exact_version() { + fn captured() -> &'static std::sync::Mutex> { + static REQUEST: OnceLock>> = + OnceLock::new(); + REQUEST.get_or_init(|| std::sync::Mutex::new(None)) + } + fn capture(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture { + *captured().lock().expect("read-repair request capture must be available") = Some(request); + Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) }) + } + + for (reason, expected_scan) in [ + ("missing_shards", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)), + ("decode_error", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)), + ("metadata_read_error", None), + ] { + let object = format!("reader-failure-{}", Uuid::new_v4()); + let version = Uuid::new_v4().to_string(); + submit_read_repair_heal_with_submitter( + ReadRepairHealSubmission { + bucket: "reader-failure", + object: &object, + version_id: Some(&version), + pool_index: 1, + set_index: 2, + part_number: Some(1), + reason, + mrf_intent: None, + }, + capture, + ) + .await; + let request = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Some(request) = captured().lock().expect("captured request must be readable").take() { + break request; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("the actual asynchronous admission must be captured"); + assert_eq!(request.scan_mode, expected_scan, "reader damage needs payload verification"); + assert_eq!(request.source, HealRequestSource::ReadRepair); + assert_eq!(request.bucket, "reader-failure"); + assert_eq!(request.object_prefix.as_deref(), Some(object.as_str())); + assert_eq!(request.object_version_id.as_deref(), Some(version.as_str())); + assert_eq!((request.pool_index, request.set_index), (Some(1), Some(2))); + assert_eq!(request.recreate_missing, None, "storage evidence decides absence"); + let key = ReadRepairHealCacheKey::new("reader-failure", &object, Some(&version), 1, 2); + release_read_repair_heal_reservation(&key).await; + } + } + // ======================================================================== // backlog#898 — groups A/C: pure old-data-dir cleanup classification. // ======================================================================== diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 6170731c7..9badefaf1 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -912,6 +912,8 @@ pub(crate) use ops::heal::AbsenceProofRequest; #[cfg(test)] pub(crate) use ops::heal::DanglingDeleteFailure; pub(crate) use ops::heal::HealedObjectAbsence; +#[cfg(test)] +pub(crate) use ops::heal::ReadRepairPauseScope; #[cfg(test)] pub(crate) use ops::hermetic_set_disks_isolated; diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index da7786df0..6bf0cb4ab 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -168,6 +168,7 @@ enum ReadRepairPausePhase { Data, PartStaging(usize), Commit, + IncarnationCommit, } #[cfg(test)] @@ -183,7 +184,7 @@ struct ReadRepairPause { static READ_REPAIR_PAUSES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); #[cfg(test)] -struct ReadRepairPauseScope { +pub(crate) struct ReadRepairPauseScope { bucket: String, object: String, phase: ReadRepairPausePhase, @@ -195,6 +196,13 @@ impl ReadRepairPauseScope { Self::install_at(bucket, object, ReadRepairPausePhase::Commit) } + pub(crate) fn install_incarnation_commit( + bucket: &str, + object: &str, + ) -> (Self, std::sync::Arc, std::sync::Arc) { + Self::install_at(bucket, object, ReadRepairPausePhase::IncarnationCommit) + } + fn install_at( bucket: &str, object: &str, @@ -1415,7 +1423,13 @@ impl SetDisks { ); } - if read_repair_uses_shared_lock || preserve_unrepairable { + // A no-remove request cannot turn surviving same-version + // data into dangling cleanup or its grace deferral. + let preserve_surviving_data = !opts.remove + && data_errs_by_disk + .values() + .any(|parts| !parts.is_empty() && parts.iter().all(|status| *status == CHECK_PART_SUCCESS)); + if read_repair_uses_shared_lock || preserve_unrepairable || preserve_surviving_data { return Ok((result, Some(cannot_heal_err))); } @@ -1958,6 +1972,10 @@ impl SetDisks { // MinIO stops on the first RenameData error. RustFS intentionally // continues per target, but reports any residue after all attempts // so successful repairs survive and failed targets remain retryable. + #[cfg(test)] + if bucket_heal_scope.is_some() { + pause_read_repair(bucket, object, ReadRepairPausePhase::IncarnationCommit).await; + } let mut rename_attempts = 0usize; let mut rename_successes = 0usize; let mut healed_disks = vec![None; out_dated_disks.len()]; @@ -4132,6 +4150,151 @@ mod heal_result_report_tests { } } + #[tokio::test(flavor = "current_thread")] + #[serial_test::serial] + async fn deep_heal_preserves_below_quorum_external_parts_without_remove() { + use tokio::io::AsyncReadExt; + + for grace in ["3600", "0"] { + let scenario = async { + for versioned in [false, true] { + let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-below-read-quorum"; + let object = "external.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create each bucket volume"); + } + let body = vec![0x5a; 1024 * 1024]; + let version = versioned.then(|| Uuid::new_v4().to_string()); + let put_options = ObjectOptions { + no_lock: true, + versioned, + version_id: version.clone(), + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; + let mut reader = PutObjReader::from_vec(body.clone()); + set.put_object(bucket, object, &mut reader, &put_options) + .await + .expect("commit every original external shard"); + if versioned { + let mut newer = PutObjReader::from_vec(vec![0xa5; body.len()]); + set.put_object( + bucket, + object, + &mut newer, + &ObjectOptions { + version_id: Some(Uuid::new_v4().to_string()), + ..put_options.clone() + }, + ) + .await + .expect("commit a distinct current version above the historical target"); + } + let version_id = version.as_deref().unwrap_or(""); + let source = disks[3] + .read_version("", bucket, object, version_id, &ReadOptions::default()) + .await + .expect("read the exact source version"); + assert_eq!(source.erasure.data_blocks, 2, "fixture must have R=2"); + let part_path = format!("{object}/{}/part.1", source.data_dir.expect("external data directory")); + let mut metadata = Vec::new(); + let mut original_parts = Vec::new(); + for disk in &disks { + metadata.push( + disk.read_all(bucket, &format!("{object}/xl.meta")) + .await + .expect("original metadata"), + ); + original_parts.push(disk.read_all(bucket, &part_path).await.expect("original external part")); + } + for dir in temp_dirs.iter().take(3) { + tokio::fs::remove_file(dir.path().join(bucket).join(&part_path)) + .await + .expect("leave exactly R-1 external shards"); + } + let opts = HealOpts { + no_lock: true, + scan_mode: HealScanMode::Deep, + remove: false, + ..Default::default() + }; + let (result, error) = set + .heal_object(bucket, object, version_id, &opts) + .await + .expect("negative deep heal"); + assert!( + matches!(error, Some(DiskError::ErasureReadQuorum)), + "a no-remove heal with surviving same-version data must retain the quorum failure, not classify cleanup grace: {error:?}" + ); + assert!(!result.integrity_verified, "below-R data cannot carry a positive integrity proof"); + for (index, disk) in disks.iter().enumerate() { + assert_eq!( + disk.read_all(bucket, &format!("{object}/xl.meta")) + .await + .expect("retained metadata"), + metadata[index], + "negative heal must preserve every original metadata byte" + ); + if index < 3 { + assert!( + disk.read_all(bucket, &part_path).await.is_err(), + "below-R heal cannot invent a missing shard" + ); + } else { + assert_eq!(disk.read_all(bucket, &part_path).await.expect("surviving shard"), original_parts[index]); + } + } + disks[0] + .write_all(bucket, &part_path, original_parts[0].clone()) + .await + .expect("restore one original source"); + let (_, error) = set + .heal_object(bucket, object, version_id, &opts) + .await + .expect("fresh heal at exactly R"); + assert!(error.is_none(), "exactly R original sources must repair independently: {error:?}"); + let mut restored = set + .get_object_reader(bucket, object, None, Default::default(), &put_options) + .await + .expect("read the repaired exact version"); + let mut actual = Vec::new(); + restored + .stream + .read_to_end(&mut actual) + .await + .expect("read every repaired payload byte"); + assert_eq!(actual, body, "later successful repair cannot hide data loss"); + if grace == "0" { + drop(restored); + for dir in temp_dirs.iter().take(3) { + tokio::fs::remove_file(dir.path().join(bucket).join(&part_path)) + .await + .expect("prepare the explicit removal control"); + } + let (_, error) = set + .heal_object(bucket, object, version_id, &HealOpts { remove: true, ..opts }) + .await + .expect("explicit dangling cleanup control"); + assert!( + error.is_none(), + "explicit removal with zero grace must still clean dangling data: {error:?}" + ); + for disk in &disks { + assert!( + disk.read_version("", bucket, object, version_id, &ReadOptions::default()) + .await + .is_err(), + "explicit cleanup must remove the target version" + ); + } + } + } + }; + temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some(grace))], scenario).await; + } + } + #[tokio::test(flavor = "current_thread")] #[serial_test::serial] async fn heal_writer_failures_emit_one_aggregate_warning_per_object() { @@ -6092,6 +6255,105 @@ mod heal_result_report_tests { } } + #[tokio::test] + #[serial_test::serial] + async fn deep_heal_rebuilds_half_truncated_inline_metadata() { + use tokio::io::AsyncReadExt; + + let (dirs, disks, set) = hermetic_set_disks_isolated(4).await; + let bucket = "heal-half-truncated-inline-metadata"; + let object = "object.bin"; + for disk in &disks { + disk.make_volume(bucket).await.expect("create fixture volume"); + } + let payload: Vec = (0..4095) + .map(|index| u8::try_from(index % 251).expect("payload byte")) + .collect(); + let options = ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }; + set.put_object(bucket, object, &mut PutObjReader::from_vec(payload.clone()), &options) + .await + .expect("persist every inline shard before physical injection"); + let read_options = ReadOptions { + read_data: true, + ..Default::default() + }; + let mut before = Vec::new(); + let mut metadata = Vec::new(); + for (dir, disk) in dirs.iter().zip(&disks) { + before.push( + tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("read original physical metadata"), + ); + let info = disk + .read_version("", bucket, object, "", &read_options) + .await + .expect("read original inline shard"); + assert!(info.data.as_ref().is_some_and(|bytes| !bytes.is_empty())); + metadata.push(info); + } + let target = dirs[0].path().join(bucket).join(object).join(STORAGE_FORMAT_FILE); + let truncated = &before[0][..before[0].len() / 2]; + tokio::fs::write(&target, truncated) + .await + .expect("truncate exact original metadata at half length"); + + let (result, error) = set + .heal_object( + bucket, + object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("heal one truncated metadata replica with three healthy sources"); + assert!(error.is_none(), "truncated inline metadata must be repairable: {error:?}"); + assert_eq!(result.before.drives[0].state, DriveState::Corrupt.to_string()); + assert_eq!(result.drives_healed(), Some(1), "the damaged replica must actually be rebuilt"); + assert!( + result + .after + .drives + .iter() + .all(|drive| drive.state == DriveState::Ok.to_string()) + ); + let repaired = disks[0] + .read_version("", bucket, object, "", &read_options) + .await + .expect("repaired target metadata must parse"); + assert!(repaired.equals(&metadata[0]), "restore the original logical metadata"); + assert_eq!(repaired.erasure.index, metadata[0].erasure.index); + assert_eq!(repaired.data, metadata[0].data, "restore the original encoded inline shard"); + assert_eq!(repaired.version_id, metadata[0].version_id); + assert_eq!(repaired.data_dir, metadata[0].data_dir); + for (dir, original) in dirs.iter().zip(&before).skip(1) { + assert_eq!( + tokio::fs::read(dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("read healthy metadata after heal"), + *original, + "healthy metadata must remain byte-identical" + ); + } + let mut reader = set + .get_object_reader(bucket, object, None, Default::default(), &options) + .await + .expect("read reconstructed object"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("read complete reconstructed payload"); + assert_eq!(body, payload); + } + #[tokio::test] async fn deep_heal_inline_bitrot_preserves_quorum_and_dry_run_boundaries() { for corrupt_shards in [2, 3] { @@ -7817,6 +8079,1804 @@ mod heal_result_report_tests { assert_eq!(survivor.size, 1024 * 1024, "survivor size must be intact"); } + #[derive(Clone, Copy)] + enum VersionCommitMutation { + Append, + DeleteHistorical, + DeleteCurrent, + } + + struct VersionCommitFixture { + version: Uuid, + data_dir: Uuid, + body: Vec, + parts: Vec>, + } + + async fn explicit_version_commit_fence(mutation: VersionCommitMutation) { + use crate::object_api::{ShardIntegrityWriteMode, WriteCompletion}; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + for (disk_count, parity) in [(4, 2), (16, 4)] { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(disk_count, 0, parity).await; + let label = match mutation { + VersionCommitMutation::Append => "append", + VersionCommitMutation::DeleteHistorical => "delete-historical", + VersionCommitMutation::DeleteCurrent => "delete-current", + }; + let prefix = if matches!(mutation, VersionCommitMutation::Append) { + "native_g02" + } else { + "native_g03" + }; + let bucket = format!("version-commit-{label}-{disk_count}"); + let object = "object.bin"; + set.make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create isolated versioned bucket"); + let options = ObjectOptions { + versioned: true, + write_completion: WriteCompletion::TailDrained, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + ..Default::default() + }; + let mut versions = Vec::new(); + for byte in [0x11, 0x22, 0x33] { + let body = vec![byte; 1024 * 1024 + 123]; + let info = set + .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &options) + .await + .expect("ACK original version after its full rename tail"); + let version = info.version_id.expect("independent version id"); + let meta = disks[0] + .read_version("", &bucket, object, &version.to_string(), &ReadOptions::default()) + .await + .expect("read original version metadata"); + assert!(meta.data.is_none(), "native version fence must reconstruct external data"); + assert_eq!(meta.erasure.data_blocks, disk_count - parity); + assert_eq!(meta.erasure.parity_blocks, parity); + let data_dir = meta.data_dir.expect("actual external generation"); + let mut parts = Vec::new(); + for dir in &dirs { + parts.push( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("every original encoded shard after ACK"), + ); + } + versions.push(VersionCommitFixture { + version, + data_dir, + body, + parts, + }); + } + let selected = if matches!(mutation, VersionCommitMutation::DeleteCurrent) { + 2 + } else { + 0 + }; + let selected_version = versions[selected].version; + let selected_dir = versions[selected].data_dir; + let missing = dirs[0] + .path() + .join(&bucket) + .join(object) + .join(selected_dir.to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing) + .await + .expect("remove exact selected-version shard"); + let (_pause, arrived, release) = ReadRepairPauseScope::install(&bucket, object); + let heal_set = set.clone(); + let heal_bucket = bucket.clone(); + let heal = tokio::spawn(async move { + heal_set + .heal_object( + &heal_bucket, + object, + &selected_version.to_string(), + &HealOpts { + scan_mode: HealScanMode::Deep, + read_repair: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), arrived.notified()) + .await + .expect("actual selected-version reconstruction reaches final commit fence"); + assert!(!heal.is_finished() && !missing.exists()); + println!("{prefix}: geometry={disk_count}/{parity} mutation={label} reconstruction_paused_before_final_rename"); + + if matches!(mutation, VersionCommitMutation::Append) { + let body = vec![0x44; 1024 * 1024 + 123]; + let info = set + .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &options) + .await + .expect("new version ACK while historical reconstruction is paused"); + let version = info.version_id.expect("new acknowledged version id"); + assert!(versions.iter().all(|v| v.version != version)); + let meta = disks[0] + .read_version("", &bucket, object, &version.to_string(), &ReadOptions::default()) + .await + .expect("new physical version metadata"); + let data_dir = meta.data_dir.expect("new external generation"); + let mut parts = Vec::new(); + for dir in &dirs { + parts.push( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("new version ACK has every physical shard"), + ); + } + versions.push(VersionCommitFixture { + version, + data_dir, + body, + parts, + }); + } else { + set.delete_object( + &bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(selected_version.to_string()), + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + .expect("exact version delete ACK before releasing stale reconstruction"); + } + // Commit pause has dropped the Heal read guard. Acquire this key's + // write fence only after the foreground ACK, draining its physical + // cleanup before snapshots; release it before resuming the Heal. + let namespace = set.new_ns_lock(&bucket, object).await.expect("foreground drain fence"); + let drained = namespace + .get_write_lock(Duration::from_secs(30)) + .await + .expect("foreground physical tail drains"); + let mut committed_metadata = Vec::new(); + for dir in &dirs { + committed_metadata.push( + tokio::fs::read(dir.path().join(&bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("foreground committed metadata on every drive"), + ); + if !matches!(mutation, VersionCommitMutation::Append) { + assert!(!dir.path().join(&bucket).join(object).join(selected_dir.to_string()).exists()); + } + } + drop(drained); + assert!(!heal.is_finished()); + println!( + "{prefix}: geometry={disk_count}/{parity} mutation={label} foreground_ack_and_physical_tail_verified_before_release" + ); + release.notify_one(); + let outcome = tokio::time::timeout(Duration::from_secs(30), heal) + .await + .expect("Heal leaves its released final fence") + .expect("Heal task joins"); + if matches!(mutation, VersionCommitMutation::Append) { + assert!(outcome.expect("historical version remains healable").1.is_none()); + } else { + assert!( + matches!(outcome, Err(DiskError::FileVersionNotFound)), + "deleted version must remain typed absent: {outcome:?}" + ); + } + for (slot, dir) in dirs.iter().enumerate() { + if slot != 0 || !matches!(mutation, VersionCommitMutation::Append) { + assert_eq!( + tokio::fs::read(dir.path().join(&bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("metadata after Heal"), + committed_metadata[slot], + "Heal must preserve every committed successor metadata byte outside its repaired target" + ); + } + if !matches!(mutation, VersionCommitMutation::Append) { + assert!(!dir.path().join(&bucket).join(object).join(selected_dir.to_string()).exists()); + } + for version in &versions { + if !matches!(mutation, VersionCommitMutation::Append) && version.version == selected_version { + assert!(matches!( + disks[slot] + .read_version("", &bucket, object, &version.version.to_string(), &ReadOptions::default()) + .await, + Err(DiskError::FileVersionNotFound) + )); + continue; + } + let meta = disks[slot] + .read_version("", &bucket, object, &version.version.to_string(), &ReadOptions::default()) + .await + .expect("every acknowledged surviving version metadata"); + assert_eq!(meta.data_dir, Some(version.data_dir)); + assert_eq!(meta.size, version.body.len() as i64); + assert_eq!( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(object) + .join(version.data_dir.to_string()) + .join("part.1") + ) + .await + .expect("physical shard before any GET"), + version.parts[slot], + "every surviving version must retain its exact original encoded shard" + ); + } + } + for version in &versions { + if !matches!(mutation, VersionCommitMutation::Append) && version.version == selected_version { + continue; + } + let mut reader = set + .get_object_reader( + &bucket, + object, + None, + Default::default(), + &ObjectOptions { + versioned: true, + version_id: Some(version.version.to_string()), + ..Default::default() + }, + ) + .await + .expect("read exact surviving version after physical proof"); + let mut actual = Vec::new(); + reader + .read_to_end(&mut actual) + .await + .expect("read complete surviving version"); + assert_eq!(actual, version.body); + } + let current = if matches!(mutation, VersionCommitMutation::DeleteCurrent) { + &versions[1] + } else { + versions.last().unwrap() + }; + let mut reader = set + .get_object_reader( + &bucket, + object, + None, + Default::default(), + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("read exact last surviving current generation"); + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.expect("read full current generation"); + assert_eq!(actual, current.body); + println!( + "{prefix}: geometry={disk_count}/{parity} mutation={label} exact_version_outcome_physical_generations_and_full_bodies_verified" + ); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn native_explicit_version_commit_preserves_acknowledged_new_version() { + explicit_version_commit_fence(VersionCommitMutation::Append).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn native_explicit_version_commit_refuses_deleted_historical_and_current_versions() { + explicit_version_commit_fence(VersionCommitMutation::DeleteHistorical).await; + explicit_version_commit_fence(VersionCommitMutation::DeleteCurrent).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn native_marker_and_batch_delete_fence_preserves_acknowledged_version_set() { + use crate::object_api::{ShardIntegrityWriteMode, WriteCompletion}; + use crate::storage_api_contracts::object::ObjectToDelete; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + for (disk_count, parity) in [(4, 2), (16, 4)] { + for mutation in ["create-marker", "delete-marker", "batch-version-delete", "unversioned-delete"] { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(disk_count, 0, parity).await; + let bucket = format!("marker-fence-{mutation}-{disk_count}"); + let object = "object.bin"; + let versioned = mutation != "unversioned-delete"; + set.make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: versioned, + ..Default::default() + }, + ) + .await + .expect("create isolated marker fence bucket"); + let options = ObjectOptions { + versioned, + write_completion: WriteCompletion::TailDrained, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }; + let mut versions = Vec::new(); + for byte in if versioned { vec![0x51, 0x62, 0x73] } else { vec![0x51] } { + let body = vec![byte; 1024 * 1024 + 123]; + let info = set + .put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &options) + .await + .expect("ACK complete original external version"); + let version = info.version_id.unwrap_or(Uuid::nil()); + assert_eq!(!version.is_nil(), versioned); + let meta = disks[0] + .read_version("", &bucket, object, &version.to_string(), &ReadOptions::default()) + .await + .expect("read independent acknowledged metadata"); + assert!(meta.data.is_none()); + let data_dir = meta.data_dir.expect("external data generation"); + let mut parts = Vec::new(); + for dir in &dirs { + parts.push( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("full encoded original shard after ACK"), + ); + } + versions.push(VersionCommitFixture { + version, + data_dir, + body, + parts, + }); + } + let mut marker = None; + if mutation == "delete-marker" { + marker = Some( + set.delete_object(&bucket, object, options.clone()) + .await + .expect("ACK original current marker") + .version_id + .expect("real marker identity"), + ); + } + let selected = versions[0].version; + let missing = dirs[0] + .path() + .join(&bucket) + .join(object) + .join(versions[0].data_dir.to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing) + .await + .expect("remove selected original data shard"); + let (_pause, arrived, release) = ReadRepairPauseScope::install(&bucket, object); + let heal_set = set.clone(); + let heal_bucket = bucket.clone(); + let heal = tokio::spawn(async move { + heal_set + .heal_object( + &heal_bucket, + object, + &selected.to_string(), + &HealOpts { + scan_mode: HealScanMode::Deep, + read_repair: true, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), arrived.notified()) + .await + .expect("real original reconstruction pauses before final metadata rename"); + assert!(!heal.is_finished() && !missing.exists()); + println!( + "native_g04: geometry={disk_count}/{parity} mutation={mutation} reconstruction_paused_before_final_rename" + ); + let mut removed = Vec::new(); + match mutation { + "create-marker" => { + marker = Some( + set.delete_object(&bucket, object, options.clone()) + .await + .expect("ACK marker creation before stale Heal resumes") + .version_id + .expect("new marker identity"), + ); + } + "delete-marker" => { + let version = marker.expect("fixture marker identity"); + let ack = set + .delete_object( + &bucket, + object, + ObjectOptions { + version_id: Some(version.to_string()), + ..options.clone() + }, + ) + .await + .expect("ACK exact current marker removal before stale Heal resumes"); + assert_eq!(ack.version_id, Some(version)); + } + "batch-version-delete" => { + removed.extend([versions[0].version, versions[1].version]); + let requests = removed + .iter() + .map(|version| ObjectToDelete { + object_name: object.to_owned(), + version_id: Some(*version), + ..Default::default() + }) + .collect(); + let (deleted, errors) = set.delete_objects(&bucket, requests, options.clone()).await; + assert_eq!(deleted.len(), removed.len()); + assert_eq!(errors.len(), removed.len()); + for ((ack, error), version) in deleted.iter().zip(&errors).zip(&removed) { + assert!(error.is_none(), "every batch item must ACK separately: {error:?}"); + assert_eq!(ack.object_name, object); + assert_eq!(ack.version_id, Some(*version)); + assert!(!ack.delete_marker); + } + } + "unversioned-delete" => { + set.delete_object(&bucket, object, options.clone()) + .await + .expect("ACK unversioned delete"); + removed.push(selected); + } + _ => unreachable!(), + } + if let Some(marker) = marker { + assert!( + !marker.is_nil() && versions.iter().all(|version| version.version != marker), + "marker ACK must have its own real version identity" + ); + } + // Drain the actual foreground metadata/cleanup tail under this + // key's write fence before releasing the older reconstruction. + let namespace = set.new_ns_lock(&bucket, object).await.expect("foreground drain fence"); + let drained = namespace + .get_write_lock(Duration::from_secs(30)) + .await + .expect("foreground tail drains"); + let mut committed = Vec::new(); + for dir in &dirs { + let metadata = dir.path().join(&bucket).join(object).join(STORAGE_FORMAT_FILE); + committed.push(match tokio::fs::read(metadata).await { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound && !versioned => None, + Err(error) => panic!("foreground metadata cannot be read: {error}"), + }); + for version in &versions { + if removed.contains(&version.version) { + assert!( + !dir.path() + .join(&bucket) + .join(object) + .join(version.data_dir.to_string()) + .exists() + ); + } + } + } + drop(drained); + assert!(!heal.is_finished()); + println!( + "native_g04: geometry={disk_count}/{parity} mutation={mutation} exact_foreground_acks_and_physical_tail_verified_before_release" + ); + release.notify_one(); + let outcome = tokio::time::timeout(Duration::from_secs(30), heal) + .await + .expect("stale Heal leaves the released fence") + .expect("Heal task joins"); + if removed.contains(&selected) { + assert!( + matches!(outcome, Err(DiskError::FileVersionNotFound)), + "deleted target must remain typed absent: {outcome:?}" + ); + } else { + assert!(outcome.expect("surviving explicit data version remains healable").1.is_none()); + } + let mut expected_versions: std::collections::HashSet<_> = versions + .iter() + .filter(|version| !removed.contains(&version.version)) + .map(|version| Some(version.version)) + .collect(); + if mutation == "create-marker" { + expected_versions.insert(marker); + } + for (slot, dir) in dirs.iter().enumerate() { + let metadata = dir.path().join(&bucket).join(object).join(STORAGE_FORMAT_FILE); + if !expected_versions.is_empty() { + let mut decoded = rustfs_filemeta::FileMeta::default(); + decoded + .unmarshal_msg(&tokio::fs::read(&metadata).await.expect("complete physical metadata")) + .expect("decode final version inventory independently"); + assert_eq!( + decoded.versions.len(), + expected_versions.len(), + "no duplicate or invented physical versions" + ); + let actual: std::collections::HashSet<_> = + decoded.versions.iter().map(|version| version.header.version_id).collect(); + assert_eq!(actual, expected_versions, "exact data and marker version set on every member"); + } + if removed.contains(&selected) || slot != 0 { + let after = tokio::fs::read(&metadata).await; + if let Some(expected) = &committed[slot] { + assert_eq!(&after.expect("committed metadata survives"), expected); + } else { + assert_eq!( + after.expect_err("deleted unversioned metadata must not resurrect").kind(), + std::io::ErrorKind::NotFound + ); + } + } + if let Some(marker) = marker { + let observed = disks[slot] + .read_version("", &bucket, object, &marker.to_string(), &ReadOptions::default()) + .await; + if mutation == "create-marker" { + let info = observed.expect("new marker survives on every disk"); + assert!(info.deleted); + assert_eq!(info.version_id, Some(marker)); + let current = disks[slot] + .read_version("", &bucket, object, "", &ReadOptions::default()) + .await + .expect("read current marker"); + assert!(current.deleted); + assert_eq!(current.version_id, Some(marker)); + } else { + assert!(matches!(observed, Err(DiskError::FileVersionNotFound))); + } + } + for version in &versions { + let observed = disks[slot] + .read_version("", &bucket, object, &version.version.to_string(), &ReadOptions::default()) + .await; + let part = dir + .path() + .join(&bucket) + .join(object) + .join(version.data_dir.to_string()) + .join("part.1"); + if removed.contains(&version.version) { + assert!(observed.is_err()); + assert!(!part.parent().unwrap().exists(), "deleted data directory must not resurrect"); + } else { + assert_eq!(observed.expect("surviving exact version").data_dir, Some(version.data_dir)); + assert_eq!(tokio::fs::read(part).await.expect("surviving part before any GET"), version.parts[slot]); + } + } + } + for version in &versions { + if removed.contains(&version.version) { + continue; + } + let mut reader = set + .get_object_reader( + &bucket, + object, + None, + Default::default(), + &ObjectOptions { + versioned, + version_id: Some(version.version.to_string()), + ..Default::default() + }, + ) + .await + .expect("read exact surviving original version"); + let mut body = Vec::new(); + reader.read_to_end(&mut body).await.expect("complete surviving body"); + assert_eq!(body, version.body); + } + if mutation == "create-marker" { + // The storage info API exposes marker identity to its S3 + // caller; it must not substitute the previous live data. + let current = set + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned, + ..Default::default() + }, + ) + .await + .expect("current storage info retains marker identity"); + assert!(current.delete_marker && current.is_latest); + assert_eq!(current.version_id, marker); + } else if !versioned { + let absent = set + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned, + ..Default::default() + }, + ) + .await; + assert!( + matches!(absent, Err(Error::ObjectNotFound(_, _))), + "current deleted object must remain typed absent: {absent:?}" + ); + } else { + let expected = versions.last().expect("surviving current version"); + let mut reader = set + .get_object_reader( + &bucket, + object, + None, + Default::default(), + &ObjectOptions { + versioned, + ..Default::default() + }, + ) + .await + .expect("current read after marker or batch deletion"); + assert_eq!(reader.object_info.version_id, Some(expected.version)); + let mut body = Vec::new(); + reader.read_to_end(&mut body).await.expect("full current body"); + assert_eq!(body, expected.body); + } + println!( + "native_g04: geometry={disk_count}/{parity} mutation={mutation} marker_set_deleted_directories_surviving_shards_and_full_bodies_verified" + ); + } + } + } + + #[tokio::test] + #[serial_test::serial] + async fn native_multipart_continue_complete_abort_fences_stale_heal() { + use crate::object_api::{ShardIntegrityWriteMode, WriteCompletion}; + use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations as _}; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + for (disk_count, parity) in [(4, 2), (16, 4)] { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(disk_count, 0, parity).await; + let bucket = format!("multipart-heal-fence-{disk_count}"); + let object = "object.bin"; + set.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create isolated MPU fence bucket"); + let options = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + ..Default::default() + }; + let first_size = 5 * 1024 * 1024 + 17; + let middle_size = 5 * 1024 * 1024 + 93; + let last_size = 1024 * 1024 + 47; + let original = set + .new_multipart_upload(&bucket, object, &options) + .await + .expect("original multipart upload"); + let mut original_parts = Vec::new(); + for (number, byte, size) in [(1, 0x21, first_size), (2, 0x32, middle_size), (3, 0x43, last_size)] { + let part = set + .put_object_part( + &bucket, + object, + &original.upload_id, + number, + &mut PutObjReader::from_vec(vec![byte; size]), + &options, + ) + .await + .expect("ACK original multipart part"); + assert!(part.etag.as_ref().is_some_and(|value| !value.is_empty())); + original_parts.push(CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + let original_ack = set + .clone() + .complete_multipart_upload(&bucket, object, &original.upload_id, original_parts, &options) + .await + .expect("ACK original completed multipart object"); + let old = disks[0] + .read_version("", &bucket, object, "", &ReadOptions::default()) + .await + .expect("original external metadata"); + assert_eq!(old.parts.len(), 3); + let old_dir = old.data_dir.expect("original external generation"); + let mut original_reader = set + .get_object_reader(&bucket, object, None, Default::default(), &options) + .await + .expect("read complete committed multipart baseline before fault"); + let mut original_body = Vec::new(); + original_reader + .read_to_end(&mut original_body) + .await + .expect("full original multipart body"); + assert_eq!( + original_body, + [vec![0x21; first_size], vec![0x32; middle_size], vec![0x43; last_size]].concat() + ); + drop(original_reader); + let mut old_metadata = Vec::new(); + let mut old_shards = Vec::new(); + for dir in &dirs { + let mut parts = Vec::new(); + for number in [1, 2, 3] { + parts.push( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(object) + .join(old_dir.to_string()) + .join(format!("part.{number}")), + ) + .await + .expect("original complete encoded multipart part"), + ); + } + old_shards.push(parts); + old_metadata.push( + tokio::fs::read(dir.path().join(&bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("original metadata after full-tail completion"), + ); + } + let active = set + .new_multipart_upload(&bucket, object, &options) + .await + .expect("pending upload A"); + let cancelled = set + .new_multipart_upload(&bucket, object, &options) + .await + .expect("pending upload B"); + assert_ne!(active.upload_id, cancelled.upload_id); + assert_ne!(active.upload_id, original.upload_id); + let first_body = vec![0x63; first_size]; + let middle_body = vec![0x69; middle_size]; + let last_body = vec![0x74; last_size]; + let first = set + .put_object_part( + &bucket, + object, + &active.upload_id, + 1, + &mut PutObjReader::from_vec(first_body.clone()), + &options, + ) + .await + .expect("ACK A first part"); + let middle = set + .put_object_part( + &bucket, + object, + &active.upload_id, + 2, + &mut PutObjReader::from_vec(middle_body.clone()), + &options, + ) + .await + .expect("ACK A second initial part"); + let mut other_parts = Vec::new(); + for (number, byte, size) in [(1, 0x85, first_size), (2, 0x96, middle_size), (3, 0xa7, last_size)] { + other_parts.push( + set.put_object_part( + &bucket, + object, + &cancelled.upload_id, + number, + &mut PutObjReader::from_vec(vec![byte; size]), + &options, + ) + .await + .expect("ACK B independent pending part"), + ); + } + assert_ne!(first.etag, other_parts[0].etag); + let (active_meta, _) = set + .check_upload_id_exists(&bucket, object, &active.upload_id, false) + .await + .expect("independent active upload metadata"); + let active_dir = active_meta.data_dir.expect("A physical staging generation"); + let active_path = SetDisks::get_multipart_upload_dir(&bucket, object, &active.upload_id, false); + let cancelled_path = SetDisks::get_multipart_upload_dir(&bucket, object, &cancelled.upload_id, false); + let mut active_original_shards = Vec::new(); + for dir in &dirs { + let mut parts = Vec::new(); + for number in [1, 2] { + parts.push( + tokio::fs::read( + dir.path() + .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET) + .join(&active_path) + .join(active_dir.to_string()) + .join(format!("part.{number}")), + ) + .await + .expect("A original acknowledged staging shard"), + ); + } + active_original_shards.push(parts); + } + println!("native_g05: geometry={disk_count}/{parity} committed_multipart_and_two_distinct_pending_uploads_seeded"); + let missing = dirs[0] + .path() + .join(&bucket) + .join(object) + .join(old_dir.to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing) + .await + .expect("remove original committed multipart shard"); + let (_pause, arrived, release) = ReadRepairPauseScope::install(&bucket, object); + let heal_set = set.clone(); + let heal_bucket = bucket.clone(); + let heal = tokio::spawn(async move { + heal_set + .heal_object( + &heal_bucket, + object, + "", + &HealOpts { + read_repair: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), arrived.notified()) + .await + .expect("real MPU reconstruction reaches final rename pause"); + assert!(!heal.is_finished() && !missing.exists()); + println!("native_g05: geometry={disk_count}/{parity} real_committed_object_heal_paused_before_final_rename"); + let last = set + .put_object_part( + &bucket, + object, + &active.upload_id, + 3, + &mut PutObjReader::from_vec(last_body.clone()), + &options, + ) + .await + .expect("ACK continued A part during paused Heal"); + set.abort_multipart_upload(&bucket, object, &cancelled.upload_id, &options) + .await + .expect("ACK B abort during paused Heal"); + let gone = set + .list_object_parts(&bucket, object, &cancelled.upload_id, None, 1000, &options) + .await; + assert!( + matches!(gone, Err(Error::InvalidUploadID(ref b, ref o, ref id)) if b==&bucket && o==object && id==&cancelled.upload_id) + ); + let pending = set + .list_object_parts(&bucket, object, &active.upload_id, None, 1000, &options) + .await + .expect("B abort cannot remove A"); + assert!(!pending.is_truncated); + assert_eq!(pending.upload_id, active.upload_id); + assert_eq!(pending.parts.len(), 3); + for (actual, expected) in pending.parts.iter().zip([&first, &middle, &last]) { + assert_eq!(actual.part_num, expected.part_num); + assert_eq!(actual.etag, expected.etag); + assert_eq!(actual.size, expected.size); + } + for (slot, dir) in dirs.iter().enumerate() { + assert_eq!( + tokio::fs::read(dir.path().join(&bucket).join(object).join(STORAGE_FORMAT_FILE)) + .await + .expect("pending parts cannot publish an object"), + old_metadata[slot] + ); + for number in [1, 2, 3] { + let path = dir + .path() + .join(&bucket) + .join(object) + .join(old_dir.to_string()) + .join(format!("part.{number}")); + if slot == 0 && number == 1 { + assert!(!path.exists(), "paused Heal cannot have published its missing old shard"); + } else { + assert_eq!( + tokio::fs::read(path) + .await + .expect("Abort preserves committed data before Complete"), + old_shards[slot][number - 1] + ); + } + } + for number in [1, 2] { + assert_eq!( + tokio::fs::read( + dir.path() + .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET) + .join(&active_path) + .join(active_dir.to_string()) + .join(format!("part.{number}")) + ) + .await + .expect("B abort preserves each original A shard"), + active_original_shards[slot][number - 1] + ); + } + assert!( + !dir.path() + .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET) + .join(&cancelled_path) + .exists() + ); + } + assert!(!heal.is_finished() && !missing.exists()); + println!( + "native_g05: geometry={disk_count}/{parity} continue_ack_and_other_abort_preserve_pending_parts_and_original_commit" + ); + let completed = set + .clone() + .complete_multipart_upload( + &bucket, + object, + &active.upload_id, + vec![ + CompletePart { + part_num: first.part_num, + etag: first.etag.clone(), + ..Default::default() + }, + CompletePart { + part_num: middle.part_num, + etag: middle.etag.clone(), + ..Default::default() + }, + CompletePart { + part_num: last.part_num, + etag: last.etag.clone(), + ..Default::default() + }, + ], + &options, + ) + .await + .expect("ACK A completion before releasing obsolete Heal"); + assert!(completed.etag.as_ref().is_some_and(|value| !value.is_empty())); + assert_ne!(completed.etag, original_ack.etag); + let namespace = set + .new_ns_lock(&bucket, object) + .await + .expect("completed MPU physical drain fence"); + let drained = namespace + .get_write_lock(Duration::from_secs(30)) + .await + .expect("completed MPU tail drains"); + let current = disks[0] + .read_version("", &bucket, object, "", &ReadOptions::default()) + .await + .expect("new completion metadata"); + let new_dir = current.data_dir.expect("new multipart physical generation"); + assert_ne!(new_dir, old_dir); + assert_eq!(current.parts.len(), 3); + let mut committed = Vec::new(); + let mut new_shards = Vec::new(); + for dir in &dirs { + let root = dir.path().join(&bucket).join(object); + committed.push( + tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("ACK completion metadata"), + ); + let mut parts = Vec::new(); + for number in [1, 2, 3] { + parts.push( + tokio::fs::read(root.join(new_dir.to_string()).join(format!("part.{number}"))) + .await + .expect("ACK completion publishes every encoded part"), + ); + } + new_shards.push(parts); + assert!( + !root.join(old_dir.to_string()).exists(), + "completion reclaims the old multipart generation" + ); + assert!( + !dir.path() + .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET) + .join(&active_path) + .exists() + ); + } + drop(drained); + assert!(!heal.is_finished()); + println!("native_g05: geometry={disk_count}/{parity} complete_ack_and_all_physical_tails_verified_before_release"); + release.notify_one(); + let (_, error) = tokio::time::timeout(Duration::from_secs(30), heal) + .await + .expect("stale MPU Heal exits its fence") + .expect("Heal task joins") + .expect("stale reconstruction returns its typed retry disposition"); + assert_eq!(error, Some(DiskError::Timeout)); + for (slot, dir) in dirs.iter().enumerate() { + let root = dir.path().join(&bucket).join(object); + assert_eq!( + tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("completion metadata after stale Heal"), + committed[slot] + ); + let mut decoded = rustfs_filemeta::FileMeta::default(); + decoded + .unmarshal_msg(&committed[slot]) + .expect("decode final generation independently"); + assert_eq!(decoded.versions.len(), 1, "completion must not publish another object version"); + for number in [1, 2, 3] { + assert_eq!( + tokio::fs::read(root.join(new_dir.to_string()).join(format!("part.{number}"))) + .await + .expect("new parts before any GET"), + new_shards[slot][number - 1] + ); + } + assert!(!root.join(old_dir.to_string()).exists(), "stale Heal cannot resurrect the old generation"); + for path in [&active_path, &cancelled_path] { + assert!(!dir.path().join(crate::disk::RUSTFS_META_MULTIPART_BUCKET).join(path).exists()); + } + } + for id in [&active.upload_id, &cancelled.upload_id] { + let gone = set.list_object_parts(&bucket, object, id, None, 1000, &options).await; + assert!( + matches!(gone, Err(Error::InvalidUploadID(ref b, ref o, ref upload)) if b==&bucket && o==object && upload==id) + ); + } + let mut reader = set + .get_object_reader(&bucket, object, None, Default::default(), &options) + .await + .expect("read exact completed current object"); + assert_eq!(reader.object_info.etag, completed.etag); + let mut actual = Vec::new(); + reader.read_to_end(&mut actual).await.expect("full completed multipart body"); + assert_eq!(actual, [first_body, middle_body, last_body].concat()); + println!( + "native_g05: geometry={disk_count}/{parity} stale_heal_rejected_exact_completed_parts_full_body_and_upload_retirement_verified" + ); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn native_default_heal_owner_fences_queued_put_and_version_delete() { + use crate::object_api::{ShardIntegrityWriteMode, WriteCompletion}; + use crate::set_disk::ops::object::{DeleteObjectCommitBarrier, PutObjectCommitBarrier, PutObjectCommitPause}; + use std::collections::HashMap; + use std::task::Poll; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + for (disk_count, parity) in [(4, 2), (16, 4)] { + for mutation in ["put", "version-delete"] { + let (dirs, disks, set) = hermetic_set_disks_for_pool_with_default_parity_isolated(disk_count, 0, parity).await; + let bucket = format!("default-heal-owner-{disk_count}-{mutation}"); + let object = "locked/key.bin"; + let reference = "independent/reference.bin"; + set.make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("isolated owner bucket"); + let options = ObjectOptions { + versioned: mutation == "version-delete", + write_completion: WriteCompletion::TailDrained, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + ..Default::default() + }; + let original_body = vec![0x18; 1024 * 1024 + 113]; + let successor_body = vec![0x29; 1024 * 1024 + 157]; + let original = set + .put_object(&bucket, object, &mut PutObjReader::from_vec(original_body.clone()), &options) + .await + .expect("original acknowledged version"); + let selected_version = original.version_id.map(|v| v.to_string()).unwrap_or_default(); + let current = if mutation == "version-delete" { + Some( + set.put_object(&bucket, object, &mut PutObjReader::from_vec(successor_body.clone()), &options) + .await + .expect("distinct current version remains protected"), + ) + } else { + None + }; + let mut current_parts = Vec::new(); + let current_data_dir = if let Some(current) = ¤t { + let info = disks[0] + .read_version( + "", + &bucket, + object, + ¤t.version_id.expect("retained current version").to_string(), + &ReadOptions::default(), + ) + .await + .expect("independent retained-current physical generation"); + let data_dir = info.data_dir.expect("retained current external generation"); + for dir in &dirs { + current_parts.push( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(object) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("independent retained-current shard oracle"), + ); + } + Some(data_dir) + } else { + None + }; + let mut reference_parts = HashMap::new(); + if mutation == "put" { + set.put_object(&bucket, reference, &mut PutObjReader::from_vec(successor_body.clone()), &options) + .await + .expect("independent successor payload coding oracle"); + for (slot, disk) in disks.iter().enumerate() { + let info = disk + .read_version("", &bucket, reference, "", &ReadOptions::default()) + .await + .expect("reference coding index"); + let path = dirs[slot] + .path() + .join(&bucket) + .join(reference) + .join(info.data_dir.expect("reference external data").to_string()) + .join("part.1"); + assert!( + reference_parts + .insert(info.erasure.index, tokio::fs::read(path).await.expect("reference encoded part")) + .is_none() + ); + } + } + let old = disks[0] + .read_version("", &bucket, object, &selected_version, &ReadOptions::default()) + .await + .expect("selected original generation"); + assert_eq!((old.erasure.data_blocks, old.erasure.parity_blocks), (disk_count - parity, parity)); + let old_dir = old.data_dir.expect("original external generation"); + let mut original_parts = Vec::new(); + let mut original_metadata = Vec::new(); + for dir in &dirs { + let root = dir.path().join(&bucket).join(object); + original_parts.push( + tokio::fs::read(root.join(old_dir.to_string()).join("part.1")) + .await + .expect("original encoded part"), + ); + original_metadata.push( + tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("original complete version metadata"), + ); + } + let namespace = set.new_ns_lock(&bucket, object).await.expect("same actual object namespace"); + let mut available_reader = Box::pin(namespace.get_read_lock(Duration::from_secs(30))); + assert!( + matches!(futures::poll!(available_reader.as_mut()), Poll::Ready(Ok(_))), + "positive control: an unowned local namespace admits a reader immediately" + ); + drop(available_reader); + let missing = dirs[0] + .path() + .join(&bucket) + .join(object) + .join(old_dir.to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing) + .await + .expect("one original external shard missing"); + let (pause, arrived, release) = + ReadRepairPauseScope::install_at(&bucket, object, super::ReadRepairPausePhase::PartStaging(1)); + let heal_set = set.clone(); + let heal_bucket = bucket.clone(); + let heal_version = selected_version.clone(); + let heal = tokio::spawn(async move { + heal_set + .heal_object( + &heal_bucket, + object, + &heal_version, + &HealOpts { + no_lock: false, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), arrived.notified()) + .await + .expect("default Deep Heal owns namespace during actual staging"); + let mut blocked_reader = Box::pin(namespace.get_read_lock(Duration::from_secs(30))); + assert!( + matches!(futures::poll!(blocked_reader.as_mut()), Poll::Pending), + "default Heal must own an exclusive namespace before any queued writer exists" + ); + drop(blocked_reader); + println!( + "native_a09: geometry={disk_count}/{parity} mutation={mutation} default_deep_heal_staged_exclusive_owner_read_blocked" + ); + let put_before = (mutation == "put") + .then(|| PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::BeforeNamespace)); + let put_commit = (mutation == "put") + .then(|| PutObjectCommitBarrier::install(&bucket, object, PutObjectCommitPause::AfterNamespace)); + let mut delete_before = + (mutation == "version-delete").then(|| DeleteObjectCommitBarrier::install_before_namespace(&bucket, object)); + let foreground_set = set.clone(); + let foreground_bucket = bucket.clone(); + let foreground_options = options.clone(); + let body = successor_body.clone(); + let version = selected_version.clone(); + let mut foreground = tokio::spawn(async move { + if mutation == "put" { + foreground_set + .put_object(&foreground_bucket, object, &mut PutObjReader::from_vec(body), &foreground_options) + .await + } else { + foreground_set + .delete_object( + &foreground_bucket, + object, + ObjectOptions { + version_id: Some(version), + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..foreground_options + }, + ) + .await + } + }); + let delete_commit; + if let Some(before) = &put_before { + before.wait_until_paused().await; + before.release_and_wait_until_namespace_pending().await; + assert!(!before.namespace_acquired()); + delete_commit = None; + } else { + let before = delete_before.take().expect("actual DELETE before-namespace observer"); + before.wait_until_paused().await; + before.release_and_wait_until_namespace_pending().await; + assert!(!before.namespace_acquired()); + drop(before); + delete_commit = Some(DeleteObjectCommitBarrier::install(&bucket, object)); + } + assert!(!foreground.is_finished() && !heal.is_finished() && !missing.exists()); + for (slot, dir) in dirs.iter().enumerate() { + let root = dir.path().join(&bucket).join(object); + assert_eq!( + tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("queued writer cannot change original metadata"), + original_metadata[slot] + ); + if slot != 0 { + assert_eq!( + tokio::fs::read(root.join(old_dir.to_string()).join("part.1")) + .await + .expect("surviving original part"), + original_parts[slot] + ); + } + } + println!( + "native_a09: geometry={disk_count}/{parity} mutation={mutation} actual_foreground_namespace_future_pending_original_bytes_unchanged" + ); + release.notify_one(); + let (result, error) = tokio::time::timeout(Duration::from_secs(30), heal) + .await + .expect("default Heal completes before queued foreground publishes") + .expect("Heal joins") + .expect("actual default Heal result"); + assert_eq!(error, None); + assert!(result.integrity_verified); + drop(pause); + if let Some(commit) = &put_commit { + tokio::select! { + _ = commit.wait_until_paused() => {}, + early = &mut foreground => panic!("PUT must reach its real post-lock barrier before publishing: {early:?}"), + } + } + if let Some(commit) = &delete_commit { + tokio::select! { + _ = commit.wait_until_paused() => {}, + early = &mut foreground => panic!("DELETE must reach its real pre-commit barrier after the Heal owner drains: {early:?}"), + } + } + assert!(!foreground.is_finished()); + for (slot, dir) in dirs.iter().enumerate() { + assert_eq!( + tokio::fs::read(dir.path().join(&bucket).join(object).join(old_dir.to_string()).join("part.1")) + .await + .expect("Heal physically reconstructs before foreground commit"), + original_parts[slot] + ); + } + println!( + "native_a09: geometry={disk_count}/{parity} mutation={mutation} heal_reconstructed_before_foreground_commit" + ); + if let Some(commit) = &put_commit { + commit.release(); + } + if let Some(commit) = &delete_commit { + commit.release(); + } + let ack = tokio::time::timeout(Duration::from_secs(30), foreground) + .await + .expect("queued foreground ACK after Heal releases ownership") + .expect("foreground joins") + .expect("real foreground ACK"); + let drained = namespace + .get_write_lock(Duration::from_secs(30)) + .await + .expect("ACK physical tail drain"); + for (slot, disk) in disks.iter().enumerate() { + let root = dirs[slot].path().join(&bucket).join(object); + assert!( + !root.join(old_dir.to_string()).exists(), + "overwritten/deleted original generation cannot remain" + ); + let raw = tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("final single acknowledged version metadata"); + let mut decoded = rustfs_filemeta::FileMeta::default(); + decoded.unmarshal_msg(&raw).expect("decode actual final version population"); + assert_eq!(decoded.versions.len(), 1); + let final_info = disk + .read_version("", &bucket, object, "", &ReadOptions::default()) + .await + .expect("final current generation"); + let data_dir = final_info.data_dir.expect("external successor generation"); + let part = tokio::fs::read(root.join(data_dir.to_string()).join("part.1")) + .await + .expect("final part before any GET"); + if mutation == "put" { + assert_ne!(data_dir, old_dir); + assert_eq!(part, reference_parts[&final_info.erasure.index]); + } else { + assert_eq!(Some(data_dir), current_data_dir); + assert_eq!( + part, current_parts[slot], + "exact version DELETE must preserve every retained-current shard" + ); + assert_eq!(ack.version_id, original.version_id); + assert_eq!(final_info.version_id, current.as_ref().expect("retained current ACK").version_id); + assert!(matches!( + disk.read_version("", &bucket, object, &selected_version, &ReadOptions::default()) + .await, + Err(DiskError::FileVersionNotFound) + )); + } + } + drop(drained); + let final_options = if let Some(current) = ¤t { + ObjectOptions { + versioned: true, + version_id: Some(current.version_id.expect("current version").to_string()), + suppress_read_repair: true, + ..options.clone() + } + } else { + ObjectOptions { + suppress_read_repair: true, + ..options.clone() + } + }; + let mut reader = set + .get_object_reader(&bucket, object, None, Default::default(), &final_options) + .await + .expect("full acknowledged successor/current body"); + let mut actual = Vec::new(); + reader + .read_to_end(&mut actual) + .await + .expect("full body with incidental repair suppressed"); + assert_eq!(actual, successor_body); + assert_eq!( + reader.object_info.etag, + if let Some(current) = ¤t { + current.etag.clone() + } else { + ack.etag.clone() + } + ); + println!( + "native_a09: geometry={disk_count}/{parity} mutation={mutation} foreground_ack_final_versions_parts_and_full_body_verified" + ); + } + } + } + + #[tokio::test] + #[serial_test::serial] + async fn native_copy_and_metadata_mutations_preserve_acknowledged_state_during_heal() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks_with_count}; + use crate::object_api::{ShardIntegrityWriteMode, WriteCompletion}; + use std::collections::HashMap; + use std::time::Duration; + use tokio::io::AsyncReadExt; + + for (disk_count, parity) in [(4, 2), (16, 4)] { + for mutation in ["copy-source", "copy-destination", "replace-metadata", "update-tags"] { + let (dirs, store) = isolated_store_over_temp_disks_with_count(disk_count).await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let bucket = format!("copy-heal-fence-{disk_count}-{mutation}"); + let source = "source/actual key.bin"; + let destination = "destination/actual key.bin"; + let control = "unrelated/control.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("isolated copy bucket"); + let options = ObjectOptions { + write_completion: WriteCompletion::TailDrained, + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + ..Default::default() + }; + let source_body = vec![0x31; 1024 * 1024 + 113]; + let destination_body = vec![0x42; 1024 * 1024 + 137]; + let control_body = vec![0x53; 1024 * 1024 + 157]; + let mut seed_acks = HashMap::new(); + for (key, body, role) in [ + (source, &source_body, "source"), + (destination, &destination_body, "destination"), + (control, &control_body, "control"), + ] { + let mut opts = options.clone(); + opts.user_defined = HashMap::from([ + ("content-type".to_owned(), format!("application/{role}")), + ("x-amz-meta-role".to_owned(), role.to_owned()), + ]); + let ack = store + .put_object(&bucket, key, &mut PutObjReader::from_vec(body.clone()), &opts) + .await + .expect("ACK complete external fixture"); + seed_acks.insert(key, ack); + } + let set = store.pools[0].disk_set[0].clone(); + let disks = set.disk_inventory().await; + let first_disk = disks[0].as_ref().expect("online first physical member"); + let original_destination = first_disk + .read_version("", &bucket, destination, "", &ReadOptions::default()) + .await + .expect("independent original COPY destination generation"); + let original_destination_dir = original_destination + .data_dir + .expect("original destination external generation"); + let heal_key = if mutation == "copy-source" { source } else { destination }; + let old = first_disk + .read_version("", &bucket, heal_key, "", &ReadOptions::default()) + .await + .expect("old healed generation metadata"); + assert_eq!((old.erasure.data_blocks, old.erasure.parity_blocks), (disk_count - parity, parity)); + assert!(!old.inline_data()); + let old_dir = old.data_dir.expect("external healed generation"); + let original_parts = futures::future::join_all(dirs.iter().map(|dir| { + tokio::fs::read( + dir.path() + .join(&bucket) + .join(heal_key) + .join(old_dir.to_string()) + .join("part.1"), + ) + })) + .await + .into_iter() + .map(|v| v.expect("original same-role part")) + .collect::>(); + let missing = dirs[0] + .path() + .join(&bucket) + .join(heal_key) + .join(old_dir.to_string()) + .join("part.1"); + tokio::fs::remove_file(&missing) + .await + .expect("inject one real missing external shard"); + let (pause, arrived, release) = ReadRepairPauseScope::install(&bucket, heal_key); + let heal_set = set.clone(); + let heal_bucket = bucket.clone(); + let heal = tokio::spawn(async move { + heal_set + .heal_object( + &heal_bucket, + heal_key, + "", + &HealOpts { + read_repair: true, + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(30), arrived.notified()) + .await + .expect("actual copy/metadata Heal reconstruction reaches final rename"); + assert!(!heal.is_finished() && !missing.exists()); + println!( + "native_g06: geometry={disk_count}/{parity} mutation={mutation} real_reconstruction_paused_before_final_rename" + ); + let ack = if matches!(mutation, "copy-source" | "copy-destination") { + // Keep the COPY source read from supplying a second repair; + // the separately paused production Heal owns this oracle. + let read_options = ObjectOptions { + suppress_read_repair: true, + ..options.clone() + }; + let reader = store + .get_object_reader(&bucket, source, None, Default::default(), &read_options) + .await + .expect("actual COPY source reader"); + let mut info = reader.object_info.clone(); + let size = info.get_actual_size().expect("COPY source logical size"); + assert_eq!(size, source_body.len() as i64); + let hash = rustfs_rio::HashReader::from_stream(reader.stream, size, size, None, None, false) + .expect("actual streamed COPY source"); + info.put_object_reader = Some(PutObjReader::new(hash)); + let ack = tokio::time::timeout( + Duration::from_secs(30), + store.copy_object(&bucket, source, &bucket, destination, &mut info, &read_options, &options), + ) + .await + .expect("COPY ACK before paused Heal release") + .expect("production cross-key COPY ACK"); + assert_eq!(ack.etag, seed_acks[source].etag); + ack + } else if mutation == "replace-metadata" { + let mut info = store + .get_object_info(&bucket, destination, &options) + .await + .expect("self-COPY source metadata"); + info.metadata_only = true; + info.user_defined = Arc::new(HashMap::from([ + ("content-type".to_owned(), "application/replaced".to_owned()), + ("x-amz-meta-race".to_owned(), "metadata-replaced".to_owned()), + ])); + tokio::time::timeout( + Duration::from_secs(30), + store.copy_object(&bucket, destination, &bucket, destination, &mut info, &options, &options), + ) + .await + .expect("self metadata COPY ACK before Heal release") + .expect("production self-COPY metadata ACK") + } else { + tokio::time::timeout( + Duration::from_secs(30), + store.put_object_tags(&bucket, destination, "race=updated&nonce=g06", &options), + ) + .await + .expect("tag update ACK before Heal release") + .expect("production tag update ACK") + }; + assert!(ack.etag.as_ref().is_some_and(|v| !v.is_empty())); + let namespace = set.new_ns_lock(&bucket, destination).await.expect("COPY/mutation namespace"); + let drained = namespace + .get_write_lock(Duration::from_secs(30)) + .await + .expect("foreground physical tails drain"); + let mut acknowledged = Vec::new(); + for key in [source, destination, control] { + let current = first_disk + .read_version("", &bucket, key, "", &ReadOptions::default()) + .await + .expect("foreground committed metadata"); + let data_dir = current.data_dir.expect("external foreground generation"); + let mut members = Vec::new(); + for dir in &dirs { + let root = dir.path().join(&bucket).join(key); + let meta = tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("foreground metadata bytes"); + let part_path = root.join(data_dir.to_string()).join("part.1"); + let part = if part_path.exists() { + Some(tokio::fs::read(part_path).await.expect("foreground part bytes")) + } else { + None + }; + members.push((meta, part)); + } + acknowledged.push((key, data_dir, members)); + } + if matches!(mutation, "copy-source" | "copy-destination") { + assert_ne!(acknowledged[1].1, original_destination_dir); + assert!(dirs.iter().all(|dir| { + !dir.path() + .join(&bucket) + .join(destination) + .join(original_destination_dir.to_string()) + .exists() + })); + } + if mutation != "copy-destination" { + assert_eq!(acknowledged.iter().find(|row| row.0 == heal_key).expect("healed key").1, old_dir); + assert!(!missing.exists(), "foreground metadata/source read must not supply the missing repair"); + } + drop(drained); + assert!(!heal.is_finished()); + println!( + "native_g06: geometry={disk_count}/{parity} mutation={mutation} foreground_ack_and_all_physical_tails_verified_before_release" + ); + release.notify_one(); + let (_, error) = tokio::time::timeout(Duration::from_secs(30), heal) + .await + .expect("copy/metadata Heal exits its fence") + .expect("Heal joins") + .expect("actual Heal result"); + assert_eq!( + error, + if mutation == "copy-source" { + None + } else { + Some(DiskError::Timeout) + } + ); + drop(pause); + for (key, data_dir, members) in &acknowledged { + for (slot, dir) in dirs.iter().enumerate() { + let root = dir.path().join(&bucket).join(key); + let metadata = tokio::fs::read(root.join(STORAGE_FORMAT_FILE)) + .await + .expect("metadata after Heal release"); + if *key != heal_key || mutation != "copy-source" { + assert_eq!(metadata, members[slot].0, "stale Heal must preserve foreground metadata exactly"); + } + let part_path = root.join(data_dir.to_string()).join("part.1"); + match &members[slot].1 { + Some(expected) => { + assert_eq!(tokio::fs::read(part_path).await.expect("acknowledged part survives Heal"), *expected) + } + None if mutation == "copy-source" => assert_eq!( + tokio::fs::read(part_path) + .await + .expect("real source Heal rebuilds before any GET"), + original_parts[slot] + ), + None => assert!(!part_path.exists(), "stale Heal cannot publish its obsolete staged part"), + } + } + } + if matches!(mutation, "replace-metadata" | "update-tags") { + let (_, error) = set + .heal_object( + &bucket, + destination, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + ..Default::default() + }, + ) + .await + .expect("independent current-metadata Heal after stale refusal"); + assert_eq!(error, None); + for (slot, dir) in dirs.iter().enumerate() { + assert_eq!( + tokio::fs::read( + dir.path() + .join(&bucket) + .join(destination) + .join(old_dir.to_string()) + .join("part.1") + ) + .await + .expect("fresh Heal rebuilds original data with acknowledged new metadata"), + original_parts[slot] + ); + } + } + for (key, expected) in [ + (source, &source_body), + ( + destination, + if matches!(mutation, "copy-source" | "copy-destination") { + &source_body + } else { + &destination_body + }, + ), + (control, &control_body), + ] { + let mut reader = store + .get_object_reader(&bucket, key, None, Default::default(), &options) + .await + .expect("full final COPY/mutation/control read"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("read complete acknowledged body"); + assert_eq!(&body, expected); + if !(key == destination && mutation == "replace-metadata") { + let role = + if key == source || key == destination && matches!(mutation, "copy-source" | "copy-destination") { + "source" + } else if key == destination { + "destination" + } else { + "control" + }; + assert_eq!(reader.object_info.user_defined.get("x-amz-meta-role").map(String::as_str), Some(role)); + assert_eq!(reader.object_info.user_defined.get("content-type"), Some(&format!("application/{role}"))); + } + if key == destination && mutation == "replace-metadata" { + let metadata = reader.object_info.user_defined.as_ref(); + assert_eq!(metadata.get("x-amz-meta-race").map(String::as_str), Some("metadata-replaced")); + assert_eq!(metadata.get("content-type").map(String::as_str), Some("application/replaced")); + assert!(!metadata.contains_key("x-amz-meta-role")); + } + } + for key in [source, destination, control] { + let expected = if key == destination && mutation == "update-tags" { + "race=updated&nonce=g06" + } else { + "" + }; + assert_eq!( + store + .get_object_tags(&bucket, key, &options) + .await + .expect("exact acknowledged and control tags after Heal"), + expected + ); + } + println!( + "native_g06: geometry={disk_count}/{parity} mutation={mutation} exact_copy_metadata_tags_generations_and_full_bodies_verified" + ); + } + } + } + #[tokio::test] #[serial_test::serial] async fn read_repair_commit_aborts_when_object_changes_before_final_rename() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index b790bc539..5934516a3 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -8200,13 +8200,23 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) { } } +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DeleteObjectCommitPause { + BeforeNamespace, + BeforeCommit, + AfterPublish, +} + #[cfg(test)] struct DeleteObjectCommitBarrierState { bucket: String, object: String, - pause_after_publish: bool, + pause: DeleteObjectCommitPause, arrived: tokio::sync::Notify, release: tokio::sync::Notify, + namespace_pending: tokio::sync::Notify, + namespace_acquired: std::sync::atomic::AtomicBool, } #[cfg(test)] @@ -8221,21 +8231,27 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock Self { - Self::install_with_mode(bucket, object, false) + Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeCommit) + } + + pub(crate) fn install_before_namespace(bucket: &str, object: &str) -> Self { + Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeNamespace) } #[cfg(feature = "test-util")] pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self { - Self::install_with_mode(bucket, object, true) + Self::install_with_mode(bucket, object, DeleteObjectCommitPause::AfterPublish) } - fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self { + fn install_with_mode(bucket: &str, object: &str, pause: DeleteObjectCommitPause) -> Self { let state = Arc::new(DeleteObjectCommitBarrierState { bucket: bucket.to_string(), object: object.to_string(), - pause_after_publish, + pause, arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), + namespace_pending: tokio::sync::Notify::new(), + namespace_acquired: std::sync::atomic::AtomicBool::new(false), }); let mut slot = DELETE_OBJECT_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -8255,6 +8271,19 @@ impl DeleteObjectCommitBarrier { pub(crate) fn release(&self) { self.state.release.notify_one(); } + + pub(crate) async fn release_and_wait_until_namespace_pending(&self) { + assert_eq!(self.state.pause, DeleteObjectCommitPause::BeforeNamespace); + let pending = self.state.namespace_pending.notified(); + self.release(); + tokio::time::timeout(Duration::from_secs(5), pending) + .await + .expect("delete object must poll its actual namespace acquisition to Pending"); + } + + pub(crate) fn namespace_acquired(&self) -> bool { + self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire) + } } #[cfg(test)] @@ -8271,6 +8300,43 @@ impl Drop for DeleteObjectCommitBarrier { } } +#[cfg(test)] +async fn pause_delete_object_before_namespace(bucket: &str, object: &str) { + let barrier = DELETE_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete object commit barrier mutex should not poison") + .as_ref() + .filter(|barrier| { + barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace + }) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + +#[cfg(test)] +fn notify_delete_object_namespace(bucket: &str, object: &str, acquired: bool) { + let barrier = DELETE_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete object commit barrier mutex should not poison") + .as_ref() + .filter(|barrier| { + barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace + }) + .cloned(); + if let Some(barrier) = barrier { + if acquired { + barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release); + } else { + barrier.namespace_pending.notify_one(); + } + } +} + #[cfg(test)] async fn pause_delete_object_commit(bucket: &str, object: &str) { let barrier = DELETE_OBJECT_COMMIT_BARRIER @@ -8278,7 +8344,9 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) { .lock() .expect("delete object commit barrier mutex should not poison") .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish) + .filter(|barrier| { + barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeCommit + }) .cloned(); if let Some(barrier) = barrier { barrier.arrived.notify_one(); @@ -8293,7 +8361,9 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) { .lock() .expect("delete object commit barrier mutex should not poison") .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish) + .filter(|barrier| { + barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::AfterPublish + }) .cloned(); if let Some(barrier) = barrier { barrier.arrived.notify_one(); @@ -10669,7 +10739,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { // Guard lock for single object delete let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock { - Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?)) + #[cfg(test)] + let guard = { + pause_delete_object_before_namespace(bucket, object).await; + let guard = self + .acquire_write_lock_diag_with_pending_hook("delete_object", bucket, object, || { + notify_delete_object_namespace(bucket, object, false); + }) + .await?; + notify_delete_object_namespace(bucket, object, true); + guard + }; + #[cfg(not(test))] + let guard = self.acquire_write_lock_diag("delete_object", bucket, object).await?; + Some(Arc::new(guard)) } else { None }; @@ -21607,6 +21690,206 @@ mod put_object_tmp_cleanup_tests { .await; } + #[tokio::test] + async fn deep_heal_preserves_committed_generation_during_old_data_cleanup() { + use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode}; + + for (disk_count, parity, read_repair) in [(4, 2, false), (4, 2, true), (16, 4, false), (16, 4, true)] { + let (dirs, disks, set) = super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated( + disk_count, 0, parity, + ) + .await; + let bucket = "native-heal-commit-cleanup"; + let object = format!("generation-{disk_count}-{parity}-{read_repair}.bin"); + for disk in &disks { + disk.make_volume(bucket).await.expect("create actual fixture volume"); + } + let options = ObjectOptions { + write_completion: crate::object_api::WriteCompletion::TailDrained, + shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected), + ..Default::default() + }; + set.put_object(bucket, &object, &mut PutObjReader::from_vec(vec![0x17; TEST_OBJECT_SIZE]), &options) + .await + .expect("acknowledge original generation on every disk"); + let old = disks[0] + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("read actual original generation"); + let old_dir = old.data_dir.expect("original external data directory"); + let old_part = dirs[0] + .path() + .join(bucket) + .join(&object) + .join(old_dir.to_string()) + .join("part.1"); + assert!(tokio::fs::try_exists(&old_part).await.expect("original part exists")); + + // Cleanup slots follow the writer's shuffled erasure distribution, + // not the physical fixture order. Pause every real cleanup slot so + // no physical old shard can disappear before the proof below. + let barriers: Vec<_> = (0..disk_count) + .map(|slot| rename_fanout_barrier::arm(&object, slot, rename_fanout_barrier::PHASE_CLEANUP)) + .collect(); + let writer_set = Arc::clone(&set); + let writer_object = object.clone(); + let writer_options = options.clone(); + let payload = vec![0xa3; TEST_OBJECT_SIZE]; + let writer_payload = payload.clone(); + let writer = tokio::spawn(async move { + writer_set + .put_object(bucket, &writer_object, &mut PutObjReader::from_vec(writer_payload), &writer_options) + .await + }); + tokio::time::timeout(Duration::from_secs(30), async { + for barrier in &barriers { + barrier.wait_until_paused().await; + } + }) + .await + .expect("overwrite commits before its real old-data cleanup pause"); + assert!(!writer.is_finished(), "tail-drained ACK must remain pending at cleanup"); + assert!( + tokio::fs::try_exists(&old_part) + .await + .expect("old source still present at pause") + ); + + let mut committed = Vec::new(); + let mut healthy_metadata = Vec::new(); + for disk in &disks { + let info = disk + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("read independently committed physical generation without GET"); + assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (disk_count - parity, parity)); + assert!(info.data.is_none(), "use actual external parts"); + assert_ne!(info.data_dir, Some(old_dir)); + committed.push(info); + healthy_metadata.push( + disk.read_all(bucket, &format!("{object}/xl.meta")) + .await + .expect("committed raw metadata"), + ); + } + let new_dir = committed[0].data_dir.expect("new committed data directory"); + assert!(committed.iter().all(|info| info.data_dir == Some(new_dir))); + let new_part = dirs[0] + .path() + .join(bucket) + .join(&object) + .join(new_dir.to_string()) + .join("part.1"); + let expected_part = tokio::fs::read(&new_part).await.expect("exact new source bytes"); + let mut committed_parts = Vec::new(); + for dir in &dirs { + let original = dir.path().join(bucket).join(&object).join(old_dir.to_string()).join("part.1"); + assert!( + tokio::fs::try_exists(original) + .await + .expect("every old cleanup slot is paused") + ); + committed_parts.push( + tokio::fs::read(dir.path().join(bucket).join(&object).join(new_dir.to_string()).join("part.1")) + .await + .expect("every new encoded shard before fault"), + ); + } + tokio::fs::remove_file(&new_part) + .await + .expect("remove only the newly committed target shard"); + println!( + "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} old_cleanup_paused new_generation_committed target_part_missing" + ); + + let (result, error) = tokio::time::timeout( + Duration::from_secs(30), + set.heal_object( + bucket, + &object, + "", + &HealOpts { + scan_mode: HealScanMode::Deep, + read_repair, + ..Default::default() + }, + ), + ) + .await + .expect("real locked Heal must progress during old cleanup") + .expect("complete actual deep Heal"); + assert!(error.is_none(), "new generation must remain repairable: {error:?}"); + assert_eq!(result.drives_healed(), Some(1)); + // Shared data-phase repair deliberately does not certify full integrity. + assert_eq!(result.integrity_verified, !read_repair); + assert_eq!(tokio::fs::read(&new_part).await.expect("rebuilt part before any GET"), expected_part); + let repaired = disks[0] + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("read actual repaired generation"); + assert_eq!(repaired.data_dir, Some(new_dir)); + assert_eq!(repaired.mod_time, committed[0].mod_time); + for (disk, expected) in disks.iter().zip(&healthy_metadata).skip(1) { + assert_eq!( + disk.read_all(bucket, &format!("{object}/xl.meta")) + .await + .expect("healthy raw metadata after Heal"), + *expected + ); + } + println!( + "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} locked_heal_completed new_generation_part_rebuilt_before_get" + ); + + for barrier in &barriers { + barrier.release(); + } + let acknowledged = tokio::time::timeout(Duration::from_secs(30), writer) + .await + .expect("cleanup drains") + .expect("writer joins") + .expect("overwrite ACK remains successful"); + assert_eq!(acknowledged.size, i64::try_from(payload.len()).expect("fixture size fits i64")); + drop(barriers); + for (slot, dir) in dirs.iter().enumerate() { + let root = dir.path().join(bucket).join(&object); + assert!( + !tokio::fs::try_exists(root.join(old_dir.to_string())) + .await + .expect("old cleanup completed") + ); + assert_eq!( + tokio::fs::read(root.join(new_dir.to_string()).join("part.1")) + .await + .expect("new shard survives cleanup before GET"), + committed_parts[slot], + "old cleanup must preserve every exact committed encoded shard" + ); + } + let mut reader = set + .get_object_reader(bucket, &object, None, HeaderMap::new(), &options) + .await + .expect("read final acknowledged generation"); + let mut body = Vec::new(); + reader + .stream + .read_to_end(&mut body) + .await + .expect("read complete final payload"); + assert_eq!(body, payload); + for disk in &disks { + let final_info = disk + .read_version("", bucket, &object, "", &ReadOptions::default()) + .await + .expect("final physical generation"); + assert_eq!(final_info.data_dir, Some(new_dir), "old cleanup must not revert the healed generation"); + } + println!( + "native_g01: geometry={disk_count}/{parity} read_repair={read_repair} overwrite_ack_preserved final_physical_generation_and_full_body_verified" + ); + } + } + #[tokio::test] async fn committed_put_releases_namespace_lock_before_old_data_cleanup() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index a7aab3d56..f222c015f 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -1818,7 +1818,10 @@ impl SetDisks { "Create bitrot reader failed read quorum" ); record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason); - return Err(to_object_err(read_err.into(), vec![bucket, object])); + // The metadata selected this object, but the part cannot + // be reconstructed. A dominant disk I/O/not-found error + // must not erase the object's read-quorum failure. + return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object])); } let reason = GetObjectFailureReason::ReadQuorum; error!( @@ -1843,7 +1846,7 @@ impl SetDisks { "Create bitrot reader did not have enough disks" ); record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason); - return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors))); + return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object])); } // Check if we have missing shards even though we can read successfully @@ -2378,10 +2381,10 @@ impl SetDisks { if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) { let reason = classify_disk_error(&read_err); record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason); - return Err(to_object_err(read_err.into(), vec![bucket, object])); + return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object])); } record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum); - return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors))); + return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object])); } let missing_shards = reader_setup.completed_failed_shards(); @@ -8192,6 +8195,105 @@ mod tests { (data_dir, files) } + #[tokio::test] + #[serial_test::serial] + async fn external_missing_shards_keep_typed_read_quorum() { + use tokio::io::AsyncReadExt; + + let payload = vec![0x37; 256 * 1024]; + for readable_shards in [1, 2, 4] { + let erasure = coding::Erasure::new(4, 2, 1024 * 1024); + let mut fi = codec_streaming_test_fileinfo(i64::try_from(payload.len()).expect("fixture size fits i64"), 1); + fi.erasure.data_blocks = erasure.data_shards; + fi.erasure.parity_blocks = erasure.parity_shards; + fi.erasure.block_size = erasure.block_size; + fi.erasure.distribution = (1..=erasure.total_shard_count()).collect(); + let (dirs, disks) = local_test_disks(erasure.total_shard_count(), CODEC_STREAMING_TEST_BUCKET).await; + let (data_dir, files) = codec_streaming_external_files(&erasure, &payload, &disks).await; + fi.data_dir = Some(data_dir); + for directory in dirs.iter().skip(readable_shards) { + tokio::fs::remove_file( + directory + .path() + .join(CODEC_STREAMING_TEST_BUCKET) + .join(CODEC_STREAMING_TEST_OBJECT) + .join(data_dir.to_string()) + .join("part.1"), + ) + .await + .expect("remove real part while retaining its selected metadata"); + } + + let mut output = Vec::new(); + let legacy = SetDisks::get_object_with_fileinfo( + CODEC_STREAMING_TEST_BUCKET, + CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), + 0, + fi.size, + &mut output, + fi.clone(), + files.clone(), + &disks, + 0, + 0, + false, + true, + false, + false, + GET_OBJECT_PATH_LEGACY_DUPLEX, + "plain_single_part", + "le_1mib", + ) + .await; + let streaming = SetDisks::get_object_mid_size_reader_with_fileinfo( + CODEC_STREAMING_TEST_BUCKET, + CODEC_STREAMING_TEST_OBJECT, + Arc::new(ErasureCache::new()), + &fi, + &files, + &disks, + 0, + 0, + false, + true, + "plain_single_part", + "le_1mib", + false, + ) + .await; + if readable_shards < erasure.data_shards { + for error in [ + legacy.expect_err("legacy read must reject quorum minus one"), + streaming.err().expect("streaming setup must reject quorum minus one"), + ] { + assert_eq!( + error, + Error::InsufficientReadQuorum( + CODEC_STREAMING_TEST_BUCKET.to_owned(), + CODEC_STREAMING_TEST_OBJECT.to_owned(), + ) + ); + } + assert!(output.is_empty(), "quorum rejection must precede all response bytes"); + } else { + legacy.expect("legacy reader retains reconstructable quorum"); + assert_eq!(output, payload, "legacy read returns every original byte at quorum"); + let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) = + streaming.expect("streaming quorum remains readable") + else { + panic!("mid-size quorum must yield an actual reader"); + }; + let mut decoded = Vec::new(); + reader + .read_to_end(&mut decoded) + .await + .expect("streaming reader returns the complete body"); + assert_eq!(decoded, payload, "streaming read returns every original byte at quorum"); + } + } + } + #[test] #[serial] fn payload_read_repair_is_suppressed_only_for_read_only_requests() { diff --git a/crates/ecstore/src/store/heal.rs b/crates/ecstore/src/store/heal.rs index cba03677e..161545da8 100644 --- a/crates/ecstore/src/store/heal.rs +++ b/crates/ecstore/src/store/heal.rs @@ -2167,6 +2167,210 @@ mod tests { shutdown.cancel(); } + #[tokio::test] + #[serial_test::serial] + async fn bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit() { + use crate::set_disk::ReadRepairPauseScope; + + let (_root, store, shutdown) = multi_pool_heal_store().await; + let bucket = format!("heal-commit-owner-{}", Uuid::new_v4().simple()); + let object = "history.bin"; + let version = Uuid::new_v4().to_string(); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("original bucket"); + let old = store + .bucket_incarnation_id_from_disk(&bucket) + .await + .expect("original identity"); + let set = store.pools[0].get_disks(0); + let mut reader = PutObjReader::from_vec(vec![0x17; 1024 * 1024]); + set.put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + versioned: true, + version_id: Some(version.clone()), + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("original acknowledged version"); + let missing = set.disks.read().await[0].clone().expect("repair target"); + let healthy = set.disks.read().await[1].clone().expect("healthy target"); + let baseline = healthy + .read_all(&bucket, &format!("{object}/xl.meta")) + .await + .expect("healthy metadata baseline"); + let original = healthy + .read_version("", &bucket, object, &version, &Default::default()) + .await + .expect("original version metadata"); + assert!(original.data.is_none(), "the commit barrier fixture must use actual external data shards"); + missing + .delete( + &bucket, + object, + DeleteOptions { + recursive: true, + immediate: true, + ..Default::default() + }, + ) + .await + .expect("remove one physical shard"); + + // The scope pauses the real repair after reconstruction, immediately + // before its generation check and final shard rename. + let (_pause, arrived, release) = ReadRepairPauseScope::install_incarnation_commit(&bucket, object); + let (drained_tx, drained_rx) = tokio::sync::oneshot::channel(); + let worker_store = store.clone(); + let worker_bucket = bucket.clone(); + let worker_version = version.clone(); + let opts = HealOpts { + pool: Some(0), + set: Some(0), + scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep, + ..Default::default() + }; + let mut waiter = tokio::spawn(async move { + worker_store + .run_bucket_heal_at_incarnation(&worker_bucket, old, &opts, move |store, bucket, opts| async move { + let result = store.heal_object_with_proof(&bucket, object, &worker_version, &opts).await?; + drained_tx + .send(result) + .expect("retain the actual storage result after caller cancellation"); + Ok(()) + }) + .await + }); + tokio::select! { + paused = tokio::time::timeout(std::time::Duration::from_secs(10), arrived.notified()) => { + paused.expect("real missing-shard repair reaches the final commit pause"); + } + completed = &mut waiter => panic!("repair ended before the actual final commit barrier: {completed:?}"), + } + assert!( + missing.read_xl(&bucket, object, false).await.is_err(), + "final shard is not published at the pause" + ); + println!("incarnation_commit_barrier: final_rename_paused"); + waiter.abort(); + assert!(waiter.await.expect_err("caller aborted").is_cancelled()); + let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket)); + assert!( + futures::poll!(&mut writer).is_pending(), + "bucket deletion/recreation cannot acquire its generation writer while the cancelled repair still owns commit" + ); + println!("incarnation_commit_barrier: caller_aborted_writer_pending"); + release.notify_one(); + let result = tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx) + .await + .expect("actual repair drains after release") + .expect("retained repair result"); + assert!( + result.error.is_none(), + "physical repair must succeed before lifecycle handoff: {:?}", + result.error + ); + let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer) + .await + .expect("generation writer acquires only after physical repair drains") + .expect("generation writer"); + assert!( + missing.read_xl(&bucket, object, false).await.is_ok(), + "the physical shard is published before writer admission" + ); + let repaired = missing + .read_version("", &bucket, object, &version, &Default::default()) + .await + .expect("repaired exact version metadata"); + assert_eq!(repaired.version_id, original.version_id); + assert_eq!(repaired.data_dir, original.data_dir); + let checked = missing + .check_parts(&bucket, object, &repaired) + .await + .expect("verify the published physical shard"); + assert_eq!( + checked.results, + vec![crate::disk::CHECK_PART_SUCCESS], + "the actual published part passes bitrot verification" + ); + assert_eq!( + healthy + .read_all(&bucket, &format!("{object}/xl.meta")) + .await + .expect("healthy after repair"), + baseline, + "repair does not overwrite the surviving version" + ); + println!("incarnation_commit_barrier: repair_drained_writer_acquired"); + drop(guard); + + set.delete_object( + &bucket, + object, + ObjectOptions { + versioned: true, + version_id: Some(version.clone()), + ..Default::default() + }, + ) + .await + .expect("normal explicit deletion of the old version"); + store + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("normal empty bucket deletion"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("same-name successor bucket"); + let new = store + .bucket_incarnation_id_from_disk(&bucket) + .await + .expect("successor identity"); + assert_ne!(old, new); + let successor_version = Uuid::new_v4().to_string(); + let mut successor_reader = PutObjReader::from_vec(vec![0x29; 512 * 1024]); + set.put_object( + &bucket, + object, + &mut successor_reader, + &ObjectOptions { + versioned: true, + version_id: Some(successor_version), + write_completion: crate::object_api::WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("acknowledged successor version"); + let successor = healthy + .read_all(&bucket, &format!("{object}/xl.meta")) + .await + .expect("successor metadata"); + assert!( + store + .heal_object_at_incarnation(&bucket, object, &version, old, &opts) + .await + .is_err() + ); + assert_eq!( + healthy + .read_all(&bucket, &format!("{object}/xl.meta")) + .await + .expect("successor after obsolete admission"), + successor, + "obsolete admission cannot change the same-name successor" + ); + println!("incarnation_commit_barrier: successor_rejects_obsolete_admission"); + shutdown.cancel(); + } + #[tokio::test] #[serial_test::serial] async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() { diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 7ffbad04e..91d67326c 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -2429,6 +2429,52 @@ mod test { } } + #[test] + fn truncated_inline_xlmeta_is_file_corrupt_only_for_supported_formats() { + let mut meta = FileMeta::new(); + meta.data + .replace("fixture", vec![0x5a; 4095]) + .expect("seed real inline framing"); + let inline_len = meta.data.as_slice().len(); + let original = meta.marshal_msg().expect("serialize valid inline metadata"); + let inline_start = original.len() - inline_len; + let parsed = FileMeta::load(&original).expect("complete inline metadata must decode"); + assert_eq!(parsed.data, meta.data); + for cut in inline_start + 1..original.len() { + assert_eq!( + FileMeta::load(&original[..cut]).expect_err("incomplete inline framing must fail"), + Error::FileCorrupt, + "inline truncation at byte {cut} must remain repairable" + ); + } + + for inline_version in [0, 2] { + let mut unsupported = original[..original.len() / 2].to_vec(); + unsupported[inline_start] = inline_version; + assert_ne!( + FileMeta::load(&unsupported).expect_err("malformed unsupported inline format must fail"), + Error::FileCorrupt, + "unknown inline version must not authorize repair" + ); + } + for (header_ver, meta_ver) in [ + (XL_HEADER_VERSION + 1, XL_META_VERSION), + (XL_HEADER_VERSION, XL_META_VERSION + 1), + ] { + let mut headers = Vec::new(); + rmp::encode::write_uint(&mut headers, u64::from(header_ver)).expect("write future header"); + rmp::encode::write_uint(&mut headers, u64::from(meta_ver)).expect("write future metadata version"); + rmp::encode::write_uint(&mut headers, 0).expect("write empty version count"); + let mut unsupported = build_xl_buffer(&headers); + unsupported.extend_from_slice(&meta.data.as_slice()[..inline_len / 2]); + assert_ne!( + FileMeta::load(&unsupported).expect_err("unsupported metadata with truncated inline data must fail"), + Error::FileCorrupt, + "future metadata schema must not authorize repair" + ); + } + } + #[test] fn truncated_xlmeta_index_and_format_reads_are_file_corrupt() { let buf = FileMeta::default() diff --git a/crates/filemeta/src/filemeta/codec.rs b/crates/filemeta/src/filemeta/codec.rs index a5342e548..44a32a5f7 100644 --- a/crates/filemeta/src/filemeta/codec.rs +++ b/crates/filemeta/src/filemeta/codec.rs @@ -188,7 +188,13 @@ impl FileMeta { if !buf.is_empty() { InlineData::validate_slice(buf).map_err(|e| { error!("data validation failed: {}", e); - e + // Validation only examines bytes already read. Classify damage + // as repairable corruption only for formats we understand. + if InlineData::version_ok_slice(buf) && Self::decode_xl_headers(meta).is_ok() { + Error::FileCorrupt + } else { + e + } })?; } diff --git a/crates/filemeta/src/filemeta_inline.rs b/crates/filemeta/src/filemeta_inline.rs index 998ee9593..1c7cedacc 100644 --- a/crates/filemeta/src/filemeta_inline.rs +++ b/crates/filemeta/src/filemeta_inline.rs @@ -168,11 +168,11 @@ impl InlineData { self.0.as_slice() } pub fn version_ok(&self) -> bool { - if self.0.is_empty() { - return true; - } + Self::version_ok_slice(&self.0) + } - self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER + pub(crate) fn version_ok_slice(raw: &[u8]) -> bool { + raw.first().is_none_or(|version| *version > 0 && *version <= INLINE_DATA_VER) } pub fn after_version(&self) -> &[u8] { diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index d212a5cf5..b9802655b 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -196,6 +196,20 @@ impl HealChannelProcessor { .map_err(Error::other) } + /// Query a local automatic task; other tokens require coordinator routing. + pub async fn execute_local_automatic_query( + &self, + heal_path: &str, + client_token: &str, + since_seq: Option, + ) -> Result> { + self.heal_manager + .get_local_automatic_task_report(heal_path, client_token, since_seq) + .await? + .map(|report| Self::query_report_response(client_token.to_owned(), Ok(report))) + .transpose() + } + /// Execute cancellation directly against the manager. pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result { let (response_tx, response_rx) = oneshot::channel(); @@ -463,6 +477,13 @@ impl HealChannelProcessor { .await }; + let response = Self::query_report_response(client_token, report)?; + let _ = response_tx.send(Ok(response.clone())); + self.publish_response(response); + Ok(()) + } + + fn query_report_response(client_token: String, report: Result) -> Result { let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone()); let settings = report .as_ref() @@ -581,27 +602,21 @@ impl HealChannelProcessor { 0, ), Err(crate::Error::InvalidClientToken) => { - let response = HealChannelResponse { + return Ok(HealChannelResponse { request_id: client_token, success: false, data: None, error: Some("invalid heal client token".to_string()), - }; - let _ = response_tx.send(Ok(response.clone())); - self.publish_response(response); - return Ok(()); + }); } Err(err) => { let error_text = err.to_string(); - let response = HealChannelResponse { + return Ok(HealChannelResponse { request_id: client_token, success: false, data: None, error: Some(error_text), - }; - let _ = response_tx.send(Ok(response.clone())); - self.publish_response(response); - return Ok(()); + }); } }; @@ -618,17 +633,12 @@ impl HealChannelProcessor { }, )?; - let response = HealChannelResponse { + Ok(HealChannelResponse { request_id: client_token, success: true, data: Some(data), error: detail, - }; - - let _ = response_tx.send(Ok(response.clone())); - self.publish_response(response); - - Ok(()) + }) } /// Process cancel request diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index c5114ef4d..5b4fbcec0 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -234,6 +234,8 @@ fn record_displaced_terminal( request: &HealRequest, ) -> Arc { let terminal = Arc::new(CompletedHealStatus { + task_id: request.id.clone(), + source: request.source, outcome: None, progress: None, retained_bytes: std::sync::OnceLock::new(), @@ -669,7 +671,10 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES { return None; } - if task.has_batch_failure() && !retryable_batch_failure { + // Admin traversals already exhaust their per-object retries. Restarting + // the whole traversal under the same token discards those receipts and + // reports previously repaired objects as unchanged on the next scan. + if task.has_batch_failure() && (task.source == HealRequestSource::Admin || !retryable_batch_failure) { return None; } @@ -1071,9 +1076,9 @@ pub struct HealManager { /// cascade without re-locking. enum TaskStateLookup { Active(Arc), - Retrying(HealTaskStatus, HealOptions), + Retrying(HealTaskStatus, HealOptions, HealRequestSource), Completed(Arc), - Queued(HealOptions), + Queued(HealOptions, HealRequestSource), /// A durable root heal record that cannot be admitted or quarantined; /// it answers status queries with the reason until an operator repairs it. Unrunnable(String), @@ -2586,6 +2591,23 @@ impl HealManager { /// completed entry. `heal_path` additionally constrains the map matches /// the way the `*_for_path` variants always have. async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result { + let state = self.lookup_memory_task_state(canonical_task_id, heal_path).await; + if !matches!(state, TaskStateLookup::NotFound) { + return Ok(state); + } + if let Some(completed) = self.root_recovery.completed(canonical_task_id).await? + && heal_path.is_none_or(|path| heal_type_matches_path(&completed.heal_type, path)) + { + return Ok(TaskStateLookup::Completed(Arc::new(completed))); + } + // A durable record that cannot run still answers with its reason. + if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await { + return Ok(TaskStateLookup::Unrunnable(reason)); + } + Ok(TaskStateLookup::NotFound) + } + + async fn lookup_memory_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup { let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path)); { @@ -2594,7 +2616,7 @@ impl HealManager { .get(canonical_task_id) .filter(|task| matches_path(&task.heal_type)) { - return Ok(TaskStateLookup::Active(Arc::clone(task))); + return TaskStateLookup::Active(Arc::clone(task)); } } @@ -2604,7 +2626,7 @@ impl HealManager { .get(canonical_task_id) .filter(|retrying| matches_path(&retrying.request.heal_type)) { - return Ok(TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone())); + return TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone(), retrying.request.source); } } @@ -2617,7 +2639,7 @@ impl HealManager { prune_completed_heal_statuses(&mut completed_heals); if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) { if completed_status_is_retrying(&completed.status) { - return Ok(TaskStateLookup::Completed(Arc::clone(completed))); + return TaskStateLookup::Completed(Arc::clone(completed)); } terminal_completed = Some(Arc::clone(completed)); } @@ -2626,7 +2648,7 @@ impl HealManager { { let queue = self.heal_queue.lock().await; if let Some(request) = queue.request_matching_id_and_path(canonical_task_id, heal_path) { - return Ok(TaskStateLookup::Queued(request.options.clone())); + return TaskStateLookup::Queued(request.options.clone(), request.source); } } @@ -2639,22 +2661,10 @@ impl HealManager { .cloned(); } - if terminal_completed.is_none() - && let Some(completed) = self.root_recovery.completed(canonical_task_id).await? - && matches_path(&completed.heal_type) - { - terminal_completed = Some(Arc::new(completed)); - } - if let Some(completed) = terminal_completed { - return Ok(TaskStateLookup::Completed(completed)); + return TaskStateLookup::Completed(completed); } - // A durable root record that can neither run nor be quarantined still - // answers status queries with its reason. - if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await { - return Ok(TaskStateLookup::Unrunnable(reason)); - } - Ok(TaskStateLookup::NotFound) + TaskStateLookup::NotFound } async fn publish_admin_terminal( @@ -2678,6 +2688,8 @@ impl HealManager { ) -> Result { let previous = self.completed_heals.lock().await.get(task_id).cloned(); let mut completed = CompletedHealStatus { + task_id: task_id.to_owned(), + source, outcome: None, progress: None, retained_bytes: std::sync::OnceLock::new(), @@ -2713,9 +2725,9 @@ impl HealManager { let canonical_task_id = self.canonical_task_id(task_id).await; match self.lookup_task_state(&canonical_task_id, None).await? { TaskStateLookup::Active(task) => Ok(task.get_status().await), - TaskStateLookup::Retrying(status, _) => Ok(status), + TaskStateLookup::Retrying(status, _, _) => Ok(status), TaskStateLookup::Completed(completed) => Ok(completed.status.clone()), - TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending), + TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending), TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }), TaskStateLookup::NotFound => Err(Error::TaskNotFound { task_id: task_id.to_string(), @@ -2734,9 +2746,9 @@ impl HealManager { let canonical_task_id = self.canonical_task_id(task_id).await; match self.lookup_task_state(&canonical_task_id, None).await? { TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await), - TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)), + TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)), TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)), - TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)), + TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)), TaskStateLookup::Unrunnable(error) => { Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default())) } @@ -2750,6 +2762,42 @@ impl HealManager { self.get_task_report_for_path_since(heal_path, task_id, None).await } + /// Resolve a node-owned automatic task without granting local control over + /// Admin tasks. The source and report come from the same state snapshot. + pub async fn get_local_automatic_task_report( + &self, + heal_path: &str, + task_id: &str, + since: Option, + ) -> Result> { + let canonical_task_id = self.canonical_task_id(task_id).await; + if canonical_task_id != task_id { + return Ok(None); + } + let state = self.lookup_memory_task_state(&canonical_task_id, Some(heal_path)).await; + let source = match &state { + TaskStateLookup::Active(task) => task.source, + TaskStateLookup::Retrying(_, _, source) | TaskStateLookup::Queued(_, source) => *source, + TaskStateLookup::Completed(completed) if completed.task_id == task_id => completed.source, + TaskStateLookup::Completed(_) => return Ok(None), + TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None), + }; + if !matches!( + source, + HealRequestSource::Scanner | HealRequestSource::Mrf | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair + ) { + return Ok(None); + } + let report = match state { + TaskStateLookup::Active(task) => active_task_report(&task, since).await, + TaskStateLookup::Retrying(status, options, _) => empty_task_report_with_options(status, options), + TaskStateLookup::Completed(completed) => completed_task_report(&completed, since), + TaskStateLookup::Queued(options, _) => empty_task_report_with_options(HealTaskStatus::Pending, options), + TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None), + }; + Ok(Some(report)) + } + /// Incremental variant of [`Self::get_task_report_for_path`] (HS-06). pub async fn get_task_report_for_path_since( &self, @@ -2760,9 +2808,9 @@ impl HealManager { let canonical_task_id = self.canonical_task_id(task_id).await; match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? { TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await), - TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)), + TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)), TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)), - TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)), + TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)), TaskStateLookup::Unrunnable(error) => { Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default())) } @@ -2786,9 +2834,9 @@ impl HealManager { let canonical_task_id = self.canonical_task_id(task_id).await; match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? { TaskStateLookup::Active(task) => Ok(task.get_status().await), - TaskStateLookup::Retrying(status, _) => Ok(status), + TaskStateLookup::Retrying(status, _, _) => Ok(status), TaskStateLookup::Completed(completed) => Ok(completed.status.clone()), - TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending), + TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending), TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }), TaskStateLookup::NotFound => { if self.path_has_task(heal_path).await { diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index f50bb49a5..4e1a2bcf8 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -109,7 +109,9 @@ pub(super) enum QueuePushOutcome { #[derive(Debug, Clone)] pub(super) struct CompletedHealStatus { + pub(super) task_id: String, pub(super) heal_type: HealType, + pub(super) source: HealRequestSource, /// Options used to execute the task, retained for token-scoped status. pub(super) options: HealOptions, pub(super) status: HealTaskStatus, @@ -137,6 +139,7 @@ impl CompletedHealStatus { fn measure_retained_bytes(&self) -> usize { let mut bytes = size_of::(); let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + add(self.task_id.capacity()); add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes())); match &self.heal_type { HealType::Cluster => {} @@ -258,7 +261,9 @@ impl CompletedHealStatus { outcome.finish(Some(crate::heal::outcome::HealAbortReason::Cancelled)); } let mut snapshot = Self { + task_id: task.id.clone(), heal_type: task.heal_type.clone(), + source: task.source, options: task.options.clone(), status, progress: Some(task.get_progress().await), diff --git a/crates/heal/src/heal/manager/root_recovery.rs b/crates/heal/src/heal/manager/root_recovery.rs index af34ba916..e080c7876 100644 --- a/crates/heal/src/heal/manager/root_recovery.rs +++ b/crates/heal/src/heal/manager/root_recovery.rs @@ -460,6 +460,8 @@ impl RootHealTerminal { fn into_completed(self) -> CompletedHealStatus { CompletedHealStatus { + task_id: self.task_id, + source: HealRequestSource::Admin, outcome: None, progress: self.progress, retained_bytes: std::sync::OnceLock::new(), diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 1b168726e..6c6ce4255 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -118,6 +118,8 @@ struct MockStorage; fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus { CompletedHealStatus { + task_id: "fixture-token".to_string(), + source: HealRequestSource::Internal, outcome: None, heal_type: HealType::Cluster, options: HealOptions::default(), @@ -3071,6 +3073,181 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() { ); } +#[tokio::test] +async fn local_automatic_query_preserves_source_path_token_and_state() { + use crate::heal::channel::HealChannelProcessor; + use crate::heal::outcome::{ + HealExecutionOutcome, HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome, + HealTraversalCoverage, + }; + + for source in [ + HealRequestSource::Scanner, + HealRequestSource::Mrf, + HealRequestSource::AutoHeal, + HealRequestSource::ReadRepair, + HealRequestSource::Admin, + HealRequestSource::Internal, + ] { + for state in ["queued", "active", "retrying", "completed", "displaced"] { + let storage: Arc = Arc::new(MockStorage); + let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None)); + let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + request.source = source; + let token = request.id.clone(); + let task = Arc::new(HealTask::from_request(request.clone(), storage)); + let mut snapshot = CompletedHealStatus::snapshot(&task, HealTaskStatus::Completed).await; + assert_eq!(snapshot.source, source, "completion must retain actual task source"); + let mut outcome = HealTaskOutcome::default(); + outcome.execution = HealExecutionOutcome::Completed; + outcome.coverage = HealTraversalCoverage::Complete; + outcome.record(HealObjectOutcome { + identity: HealObjectIdentity { + kind: HealObjectKind::Object, + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + bucket_incarnation_id: Some(uuid::Uuid::nil()), + pool_index: Some(0), + set_index: Some(0), + }, + disposition: HealObjectDisposition::VerifiedHealthy, + detail: None, + }); + snapshot.outcome = Some(Arc::new(outcome)); + snapshot.seqed_items = vec![( + 3, + HealResultItem { + bucket: "bucket".to_string(), + object: "object".to_string(), + ..Default::default() + }, + )]; + snapshot.min_seq = 3; + snapshot.next_seq = 4; + let expected_status = match state { + "queued" => { + manager.heal_queue.lock().await.push(request.clone()); + HealTaskStatus::Pending + } + "active" => { + manager.active_heals.lock().await.insert(token.clone(), task); + HealTaskStatus::Pending + } + "retrying" => { + manager.retrying_heals.lock().await.insert( + token.clone(), + RetryingHeal { + request: request.clone(), + error: "transient".to_string(), + cancel_token: CancellationToken::new(), + }, + ); + HealTaskStatus::Retrying { + error: "transient".to_string(), + retry_attempt: request.retry_attempts, + } + } + "completed" => { + manager.completed_heals.lock().await.insert(token.clone(), Arc::new(snapshot)); + HealTaskStatus::Completed + } + "displaced" => { + let terminal = record_displaced_terminal(&manager.displaced_terminals, &request); + terminal.status.clone() + } + _ => unreachable!("fixed test states"), + }; + let automatic = !matches!(source, HealRequestSource::Admin | HealRequestSource::Internal); + let report = manager + .get_local_automatic_task_report("bucket/object", &token, Some(2)) + .await + .expect("local source lookup"); + assert_eq!(report.is_some(), automatic, "source={source:?} state={state}"); + if let Some(report) = report { + assert_eq!(report.status, expected_status); + if state == "completed" { + let outcome = report.outcome.as_ref().expect("retained structured outcome"); + assert_eq!(outcome.objects.len(), 1); + assert_eq!(outcome.objects[0].identity.object, "object"); + assert_eq!(outcome.objects[0].identity.pool_index, Some(0)); + assert_eq!(outcome.objects[0].identity.set_index, Some(0)); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::VerifiedHealthy); + assert_eq!(report.min_seq, 3); + assert_eq!(report.next_seq, 4); + assert_eq!(report.result_items.len(), 1); + assert_eq!(report.result_items[0].object, "object"); + } + let processor = HealChannelProcessor::new(manager.clone()); + let local = processor + .execute_local_automatic_query("bucket/object", &token, Some(2)) + .await + .expect("local response") + .expect("automatic task response"); + let direct = processor + .execute_query_request_since("bucket/object".to_string(), token.clone(), Some(2)) + .await + .expect("original query response"); + assert!(local.success); + assert_eq!(local.request_id, direct.request_id); + assert_eq!(local.data, direct.data, "same report encoding and cursor contract"); + assert_eq!(local.error, direct.error); + let coordinator = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None)); + let missing = HealChannelProcessor::new(coordinator) + .execute_query_request("bucket/object".to_string(), token.clone()) + .await + .expect("coordinator query"); + let body: serde_json::Value = + serde_json::from_slice(missing.data.as_deref().expect("coordinator response")).expect("coordinator JSON"); + assert_eq!(body["summary"], "notFound", "a remote manager does not own the automatic task"); + if state == "completed" { + let completed = manager + .completed_heals + .lock() + .await + .get(&token) + .cloned() + .expect("original completion"); + manager + .completed_heals + .lock() + .await + .insert("retired-alias".to_string(), completed); + assert!( + manager + .get_local_automatic_task_report("bucket/object", "retired-alias", None) + .await + .expect("retired alias lookup") + .is_none() + ); + } + } + assert!( + manager + .get_local_automatic_task_report("bucket/other", &token, None) + .await + .expect("wrong path lookup") + .is_none() + ); + assert!( + manager + .get_local_automatic_task_report("bucket/object", "unknown", None) + .await + .expect("unknown token lookup") + .is_none() + ); + manager.insert_task_alias("admin-alias", &token).await; + assert!( + manager + .get_local_automatic_task_report("bucket/object", "admin-alias", None) + .await + .expect("alias lookup") + .is_none() + ); + } + } +} + #[tokio::test] async fn test_queued_duplicate_token_can_query_and_cancel_original_request() { let storage: Arc = Arc::new(MockStorage); @@ -3886,6 +4063,38 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab } } +#[tokio::test] +async fn admin_exhausted_batches_do_not_restart_the_traversal() { + for heal_type in [ + HealType::Bucket { + bucket: "bucket".to_owned(), + }, + HealType::Prefix { + bucket: "bucket".to_owned(), + prefix: "prefix/".to_owned(), + }, + HealType::Cluster, + ] { + let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::Normal); + request.source = HealRequestSource::Admin; + let task = HealTask::from_request(request, Arc::new(MockStorage)); + let result = Err(task + .record_batch_failure(BatchHealFailure { + scope: "completed-admin-traversal".to_owned(), + failed: 1, + retryable: 1, + permanent: 0, + first_object: "exhausted-object".to_owned(), + first_error: Error::Storage(EcstoreError::SlowDown).to_string(), + }) + .await); + assert!( + retry_request_for_result_with_budget(&task, &result).await.is_none(), + "the exhausted admin batch must retain its original object receipts" + ); + } +} + #[tokio::test] async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() { for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] { @@ -3968,6 +4177,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> Arc::new(CompletedHealStatus { progress: None, outcome: None, + source: request.source, + task_id: request.id.clone(), retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type, options: request.options.clone(), @@ -4816,6 +5027,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { Arc::new(CompletedHealStatus { progress: None, outcome: None, + source: request.source, + task_id: request.id.clone(), retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), options: request.options.clone(), @@ -4855,6 +5068,8 @@ async fn test_get_task_status_reads_recent_completed_status() { Arc::new(CompletedHealStatus { progress: None, outcome: None, + source: HealRequestSource::Internal, + task_id: "completed-token".to_string(), retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Bucket { bucket: "bucket".to_string(), @@ -4888,6 +5103,8 @@ async fn test_get_task_report_for_path_reads_completed_items() { Arc::new(CompletedHealStatus { progress: None, outcome: None, + source: HealRequestSource::Internal, + task_id: "completed-token".to_string(), retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Object { bucket: "bucket".to_string(), diff --git a/crates/heal/src/heal/manager/tests/root_recovery.rs b/crates/heal/src/heal/manager/tests/root_recovery.rs index f40368ac4..ea08d7974 100644 --- a/crates/heal/src/heal/manager/tests/root_recovery.rs +++ b/crates/heal/src/heal/manager/tests/root_recovery.rs @@ -117,6 +117,8 @@ fn admin_request(heal_type: HealType) -> HealRequest { fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus { CompletedHealStatus { + task_id: "admin-fixture".to_string(), + source: HealRequestSource::Admin, outcome: None, heal_type: heal_type.clone(), options: HealOptions::default(), @@ -1239,6 +1241,8 @@ async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() { .expect("durable bucket responsibility"); let completed = CompletedHealStatus { outcome: None, + source: request.source, + task_id: request.id.clone(), heal_type: request.heal_type.clone(), options: request.options.clone(), status: HealTaskStatus::Completed, diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 64b504d78..9e8c36ee2 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -37,7 +37,7 @@ use std::{ collections::VecDeque, future::Future, sync::{ - Arc, + Arc, OnceLock, atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }, time::{Duration, Instant, SystemTime}, @@ -59,6 +59,19 @@ pub(crate) struct OutcomeFinishTestHook { pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock>>> = std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None)); +#[cfg(test)] +static STORAGE_RECEIPT_TEST_HOOK: std::sync::LazyLock>>> = + std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None)); + +#[cfg(test)] +async fn pause_storage_receipt(task_id: &str) { + let hook = STORAGE_RECEIPT_TEST_HOOK.lock().await.clone(); + if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) { + hook.reached.notify_one(); + hook.release.notified().await; + } +} + #[cfg(test)] async fn pause_outcome_finish(task_id: &str) { let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone(); @@ -436,6 +449,9 @@ pub struct HealTask { replacement_start_retry_count: Arc, /// Set when the locked storage read classified the single target as absent. object_absent: Arc, + /// The first resolved incarnation belongs to this task even if storage + /// fails or the bucket is recreated before its terminal outcome is recorded. + single_object_incarnation: Arc>>, /// Task status pub status: Arc>, /// Progress tracking @@ -508,6 +524,7 @@ impl HealTask { replacement_running: Arc::new(AtomicBool::new(false)), replacement_start_retry_count: Arc::new(AtomicU32::new(0)), object_absent: Arc::new(AtomicBool::new(false)), + single_object_incarnation: Arc::new(OnceLock::new()), status: Arc::new(RwLock::new(HealTaskStatus::Pending)), progress: Arc::new(RwLock::new(HealProgress::new())), result_items: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_RETAINED_HEAL_RESULT_ITEMS))), @@ -669,7 +686,9 @@ impl HealTask { } => (bucket, object, Some(version_id.as_str())), _ => return None, }; - Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index)) + let mut identity = self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index); + identity.bucket_incarnation_id = self.single_object_incarnation.get().copied().flatten(); + Some(identity) } pub(super) async fn record_verified_storage_receipt( diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 563051182..9db89dffe 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -306,6 +306,8 @@ impl HealTask { let mut progress = self.progress.write().await; progress.update_stage(3, 3); } + // Keep storage/quorum identity for the cluster retry classifier. + // The structured log above already carries the bucket context. Err(e) } } @@ -790,21 +792,27 @@ impl HealTask { match unavailable_recreate_error(&storage_result.item, &heal_opts) { Some(error) => Some(error), None => { - telemetry_unknown |= !increment_counter(&mut healed); - telemetry_unknown |= !add_bytes( - &mut bytes, - u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX), - ); + #[cfg(test)] + super::pause_storage_receipt(&self.id).await; recorded_authoritative_outcome = self.record_verified_storage_receipt(&identity, storage_result.receipt).await; - if !recorded_authoritative_outcome - && !heal_opts.dry_run - && !storage_result.item.detail.is_empty() - { - detail = Some(storage_result.item.detail.clone()); + if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() { + Some(Error::TaskCancelled) + } else { + telemetry_unknown |= !increment_counter(&mut healed); + telemetry_unknown |= !add_bytes( + &mut bytes, + u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX), + ); + if !recorded_authoritative_outcome + && !heal_opts.dry_run + && !storage_result.item.detail.is_empty() + { + detail = Some(storage_result.item.detail.clone()); + } + self.record_result_item(storage_result.item).await; + None } - self.record_result_item(storage_result.item).await; - None } } } diff --git a/crates/heal/src/heal/task/heal_object.rs b/crates/heal/src/heal/task/heal_object.rs index b03026cf4..a4949305f 100644 --- a/crates/heal/src/heal/task/heal_object.rs +++ b/crates/heal/src/heal/task/heal_object.rs @@ -220,6 +220,11 @@ impl HealTask { let mut expected_identity = self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index); expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?; + if matches!(self.heal_type, HealType::Object { .. }) { + expected_identity.bucket_incarnation_id = *self + .single_object_incarnation + .get_or_init(|| expected_identity.bucket_incarnation_id); + } let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts); let heal_result = if self.source == HealRequestSource::ReadRepair { diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 1d7baaab5..60af25642 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -306,6 +306,53 @@ mod canonical_outcome { assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2); } + #[tokio::test] + async fn bucket_cancellation_after_storage_success_keeps_typed_cancelled_responsibility() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_receipts: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]), + )])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + let task = Arc::new(bucket_task(storage.clone())); + let hook = Arc::new(OutcomeFinishTestHook { + task_id: task.id.clone(), + reached: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + *STORAGE_RECEIPT_TEST_HOOK.lock().await = Some(hook.clone()); + let runner = { + let task = task.clone(); + tokio::spawn(async move { task.execute().await }) + }; + tokio::time::timeout(Duration::from_secs(10), hook.reached.notified()) + .await + .expect("storage success must reach the receipt boundary"); + assert_eq!(storage.healed_objects.lock().expect("storage mutations").as_slice(), ["object-a"]); + task.cancel().await.expect("cancel before receipt recording"); + hook.release.notify_one(); + let result = tokio::time::timeout(Duration::from_secs(10), runner) + .await + .expect("cancelled worker must stop") + .expect("worker must not panic"); + STORAGE_RECEIPT_TEST_HOOK.lock().await.take(); + assert!(matches!(result, Err(Error::TaskCancelled))); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Cancelled)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 1); + assert_eq!(outcome.counters.healed, 0); + assert_eq!(outcome.counters.unchanged, 0); + assert_eq!(outcome.counters.unknown, 0); + assert_eq!(outcome.counters.skipped, 1); + assert_eq!(outcome.objects.len(), 1); + assert_eq!(outcome.objects[0].identity.object, "object-a"); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled); + } + #[tokio::test] async fn bucket_heal_rejects_stale_receipts_without_double_recording() { let expected_incarnation = Uuid::new_v4(); @@ -813,6 +860,85 @@ mod canonical_outcome { ); } + #[tokio::test(start_paused = true)] + async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() { + for (failures, succeeds) in [(1, true), (4, false)] { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])), + typed_bucket_heal_errors: Mutex::new(HashMap::from([( + "bucket-a".to_string(), + (0..failures) + .map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum)) + .collect(), + )])), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + let result = task.execute().await; + assert_eq!( + result.is_ok(), + succeeds, + "original cluster task must honor its bounded bucket retry budget" + ); + let outcome = task.get_outcome().await; + if succeeds { + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + } else { + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure"); + assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0)); + } + let calls = storage.bucket_heal_calls.lock().expect("original bucket calls"); + assert_eq!( + calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(), + if succeeds { 2 } else { 4 } + ); + assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited"); + } + } + + #[tokio::test] + async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() { + for error in [ + Error::Storage(EcstoreError::ErasureWriteQuorum), + Error::Disk(DiskError::FileAccessDenied), + ] { + let retryable = error.is_recoverable_heal(); + let storage = Arc::new(MockStorage { + typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])), + ..Default::default() + }); + let task = bucket_task(storage.clone()); + let observed = task + .heal_bucket("bucket-a") + .await + .expect_err("bucket prepass must retain the storage error"); + assert!(matches!( + observed, + Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied) + )); + assert_eq!(observed.is_recoverable_heal(), retryable); + assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]); + assert!( + storage.heal_object_calls.lock().expect("object calls").is_empty(), + "failed prepass must not heal objects" + ); + } + } + #[tokio::test(start_paused = true)] async fn cancellation_and_deadline_leave_partial_coverage() { for cancel in [false, true] { @@ -1571,6 +1697,7 @@ struct MockStorage { listed_buckets: Mutex>>, bucket_discovery_incomplete: bool, bucket_heal_errors: Mutex>>, + typed_bucket_heal_errors: Mutex>>, bucket_heal_calls: Mutex>, delete_recovery_results: Mutex>>>, delete_recovery_calls: Mutex>, @@ -1819,6 +1946,127 @@ async fn failed_object_heal_rejects_matching_positive_storage_receipt() { | HealObjectDisposition::AuthoritativelyAbsent ) })); + assert_eq!(outcome.objects.len(), 1); + assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation)); +} + +#[tokio::test] +async fn object_failure_retains_incarnation_for_inner_and_outer_storage_errors() { + for failure in [ + MockHealObjectOutcome::PermissionDenied, + MockHealObjectOutcome::RetryableReadQuorum, + MockHealObjectOutcome::RetryableLockTimeout, + MockHealObjectOutcome::ErrOther("backend failed"), + ] { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + bucket_incarnation_id: Mutex::new(Some(incarnation)), + heal_object_outcome: Mutex::new(Some(failure)), + ..Default::default() + }); + let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), Some("version-a".to_owned())); + request.options.pool_index = Some(2); + request.options.set_index = Some(3); + let task = HealTask::from_request(request, storage); + task.execute() + .await + .expect_err("the failed storage operation must remain failed"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.objects.len(), 1); + assert_eq!( + (outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures), + (1, 1, 1) + ); + assert_eq!((outcome.counters.healed, outcome.counters.unchanged, outcome.counters.unknown), (0, 0, 0)); + let object = &outcome.objects[0]; + assert_eq!(object.identity.bucket_incarnation_id, Some(incarnation)); + assert_eq!(object.identity.bucket, "bucket-a"); + assert_eq!(object.identity.object, "object-a"); + assert_eq!(object.identity.version_id.as_deref(), Some("version-a")); + assert_eq!((object.identity.pool_index, object.identity.set_index), (Some(2), Some(3))); + assert!(matches!(object.disposition, HealObjectDisposition::Failed(_))); + } +} + +#[tokio::test(start_paused = true)] +async fn object_abort_keeps_pre_storage_incarnation_after_bucket_recreation() { + for cancel in [false, true] { + let original = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + bucket_incarnation_id: Mutex::new(Some(original)), + block_heal_object: Mutex::new(true), + ..Default::default() + }); + let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None); + request.options.timeout = Some(Duration::from_secs(10)); + let task = HealTask::from_request(request, storage.clone()); + let execution = task.execute(); + tokio::pin!(execution); + assert!(futures::poll!(&mut execution).is_pending()); + assert_eq!(storage.heal_object_calls.lock().expect("storage boundary").as_slice(), ["object-a"]); + *storage.bucket_incarnation_id.lock().expect("recreated bucket") = Some(Uuid::new_v4()); + if cancel { + task.cancel().await.expect("cancel blocked storage"); + } else { + tokio::time::advance(Duration::from_secs(11)).await; + } + let result = execution.await; + assert!(matches!( + (&result, cancel), + (Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false) + )); + let outcome = task.get_outcome().await; + assert_eq!(outcome.objects.len(), 1); + assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original)); + assert_eq!((outcome.counters.processed, outcome.counters.skipped), (1, 1)); + assert_eq!((outcome.counters.failed, outcome.counters.healed, outcome.counters.unknown), (0, 0, 0)); + assert_eq!( + outcome.objects[0].disposition, + if cancel { + HealObjectDisposition::Cancelled + } else { + HealObjectDisposition::Deferred { + reason: HealDeferredReason::Deadline, + retry_not_before: None, + } + } + ); + } +} + +#[tokio::test] +async fn object_unverified_outcome_keeps_original_incarnation_without_claiming_repair() { + let original = Uuid::new_v4(); + let successor = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + bucket_incarnation_id: Mutex::new(Some(original)), + bucket_incarnation_after_object_heal: Mutex::new(Some(successor)), + ..Default::default() + }); + let task = HealTask::from_request(HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None), storage.clone()); + task.execute().await.expect("legacy storage keeps its unverified result"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original)); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown); + assert_eq!((outcome.counters.healed, outcome.counters.unknown), (0, 1)); + assert_eq!(*storage.bucket_incarnation_id.lock().expect("successor"), Some(successor)); +} + +#[tokio::test] +async fn object_dry_run_does_not_adopt_bucket_incarnation() { + let storage = Arc::new(MockStorage { + bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())), + ..Default::default() + }); + let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None); + request.options.dry_run = true; + let task = HealTask::from_request(request, storage); + task.execute().await.expect("dry-run observation"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.objects.len(), 1); + assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, None); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved); + assert_eq!(outcome.counters.healed, 0); } #[tokio::test] @@ -2418,6 +2666,15 @@ impl HealStorageAPI for MockStorage { async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result { self.bucket_heal_calls.lock().unwrap().push(bucket.to_string()); self.bucket_heal_opts.lock().unwrap().push(*opts); + if let Some(error) = self + .typed_bucket_heal_errors + .lock() + .unwrap() + .get_mut(bucket) + .and_then(VecDeque::pop_front) + { + return Err(error); + } if let Some(error) = self .bucket_heal_errors .lock() diff --git a/crates/heal/tests/shard_identity_receipt_test.rs b/crates/heal/tests/shard_identity_receipt_test.rs index aa0f59a8d..a78c55d90 100644 --- a/crates/heal/tests/shard_identity_receipt_test.rs +++ b/crates/heal/tests/shard_identity_receipt_test.rs @@ -550,3 +550,159 @@ fn admin_normal_protected_repairs_report_repaired_for_data_and_parity() { .join() .expect("heal fixture result"); } + +#[test] +#[serial] +fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() { + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("mixed batch runtime") + .block_on(async { + use rustfs_heal::heal::{ + manager::{HealConfig, HealManager}, + outcome::{HealExecutionOutcome, HealFailureClass, HealTraversalCoverage}, + task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType}, + }; + use rustfs_heal_contracts::heal_channel::HealRequestSource; + use std::sync::Arc; + + let root = tempfile::tempdir().expect("mixed batch fixture"); + let env = TestECStoreEnv::builder().base_dir(root.path()).build().await; + let bucket = "admin-mixed-batch"; + env.make_bucket(bucket, false).await; + let set = env.ecstore.pools[0].get_disks(0); + let disks = set.disks.read().await.iter().flatten().cloned().collect::>(); + let mut originals = Vec::new(); + for (object, missing, byte) in [("healthy", 0, 0x11), ("repaired", 1, 0x52), ("failed", 3, 0xa3)] { + env.ecstore + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(vec![byte; 5 * 1024 * 1024 + 123]), + &ObjectOptions { + shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected), + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) + .await + .expect("commit complete fixture"); + for (slot, disk) in disks.iter().enumerate() { + let meta = disk + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .expect("fixture metadata"); + assert_eq!(meta.erasure.data_blocks, 2, "failed fixture must retain R-1 shards"); + let path = env.disk_paths[slot] + .join(bucket) + .join(object) + .join(meta.data_dir.expect("external shard").to_string()) + .join("part.1"); + let bytes = tokio::fs::read(&path).await.expect("original part"); + if slot < missing { + tokio::fs::remove_file(&path).await.expect("inject missing part"); + } + originals.push((object, path, bytes, slot < missing)); + } + } + let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + let manager = HealManager::new( + storage.clone(), + Some(HealConfig { + enable_auto_heal: false, + ..Default::default() + }), + ); + manager.start().await.expect("start mixed batch manager"); + let mut request = HealRequest::new( + HealType::Bucket { + bucket: bucket.to_owned(), + }, + HealOptions { + recursive: true, + scan_mode: HealScanMode::Deep, + pool_index: Some(0), + set_index: Some(0), + timeout: Some(Duration::from_secs(150)), + ..Default::default() + }, + HealPriority::High, + ); + request.source = HealRequestSource::Admin; + let token = request.id.clone(); + manager.submit_heal_request(request).await.expect("admit original mixed task"); + let report = tokio::time::timeout(Duration::from_secs(150), async { + loop { + let report = manager.get_task_report(&token).await.expect("same-token report"); + if matches!( + report.status, + HealTaskStatus::Completed | HealTaskStatus::Failed { .. } | HealTaskStatus::Timeout + ) { + break report; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("mixed task terminal deadline"); + manager.stop().await.expect("stop mixed task manager"); + for (object, path, original, missing) in originals { + if object == "failed" && missing { + assert!(!path.exists(), "below-R failure must not invent a shard"); + } else { + assert_eq!(tokio::fs::read(&path).await.expect("preserved or repaired shard"), original); + } + } + assert!(matches!(report.status, HealTaskStatus::Failed { .. }), "{report:?}"); + let outcome = report.outcome.as_ref().expect("canonical mixed outcome"); + assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!( + ( + outcome.counters.processed, + outcome.counters.healed, + outcome.counters.unchanged, + outcome.counters.failed, + outcome.counters.skipped, + outcome.counters.unknown, + outcome.counters.attempt_failures + ), + (3, 1, 1, 1, 0, 0, 4), + "{outcome:?}" + ); + for (object, disposition) in [ + ("healthy", HealObjectDisposition::VerifiedHealthy), + ("repaired", HealObjectDisposition::Repaired), + ("failed", HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)), + ] { + let receipt = outcome + .objects + .iter() + .find(|item| item.identity.object == object) + .expect("exact object receipt"); + assert_eq!(receipt.disposition, disposition); + } + drop(manager); + let restarted = HealManager::new( + storage, + Some(HealConfig { + enable_auto_heal: false, + ..Default::default() + }), + ); + restarted.start().await.expect("restart with retained failed batch"); + let restored = restarted.get_task_report(&token).await.expect("original token after restart"); + assert_eq!(restored.status, report.status); + assert_eq!(restored.outcome, report.outcome); + assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay"); + restarted.stop().await.expect("stop restarted mixed batch manager"); + }); + }) + .expect("mixed batch thread") + .join() + .expect("mixed batch result"); +} diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 92eba6042..77e95667e 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -5440,8 +5440,8 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met .await .expect_err("truncated primary must fail in the real object reader"); assert!( - error.to_string().contains("InlineData value out of range"), - "unexpected truncated-primary error: {error}" + matches!(error, EcstoreError::FileCorrupt), + "unexpected truncated-primary error: {error:?}" ); assert_eq!( read_config_with_revision(store.clone(), &backup_path) diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index bf3138f2b..2ed59f070 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -21,10 +21,10 @@ use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; use crate::data_usage_define::{ - DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap, - DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH, - PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path, - scan_frontier_covers_path, + DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageHash, + DataUsageHashMap, DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, + MAX_DATA_USAGE_CACHE_DEPTH, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, + SizeSummary, hash_path, scan_frontier_covers_path, }; use crate::error::ScannerError; use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter}; @@ -1608,7 +1608,7 @@ impl FolderScanner { let candidate_type = pending_scanner_heal_candidate_type(kind); let priority = request.priority; let scan_mode = request.scan_mode.unwrap_or(self.scan_mode); - let result = match send_scanner_heal_request(candidate_type, request).await { + let result = match send_scanner_heal_request(candidate_type, request, scanner_disk_heal_scope(&self.local_disk)).await { Ok(result) => result, Err(err) => { self.update_pending_scanner_heal_after_admission( @@ -2269,6 +2269,7 @@ impl FolderScanner { lifecycle: active_life_cycle.clone(), object_lock: active_object_lock.clone(), replication: active_replication.clone(), + heal_scope: None, heal_enabled, heal_bitrot: self.scan_mode == HealScanMode::Deep, debug: self.data_usage_scanner_debug, @@ -3472,6 +3473,7 @@ pub(crate) async fn scan_data_folder_scoped( mod item_actions; mod ledger; +pub(crate) use item_actions::scanner_disk_heal_scope; use item_actions::*; pub use item_actions::{GetSizeFn, ScannerItem}; use ledger::*; diff --git a/crates/scanner/src/scanner_folder/item_actions.rs b/crates/scanner/src/scanner_folder/item_actions.rs index 7ffc4c9c4..708d9faa9 100644 --- a/crates/scanner/src/scanner_folder/item_actions.rs +++ b/crates/scanner/src/scanner_folder/item_actions.rs @@ -474,10 +474,23 @@ pub(super) fn record_heal_candidate_admission( } } +pub(crate) fn scanner_disk_heal_scope(disk: &Disk) -> Option { + let location = disk.get_disk_location(); + location.valid().then_some(())?; + Some(DataUsageCacheSource::new(location.pool_idx?, location.set_idx?)) +} + pub(super) async fn send_scanner_heal_request( candidate_type: &'static str, - request: HealChannelRequest, + mut request: HealChannelRequest, + scope: Option, ) -> Result { + if request.object_prefix.as_deref().is_some_and(|object| !object.is_empty()) { + let scope = scope + .ok_or_else(|| ScannerError::Other("scanner object heal requires the scanning disk's pool and set".to_string()))?; + request.pool_index = Some(scope.pool_index); + request.set_index = Some(scope.set_index); + } let priority = request.priority; let trace_context = scanner_heal_candidate_trace_context(&request); let candidate = crate::heal_completion::prepare(&request); @@ -537,6 +550,8 @@ pub struct ScannerItem { pub lifecycle: Option>, pub object_lock: Option>, pub replication: Option>, + /// Actual scanning-disk location; refreshed by the disk reader before actions. + pub heal_scope: Option, pub heal_enabled: bool, pub heal_bitrot: bool, pub debug: bool, @@ -1207,6 +1222,7 @@ impl ScannerItem { scan_mode, HealChannelPriority::Low, ), + self.heal_scope, ) .await; @@ -1368,6 +1384,7 @@ mod tests { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 80a862a3b..63eb12cbd 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -339,7 +339,10 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) { .await .expect("failed to create test directory"); - let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint"); + let mut endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); let disk = new_disk( &endpoint, &DiskOption { @@ -766,6 +769,7 @@ fn test_classify_get_size_failure_marks_metadata_heal_object_path() { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -797,6 +801,7 @@ fn test_classify_get_size_failure_records_transient_metadata_error_without_heal( lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -853,6 +858,7 @@ async fn test_heal_replication_skips_only_unversioned_data_versions() { lifecycle: None, object_lock: None, replication: Some(Arc::new(ReplicationConfig::new(None, None))), + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -2255,6 +2261,7 @@ async fn test_heal_actions_returns_actual_size_without_inline_heal() { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: true, heal_bitrot: true, debug: false, @@ -2495,7 +2502,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); let heal_starts = Arc::new(AtomicUsize::new(0)); let heal_starts_clone = heal_starts.clone(); - let healed_versions = Arc::new(Mutex::new(Vec::>::new())); + let healed_versions = Arc::new(Mutex::new(Vec::new())); let healed_versions_clone = healed_versions.clone(); let (_commands, mut heal_rx) = rustfs_heal_contracts::heal_channel::init_heal_channels() .expect("heal channels should initialize once for scanner tests"); @@ -2507,7 +2514,7 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { healed_versions_clone .lock() .expect("heal version capture lock should not be poisoned") - .push(request.object_version_id); + .push((request.object_prefix, request.object_version_id, request.pool_index, request.set_index)); let _ = response_tx.send(Ok(rustfs_heal_contracts::heal_channel::HealAdmissionReceipt { result: HealAdmissionResult::Accepted, task_id: request.id, @@ -2588,9 +2595,90 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { .lock() .expect("heal version capture lock should not be poisoned") .iter() - .any(|version| version.as_deref() == Some(orphan_version_text.as_str())), + .any(|(_, version, _, _)| version.as_deref() == Some(orphan_version_text.as_str())), "sub-quorum orphan version must be submitted as an exact heal candidate" ); + assert!( + healed_versions + .lock() + .expect("heal scope capture lock should not be poisoned") + .iter() + .filter(|(object, _, _, _)| object.is_some()) + .all(|(_, _, pool, set)| (*pool, *set) == (Some(0), Some(0))), + "high-priority discovered objects must retain the scanning disk's scope" + ); + let bucket_admission = scanner + .send_required_scanner_heal_request( + PendingScannerHealKind::Bucket, + bucket.to_string(), + None, + None, + build_bucket_heal_request(bucket.to_string(), HealChannelPriority::High), + ) + .await + .expect("submit bucket request"); + assert_eq!(bucket_admission, HealAdmissionResult::Accepted); + assert_eq!( + healed_versions.lock().expect("read captured bucket scope").last(), + Some(&(None, None, None, None)), + "bucket healing must retain its cross-pool scope" + ); + scanner.new_cache.info.name = bucket.to_string(); + scanner.new_cache.info.pending_heals = vec![pending_heal( + PendingScannerHealKind::Object, + bucket, + Some("pending-object"), + Some(&orphan_version_text), + 0, + 1, + )]; + scanner + .retry_pending_scanner_heals() + .await + .expect("retry persisted object heal"); + assert_eq!( + healed_versions.lock().expect("read retried object scope").last(), + Some(&(Some("pending-object".to_string()), Some(orphan_version_text.clone()), Some(0), Some(0))), + "persisted retries must bind the current scanning disk" + ); + let unscoped_endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("parse uninitialized endpoint"); + scanner.local_disk = new_disk( + &unscoped_endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open disk without initialized location"); + let request_count = healed_versions.lock().expect("count admitted requests").len(); + let deferred = scanner + .send_required_scanner_heal_request( + PendingScannerHealKind::Object, + bucket.to_string(), + Some("unscoped-object".to_string()), + Some(orphan_version_text.clone()), + build_object_heal_request( + bucket.to_string(), + "unscoped-object".to_string(), + Some(orphan_version_text), + HealScanMode::Deep, + HealChannelPriority::High, + ), + ) + .await + .expect("defer object with unknown disk location"); + assert_eq!(deferred, HealAdmissionResult::Full); + assert_eq!(healed_versions.lock().expect("count unchanged admissions").len(), request_count); + assert!( + scanner + .new_cache + .info + .pending_heals + .iter() + .any(|entry| entry.object.as_deref() == Some("unscoped-object")), + "an unscoped high-priority object must remain pending" + ); for (path, expected) in expected_metadata { assert_eq!( tokio::fs::read(&path) diff --git a/crates/scanner/src/scanner_io/io_disk.rs b/crates/scanner/src/scanner_io/io_disk.rs index 32285b0a9..2a08e10bf 100644 --- a/crates/scanner/src/scanner_io/io_disk.rs +++ b/crates/scanner/src/scanner_io/io_disk.rs @@ -146,6 +146,8 @@ impl ScannerIODisk for Disk { // `object_infos`. global_metrics().record_scanner_versions_scanned(object_infos.len() as u64); + // A caller-provided item or persisted usage cache cannot choose another set. + item.heal_scope = crate::scanner_folder::scanner_disk_heal_scope(self); #[cfg(test)] let _ = crate::scanner_folder::observe_test_size_dispatch_boundary("apply_actions"); item.apply_actions_cancellable( diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 80e86f1cb..d4f36c441 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -1772,6 +1772,7 @@ async fn scanner_item_object_lock_uses_cached_config() { lifecycle: None, object_lock: Some(cached.clone()), replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -4521,6 +4522,7 @@ async fn get_size_treats_missing_metadata_as_skip_file() { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -4576,6 +4578,7 @@ async fn get_size_marks_corrupt_metadata_for_heal() { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -4658,6 +4661,7 @@ async fn get_size_counts_delete_markers_separately_from_versions() { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, diff --git a/crates/scanner/tests/lifecycle/scanner_heal_scope.rs b/crates/scanner/tests/lifecycle/scanner_heal_scope.rs new file mode 100644 index 000000000..19ecdab1b --- /dev/null +++ b/crates/scanner/tests/lifecycle/scanner_heal_scope.rs @@ -0,0 +1,144 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_filemeta::{FileInfo, FileMeta}; +use rustfs_heal_contracts::heal_channel::{ + HealAdmissionReceipt, HealAdmissionResult, HealReceiptCommand, HealRequestSource, init_heal_channels, +}; +use rustfs_scanner::DataUsageCacheSource; +use rustfs_scanner::scanner_folder::ScannerItem; +use rustfs_scanner::scanner_io::ScannerIODisk; +use tokio::sync::mpsc; + +use super::storage_api::lifecycle::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk}; + +#[test] +fn scanner_object_requests_bind_the_actual_disk_scope() { + const CHILD: &str = "RUSTFS_TEST_SCANNER_SCOPE_CHILD"; + if std::env::var_os(CHILD).is_none() { + let status = std::process::Command::new(std::env::current_exe().expect("locate integration test executable")) + .args([ + "--exact", + "scanner_heal_scope::scanner_object_requests_bind_the_actual_disk_scope", + "--nocapture", + ]) + .env(CHILD, "1") + .status() + .expect("run scanner scope regression in an isolated process"); + assert!(status.success(), "isolated scanner scope regression failed: {status}"); + return; + } + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build isolated scanner runtime") + .block_on(check_scanner_object_scope()); +} + +async fn check_scanner_object_scope() { + let (_commands, mut receipts) = init_heal_channels().expect("initialize isolated scanner request channels"); + let (captured, mut requests) = mpsc::unbounded_channel(); + let responder = tokio::spawn(async move { + while let Some(command) = receipts.recv().await { + let HealReceiptCommand { request, response_tx } = command; + let receipt = HealAdmissionReceipt { + result: HealAdmissionResult::Accepted, + task_id: request.id.clone(), + }; + captured.send(request).expect("retain scanner request"); + let _ = response_tx.send(Ok(receipt)); + } + }); + + for (pool, set, disk_index) in [ + (Some(0), Some(0), Some(1)), + (Some(2), Some(3), Some(4)), + (None, Some(3), Some(4)), + (Some(2), None, Some(4)), + (Some(2), Some(3), None), + (None, None, None), + ] { + for version_id in [None, Some(uuid::Uuid::from_u128(42))] { + let root = tempfile::tempdir().expect("create scanner disk"); + let bucket = "scanner-scope-bucket"; + let object = "object"; + let path = root.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE); + tokio::fs::create_dir_all(path.parent().expect("metadata parent")) + .await + .expect("create object directory"); + let mut info = FileInfo::new(object, 4, 2); + info.volume = bucket.to_string(); + info.name = object.to_string(); + info.size = 1; + info.version_id = version_id; + info.data_dir = Some(uuid::Uuid::new_v4()); + info.erasure.index = 1; + info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1).expect("valid object timestamp")); + info.add_object_part(1, "fixture-etag".to_string(), 1, info.mod_time, 1, None, None); + let mut meta = FileMeta::new(); + meta.add_version(info).expect("encode original null version"); + let original = meta.marshal_msg().expect("marshal original metadata"); + tokio::fs::write(&path, &original).await.expect("write original metadata"); + let mut endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("parse scanning disk endpoint"); + if let Some(pool) = pool { + endpoint.set_pool_index(pool); + } + if let Some(set) = set { + endpoint.set_set_index(set); + } + if let Some(disk_index) = disk_index { + endpoint.set_disk_index(disk_index); + } + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open actual scanning disk"); + let item = ScannerItem { + path: path.to_string_lossy().to_string(), + bucket: bucket.to_string(), + prefix: object.to_string(), + object_name: STORAGE_FORMAT_FILE.to_string(), + file_type: tokio::fs::metadata(&path).await.expect("stat original metadata").file_type(), + lifecycle: None, + object_lock: None, + replication: None, + // This foreign caller-provided scope must never reach admission. + heal_scope: Some(DataUsageCacheSource::new(99, 98)), + heal_enabled: true, + heal_bitrot: true, + debug: false, + }; + disk.get_size_with_tier_names(item, &[]).await.expect("scan original object"); + if let (Some(pool), Some(set), Some(_)) = (pool, set, disk_index) { + let request = requests.try_recv().expect("scanner submitted one real object request"); + assert_eq!(request.pool_index, Some(pool)); + assert_eq!(request.set_index, Some(set)); + assert_eq!(request.bucket, bucket); + assert_eq!(request.object_prefix.as_deref(), Some(object)); + assert_eq!(request.object_version_id, version_id.map(|value| value.to_string())); + assert_eq!(request.source, HealRequestSource::Scanner); + assert_eq!(request.no_lock, None); + assert_eq!(request.recreate_missing, Some(false)); + } + assert!(requests.try_recv().is_err(), "no duplicate or unscoped request may be admitted"); + assert_eq!(tokio::fs::read(&path).await.expect("read original metadata"), original); + } + } + responder.abort(); +} diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 713f00edc..109a0387e 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -39,6 +39,9 @@ use uuid::Uuid; mod storage_api; +#[path = "lifecycle/scanner_heal_scope.rs"] +mod scanner_heal_scope; + use storage_api::lifecycle::{ BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DeleteAfterObjectLockSnapshotBarrier, DiskOption, ECStore, EcstoreError, Endpoint, EndpointServerPools, Endpoints, @@ -467,6 +470,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str lifecycle, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, @@ -504,6 +508,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) { lifecycle: None, object_lock: None, replication: None, + heal_scope: None, heal_enabled: false, heal_bitrot: false, debug: false, diff --git a/docs/architecture/heal-terminal-reports.md b/docs/architecture/heal-terminal-reports.md index bc56925b9..573045c89 100644 --- a/docs/architecture/heal-terminal-reports.md +++ b/docs/architecture/heal-terminal-reports.md @@ -4,6 +4,10 @@ Admin heal tokens retain their terminal status for ten minutes from the original ## Publication and Recovery +Authenticated token queries first inspect the receiving node's in-memory automatic tasks. An original Scanner, MRF, automatic-Heal, or read-repair token can return its path-bound snapshot on that node, including the retained outcome and incremental cursor. The task source and original ID remain attached to the completion snapshot; aliases cannot enter this local query path. The local lookup never reads administrator recovery records. + +Administrator tokens and unknown local identities continue through the cluster coordinator. Starting and cancelling tasks retain their coordinator route. Querying another node does not discover or aggregate a remote automatic task, and this path does not extend automatic-task retention across process restart. + The original schema-1 `terminal-root-heal-.json` remains the commit marker. Its separate `heal-terminal-report-.json` report has its own schema version and embeds the exact terminal identity. Both files belong to the same coordinator disk as the pending intent. Publication writes the report through the storage owner's conditional file update, writes the terminal marker, and then conditionally removes the pending intent. Recovery never treats a report without its marker as committed. A committed terminal continues to suppress stale pending work even when its report is unreadable. An initial publication failure leaves the existing responsibility available for recovery. diff --git a/docs/testing/README.md b/docs/testing/README.md index cc09fb399..32dadc8a7 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -34,6 +34,8 @@ The [Pool layout compatibility reference](pool-layout-compatibility.md) defines The [Scanner/Heal topology protection gates](scanner-heal-topology-gates.md) map G-EXP, G-REB and G-DEC and their fault windows to the `scanner_heal_topology_` cases, their recorded inventory, and the basis of their timeouts. +The [immutable validation candidate reference](validation-candidates.md) describes isolated package builds and exact candidate resolution for the current auto-testing fleet. + The [distributed multipart RPC delay gates](multipart-rpc-delay.md) define authenticated method-entry barriers, latency and peer-loss oracles, and the boundary between hook evidence and production incident attribution. ## Naming conventions diff --git a/docs/testing/validation-candidates.md b/docs/testing/validation-candidates.md new file mode 100644 index 000000000..23bd993d2 --- /dev/null +++ b/docs/testing/validation-candidates.md @@ -0,0 +1,52 @@ +# Immutable validation candidates + +Canonical validation on the current auto-testing fleet consumes one exact +package, source commit and candidate-bound tool build. A successful package +build is provenance evidence, not a canonical test PASS. + +The scheduled Functional Chain continues to resolve successful Nightly GNU +Build attempts from `main`. Optional manual validation can also resolve a +validation package built without publishing nightly aliases or release assets. + +## Build a validation package + +Dispatch **RustFS Functional Chain** from the desired driver revision with +only `validation_package_sha` set to the full 40-character RustFS source SHA. +Keep candidate, case, helper, tools and cleanup inputs empty or false. The +request validator rejects mixed package/fleet execution before building. + +The package job calls `rustfs-validation-package.yml`, checks out the requested +source, confirms its Git SHA and builds the ordinary release binary and DEB. +This job uses build runners and does not acquire or mutate the Heal fleet. +The same workflow can be dispatched directly with its `source_sha` input when +it is available on the repository's default branch. + +Publication requires complete R2 configuration, conditional `PutObject` with +`If-None-Match: *`, and a public download whose SHA-256 matches the local DEB. +Failure emits no candidate manifest. An existing object is never overwritten. +The only object path is: + +```text +artifacts/rustfs/packages/validation/runs////rustfs.deb +``` + +The schema-3 artifact `validation-candidate--` records the +workflow SHA independently of the exact source SHA and source ref, the +`validation` channel, build run/attempt, immutable URL and package checksum. +The resolver verifies the successful manual run, repository/workflow identity, +exact artifact name and digest, archive members, schema and immutable URL. +The ordinary scheduled chain does not enable validation package resolution. + +## Validate the candidate + +After the package run succeeds, dispatch Functional Chain with its exact +`build_run_id` and `build_run_attempt`, a canonical `validation_case`, and an +exact `auto_testing_sha`. Heal also requires a new tool build verified against +that source and helper revision. Reassess source-bound contracts and native +prerequisites when the source changes; an older tool artifact does not certify +a new candidate. + +The fleet lane retains the shared environment lock, per-case evidence and +cleanup gate. Record new results separately. A package build, native test, +diagnostic replay, clean recovery or partial topology result cannot replace +complete original source assertions or rewrite a retained canonical FAIL. diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 9501b07fe..d1cdfdbe3 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -1051,6 +1051,21 @@ async fn route_cluster_heal_control( }) } +async fn query_heal_status_on_owner( + local: L, + coordinator: C, +) -> S3Result +where + L: Future>>, + C: FnOnce() -> CF, + CF: Future>, +{ + match local.await? { + Some(response) => Ok(response), + None => coordinator().await, + } +} + async fn execute_after_heal_start_preflight( endpoints: &EndpointServerPools, options: &HealOpts, @@ -1514,20 +1529,41 @@ impl Operation for HealHandler { let heal_path = encode_heal_control_path(&hip.bucket, &hip.obj_prefix); if !hip.client_token.is_empty() && !hip.force_start && !hip.force_stop { let client_token = hip.client_token.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); - let context = app_context - .clone() - .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?; - let route = prepare_heal_control_route(&context)?; - let envelope = rustfs_protos::heal_control::Envelope::query( - request_id.clone(), - new_heal_control_metadata(&route)?, - heal_path, - client_token.clone(), - hip.since_seq, + // Automatic workers run on their owning node. Only an exact + // path/token/source snapshot can answer locally; Admin tokens and + // unknown identities retain the coordinator authority boundary. + let response = query_heal_status_on_owner( + async { + let Some(manager) = rustfs_heal::get_heal_manager() else { + return Ok(None); + }; + let processor = rustfs_heal::HealChannelProcessor::new(manager.clone()); + timeout( + PEER_HEAL_STATUS_TIMEOUT, + processor.execute_local_automatic_query(&heal_path, &client_token, hip.since_seq), + ) + .await + .map_err(|_| cluster_heal_control_unavailable("local_task_query_expired"))? + .map_err(|_| cluster_heal_control_unavailable("local_task_query_failed")) + }, + || async { + let request_id = uuid::Uuid::new_v4().to_string(); + let context = app_context + .clone() + .ok_or_else(|| cluster_heal_control_unavailable("app_context_unavailable"))?; + let route = prepare_heal_control_route(&context)?; + let envelope = rustfs_protos::heal_control::Envelope::query( + request_id.clone(), + new_heal_control_metadata(&route)?, + heal_path.clone(), + client_token.clone(), + hip.since_seq, + ) + .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?; + submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token.clone()).await + }, ) - .map_err(|err| s3_error!(InternalError, "encode heal control query failed: {err}"))?; - let response = submit_cluster_heal_channel_command(context, route, envelope, &request_id, client_token).await?; + .await?; if !response.success { return Err(s3_error!( InternalError, @@ -1995,6 +2031,54 @@ mod tests { use std::sync::atomic::{AtomicBool, Ordering}; use uuid::Uuid; + #[tokio::test] + async fn local_automatic_heal_query_preserves_owner_and_coordinator_errors() { + let response = || rustfs_heal_contracts::heal_channel::HealChannelResponse { + request_id: "original-token".to_string(), + success: true, + data: Some(b"retained-owner-report".to_vec()), + error: None, + }; + let called = AtomicBool::new(false); + let local = super::query_heal_status_on_owner(async { Ok(Some(response())) }, || async { + called.store(true, Ordering::SeqCst); + Ok(response()) + }) + .await + .expect("local automatic owner response"); + assert!(!called.load(Ordering::SeqCst)); + assert_eq!(local.request_id, "original-token"); + assert_eq!(local.data.as_deref(), Some(b"retained-owner-report".as_slice())); + + let fallback = super::query_heal_status_on_owner(async { Ok(None) }, || async { + called.store(true, Ordering::SeqCst); + Ok(response()) + }) + .await + .expect("unknown local identity uses coordinator"); + assert!(called.swap(false, Ordering::SeqCst)); + assert_eq!(fallback.data, local.data); + + let error = super::query_heal_status_on_owner( + async { Err(super::admin_error(S3ErrorCode::InternalError, "local lookup failed")) }, + || async { + called.store(true, Ordering::SeqCst); + Ok(response()) + }, + ) + .await + .expect_err("local failure must not become coordinator success"); + assert_eq!(error.code(), &S3ErrorCode::InternalError); + assert!(!called.load(Ordering::SeqCst)); + + let error = super::query_heal_status_on_owner(async { Ok(None) }, || async { + Err(super::admin_error(S3ErrorCode::ServiceUnavailable, "coordinator unavailable")) + }) + .await + .expect_err("coordinator error must remain visible"); + assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable); + } + #[test] fn mrf_legacy_only_refresh_accepts_an_unobserved_incarnation() { for action in ["refresh", "recheck", "acceptUnverifiedRisk"] { diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index 8fa6fb4c4..8b8523aad 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -3142,7 +3142,10 @@ impl DefaultObjectUsecase { // resolver; only the response headers, authorization and audit // summary are derived here, without a second KMS unwrap. Some(headers) => ( - Some(headers.server_side_encryption), + headers + .sse_customer_algorithm + .is_none() + .then_some(headers.server_side_encryption), headers.sse_customer_algorithm, headers.sse_customer_key_md5, headers.ssekms_key_id, diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs index c997237d5..520437cd0 100644 --- a/rustfs/src/app/object/head.rs +++ b/rustfs/src/app/object/head.rs @@ -508,12 +508,13 @@ impl DefaultObjectUsecase { })?; let metadata_map = info.user_defined.clone(); - let server_side_encryption = metadata_map - .get("x-amz-server-side-encryption") - .map(|v| ServerSideEncryption::from(v.clone())); let sse_customer_algorithm = metadata_map .get("x-amz-server-side-encryption-customer-algorithm") .map(|v| SSECustomerAlgorithm::from(v.clone())); + let server_side_encryption = metadata_map + .get("x-amz-server-side-encryption") + .filter(|_| sse_customer_algorithm.is_none()) + .map(|v| ServerSideEncryption::from(v.clone())); let sse_customer_key_md5 = metadata_map.get("x-amz-server-side-encryption-customer-key-md5").cloned(); let sse_kms_key_id = metadata_map.get("x-amz-server-side-encryption-aws-kms-key-id").cloned(); let storage_class = response_storage_class(&info, &metadata_map); diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index ca95df22c..59e6c4e97 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1436,7 +1436,10 @@ impl DefaultObjectUsecase { let output = PutObjectOutput { e_tag, - server_side_encryption: committed.effective_sse.clone(), + server_side_encryption: committed + .effective_sse + .clone() + .filter(|_| committed.sse_customer_algorithm.is_none()), sse_customer_algorithm: committed.sse_customer_algorithm.clone(), sse_customer_key_md5: committed.sse_customer_key_md5.clone(), ssekms_key_id: committed.effective_kms_key_id.clone(), @@ -3967,6 +3970,12 @@ mod tests { async fn sse_c_put_is_accepted_on_a_bucket_with_default_encryption() { use md5::{Digest as _, Md5}; + let context = crate::storage::storage_api::bootstrap_instance_ctx(); + assert!( + context.object_encryption_resolver().is_some(), + "real SSE-C reads need the production encryption resolver" + ); + for (algorithm, kms_key_id, prefix) in [ (ServerSideEncryption::AES256, None, "ssec-over-aes256-default"), (ServerSideEncryption::AWS_KMS, Some("bucket-key"), "ssec-over-kms-default"), @@ -3988,14 +3997,69 @@ mod tests { .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) .sse_customer_algorithm(Some("AES256".to_string())) .sse_customer_key(Some(base64_simd::STANDARD.encode_to_string(customer_key))) - .sse_customer_key_md5(Some(key_md5)) + .sse_customer_key_md5(Some(key_md5.clone())) .build() .expect("SSE-C PUT input must build"); - DefaultObjectUsecase::from_global() + let usecase = DefaultObjectUsecase::from_global(); + let response = usecase .execute_put_object(&FS::new(), build_request(input, Method::PUT)) .await .unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}")); + assert_eq!(response.output.server_side_encryption, None); + assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256")); + assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str())); + + let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key); + let mut read_headers = HeaderMap::new(); + read_headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256")); + read_headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&encoded_key).expect("encoded customer key is a valid header"), + ); + read_headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&key_md5).expect("customer key MD5 is a valid header"), + ); + let head_input = HeadObjectInput::builder() + .bucket(bucket.clone()) + .key("ledger.csv".to_string()) + .sse_customer_algorithm(Some("AES256".to_string())) + .sse_customer_key(Some(encoded_key.clone())) + .sse_customer_key_md5(Some(key_md5.clone())) + .build() + .expect("SSE-C HEAD input must build"); + let mut head_request = build_request(head_input, Method::HEAD); + head_request.headers = read_headers.clone(); + let head = Box::pin(usecase.execute_head_object(head_request)) + .await + .expect("matching SSE-C key must authorize HEAD"); + assert_eq!(head.output.server_side_encryption, None); + assert_eq!(head.output.sse_customer_algorithm.as_deref(), Some("AES256")); + assert_eq!(head.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str())); + + let get_input = GetObjectInput::builder() + .bucket(bucket.clone()) + .key("ledger.csv".to_string()) + .sse_customer_algorithm(Some("AES256".to_string())) + .sse_customer_key(Some(encoded_key)) + .sse_customer_key_md5(Some(key_md5.clone())) + .build() + .expect("SSE-C GET input must build"); + let mut get_request = build_request(get_input, Method::GET); + get_request.headers = read_headers; + let get = Box::pin(usecase.execute_get_object(get_request)) + .await + .expect("matching SSE-C key must authorize GET"); + assert_eq!(get.output.server_side_encryption, None); + assert_eq!(get.output.sse_customer_algorithm.as_deref(), Some("AES256")); + assert_eq!(get.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str())); + let mut body = get.output.body.expect("SSE-C GET must include the decrypted body"); + let mut plaintext = Vec::new(); + while let Some(chunk) = body.next().await { + plaintext.extend_from_slice(&chunk.expect("SSE-C body must decrypt completely")); + } + assert_eq!(plaintext, payload, "header classification must preserve the complete decrypted payload"); let stored = store .get_object_info(&bucket, "ledger.csv", &ObjectOptions::default()) diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 5f6d59591..d0c1e6b2e 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -310,6 +310,7 @@ impl std::error::Error for ApiError { pub(crate) struct ApiErrorDiagnostic { pub storage_code: Option, pub io_kind: Option, + pub io_os_error: Option, pub rpc_code: Option, pub pool_metadata: Option, pub truncated: bool, @@ -333,10 +334,16 @@ impl From<&PoolMetadataError> for PoolMetadataDiagnostic { } } -impl ApiError { - pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic { +impl ApiErrorDiagnostic { + pub(crate) fn from_error(error: &(dyn std::error::Error + 'static)) -> Self { let mut diagnostic = ApiErrorDiagnostic::default(); - let mut current = std::error::Error::source(self); + // S3Error retains its payload through an inherent accessor; its + // std::error::Error implementation does not expose that source. + let mut current = if let Some(error) = error.downcast_ref::() { + error.source().map(|source| source as &(dyn std::error::Error + 'static)) + } else { + std::error::Error::source(error) + }; for _ in 0..16 { let Some(error) = current else { return diagnostic; @@ -354,8 +361,11 @@ impl ApiError { } current = if let Some(io) = error.downcast_ref::() { diagnostic.io_kind = Some(io.kind()); + diagnostic.io_os_error = io.raw_os_error(); // io::Error::source can skip the wrapped error itself. io.get_ref().map(|source| source as &(dyn std::error::Error + 'static)) + } else if let Some(error) = error.downcast_ref::() { + error.source().map(|source| source as &(dyn std::error::Error + 'static)) } else { error.source() }; @@ -363,6 +373,12 @@ impl ApiError { diagnostic.truncated = current.is_some(); diagnostic } +} + +impl ApiError { + pub(crate) fn diagnostic(&self) -> ApiErrorDiagnostic { + ApiErrorDiagnostic::from_error(self) + } /// Access-denied error with the exact message emitted by the authorization /// paths in `storage::access`; callers there match on the code only. @@ -555,6 +571,7 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option(); if matches!(upload_error, Some(UploadStreamError::Sha256Mismatch)) || error.is::() { @@ -564,6 +581,9 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option() + .is_some_and(|status| status.code() == tonic::Code::Unavailable); upload_limit_exceeded |= error.is::(); server_source_failed |= error.is::(); client_body_timed_out |= error.is::(); @@ -591,6 +611,8 @@ fn body_read_error_code(err: &(dyn std::error::Error + 'static)) -> Option(&error)); } + #[test] + fn rpc_unavailable_preserves_service_refusal_across_io_boundaries() { + let status = || tonic::Status::unavailable("private transport detail must not reach the S3 response"); + let storage = || StorageError::from(status()); + let original = storage(); + let cloned = original.clone(); + for error in [ + ApiError::from(original), + ApiError::from(cloned), + ApiError::from(StorageError::Io(IoError::other(IoError::other(status())))), + ApiError::from(IoError::other(status())), + ApiError::from(IoError::other(storage())), + ] { + assert_eq!(error.code, S3ErrorCode::ServiceUnavailable); + assert_eq!(error.message, ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable)); + assert_eq!(error.diagnostic().rpc_code, Some(tonic::Code::Unavailable)); + let s3_error = S3Error::from(error); + assert_eq!(s3_error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE)); + } + } + + #[test] + fn rpc_unavailable_mapping_does_not_match_other_codes_or_untyped_messages() { + for code in [ + tonic::Code::Cancelled, + tonic::Code::Unknown, + tonic::Code::InvalidArgument, + tonic::Code::DeadlineExceeded, + tonic::Code::NotFound, + tonic::Code::AlreadyExists, + tonic::Code::PermissionDenied, + tonic::Code::ResourceExhausted, + tonic::Code::FailedPrecondition, + tonic::Code::Aborted, + tonic::Code::OutOfRange, + tonic::Code::Unimplemented, + tonic::Code::Internal, + tonic::Code::DataLoss, + tonic::Code::Unauthenticated, + ] { + for error in [ + ApiError::from(StorageError::from(tonic::Status::new(code, "rpc unavailable"))), + ApiError::from(IoError::other(tonic::Status::new(code, "rpc unavailable"))), + ] { + assert_eq!(error.code, S3ErrorCode::InternalError, "{code:?}"); + } + } + for error in [ + ApiError::from(StorageError::Io(IoError::other("status: Unavailable"))), + ApiError::from(IoError::other("status: Unavailable")), + ] { + assert_eq!(error.code, S3ErrorCode::InternalError); + } + } + + #[test] + fn rpc_unavailable_mapping_keeps_client_digest_error_precedence() { + for sha256 in [false, true] { + let source = || { + let mut status = tonic::Status::unavailable("private transport failure"); + if sha256 { + status.set_source(std::sync::Arc::new(UploadStreamError::Sha256Mismatch)); + } else { + status.set_source(std::sync::Arc::new(rustfs_rio::BadDigest { + expected_md5: "expected".into(), + calculated_md5: "calculated".into(), + })); + } + status + }; + for error in [ + ApiError::from(StorageError::Io(IoError::other(source()))), + ApiError::from(IoError::other(source())), + ] { + assert_eq!(error.code.as_str(), if sha256 { CONTENT_SHA256_MISMATCH_CODE } else { "BadDigest" }); + } + } + } + #[test] fn body_size_limit_exceeded_maps_to_entity_too_large_across_io_boundaries() { // Shape observed in production (issue #7596): diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 36fe4fbbe..186fc1d8d 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -22,7 +22,7 @@ use super::{ update_bucket_metadata_config_if_incarnation, }; use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook; -use crate::error::ApiError; +use crate::error::{ApiError, ApiErrorDiagnostic}; use crate::storage::access::{apply_bucket_generation_guard, bucket_config_mutation_incarnation, has_bypass_governance_header}; use crate::storage::helper::OperationHelper; use crate::storage::options::get_opts; @@ -60,6 +60,32 @@ const LOG_COMPONENT_STORAGE: &str = "storage"; const LOG_SUBSYSTEM_OBJECT: &str = "object"; const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; const LOG_SUBSYSTEM_TAGGING: &str = "tagging"; +const EVENT_OBJECT_INTERNAL_ERROR: &str = "object_internal_error"; + +fn log_object_internal_error(operation: &'static str, request_id: Option<&str>, error: &S3Error) { + if error.code() != &S3ErrorCode::InternalError { + return; + } + // Generated S3 adapters serialize handler errors without retaining their + // typed sources in logs. Keep only bounded classifications at this boundary. + let diagnostic = ApiErrorDiagnostic::from_error(error); + error!( + event = EVENT_OBJECT_INTERNAL_ERROR, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_OBJECT, + result = "error", + operation, + request_id = request_id.unwrap_or("unavailable"), + storage_error_code = ?diagnostic.storage_code, + io_error_kind = ?diagnostic.io_kind, + io_os_error = diagnostic.io_os_error, + rpc_error_code = ?diagnostic.rpc_code, + pool_metadata_reason = diagnostic.pool_metadata.map(|context| context.reason), + pool_metadata_phase = diagnostic.pool_metadata.map(|context| context.phase), + source_chain_truncated = diagnostic.truncated, + "Object request returned an internal error" + ); +} use crate::app::storage_api::object_usecase::bucket::replication::{ OperatorRuleContract, ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication, @@ -649,8 +675,16 @@ impl S3 for FS { #[instrument(level = "debug", skip(self, req))] async fn delete_object(&self, req: S3Request) -> S3Result> { crate::hp_guard!("S3::delete_object"); + let request_id = req + .extensions + .get::() + .map(|ctx| ctx.request_id.clone()); let usecase = s3_api::object_usecase_for(self); - Box::pin(usecase.execute_delete_object(req)).await + let result = Box::pin(usecase.execute_delete_object(req)).await; + if let Err(error) = &result { + log_object_internal_error("DeleteObject", request_id.as_deref(), error); + } + result } #[instrument(level = "debug", skip(self))] @@ -969,8 +1003,16 @@ impl S3 for FS { )] async fn get_object(&self, req: S3Request) -> S3Result> { crate::hp_guard!("S3::get_object"); + let request_id = req + .extensions + .get::() + .map(|ctx| ctx.request_id.clone()); let usecase = s3_api::object_usecase_for(self); - usecase.execute_get_object(req).await + let result = usecase.execute_get_object(req).await; + if let Err(error) = &result { + log_object_internal_error("GetObject", request_id.as_deref(), error); + } + result } async fn get_object_acl(&self, req: S3Request) -> S3Result> { @@ -2089,6 +2131,73 @@ mod tests { use s3s::{S3, S3Error, S3ErrorCode, S3Request}; use std::sync::atomic::Ordering; + #[test] + fn object_internal_error_logs_only_correlated_bounded_classifications() { + #[derive(Clone, Default)] + struct Writer(std::sync::Arc>>); + impl std::io::Write for Writer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().expect("log buffer").extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for Writer { + type Writer = Self; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + let writer = Writer::default(); + let subscriber = tracing_subscriber::fmt() + .json() + .without_time() + .with_writer(writer.clone()) + .finish(); + let mut status = tonic::Status::internal("secret source message"); + status + .metadata_mut() + .insert("authorization", "secret-token".parse().expect("metadata")); + let error: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from(status)).into(); + let original_code = error.code().clone(); + let original_status = error.status_code(); + let original_message = error.message().map(str::to_owned); + tracing::subscriber::with_default(subscriber, || { + super::log_object_internal_error("GetObject", Some("server-owned-request"), &error); + let mut opaque = S3Error::with_message(S3ErrorCode::InternalError, "secret outer message"); + opaque.set_source(Box::new(std::io::Error::other("secret inner message"))); + super::log_object_internal_error("DeleteObject", None, &opaque); + let unavailable: S3Error = crate::error::ApiError::from(crate::storage::StorageError::from( + tonic::Status::unavailable("secret transport message"), + )) + .into(); + assert_eq!(unavailable.code(), &S3ErrorCode::ServiceUnavailable); + assert_eq!(unavailable.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE)); + super::log_object_internal_error("GetObject", None, &unavailable); + }); + let output = String::from_utf8(writer.0.lock().expect("log buffer").clone()).expect("UTF8 log"); + assert!(!output.contains("secret")); + assert!(!output.contains("authorization")); + let rows = output + .lines() + .map(|line| serde_json::from_str::(line).expect("JSON log")) + .collect::>(); + assert_eq!(rows.len(), 2); + for row in &rows { + assert_eq!(row["fields"]["event"], "object_internal_error"); + assert_eq!(row["fields"]["source_chain_truncated"], false); + } + assert_eq!(rows[0]["fields"]["request_id"], "server-owned-request"); + assert_eq!(rows[0]["fields"]["rpc_error_code"], "Some(Internal)"); + assert_eq!(rows[1]["fields"]["request_id"], "unavailable"); + assert_eq!(rows[1]["fields"]["io_error_kind"], "Some(Other)"); + assert_eq!(error.code(), &original_code); + assert_eq!(error.status_code(), original_status); + assert_eq!(error.message(), original_message.as_deref()); + } + #[test] fn website_config_rejects_missing_index_and_invalid_redirect_rules() { assert!(validate_website_configuration(&WebsiteConfiguration::default()).is_err()); diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 9c6ed79ab..7b365d86a 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -62,7 +62,7 @@ 1|crates/ecstore/src/set_disk/ops/heal.rs 8|crates/ecstore/src/set_disk/ops/multipart.rs 2|crates/ecstore/src/set_disk/ops/object.rs -3|crates/ecstore/src/set_disk/read.rs +1|crates/ecstore/src/set_disk/read.rs 5|crates/ecstore/src/store/bucket.rs 1|crates/ecstore/src/store/heal_walk.rs 10|crates/ecstore/src/store/init.rs diff --git a/scripts/initialize_validation_evidence.py b/scripts/initialize_validation_evidence.py new file mode 100644 index 000000000..c77048d0e --- /dev/null +++ b/scripts/initialize_validation_evidence.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Retain self-hosted validation evidence outside Actions' temporary cleanup.""" +import json +import os +from pathlib import Path +import re +import stat +import sys + +MINIMUM_FREE_BYTES = 10 * 1024**3 +MINIMUM_FREE_INODES = 50000 + + +def initialize(environ=None): + env = os.environ if environ is None else environ + run, attempt = env['GITHUB_RUN_ID'], env['GITHUB_RUN_ATTEMPT'] + if not all(re.fullmatch(r'[1-9][0-9]*', value) for value in (run, attempt)): + raise ValueError('Evidence requires numeric workflow run and attempt identities') + tools_only = env['VALIDATION_TOOLS_ONLY'] + if tools_only not in {'true', 'false'}: + raise ValueError('Evidence requires an explicit tools-only selection') + temporary = Path(env['RUNNER_TEMP']) + if not temporary.is_absolute() or temporary.resolve(strict=True) != temporary or not temporary.is_dir(): + raise ValueError('Runner temporary directory must be an existing canonical absolute directory') + parent = temporary if tools_only == 'true' else temporary.parent / 'rustfs-validation-evidence' + if tools_only == 'false': + parent.mkdir(mode=0o700, exist_ok=True) + info = parent.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError('Persistent evidence parent must be a private runner-owned directory') + root = parent / f'rustfs-validation-{run}-{attempt}' + root.mkdir(mode=0o700) + observations = [] + for path in dict.fromkeys((temporary, parent)): + usage = os.statvfs(path) + observations.append({'path': str(path), 'availableBytes': usage.f_bavail * usage.f_frsize, + 'availableInodes': usage.f_favail, 'minimumFreeBytes': MINIMUM_FREE_BYTES, + 'minimumFreeInodes': MINIMUM_FREE_INODES}) + ready = all(item['availableBytes'] >= MINIMUM_FREE_BYTES and item['availableInodes'] >= MINIMUM_FREE_INODES + for item in observations) + report = {'schemaVersion': 1, 'runId': run, 'runAttempt': attempt, 'evidenceRoot': str(root), + 'persistentOutsideRunnerTemp': tools_only == 'false', 'storage': observations, 'ready': ready, + 'automaticEvidenceDeletion': False, 'canonicalPassClaimed': False} + with (root / 'runner-storage-preflight.json').open('x') as stream: + json.dump(report, stream, indent=2) + stream.write(chr(10)) + with Path(env['GITHUB_ENV']).open('a') as stream: + if any(character in str(root) for character in (chr(10), chr(13))): + raise ValueError('Evidence path cannot contain environment-file delimiters') + stream.write('VALIDATION_ROOT=' + str(root) + chr(10)) + print(json.dumps(report, sort_keys=True)) + if not ready: + raise ValueError('Validation requires at least 10 GiB and 50000 free inodes on runner storage') + return root + + +if __name__ == '__main__': + try: + initialize() + except (OSError, ValueError, KeyError) as error: + print(f'Validation evidence initialization failed: {error}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/inspect_validation_runner.py b/scripts/inspect_validation_runner.py new file mode 100644 index 000000000..e1170723a --- /dev/null +++ b/scripts/inspect_validation_runner.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Inspect GitHub runner registration and queued jobs without accessing the fleet.""" + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import sys + +from resolve_functional_candidate import REPOSITORY, api, require + + +LABEL = "smoke-testing" +ACTIVE_STATUSES = ("queued", "in_progress", "pending", "waiting", "requested") + + +def validate_request(inputs): + require(isinstance(inputs, dict) and inputs.get("validation_runner_inspect_only") is True, + "runner inspection must be explicitly selected") + defaults = {"validation_runner_inspect_only": True, + "validation_retain_evidence_run_attempt": "1", + "i05_diagnostic_branch": "source-plan", "validation_tools_run_attempt": "1"} + require(all(value == defaults[key] if key in defaults else value in ("", False, None) + for key, value in inputs.items()), + "runner inspection cannot select a candidate, tools, fleet case or recovery") + + +def pages(path, field): + rows = [] + for page in range(1, 11): + separator = "&" if "?" in path else "?" + batch = api(f"{path}{separator}per_page=100&page={page}").get(field) + require(isinstance(batch, list), "GitHub inventory response is incomplete") + rows.extend(batch) + if len(batch) < 100: + return rows + raise ValueError("GitHub inventory exceeds the bounded inspection limit") + + +def collect(report): + prefix = f"repos/{REPOSITORY}/actions" + report["stage"] = "runner-inventory" + runners = pages(prefix + "/runners", "runners") + report["runners"] = [ + {"id": row["id"], "name": row["name"], "status": row["status"], + "busy": row["busy"], "labels": [label["name"] for label in row["labels"]]} + for row in runners if LABEL in {label["name"] for label in row["labels"]}] + online = [row for row in report["runners"] if row["status"] == "online"] + report["runnerAvailability"] = ( + "online-idle" if any(row["busy"] is False for row in online) else + "online-busy" if online else "offline" if report["runners"] else "unregistered") + report["stage"] = "active-workflows" + runs = {} + for status in ACTIVE_STATUSES: + for row in pages(prefix + f"/runs?status={status}", "workflow_runs"): + runs[row["id"]] = row + report["fleetJobs"] = [] + report["runsWithoutJobs"] = [] + for run_id, row in sorted(runs.items()): + report["stage"] = "workflow-jobs" + jobs = pages(prefix + f"/runs/{run_id}/jobs", "jobs") + identity = {"runId": run_id, "workflow": row["name"], "status": row["status"], + "createdAt": row["created_at"], "url": row["html_url"]} + if not jobs: + report["runsWithoutJobs"].append(identity) + for job in jobs: + if LABEL in job.get("labels", []) and job["status"] != "completed": + report["fleetJobs"].append({**identity, "jobId": job["id"], "jobName": job["name"], + "jobStatus": job["status"], "runnerName": job.get("runner_name"), + "labels": job["labels"], "jobUrl": job["html_url"]}) + report.update(stage="complete", complete=True, activeWorkflowCount=len(runs)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + require(not args.output.exists(), "retain the original runner inspection report") + report = {"schemaVersion": 1, "repository": REPOSITORY, "label": LABEL, + "observedAt": datetime.now(timezone.utc).isoformat(), "complete": False, + "runId": os.environ.get("GITHUB_RUN_ID"), "runAttempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "workflowCommit": os.environ.get("GITHUB_SHA"), + "runnerAvailability": "unknown", "stage": "request-validation", + "fleetAccessed": False, "mutated": False, + "cleanStateCertified": False, "sharedWorkflowIdleCertified": False, + "canonicalCasePassClaimed": False} + try: + validate_request(json.loads(os.environ["RUNNER_INSPECTION_INPUTS"])) + collect(report) + except Exception as error: + # Subprocess diagnostics can include configuration or credentials. + report["errorType"] = type(error).__name__ + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + json.dump(report, stream, indent=2) + stream.write("\n") + print(json.dumps({key: report[key] for key in ("complete", "stage", "runnerAvailability")})) + return 0 if report["complete"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/prepare_retained_validation_recovery.py b/scripts/prepare_retained_validation_recovery.py new file mode 100644 index 000000000..1a4c4c18c --- /dev/null +++ b/scripts/prepare_retained_validation_recovery.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Copy verified local evidence into a recovery input without claiming an old artifact.""" + +import argparse +import json +import os +from pathlib import Path +import re +import sys + +from resolve_functional_candidate import REPOSITORY, api, require +from retain_validation_evidence import inventory, prepare + + +def stage(env, report): + source = env["FAILED_RUN_ID"] + require(re.fullmatch(r"[1-9][0-9]{0,19}", source), "exact failed run required") + run = api(f"repos/{REPOSITORY}/actions/runs/{source}") + require(run.get("head_branch") == env["GITHUB_REF_NAME"] + and type(run.get("run_attempt")) is int and run["run_attempt"] > 0, + "local recovery source branch or attempt differs") + root = prepare({**env, "EVIDENCE_RETAIN_INPUTS": json.dumps({ + "validation_retain_evidence_run_id": source, + "validation_retain_evidence_run_attempt": str(run["run_attempt"])})}, report) + report.update(complete=False, stage="stable-recovery-copy") + destination = Path(env["VALIDATION_ROOT"]) / "failed-artifact" + require(destination.parent.is_absolute() and destination.parent.resolve(strict=True) == destination.parent, + "recovery evidence root is not canonical") + destination.mkdir(mode=0o700, exist_ok=False) + copied = destination / "retained-source-evidence" + files, size = inventory(root, copy_to=copied) + require(files == report["files"] and size == report["bytes"] + and inventory(copied) == (files, size), "retained evidence changed during recovery copy") + report.update(complete=True, stage="complete", recoveryInput=str(copied)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + report = {"schemaVersion": 1, "collectionRunId": os.environ["GITHUB_RUN_ID"], + "collectionAttempt": os.environ["GITHUB_RUN_ATTEMPT"], "complete": False, + "stage": "source-workflow", + "collectionKind": "runner-local-recovery-input-copy", "sourceArtifactClaimed": False, + "fleetAccessed": False, "originalEvidenceModified": False, + "sourceVerdictRewritten": False, "canonicalCasePassClaimed": False} + try: + stage(os.environ, report) + except Exception as error: + report.update(complete=False, errorType=type(error).__name__) + with args.report.open("x") as stream: + json.dump(report, stream, indent=2) + stream.write("\n") + print(json.dumps({key: report[key] for key in ("complete", "stage")})) + return 0 if report["complete"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/resolve_functional_candidate.py b/scripts/resolve_functional_candidate.py index 7acdcd863..1956cf4d3 100644 --- a/scripts/resolve_functional_candidate.py +++ b/scripts/resolve_functional_candidate.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Resolve a nightly manifest from its exact GitHub build attempt and artifact.""" +"""Resolve a candidate manifest from its exact GitHub build attempt and artifact.""" from __future__ import annotations import hashlib @@ -15,6 +15,7 @@ import zipfile REPOSITORY = "rustfs/rustfs" ROOT = Path(__file__).resolve().parents[1] MAX_ARCHIVE = 1024 * 1024 +VALIDATION_BUILD_WORKFLOWS = {".github/workflows/rustfs-validation-package.yml", ".github/workflows/rustfs-functional-chain.yml"} def require(condition, message): @@ -39,8 +40,11 @@ def validate_manifest(manifest, run): require(isinstance(manifest, dict), "manifest must be an object") common = {"schema", "source_sha", "build_run_id", "build_run_attempt", "package_url", "package_sha256"} version = manifest.get("schema") - require(type(version) is int and version in (1, 2), "unsupported candidate schema") - require(set(manifest) == (common if version == 1 else common | {"workflow_sha", "source_ref"}), "unexpected candidate fields") + require(type(version) is int and version in (1, 2, 3), "unsupported candidate schema") + expected_fields = common if version == 1 else common | {"workflow_sha", "source_ref"} + if version == 3: + expected_fields |= {"channel"} + require(set(manifest) == expected_fields, "unexpected candidate fields") require(positive(manifest["build_run_id"]) and positive(manifest["build_run_attempt"]), "invalid build identity") require((manifest["build_run_id"], manifest["build_run_attempt"]) == (run["id"], run["run_attempt"]), "candidate belongs to another build attempt") require(sha(manifest["source_sha"]) and sha(manifest["package_sha256"], 64), "invalid candidate hash") @@ -49,7 +53,11 @@ def validate_manifest(manifest, run): else: require(manifest["workflow_sha"] == run["head_sha"] and sha(manifest["workflow_sha"]), "candidate workflow SHA differs from artifact provenance") require(isinstance(manifest["source_ref"], str) and bool(re.fullmatch(r"[A-Za-z0-9_./-]{1,200}", manifest["source_ref"])), "invalid build source ref") - expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/{run['id']}/" + if version == 3: + require(manifest["channel"] == "validation", "validation candidate has the wrong publication channel") + require(manifest["source_ref"] == manifest["source_sha"], "validation candidate must bind an exact source SHA") + channel = "validation" if version == 3 else "nightly" + expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/{channel}/runs/{run['id']}/" f"{run['run_attempt']}/{manifest['package_sha256']}/rustfs.deb") require(manifest["package_url"] == expected, "package URL does not bind the run, attempt and checksum") return manifest @@ -72,19 +80,31 @@ def read_json_artifact(archive, artifact, run, expected_name, member, max_json=1 def read_manifest(archive, artifact, run): - name = f"nightly-candidate-{run['id']}-{run['run_attempt']}" - return validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run) + prefix = "validation-candidate" if run.get("path") in VALIDATION_BUILD_WORKFLOWS else "nightly-candidate" + name = f"{prefix}-{run['id']}-{run['run_attempt']}" + manifest = validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run) + if prefix == "validation-candidate": + require(manifest["schema"] == 3 and manifest["channel"] == "validation", + "validation workflow requires a schema-3 validation candidate") + else: + require(manifest["schema"] != 3, "nightly workflow cannot publish a validation candidate") + return manifest -def resolve(run_id, attempt): +def resolve(run_id, attempt, *, allow_validation=False): require(positive(run_id) and positive(attempt), "build run and attempt are required positive integers") endpoint = f"repos/{REPOSITORY}/actions/runs/{run_id}" run = api(f"{endpoint}/attempts/{attempt}") require(run.get("id") == run_id and run.get("run_attempt") == attempt, "GitHub returned a different build attempt") - require(run.get("path") == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main", "candidate must come from nightly-gnu on main") + path = run.get("path") + validation = allow_validation and path in VALIDATION_BUILD_WORKFLOWS + require(path == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main" or validation, + "candidate must come from nightly-gnu on main or an explicitly allowed validation build") require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "candidate came from another repository") - require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "nightly attempt has not completed successfully") - name = f"nightly-candidate-{run_id}-{attempt}" + require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "candidate attempt has not completed successfully") + if validation: + require(run.get("event") == "workflow_dispatch", "validation candidate must be manually dispatched") + name = f"{'validation' if validation else 'nightly'}-candidate-{run_id}-{attempt}" artifacts = [] for page in range(1, 11): batch = api(f"{endpoint}/artifacts?per_page=100&page={page}")["artifacts"] diff --git a/scripts/retain_validation_evidence.py b/scripts/retain_validation_evidence.py new file mode 100644 index 000000000..0c5dd15d7 --- /dev/null +++ b/scripts/retain_validation_evidence.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Admit an interrupted run's existing runner-local evidence for read-only upload.""" + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import sys + +from resolve_functional_candidate import REPOSITORY, api, require + + +MAX_FILES = 200000 +MAX_BYTES = 4 * 1024**3 + + +def stable(info): + return (info.st_dev, info.st_ino, info.st_mode, info.st_uid, info.st_gid, + info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +def inventory(root, *, copy_to=None): + files, total, entries = [], 0, 0 + device = root.lstat().st_dev + if copy_to is not None: + require(copy_to.is_absolute() and copy_to.parent.resolve(strict=True) == copy_to.parent, + "evidence copy parent is not canonical") + copy_to.mkdir(mode=0o700, exist_ok=False) + + def visit(fd, prefix): + nonlocal total, entries + before = os.fstat(fd) + require(stat.S_ISDIR(before.st_mode) and before.st_uid == os.getuid() and before.st_dev == device, + "evidence directory has a different owner or type") + names = sorted(os.listdir(fd)) + require(len(names) <= MAX_FILES, "evidence directory exceeds the entry budget") + for name in names: + entries += 1 + require(entries <= MAX_FILES, "evidence exceeds the entry budget") + info = os.stat(name, dir_fd=fd, follow_symlinks=False) + path = prefix + name + require(info.st_uid == os.getuid() and info.st_dev == device, "evidence entry has a different owner or device") + if stat.S_ISDIR(info.st_mode): + if copy_to is not None: + (copy_to / path).mkdir(mode=0o700, exist_ok=False) + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + try: + require(stable(os.fstat(child)) == stable(info), "evidence directory changed before inspection") + visit(child, path + "/") + finally: + os.close(child) + else: + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 + and len(files) < MAX_FILES and total + info.st_size <= MAX_BYTES, + "evidence contains links, special files or exceeds its byte/file budget") + child = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd) + target = None + try: + require(stable(os.fstat(child)) == stable(info), "evidence file changed before inspection") + if copy_to is not None: + target = os.open(copy_to / path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + digest, size = hashlib.sha256(), 0 + while chunk := os.read(child, 1024**2): + size += len(chunk) + require(size <= info.st_size, "evidence file grew during inspection") + digest.update(chunk) + if target is not None: + remaining = memoryview(chunk) + while remaining: + written = os.write(target, remaining) + require(written > 0, "evidence copy made no progress") + remaining = remaining[written:] + require(size == info.st_size and stable(os.fstat(child)) == stable(info), + "evidence file changed during inspection") + finally: + os.close(child) + if target is not None: + os.close(target) + total += size + files.append({"path": path, "size": size, "sha256": digest.hexdigest()}) + require(stable(os.stat(name, dir_fd=fd, follow_symlinks=False)) == stable(info), + "evidence entry changed during inspection") + require(sorted(os.listdir(fd)) == names + and stable(os.fstat(fd)) == stable(before), "evidence population changed during inspection") + + fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + visit(fd, "") + finally: + os.close(fd) + return files, total + + +def prepare(env, report): + inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"]) + source = inputs.get("validation_retain_evidence_run_id") + attempt = inputs.get("validation_retain_evidence_run_attempt", "1") + require(isinstance(source, str) and re.fullmatch(r"[1-9][0-9]{0,19}", source), "exact source run ID required") + require(isinstance(attempt, str) and re.fullmatch(r"[1-9][0-9]{0,9}", attempt), "exact source run attempt required") + defaults = {"validation_retain_evidence_run_id": source, + "validation_retain_evidence_run_attempt": attempt, + "i05_diagnostic_branch": "source-plan", "validation_tools_run_attempt": "1"} + require(all(value == defaults[key] if key in defaults else value in ("", False, None) + for key, value in inputs.items()), "evidence retention is exclusive with all execution modes") + require(source != env["GITHUB_RUN_ID"], "cannot retain the running collection job") + report["stage"] = "source-workflow" + run = api(f"repos/{REPOSITORY}/actions/runs/{source}/attempts/{attempt}") + require(run.get("id") == int(source) and type(run.get("run_attempt")) is int and run["run_attempt"] == int(attempt) + and run.get("repository", {}).get("full_name") == REPOSITORY + and run.get("status") == "completed" and run.get("conclusion") in {"failure", "cancelled", "timed_out"} + and run.get("event") == "workflow_dispatch" + and run.get("path") == ".github/workflows/rustfs-functional-chain.yml", + "source is not one completed interrupted validation workflow") + jobs = api(f"repos/{REPOSITORY}/actions/runs/{source}/attempts/{run['run_attempt']}/jobs?per_page=100") + require(jobs.get("total_count", 101) <= 100 + and len([job for job in jobs.get("jobs", []) if job.get("name") == "validation" + and job.get("runner_name") == env["RUNNER_NAME"]]) == 1, + "source validation did not execute on this runner") + report["sourceRun"] = {key: run[key] for key in ("id", "run_attempt", "head_sha", "status", "conclusion", "html_url")} + report["stage"] = "owned-persistent-root" + temporary = Path(env["RUNNER_TEMP"]) + require(temporary.is_absolute() and temporary.resolve(strict=True) == temporary + and temporary.is_dir(), "runner temporary path is not canonical") + parent = temporary.parent / "rustfs-validation-evidence" + root = parent / f"rustfs-validation-{source}-{run['run_attempt']}" + for path in (parent, root): + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() + and stat.S_IMODE(info.st_mode) == 0o700 and path.resolve(strict=True) == path, + "persistent evidence must be an existing private runner-owned directory") + report["stage"] = "stable-file-inventory" + files, size = inventory(root) + marker = root / "runner-storage-preflight.json" + require(any(row["path"] == marker.name for row in files) and marker.stat().st_size < 1024**2, + "source storage provenance is absent or oversized") + storage = json.loads(marker.read_text()) + require(storage.get("runId") == source and storage.get("runAttempt") == str(run["run_attempt"]) + and storage.get("evidenceRoot") == str(root) + and storage.get("persistentOutsideRunnerTemp") is True, + "evidence storage provenance belongs to another run or root") + require(not any(character in str(root) for character in ("\n", "\r")), "unsafe evidence export path") + report.update(complete=True, stage="complete", evidenceRoot=str(root), files=files, bytes=size) + return root + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + report = {"schemaVersion": 1, "collectionRunId": os.environ["GITHUB_RUN_ID"], + "collectionAttempt": os.environ["GITHUB_RUN_ATTEMPT"], "complete": False, + "observedAt": datetime.now(timezone.utc).isoformat(), + "stage": "request-validation", "fleetAccessed": False, "originalEvidenceModified": False, + "sourceVerdictRewritten": False, "canonicalCasePassClaimed": False} + root = None + try: + root = prepare(os.environ, report) + except Exception as error: + report["errorType"] = type(error).__name__ + args.report.parent.mkdir(parents=True, exist_ok=True) + with args.report.open("x") as stream: + json.dump(report, stream, indent=2) + stream.write("\n") + if root is not None: + with Path(os.environ["GITHUB_ENV"]).open("a") as stream: + stream.write("VALIDATION_RETAIN_ROOT=" + str(root) + "\n") + stream.write("VALIDATION_RETAIN_SOURCE_RUN=" + str(report["sourceRun"]["id"]) + "\n") + stream.write("VALIDATION_RETAIN_SOURCE_ATTEMPT=" + str(report["sourceRun"]["run_attempt"]) + "\n") + print(json.dumps({key: report[key] for key in ("complete", "stage")})) + return 0 if root is not None else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index f36ff671b..bc145528c 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -4,6 +4,7 @@ import copy import hashlib import io import json +import os from pathlib import Path import subprocess import tempfile @@ -13,6 +14,399 @@ import zipfile import functional_chain_evidence as evidence import resolve_functional_candidate as candidate +import initialize_validation_evidence as validation_storage + + +class ValidationStorageTests(unittest.TestCase): + def fixture(self, root, *, tools_only=False): + temporary = root / '_work/_temp' + temporary.mkdir(parents=True) + return dict(RUNNER_TEMP=str(temporary.resolve()), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', + VALIDATION_TOOLS_ONLY=str(tools_only).lower(), GITHUB_ENV=str(root / 'env')) + + def usage(self, *, available=None, inodes=None): + return mock.Mock(f_bavail=validation_storage.MINIMUM_FREE_BYTES if available is None else available, + f_frsize=1, f_favail=validation_storage.MINIMUM_FREE_INODES if inodes is None else inodes) + + def test_self_hosted_evidence_survives_temp_cleanup_and_refuses_reuse(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + env = self.fixture(root) + with mock.patch.object(validation_storage.os, 'statvfs', return_value=self.usage()): + evidence_root = validation_storage.initialize(env) + with self.assertRaises(FileExistsError): + validation_storage.initialize(env) + self.assertEqual(evidence_root, Path(env['RUNNER_TEMP']).parent / + 'rustfs-validation-evidence/rustfs-validation-77-1') + self.assertEqual(evidence_root.parent.stat().st_mode & 0o777, 0o700) + self.assertEqual(evidence_root.stat().st_mode & 0o777, 0o700) + Path(env['RUNNER_TEMP']).rmdir() + self.assertTrue((evidence_root / 'runner-storage-preflight.json').is_file()) + self.assertEqual(Path(env['GITHUB_ENV']).read_text(), 'VALIDATION_ROOT=' + str(evidence_root) + chr(10)) + + def test_each_filesystem_byte_and_inode_budget_fails_before_candidate_execution(self): + for position in (0, 1): + for kind in ('bytes', 'inodes'): + with self.subTest(position=position, kind=kind), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + env = self.fixture(root) + usage = [self.usage(), self.usage()] + usage[position] = self.usage(available=validation_storage.MINIMUM_FREE_BYTES - 1) if kind == 'bytes' \ + else self.usage(inodes=validation_storage.MINIMUM_FREE_INODES - 1) + with mock.patch.object(validation_storage.os, 'statvfs', side_effect=usage): + with self.assertRaisesRegex(ValueError, 'at least 10 GiB'): + validation_storage.initialize(env) + evidence_root = Path(Path(env['GITHUB_ENV']).read_text().strip().split('=', 1)[1]) + report = json.loads((evidence_root / 'runner-storage-preflight.json').read_text()) + self.assertFalse(report['ready']) + self.assertFalse(report['automaticEvidenceDeletion']) + self.assertEqual(len(report['storage']), 2) + + def test_tools_resolver_keeps_its_disposable_hosted_location(self): + with tempfile.TemporaryDirectory() as temp: + env = self.fixture(Path(temp), tools_only=True) + with mock.patch.object(validation_storage.os, 'statvfs', return_value=self.usage()): + root = validation_storage.initialize(env) + self.assertEqual(root.parent, Path(env['RUNNER_TEMP'])) + self.assertFalse(json.loads((root / 'runner-storage-preflight.json').read_text()) + ['persistentOutsideRunnerTemp']) + + def test_rejects_unsafe_parent_and_untrusted_identities(self): + for kind in ('symlink', 'public', 'run', 'attempt', 'selection'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + env = self.fixture(root) + parent = Path(env['RUNNER_TEMP']).parent / 'rustfs-validation-evidence' + if kind == 'symlink': + other = root / 'other' + other.mkdir(mode=0o700) + parent.symlink_to(other, target_is_directory=True) + elif kind == 'public': + parent.mkdir(mode=0o755) + parent.chmod(0o755) + else: + env[{'run': 'GITHUB_RUN_ID', 'attempt': 'GITHUB_RUN_ATTEMPT', + 'selection': 'VALIDATION_TOOLS_ONLY'}[kind]] = '../foreign' + with self.assertRaises(ValueError): + validation_storage.initialize(env) + self.assertFalse(Path(env['GITHUB_ENV']).exists()) + self.assertFalse((parent / 'rustfs-validation-77-1').exists()) + + def test_workflow_uploads_persistent_evidence_and_preflights_before_resolution(self): + source = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() + initialize = source.index(' - name: Initialize validation evidence') + self.assertLess(initialize, source.index(' - name: Resolve immutable candidate')) + self.assertIn('python3 -B scripts/initialize_validation_evidence.py', source) + self.assertIn('VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}', source) + upload = source.split(' - name: Upload validation evidence', 1)[1].split(' validation-tools:', 1)[0] + self.assertIn("if: ${{ always() && env.VALIDATION_ROOT != '' }}", upload) + self.assertIn('path: ${{ env.VALIDATION_ROOT }}', upload) + + +class RaceNativeToolsWorkflowTests(unittest.TestCase): + def chain_shell_step(self, name): + source = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() + step = source.split(' - name: ' + name + '\n', 1)[1].split('\n - name:', 1)[0] + shell = step.split(' run: |\n', 1)[1] + return '\n'.join(line[10:] for line in shell.splitlines() + if not line.strip() or line.startswith(' ')) + + def test_race_actual_workflow_selection_preserves_legacy_and_propagates_failures(self): + shell = self.chain_shell_step('Select exact-source race native evidence') + g01 = 'heal.race-unversioned-overwrite' + cases = [(False, 2, 0, 'false', 'a' * 40, g01), (True, 2, 0, 'false', 'a' * 40, g01), + (True, 0, 0, 'true', 'a' * 40, g01), (True, 1, 1, None, 'a' * 40, g01), + (True, 42, 42, None, 'a' * 40, g01)] + reviewed = [('6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca', g01)] + reviewed += [('bd0671ba781d8ebf509f4f03533228dfb44496f1', case) for case in + (g01, 'heal.race-versioned-put', 'heal.race-version-delete')] + reviewed += [('8380466f3a27c3c3ef4115084178a0ab99a78f32', case) for case in + (g01, 'heal.race-versioned-put', 'heal.race-version-delete', 'heal.race-markers-and-batch-delete')] + reviewed += [('757b751482bb970bc321e92059690a7698554358', case) for case in + ('heal.admin-nolock-safety', 'heal.race-copy-and-metadata-update', 'heal.race-multipart-complete-abort')] + for source, case in reviewed: + cases += [(False, 2, 1, None, source, case), (True, 2, 1, None, source, case), + (True, 0, 0, 'true', source, case)] + for helper, support, code, required, source, case in cases: + with self.subTest(helper=helper, support=support, source=source, case=case), tempfile.TemporaryDirectory() as name: + root = Path(name) + env = self.fixture(root, new_helper=helper) + output = root / 'output' + env.update(SUPPORT_STATUS=str(support), GITHUB_OUTPUT=str(output), CANDIDATE_SHA=source, + VALIDATION_CASE_ID=case) + result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, code, result.stderr) + self.assertEqual(output.read_text() if output.exists() else None, + 'required=' + required + '\n' if required else None) + if helper: + calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] + self.assertEqual(calls, [{'kind': 'race_native', 'args': ['supports', '--commit', source, + '--case-id', case]}]) + + def test_race_actual_admission_requires_native_and_runtime_before_exporting_directory(self): + shell = self.chain_shell_step('Verify exact-source race native and runtime evidence') + for case in ('heal.race-unversioned-overwrite', 'heal.race-versioned-put', 'heal.race-version-delete', + 'heal.race-markers-and-batch-delete', 'heal.race-multipart-complete-abort', + 'heal.admin-nolock-safety', 'heal.race-copy-and-metadata-update'): + for native_exit, runtime in [(0, 0), (42, 0), (0, 42), (0, 'missing'), (0, 'collector-only')]: + with self.subTest(case=case, native_exit=native_exit, runtime=runtime), tempfile.TemporaryDirectory() as name: + root = Path(name) + env = self.fixture(root, new_helper=True) + directory = root / 'auto-testing/scripts/validation_heal' + runtime_driver = 'nolock.py' if case == 'heal.admin-nolock-safety' else 'race.py' + (directory / 'race_native_evidence.py').write_text( + 'import os,sys\nprint("{}")\nsys.exit(int(os.environ["VERIFY_EXIT"]))\n') + if runtime == 'collector-only': + (directory / runtime_driver).write_text( + 'import argparse\np=argparse.ArgumentParser()\np.add_argument("--rc-bin", required=True)\np.parse_args()\n') + elif runtime != 'missing': + (directory / runtime_driver).write_text( + 'import json,os,pathlib,sys\n' + 'assert not pathlib.Path(os.environ["GITHUB_ENV"]).exists()\n' + 'assert sys.argv[1] == "preflight"\n' + 'assert sys.argv[-2:] == ["--case-id",os.environ["VALIDATION_CASE_ID"]]\n' + 'assert pathlib.Path(os.environ["VALIDATION_HEAL_NATIVE_DIRECTORY"],"tool-run.json").is_file()\n' + 'pathlib.Path("runtime-call.json").write_text(json.dumps(sys.argv[1:]))\n' + 'sys.exit(int(os.environ["RUNTIME_EXIT"]))\n') + validation = root / 'validation' + validation.mkdir() + (validation / 'tool-run.json').write_text('{"id":123}') + native = root / 'rustfs-validation-native-77-1' + native.mkdir() + output = root / 'env' + env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', + VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(native_exit), + RUNTIME_EXIT=str(runtime), VALIDATION_CASE_ID=case) + result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) + expected = native_exit or (runtime if isinstance(runtime, int) else 2) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes()) + self.assertEqual(output.exists(), expected == 0) + self.assertEqual((root / 'runtime-call.json').exists(), native_exit == 0 and isinstance(runtime, int)) + if expected == 0: + self.assertEqual(output.read_text(), 'VALIDATION_HEAL_NATIVE_DIRECTORY=' + str(native) + '\n') + workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() + self.assertLess(workflow.index('Verify exact-source race native and runtime'), + workflow.index('Run selected canonical validation case')) + self.assertIn("steps.race_native.outputs.required == 'true'", workflow) + self.assertIn("inputs.validation_case == 'heal.race-markers-and-batch-delete'", workflow) + self.assertIn("inputs.validation_case == 'heal.admin-nolock-safety'", workflow) + self.assertIn("inputs.validation_case == 'heal.race-copy-and-metadata-update'", workflow) + selection = workflow.split(' - name: Select exact-source race native evidence\n', 1)[1].split('\n env:', 1)[0] + self.assertIn("inputs.validation_case == 'heal.race-multipart-complete-abort'", selection) + + def test_i05_native_prerequisite_propagates_failure_and_exports_only_after_verification(self): + shell = self.chain_shell_step('Verify I05 source-plan or queued-successor native prerequisites') + for branch in ('source-plan', 'queued-successor'): + for exit_code in (0, 42): + with self.subTest(branch=branch, exit_code=exit_code), tempfile.TemporaryDirectory() as name: + root = Path(name) + env = self.fixture(root, new_helper=True) + script_name = 'lock_source.py' if branch == 'source-plan' else 'race_native_evidence.py' + script = root / 'auto-testing/scripts/validation_heal' / script_name + script.write_text('import os,sys\n' + 'assert sys.argv[1] == "--directory"\n' + 'assert "--candidate" in sys.argv\n' + 'assert sys.argv[sys.argv.index("--testing-commit") + 1] == os.environ["AUTO_TESTING_SHA"]\n' + 'if os.environ["I05_DIAGNOSTIC_BRANCH"] == "queued-successor":\n' + ' assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n' + 'else:\n' + ' assert "--case-id" not in sys.argv\n' + 'sys.exit(int(os.environ["VERIFY_EXIT"]))\n') + validation = root / 'validation'; validation.mkdir() + (validation / 'tool-run.json').write_text('{"id":123}') + native = root / 'rustfs-validation-native-77-1'; native.mkdir() + output = root / 'env' + env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', + VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code), + I05_DIAGNOSTIC_BRANCH=branch) + result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, exit_code, result.stderr) + self.assertEqual(output.exists(), exit_code == 0) + self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes()) + workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() + self.assertLess(workflow.index('Verify I05 source-plan or queued-successor native prerequisites'), + workflow.index('Run selected canonical validation case')) + download = workflow.split(' - name: Download pinned native storage evidence\n', 1)[1].split(' uses:', 1)[0] + self.assertIn("inputs.i05_diagnostic_branch != 'timeout'", download) + execution = workflow.split(' - name: Run selected canonical validation case\n', 1)[1].split(' run:', 1)[0] + self.assertIn('I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}', execution) + + def test_f07_directory_fsync_artifact_is_candidate_scoped_and_bound(self): + shell = self.chain_shell_step('Bind F07 exact-candidate directory fsync evidence') + with tempfile.TemporaryDirectory() as name: + root = Path(name) + validation = root / 'validation'; validation.mkdir() + (validation / 'tool-run.json').write_text('{"id":123}') + native = root / 'rustfs-validation-native-77-1'; native.mkdir() + output = root / 'env' + env = dict(os.environ, RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', + VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output)) + result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes()) + self.assertEqual(output.read_text(), 'VALIDATION_HEAL_NATIVE_DIRECTORY=' + str(native) + '\n') + workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() + binding = workflow.split(' - name: Bind F07 exact-candidate directory fsync evidence\n', 1)[1] + guard = binding.split(' run:', 1)[0] + self.assertIn("inputs.validation_case == 'heal.checkpoint-and-mrf-corruption'", guard) + self.assertIn("steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac'", guard) + self.assertLess(workflow.index('Bind F07 exact-candidate directory fsync evidence'), + workflow.index('Run selected canonical validation case')) + + def shell_step(self, name): + from check_test_wiring import yaml_block + source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text() + job = yaml_block(source.splitlines(), 'native-heal-regressions', 2) + start = job.index(' - name: ' + name) + end = next((index for index in range(start + 1, len(job)) + if job[index].startswith(' - name: ')), len(job)) + step = job[start:end] + return '\n'.join(line[10:] for line in step[step.index(' run: |') + 1:] + if not line.strip() or line.startswith(' ')) + + def fixture(self, root, *, new_helper): + directory = root / 'auto-testing/scripts/validation_heal' + directory.mkdir(parents=True) + stub = '''import json, os, pathlib, sys +kind = pathlib.Path(__file__).stem +with pathlib.Path('calls.jsonl').open('a') as out: + out.write(json.dumps({'kind': kind, 'args': sys.argv[1:]}) + '\\n') +if sys.argv[1] == 'supports': sys.exit(int(os.environ['SUPPORT_STATUS'])) +sys.exit(int(os.environ.get(kind.upper() + '_FAIL', '0'))) +''' + (directory / 'native_regressions.py').write_text(stub) + if new_helper: (directory / 'race_native.py').write_text(stub) + bin_dir = root / 'bin' + bin_dir.mkdir() + git = bin_dir / 'git' + git.write_text('#!/bin/sh\nprintf "%s\\n" "$FAKE_HEAD"\n') + git.chmod(0o755) + env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, CANDIDATE_SHA='a' * 40, + FAKE_HEAD='b' * 40, SUPPORT_STATUS='2', PATH=str(bin_dir) + os.pathsep + os.environ['PATH']) + env.pop('BASH_ENV', None) + return env + + def test_actual_native_build_routes_old_new_unknown_and_failed_collectors(self): + shell = self.shell_step('Build and enumerate the unmodified candidate test executable') + for mode, helper, support, failure, head, expected_code, expected_calls in [ + ('old-harness', False, 2, None, 'b', 0, [('native_regressions', 'build')]), + ('legacy', True, 2, None, 'b', 0, [('race_native', 'supports'), ('native_regressions', 'build')]), + ('race', True, 0, None, 'b', 0, [('race_native', 'supports'), ('race_native', 'build')]), + ('support-error', True, 42, None, 'b', 42, [('race_native', 'supports')]), + ('invalid-support', True, 1, None, 'b', 1, [('race_native', 'supports')]), + ('race-build-error', True, 0, 'RACE_NATIVE_FAIL', 'b', 42, [('race_native', 'supports'), ('race_native', 'build')]), + ('legacy-build-error', True, 2, 'NATIVE_REGRESSIONS_FAIL', 'b', 42, [('race_native', 'supports'), ('native_regressions', 'build')]), + ('wrong-helper', True, 0, None, 'c', 1, []), + ]: + with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode): + root = Path(temp) + env = self.fixture(root, new_helper=helper) + env.update(SUPPORT_STATUS=str(support), FAKE_HEAD=head * 40) + if failure: env[failure] = '42' + result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell], + cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, expected_code, result.stderr) + calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else [] + self.assertEqual([(row['kind'], row['args'][0]) for row in calls], expected_calls) + for row in calls: + if row['args'][0] == 'supports': + self.assertEqual(row['args'], ['supports', '--commit', 'a' * 40]) + else: + expected = ['build', '--source', 'candidate', '--commit', 'a' * 40] + if row['kind'] == 'race_native': expected += ['--testing-commit', 'b' * 40] + self.assertEqual(row['args'], expected + ['--out', 'native-tools']) + + def test_actual_native_run_rejects_ambiguous_formats_and_propagates_failures(self): + shell = self.shell_step('Run exact native regressions separately from fleet validation') + for mode, helper, manifests, failure, expected_code, expected_kind in [ + ('legacy', False, ['native-regressions.json'], None, 0, 'native_regressions'), + ('race', True, ['race-native.json'], None, 0, 'race_native'), + ('ambiguous', True, ['native-regressions.json', 'race-native.json'], None, 1, None), + ('legacy-error', True, ['native-regressions.json'], 'NATIVE_REGRESSIONS_FAIL', 42, 'native_regressions'), + ('race-error', True, ['race-native.json'], 'RACE_NATIVE_FAIL', 42, 'race_native'), + ('missing-race-helper', False, ['race-native.json'], None, 2, None), + ]: + with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode): + root = Path(temp) + env = self.fixture(root, new_helper=helper) + if failure: env[failure] = '42' + (root / 'native-tools').mkdir() + for manifest in manifests: (root / 'native-tools' / manifest).write_text('{}') + result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell], + cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, expected_code, result.stderr) + calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else [] + self.assertEqual(len(calls), int(expected_kind is not None)) + if expected_kind: + self.assertEqual(calls, [{'kind': expected_kind, 'args': ['run', '--directory', 'native-tools', '--out', 'native-evidence']}]) + + def test_native_upload_retains_both_fixed_executable_kinds(self): + source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text() + self.assertIn(' native-tools/ecstore-native-tests\n', source) + self.assertIn(' native-tools/ecstore-race-native-tests\n', source) + + +class PersistenceToolsWorkflowTests(unittest.TestCase): + def test_supplement_preserves_old_pins_and_propagates_actual_collector_failure(self): + from check_test_wiring import yaml_block + source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text() + job = yaml_block(source.splitlines(), 'native-heal-regressions', 2) + start = job.index(' - name: Run existing F07 directory fsync failure regressions') + end = job.index(' - name: Upload native executable identities and original diagnostics') + step = job[start:end] + shell = '\n'.join(line[10:] for line in step[step.index(' run: |') + 1:] + if not line.strip() or line.startswith(' ')) + for mode in ('old-harness', 'success', 'run-failure', 'verify-failure'): + with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode): + root = Path(temp) + if mode != 'old-harness': + collector = root / 'auto-testing/scripts/validation_heal/persistence_native.py' + collector.parent.mkdir(parents=True) + collector.write_text('''import json, os, pathlib, sys +with pathlib.Path('calls.jsonl').open('a') as out: out.write(json.dumps(sys.argv[1:]) + '\\n') +sys.exit(42 if os.environ['COLLECTOR_MODE'] == sys.argv[1] + '-failure' else 0) +''') + env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, COLLECTOR_MODE=mode) + env.pop('BASH_ENV', None) + result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell], + cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, mode in ('old-harness', 'success'), result.stderr) + if mode == 'old-harness': + report = json.loads((root / 'native-persistence-evidence/not-available.json').read_text()) + self.assertIs(report['complete'], False) + self.assertIs(report['requiredNativeCoverageComplete'], False) + self.assertIs(report['canonicalCasePassClaimed'], False) + self.assertFalse((root / 'calls.jsonl').exists()) + else: + calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] + self.assertEqual([call[0] for call in calls], ['run'] if mode == 'run-failure' else ['run', 'verify']) + for call in calls: self.assertEqual(call[-2:], ['--testing-commit', 'b' * 40]) + + +class InterruptedCleanupGuardTests(unittest.TestCase): + def test_actual_workflow_guard_rejects_live_success_and_unrelated_runs(self): + source = (candidate.ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text() + step = source[source.index(' - name: Audit and finalize retained validation environment'):] + guard = step.split(" python3 - <<'PY'", 1)[1].split(" PY", 1)[0] + guard = "\n".join(line[10:] for line in guard.splitlines()) + for state, conclusion, branch, workflow, accepted in ( + ("completed", "failure", "topic", "RustFS Functional Chain", True), + ("completed", "cancelled", "topic", "RustFS Functional Chain", True), + ("in_progress", "cancelled", "topic", "RustFS Functional Chain", False), + ("completed", "success", "topic", "RustFS Functional Chain", False), + ("completed", "cancelled", "other", "RustFS Functional Chain", False), + ("completed", "cancelled", "topic", "Other Workflow", False), + ): + with self.subTest(state=state, conclusion=conclusion, branch=branch, workflow=workflow), tempfile.TemporaryDirectory() as temp: + Path(temp, "failed-run.json").write_text(json.dumps({"status": state, "conclusion": conclusion, + "headBranch": branch, "workflowName": workflow})) + with mock.patch.dict(os.environ, VALIDATION_ROOT=temp, GITHUB_REF_NAME="topic"): + if accepted: + exec(compile(guard, "workflow-cleanup-guard", "exec"), {}) + else: + with self.assertRaises(ValueError): + exec(compile(guard, "workflow-cleanup-guard", "exec"), {}) class CandidateTests(unittest.TestCase): @@ -76,6 +470,98 @@ class CandidateTests(unittest.TestCase): self.assertEqual(result["artifact_id"], 789) self.assertEqual(result["manifest"]["source_sha"], "b" * 40) + def test_validation_manifest_binds_requested_source_and_private_channel(self): + manifest = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"], + "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")} + run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"} + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w") as archive: + archive.writestr("validation-candidate-123-2.json", json.dumps(manifest)) + raw = payload.getvalue() + artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False, + "size_in_bytes": len(raw), "digest": "sha256:" + hashlib.sha256(raw).hexdigest(), + "workflow_run": {"id": 123, "head_sha": "a" * 40}} + result = candidate.read_manifest(raw, artifact, run) + self.assertEqual(result["channel"], "validation") + self.assertIn("/packages/validation/runs/123/2/", result["package_url"]) + + def test_validation_resolver_requires_manual_workflow_dispatch(self): + payload, artifact = self.archive() + run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40, + "path": ".github/workflows/rustfs-validation-package.yml", "head_branch": "main", + "head_repository": {"full_name": "rustfs/rustfs"}, "event": "schedule", + "status": "completed", "conclusion": "success", "run_started_at": "2026-10-12T00:00:00Z"} + with mock.patch.object(candidate, "api", side_effect=[run]): + with self.assertRaisesRegex(ValueError, "manually dispatched"): + candidate.resolve(123, 2, allow_validation=True) + + def test_validation_workflow_rejects_legacy_manifest_schema(self): + manifest = {**self.manifest} + payload_buffer = io.BytesIO() + with zipfile.ZipFile(payload_buffer, "w") as archive: + archive.writestr("validation-candidate-123-2.json", json.dumps(manifest)) + payload = payload_buffer.getvalue() + artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False, + "size_in_bytes": len(payload), "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), + "workflow_run": {"id": 123, "head_sha": "a" * 40}} + run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"} + with self.assertRaisesRegex(ValueError, "schema-3"): + candidate.read_manifest(payload, artifact, run) + + def test_private_candidate_resolution_is_explicit_and_attempt_bound(self): + self.manifest.update(schema=3, channel="validation", source_ref=self.manifest["source_sha"], + package_url=self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")) + payload, artifact = self.archive(["validation-candidate-123-2.json"]) + artifact["name"] = "validation-candidate-123-2" + base = {**self.run, "head_branch": "cxymds/validation-runner", "head_repository": {"full_name": "rustfs/rustfs"}, + "event": "workflow_dispatch", "status": "completed", "conclusion": "success", + "run_started_at": "2026-10-12T00:00:00Z"} + for path in candidate.VALIDATION_BUILD_WORKFLOWS: + run = {**base, "path": path} + with self.subTest(path=path), mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]): + resolved = candidate.resolve(123, 2, allow_validation=True) + self.assertEqual(resolved["manifest"]["source_sha"], "b" * 40) + self.assertEqual(resolved["workflow_sha"], "a" * 40) + with mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError): + candidate.resolve(123, 2) + for field, value in (("id", 124), ("run_attempt", 1), ("event", "push"), ("status", "in_progress"), + ("conclusion", "failure"), ("path", ".github/workflows/other.yml"), + ("head_repository", {"full_name": "untrusted/rustfs"})): + run = {**base, "path": ".github/workflows/rustfs-functional-chain.yml", field: value} + with self.subTest(field=field), mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError): + candidate.resolve(123, 2, allow_validation=True) + + def test_private_manifest_rejects_alias_channel_source_and_schema_substitution(self): + valid = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"], + "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")} + for key, value in (("channel", "nightly"), ("source_ref", "main"), ("source_sha", "e" * 40), + ("package_url", self.manifest["package_url"]), ("workflow_sha", "e" * 40), + ("build_run_attempt", 3), ("schema", True)): + with self.subTest(key=key), self.assertRaises(ValueError): + candidate.validate_manifest({**valid, key: value}, self.run) + + def test_package_only_mode_rejects_conflicting_fleet_and_tool_requests(self): + from check_test_wiring import yaml_block + workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml" + job = yaml_block(workflow.read_text().splitlines(), "validation-package-request", 2) + begin = job.index(" run: |") + 1 + script = "\n".join(line[10:] for line in job[begin:] if not line.strip() or line.startswith(" ")) + import os + inputs = {"validation_package_sha": "a" * 40} + env = {**os.environ, "PACKAGE_INPUTS": json.dumps(inputs)} + result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + for field in ("build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha", "validation_tools_only", + "validation_native_tools", "validation_shared_tools", "validation_manager_tools", + "validation_cleanup_run_id", "validation_tools_run_id"): + with self.subTest(field=field): + env["PACKAGE_INPUTS"] = json.dumps({**inputs, field: "not-empty"}) + result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True) + self.assertNotEqual(result.returncode, 0) + for value in ("main", "a" * 39, "a" * 40 + "\nFORGED=value"): + env["PACKAGE_INPUTS"] = json.dumps({"validation_package_sha": value}) + self.assertNotEqual(subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True).returncode, 0) + class EvidenceTests(unittest.TestCase): def setUp(self): diff --git a/scripts/test_inspect_validation_runner.py b/scripts/test_inspect_validation_runner.py new file mode 100644 index 000000000..747c23af4 --- /dev/null +++ b/scripts/test_inspect_validation_runner.py @@ -0,0 +1,123 @@ +"""Runner diagnostics must remain read-only when the shared fleet is unavailable.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import inspect_validation_runner as inspection +from check_test_wiring import yaml_block + + +class RunnerInspectionTests(unittest.TestCase): + def request(self): + return {"validation_runner_inspect_only": True, "validation_case": "", + "validation_tools_only": False, "i05_diagnostic_branch": "source-plan", + "validation_tools_run_attempt": "1"} + + def test_workflow_default_allows_inspection_request(self): + workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml" + block = yaml_block(workflow.read_text().splitlines(), "i05_diagnostic_branch", 6) + default = next(line.split(":", 1)[1].strip() for line in block if line.strip().startswith("default:")) + inspection.validate_request({**self.request(), "i05_diagnostic_branch": default}) + + def test_exclusive_request_rejects_every_execution_mode_before_api_access(self): + inspection.validate_request(self.request()) + for key, value in [("validation_runner_inspect_only", False), + ("validation_case", "environment.inspect"), ("build_run_id", "123"), + ("validation_package_sha", "a" * 40), ("validation_tools_only", True), + ("validation_native_tools", True), ("validation_shared_tools", True), + ("validation_manager_tools", True), ("validation_cleanup_run_id", "123"), + ("auto_testing_sha", "a" * 40), ("validation_tools_run_id", "123"), + ("i05_diagnostic_branch", "queued-successor"), + ("i05_diagnostic_branch", "timeout"), + ("validation_tools_run_attempt", "2")]: + with self.subTest(key=key), self.assertRaises(ValueError): + inspection.validate_request({**self.request(), key: value}) + + def fixture(self, runners, *, unknown_jobs=False): + def api(path): + self.assertTrue(path.startswith("repos/rustfs/rustfs/actions/")) + if "/runners?" in path: + return {"runners": runners} + if "/runs?status=queued&" in path: + return {"workflow_runs": [{"id": 123, "name": "RustFS Functional Chain", + "status": "queued", "created_at": "2026-10-26T00:00:00Z", "html_url": "run-url"}]} + if "/runs?status=" in path: + return {"workflow_runs": []} + self.assertIn("/runs/123/jobs?", path) + return {"jobs": [] if unknown_jobs else [ + {"id": 456, "name": "validation", "status": "queued", "runner_name": "", + "labels": ["smoke-testing"], "html_url": "job-url", "steps": [{"secret": "excluded"}]}, + {"id": 457, "name": "native", "status": "in_progress", "labels": ["ubuntu-latest"]}, + {"id": 458, "name": "prior", "status": "completed", "labels": ["smoke-testing"]}]} + return api + + def test_offline_idle_busy_and_unregistered_are_distinct_with_only_fleet_jobs(self): + runner = {"id": 7, "name": "rustfs-smoke-testing", "status": "offline", "busy": False, + "labels": [{"name": "smoke-testing"}], "extra": {"secret": "excluded"}} + for state, busy, expected in [("offline", False, "offline"), ("online", True, "online-busy"), + ("online", False, "online-idle"), (None, False, "unregistered")]: + with self.subTest(expected=expected): + rows = [{**runner, "status": state, "busy": busy}] if state else [] + report = {} + with patch.object(inspection, "api", side_effect=self.fixture(rows)): + inspection.collect(report) + self.assertEqual(report["runnerAvailability"], expected) + self.assertTrue(report["complete"]) + self.assertEqual([row["jobId"] for row in report["fleetJobs"]], [456]) + self.assertNotIn("secret", json.dumps(report)) + + def test_jobless_workflow_is_retained_without_inventing_a_runner_assignment(self): + report = {} + with patch.object(inspection, "api", side_effect=self.fixture([], unknown_jobs=True)): + inspection.collect(report) + self.assertEqual(report["fleetJobs"], []) + self.assertEqual(report["runsWithoutJobs"][0]["runId"], 123) + + def test_inventory_pagination_is_bounded_and_cannot_claim_complete_when_truncated(self): + row = {"id": 7} + with patch.object(inspection, "api", return_value={"runners": [row] * 100}) as api: + with self.assertRaisesRegex(ValueError, "bounded inspection"): + inspection.pages("repos/rustfs/rustfs/actions/runners", "runners") + self.assertEqual(api.call_count, 10) + + def test_api_failure_keeps_partial_report_without_subprocess_diagnostics_or_fleet_claims(self): + for inputs, expected_calls in [(self.request(), 1), ({**self.request(), "validation_tools_only": True}, 0)]: + with self.subTest(inputs=inputs), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "report.json" + with patch.dict(os.environ, RUNNER_INSPECTION_INPUTS=json.dumps(inputs)), \ + patch.object(inspection.sys, "argv", ["inspect", "--output", str(output)]), \ + patch.object(inspection, "api", side_effect=subprocess.CalledProcessError( + 1, ["gh", "api"], stderr=b"secret-token")) as api: + self.assertEqual(inspection.main(), 1) + report = json.loads(output.read_text()) + self.assertFalse(report["complete"]) + for field in ("mutated", "fleetAccessed", "cleanStateCertified", + "sharedWorkflowIdleCertified", "canonicalCasePassClaimed"): + self.assertIs(report[field], False) + self.assertNotIn("secret-token", output.read_text()) + self.assertEqual(api.call_count, expected_calls) + + def test_workflow_is_hosted_and_uses_a_separate_concurrency_group(self): + workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml" + source = workflow.read_text() + job = source.split(" validation-runner-inspection:\n", 1)[1].split(" validation-package-request:", 1)[0] + self.assertIn("runs-on: ubuntu-latest", job) + self.assertIn("timeout-minutes: 10", job) + self.assertIn("RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}", job) + self.assertIn("if: ${{ always() }}", job) + self.assertNotIn("RUSTFS_SECRET_KEY", job) + concurrency = source.split("concurrency:\n", 1)[1].split("jobs:\n", 1)[0] + self.assertLess(concurrency.index("validation_runner_inspect_only"), concurrency.index("validation_package_sha")) + self.assertIn("rustfs-functional-validation-runner-inspection", concurrency) + for name in ("validation-package-request", "validation-package", "prepare", "validation"): + job = source.split(" " + name + ":\n", 1)[1].split("\n runs-on:", 1)[0].split("\n uses:", 1)[0] + self.assertIn("!inputs.validation_runner_inspect_only", job) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_nightly_candidate.py b/scripts/test_nightly_candidate.py index f7ec209b7..db8dba1af 100644 --- a/scripts/test_nightly_candidate.py +++ b/scripts/test_nightly_candidate.py @@ -221,5 +221,171 @@ SH self.assertNotIn(" overwrite: true", upload) +class ValidationPackageTests(unittest.TestCase): + def setUp(self): + # Exercise the real upload shell with the same conditional object-store + # fixture used by the nightly publication tests. + self.fixture = NightlyCandidateTests("test_success_binds_actual_package_checkout_and_attempt") + self.fixture.setUp() + self.addCleanup(self.fixture.temp.cleanup) + self.workflow = ROOT / ".github/workflows/rustfs-validation-package.yml" + job = yaml_block(self.workflow.read_text().splitlines(), "publish", 2) + start = job.index(" - name: Upload validation package to immutable R2 path") + end = job.index(" - name: Upload validation candidate manifest") + upload = job[start:end] + begin = upload.index(" run: |") + 1 + self.fixture.shell = "\n".join(line[10:] for line in upload[begin:] if not line.strip() or line.startswith(" ")) + self.fixture.shims.write_text(self.fixture.shims.read_text().replace("/packages/nightly/runs/*", "/packages/validation/runs/*")) + self.fixture.env.update(PACKAGE_FILE=self.fixture.package.name, VALIDATION_SOURCE_SHA=self.fixture.sha, + VALIDATION_SOURCE_REF=self.fixture.sha, GITHUB_SHA="f" * 40, AWS_CLI="aws") + def shell(name, end_name): + step = job[job.index(" - name: " + name):job.index(" - name: " + end_name)] + return "\n".join(line[10:] for line in step[step.index(" run: |") + 1:] + if not line.strip() or line.startswith(" ")) + self.stage_shell = shell("Validate and stage opaque package bytes", "Prepare trusted publication tools") + self.tools_shell = shell("Prepare trusted publication tools", "Upload validation package to immutable R2 path") + + def stage(self): + env = dict(os.environ, RUNNER_TEMP=str(self.fixture.root)) + env.pop("BASH_ENV", None) + return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.stage_shell], + cwd=self.fixture.root, env=env, capture_output=True, text=True) + + def test_success_uploads_only_one_immutable_object_and_exact_source(self): + result = self.fixture.run_publish() + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.fixture.manifest() + self.assertEqual(manifest["schema"], 3) + self.assertEqual(manifest["channel"], "validation") + self.assertEqual(manifest["source_sha"], self.fixture.sha) + self.assertEqual(manifest["source_ref"], self.fixture.sha) + self.assertEqual(manifest["workflow_sha"], "f" * 40) + validate_manifest(manifest, {"id": 12345, "run_attempt": 1, "head_sha": "f" * 40}) + objects = [path for path in self.fixture.store.rglob("*") if path.is_file()] + self.assertEqual(len(objects), 1) + self.assertEqual(objects[0].read_bytes(), self.fixture.package.read_bytes()) + self.assertEqual(str(objects[0].relative_to(self.fixture.store)), + f"artifacts/rustfs/packages/validation/runs/12345/1/{self.fixture.digest}/rustfs.deb") + self.assertNotIn("s3 cp", (self.fixture.root / "aws.log").read_text()) + + def test_failure_or_missing_credentials_cannot_certify_a_candidate(self): + for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download")): + with self.subTest(mode=mode): + result = self.fixture.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index)) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.fixture.output.exists()) + self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), []) + for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"): + with self.subTest(key=key): + self.assertNotEqual(self.fixture.run_publish(**{key: ""}).returncode, 0) + self.assertFalse(self.fixture.output.exists()) + + def test_stale_source_and_duplicate_objects_are_rejected(self): + for source in ("not-an-exact-sha", "e" * 40): + with self.subTest(source=source): + result = self.fixture.run_publish(VALIDATION_SOURCE_SHA=source) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.fixture.store.exists()) + result = self.fixture.run_publish() + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.fixture.manifest() + key = manifest["package_url"].removeprefix("https://dl.rustfs.com/") + stored = self.fixture.store / key + stored.write_bytes(b"preexisting conflicting object") + (self.fixture.root / "validation-candidate-12345-1.json").unlink() + result = self.fixture.run_publish() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(stored.read_bytes(), b"preexisting conflicting object") + self.assertFalse(self.fixture.output.exists()) + + def test_exact_source_and_workflow_identity_are_preserved_separately(self): + lines = self.workflow.read_text().splitlines() + build = "\n".join(yaml_block(lines, "build", 2)) + publish = "\n".join(yaml_block(lines, "publish", 2)) + source = "\n".join(yaml_block(lines, "resolve-source", 2)) + self.assertIn('[[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]', source) + self.assertIn('test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"', source) + self.assertIn("ref: ${{ needs.resolve-source.outputs.source_sha }}", build) + self.assertIn('--arg source_sha "${VALIDATION_SOURCE_SHA}"', publish) + self.assertIn('--arg workflow_sha "${GITHUB_SHA}"', publish) + self.assertNotIn("PF_TESTING_GH_TOKEN", build) + self.assertNotIn("publish_nightly_assets", build) + + def test_candidate_execution_and_publication_credentials_are_in_separate_jobs(self): + lines = self.workflow.read_text().splitlines() + build = "\n".join(yaml_block(lines, "build", 2)) + publish = "\n".join(yaml_block(lines, "publish", 2)) + self.assertIn("uses: ./.github/actions/setup", build) + self.assertIn("cargo build", build) + self.assertNotIn("secrets.", build) + self.assertNotIn("id-token: write", build) + self.assertIn("needs: [resolve-source, build]", publish) + self.assertIn("runs-on: ubuntu-latest", publish) + for forbidden in ("actions/checkout@", "actions/cache@", "uses: ./", "needs.build.outputs", "cargo ", "dpkg", "source "): + with self.subTest(forbidden=forbidden): self.assertNotIn(forbidden, publish) + name = "name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}" + self.assertIn(name, build) + self.assertIn(name, publish) + self.assertIn("VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}", publish) + self.assertIn("VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}", publish) + prepare = publish.split(" - name: Upload validation package to immutable R2 path")[0] + self.assertNotIn("secrets.", prepare) + self.assertIn("secrets.R2_SECRET_ACCESS_KEY", publish) + + def test_staging_rejects_nonregular_empty_or_extra_artifact_entries(self): + incoming = self.fixture.root / "validation-package-input" + incoming.mkdir() + package = incoming / "rustfs.deb" + for kind in ("missing", "empty", "symlink", "directory", "fifo", "extra", "directory-symlink"): + with self.subTest(kind=kind): + if kind == "empty": package.touch() + elif kind == "symlink": package.symlink_to(self.fixture.package) + elif kind == "directory": package.mkdir() + elif kind == "fifo": os.mkfifo(package) + elif kind == "extra": + package.write_bytes(b"opaque") + (incoming / "BASH_ENV").write_text("exit 1") + elif kind == "directory-symlink": + incoming.rmdir() + incoming.symlink_to(self.fixture.root, target_is_directory=True) + self.assertNotEqual(self.stage().returncode, 0) + self.assertFalse((self.fixture.root / "validation-publication").exists()) + if incoming.is_symlink(): + incoming.unlink() + incoming.mkdir() + elif package.is_dir(): package.rmdir() + else: package.unlink(missing_ok=True) + (incoming / "BASH_ENV").unlink(missing_ok=True) + + def test_staging_and_publication_never_execute_candidate_package_or_python_hooks(self): + incoming = self.fixture.root / "validation-package-input" + incoming.mkdir() + marker = self.fixture.root / "candidate-executed" + payload = f'#!/bin/bash\nprintf "%s" "$R2_SECRET_ACCESS_KEY" > "{marker}"\n'.encode() + (incoming / "rustfs.deb").write_bytes(payload) + (self.fixture.root / "sitecustomize.py").write_text(f"open({str(marker)!r}, 'w').write('python hook')") + result = self.stage() + self.assertEqual(result.returncode, 0, result.stderr) + staged = self.fixture.root / "validation-publication/rustfs.deb" + self.assertEqual(staged.read_bytes(), payload) + result = self.fixture.run_publish(PACKAGE_FILE=str(staged)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(marker.exists()) + self.assertEqual(self.fixture.manifest()["package_sha256"], hashlib.sha256(payload).hexdigest()) + + def test_cli_bootstrap_finishes_without_publication_secrets(self): + for mode in ("success", "old-cli", "bootstrap-failure", "install-failure", "broken-install"): + with self.subTest(mode=mode): + (self.fixture.root / "installed").unlink(missing_ok=True) + env = dict(self.fixture.env, FAKE_MODE=mode) + for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"): + env.pop(key) + result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", self.tools_shell], + cwd=self.fixture.root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, mode in ("success", "old-cli"), result.stderr) + self.assertFalse(self.fixture.store.exists()) + self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), []) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_retain_validation_evidence.py b/scripts/test_retain_validation_evidence.py new file mode 100644 index 000000000..0bd40afa3 --- /dev/null +++ b/scripts/test_retain_validation_evidence.py @@ -0,0 +1,203 @@ +"""Interrupted evidence is read-only and never an implicit recovery or PASS.""" + +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import retain_validation_evidence as retain +import prepare_retained_validation_recovery as recovery_input +from check_test_wiring import yaml_block + + +class RetainEvidenceTests(unittest.TestCase): + def fixture(self, directory): + temporary = Path(directory).resolve() / "_work/_temp" + temporary.mkdir(parents=True) + parent = temporary.parent / "rustfs-validation-evidence" + parent.mkdir(mode=0o700) + root = parent / "rustfs-validation-77-1" + root.mkdir(mode=0o700) + (root / "runner-storage-preflight.json").write_text(json.dumps({ + "runId": "77", "runAttempt": "1", "evidenceRoot": str(root), + "persistentOutsideRunnerTemp": True})) + (root / "original.json").write_bytes(b'{"verdict":"RUNNING"}') + env = {"RUNNER_TEMP": str(temporary), "GITHUB_RUN_ID": "88", "GITHUB_RUN_ATTEMPT": "1", + "RUNNER_NAME": "owned-runner", "GITHUB_ENV": str(temporary / "env"), + "EVIDENCE_RETAIN_INPUTS": json.dumps({"validation_retain_evidence_run_id": "77", + "i05_diagnostic_branch": "source-plan", "validation_tools_run_attempt": "1"})} + run = {"id": 77, "run_attempt": 1, "status": "completed", "conclusion": "failure", + "event": "workflow_dispatch", "repository": {"full_name": "rustfs/rustfs"}, + "path": ".github/workflows/rustfs-functional-chain.yml", "head_sha": "a" * 40, + "html_url": "source-url"} + jobs = {"total_count": 1, "jobs": [{"name": "validation", "runner_name": "owned-runner"}]} + return env, root, run, jobs + + def test_workflow_default_allows_evidence_retention(self): + workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml" + block = yaml_block(workflow.read_text().splitlines(), "i05_diagnostic_branch", 6) + default = next(line.split(":", 1)[1].strip() for line in block if line.strip().startswith("default:")) + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"]) + env["EVIDENCE_RETAIN_INPUTS"] = json.dumps({**inputs, "i05_diagnostic_branch": default}) + with patch.object(retain, "api", side_effect=[run, jobs]): + self.assertEqual(retain.prepare(env, {}), root) + + def test_existing_bytes_and_original_running_verdict_remain_unchanged(self): + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + before = {p.name: p.read_bytes() for p in root.iterdir()} + report = {} + with patch.object(retain, "api", side_effect=[run, jobs]) as api: + self.assertEqual(retain.prepare(env, report), root) + self.assertTrue(report["complete"]) + self.assertEqual({p.name: p.read_bytes() for p in root.iterdir()}, before) + self.assertEqual({row["path"]: row["sha256"] for row in report["files"]}, + {name: hashlib.sha256(raw).hexdigest() for name, raw in before.items()}) + self.assertEqual([call.args[0] for call in api.call_args_list], [ + "repos/rustfs/rustfs/actions/runs/77/attempts/1", + "repos/rustfs/rustfs/actions/runs/77/attempts/1/jobs?per_page=100"]) + + def test_mixed_execution_requests_and_unsafe_run_ids_refuse_before_api(self): + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + inputs = json.loads(env["EVIDENCE_RETAIN_INPUTS"]) + for field, value in [("validation_case", "heal.admin-healthy-noop"), + ("validation_runner_inspect_only", True), ("validation_tools_only", True), + ("build_run_id", "1"), ("validation_retain_evidence_run_id", "../77"), + ("i05_diagnostic_branch", "timeout"), + ("i05_diagnostic_branch", "queued-successor"), + ("validation_retain_evidence_run_id", "88")]: + with self.subTest(field=field, value=value), patch.object(retain, "api") as api: + altered = {**env, "EVIDENCE_RETAIN_INPUTS": json.dumps({**inputs, field: value})} + with self.assertRaises(ValueError): retain.prepare(altered, {}) + api.assert_not_called() + + def test_wrong_repository_running_source_and_different_runner_refuse(self): + for mutation in ("repo", "running", "success", "runner", "attempt"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + if mutation == "repo": run["repository"]["full_name"] = "foreign/repo" + elif mutation == "running": run["status"] = "in_progress" + elif mutation == "success": run["conclusion"] = "success" + elif mutation == "attempt": run["run_attempt"] = 2 + else: jobs["jobs"][0]["runner_name"] = "other-runner" + with patch.object(retain, "api", side_effect=[run, jobs]), self.assertRaises(ValueError): + retain.prepare(env, {}) + + def test_links_special_files_and_byte_budget_do_not_escape_the_original_root(self): + for mutation in ("symlink", "hardlink", "fifo", "budget", "public-root"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + if mutation == "symlink": (root / "foreign").symlink_to(Path(directory) / "secret") + elif mutation == "hardlink": os.link(root / "original.json", root / "alias") + elif mutation == "fifo": os.mkfifo(root / "fifo") + elif mutation == "public-root": root.chmod(0o755) + with patch.object(retain, "api", side_effect=[run, jobs]), \ + patch.object(retain, "MAX_BYTES", 1 if mutation == "budget" else retain.MAX_BYTES), \ + self.assertRaises(ValueError): + retain.prepare(env, {}) + + def test_file_changes_and_foreign_storage_provenance_are_rejected(self): + for mutation in ("write", "marker"): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + if mutation == "marker": + (root / "runner-storage-preflight.json").write_text('{"runId":"foreign"}') + real = os.read + changed = [] + def read(fd, size): + raw = real(fd, size) + if mutation == "write" and not changed: + changed.append(True) + with (root / "original.json").open("ab") as stream: stream.write(b"changed") + return raw + with patch.object(retain, "api", side_effect=[run, jobs]), patch.object(retain.os, "read", side_effect=read), \ + self.assertRaises(ValueError): + retain.prepare(env, {}) + + def test_cli_exports_only_admitted_root_and_emits_no_case_or_recovery_claim(self): + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + output = Path(directory) / "receipt.json" + with patch.dict(os.environ, env), patch.object(retain, "api", side_effect=[run, jobs]), \ + patch.object(retain.sys, "argv", ["retain", "--report", str(output)]): + self.assertEqual(retain.main(), 0) + value = json.loads(output.read_text()) + for field in ("fleetAccessed", "originalEvidenceModified", "sourceVerdictRewritten", "canonicalCasePassClaimed"): + self.assertIs(value[field], False) + self.assertIn("VALIDATION_RETAIN_ROOT=" + str(root), Path(env["GITHUB_ENV"]).read_text()) + + def test_cli_keeps_incomplete_receipt_without_exporting_a_missing_original_root(self): + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + env["EVIDENCE_RETAIN_INPUTS"] = '{"validation_retain_evidence_run_id":"99"}' + run["id"] = 99 + output = Path(directory) / "receipt.json" + with patch.dict(os.environ, env), patch.object(retain, "api", side_effect=[run, jobs]), \ + patch.object(retain.sys, "argv", ["retain", "--report", str(output)]): + self.assertEqual(retain.main(), 1) + value = json.loads(output.read_text()) + self.assertFalse(value["complete"]) + self.assertEqual(value["errorType"], "FileNotFoundError") + self.assertFalse(Path(env["GITHUB_ENV"]).exists()) + + def test_recovery_copy_keeps_original_bytes_and_separate_collection_identity(self): + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + destination = Path(directory).resolve() / 'new-recovery'; destination.mkdir(mode=0o700) + env.update(FAILED_RUN_ID='77', GITHUB_REF_NAME='topic', VALIDATION_ROOT=str(destination)) + run['head_branch'] = 'topic' + nested = root / 'nested'; nested.mkdir(); (nested / 'receipt').write_bytes(b'ACK') + before = retain.inventory(root) + report = {'collectionRunId': '88', 'sourceArtifactClaimed': False} + with patch.object(recovery_input, 'api', return_value=run), patch.object(retain, 'api', side_effect=[run, jobs]): + recovery_input.stage(env, report) + self.assertTrue(report['complete']) + self.assertEqual(report['collectionRunId'], '88') + self.assertEqual(report['sourceRun']['id'], 77) + self.assertIs(report['sourceArtifactClaimed'], False) + copied = Path(report['recoveryInput']) + self.assertEqual(retain.inventory(copied), before) + self.assertEqual(retain.inventory(root), before) + self.assertNotEqual((copied / 'original.json').stat().st_ino, (root / 'original.json').stat().st_ino) + + def test_recovery_copy_refuses_changed_source_and_foreign_branch(self): + for mutation in ('drift', 'branch'): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + destination = Path(directory).resolve() / 'recovery'; destination.mkdir(mode=0o700) + env.update(FAILED_RUN_ID='77', GITHUB_REF_NAME='topic', VALIDATION_ROOT=str(destination)) + run['head_branch'] = 'foreign' if mutation == 'branch' else 'topic' + original = retain.prepare + def changed(environment, report): + source = original(environment, report) + (source / 'original.json').write_bytes(b'new bytes after admission') + return source + with patch.object(recovery_input, 'api', return_value=run), \ + patch.object(retain, 'api', side_effect=[run, jobs]), \ + patch.object(recovery_input, 'prepare', side_effect=changed), self.assertRaises(ValueError): + recovery_input.stage(env, {}) + + def test_copy_does_not_follow_a_symlink_swapped_after_directory_listing(self): + with tempfile.TemporaryDirectory() as directory: + env, root, run, jobs = self.fixture(directory) + secret = Path(directory) / 'private'; secret.write_bytes(b'outside-root-secret') + destination = Path(directory).resolve() / 'copy' + original_stat = os.stat + def swapped(path, *args, **kwargs): + info = original_stat(path, *args, **kwargs) + if path == 'original.json' and kwargs.get('follow_symlinks') is False: + (root / path).unlink(); (root / path).symlink_to(secret) + return info + with patch.object(retain.os, 'stat', side_effect=swapped), self.assertRaises(OSError): + retain.inventory(root, copy_to=destination) + self.assertFalse((destination / 'original.json').exists()) + + +if __name__ == "__main__": + unittest.main()