ci(validation): dispatch isolated packages through the driver

test(validation): verify isolated package publication failures
test(validation): enforce private candidate manifest schema
2026-10-27 22:37:13 +00:00 · 2026-10-12 12:29:11 +08:00 · 2026-10-12 12:22:53 +08:00 · 2026-10-12 12:18:15 +08:00 · 2026-10-12 12:06:59 +08:00 · 2026-10-12 11:36:29 +08:00
19 changed files with 1311 additions and 40 deletions
@@ -20,13 +20,65 @@ on:
  workflow_dispatch:
    inputs:
      build_run_id:
-        description: Successful nightly build run on main
+        description: Successful nightly or validation package run; empty for package-only build
        type: string
-        required: true
+        required: false
+        default: ''
      build_run_attempt:
        description: Exact successful build attempt
        type: string
-        required: true
+        required: false
+        default: ''
+      validation_package_sha:
+        description: Exact source SHA for an isolated package-only build
+        type: string
+        required: false
+        default: ''
+      validation_case:
+        description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain
+        type: string
+        required: false
+        default: ''
+      auto_testing_sha:
+        description: Exact auto-testing commit for a validation case
+        type: string
+        required: false
+        default: ''
+      validation_tools_only:
+        description: Build candidate-bound validation tools without running a fleet case
+        type: boolean
+        required: false
+        default: false
+      validation_native_tools:
+        description: Include exact-source native regressions in a tools-only build
+        type: boolean
+        required: false
+        default: false
+      validation_shared_tools:
+        description: Include the controlled C10 shared-reference encoder in a tools-only build
+        type: boolean
+        required: false
+        default: false
+      validation_manager_tools:
+        description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build
+        type: boolean
+        required: false
+        default: false
+      validation_cleanup_run_id:
+        description: Failed validation run whose supported retained environment needs evidence-based finalization
+        type: string
+        required: false
+        default: ''
+      validation_tools_run_id:
+        description: Successful tool build run; required by Heal physical metadata checks
+        type: string
+        required: false
+        default: ''
+      validation_tools_run_attempt:
+        description: Exact tool build attempt
+        type: string
+        required: false
+        default: '1'
  workflow_run:
    workflows: ["Nightly GNU Build"]
    types: [completed]
@@ -36,12 +88,47 @@ permissions:
  actions: read

 concurrency:
-  group: rustfs-functional-chain-runs
+  group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }}
  cancel-in-progress: false

 jobs:
+  validation-package-request:
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Validate exclusive package-only request
+        env:
+          PACKAGE_INPUTS: ${{ toJSON(inputs) }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          import re
+          value = json.loads(os.environ["PACKAGE_INPUTS"])
+          if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None:
+              raise ValueError("package build requires an exact source SHA")
+          exclusive = ("build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha",
+                       "validation_tools_only", "validation_native_tools", "validation_shared_tools",
+                       "validation_manager_tools", "validation_cleanup_run_id", "validation_tools_run_id")
+          if any(value.get(name) for name in exclusive):
+              raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution")
+          PY
+
+  validation-package:
+    needs: validation-package-request
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' }}
+    uses: ./.github/workflows/rustfs-validation-package.yml
+    with:
+      source_sha: ${{ inputs.validation_package_sha }}
+    secrets:
+      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+      R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BUCKET }}
+
  prepare:
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }}
    runs-on: smoke-testing
    timeout-minutes: 10
    outputs:
@@ -89,6 +176,282 @@ jobs:
            echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
          fi

+  validation:
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '') }}
+    runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }}
+    timeout-minutes: 90
+    outputs:
+      source_sha: ${{ steps.candidate.outputs.source_sha }}
+    concurrency:
+      group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }}
+      cancel-in-progress: false
+    env:
+      RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }}
+      RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }}
+      RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }}
+      RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }}
+      RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }}
+    steps:
+      - name: Checkout candidate resolver
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Initialize validation evidence
+        run: |
+          set -euo pipefail
+          umask 077
+          VALIDATION_ROOT="${RUNNER_TEMP}/rustfs-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          mkdir -- "${VALIDATION_ROOT}"
+          printf 'VALIDATION_ROOT=%s\n' "${VALIDATION_ROOT}" >> "${GITHUB_ENV}"
+
+      - name: Resolve immutable candidate
+        id: candidate
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+          CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
+          SELECTED_CASE: ${{ inputs.validation_case }}
+          TOOLS_ONLY: ${{ inputs.validation_tools_only }}
+          NATIVE_TOOLS: ${{ inputs.validation_native_tools }}
+          SHARED_TOOLS: ${{ inputs.validation_shared_tools }}
+          MANAGER_TOOLS: ${{ inputs.validation_manager_tools }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          import re
+          from scripts.resolve_functional_candidate import resolve
+
+          revision = os.environ["AUTO_TESTING_SHA"]
+          cleanup = os.environ["CLEANUP_RUN_ID"]
+          if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("tools-only execution cannot select a fleet case or cleanup")
+          if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup")
+          if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup")
+          if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
+              raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
+          if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
+              raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
+          if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
+              raise ValueError("validation requires an exact auto-testing commit SHA")
+          candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True)
+          manifest = candidate["manifest"]
+          root = Path(os.environ["VALIDATION_ROOT"])
+          (root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n")
+          (root / "candidate.json").write_text(json.dumps({
+              "package_url": manifest["package_url"],
+              "package_sha256": manifest["package_sha256"],
+              "source_sha": manifest["source_sha"],
+          }, sort_keys=True) + "\n")
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write("source_sha=" + manifest["source_sha"] + "\n")
+          PY
+
+      - name: Checkout exact auto-testing revision
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Install validation Python client
+        if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
+        run: |
+          set -euo pipefail
+          python3 -m venv "${RUNNER_TEMP}/rustfs-validation-python"
+          "${RUNNER_TEMP}/rustfs-validation-python/bin/python" -m pip install \
+            -r auto-testing/scripts/validation_ilm/requirements.txt
+          printf '%s\n' "${RUNNER_TEMP}/rustfs-validation-python/bin" >> "${GITHUB_PATH}"
+
+      - name: Validate Heal tool build identity
+        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }}
+          TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }}
+        run: |
+          set -euo pipefail
+          [[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
+          [[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
+          gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \
+            > "${VALIDATION_ROOT}/tool-run.json"
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text())
+          if (value.get("conclusion") != "success" or value.get("status") != "completed"
+                  or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"])
+                  or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}):
+              raise ValueError("Heal tool run is not an exact successful validation build attempt")
+          PY
+
+      - name: Download pinned Heal metadata inspector
+        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify Heal metadata inspector
+        if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
+        run: |
+          set -euo pipefail
+          TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          python3 -B auto-testing/scripts/validation_heal/inspector.py \
+            --directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            > "${VALIDATION_ROOT}/tool-manifest.json"
+          printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Download pinned C07 native authority evidence
+        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify C07 native source and exact test evidence
+        if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
+        run: |
+          set -euo pipefail
+          NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/authority.py \
+            --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            > "${VALIDATION_ROOT}/native-authority-manifest.json"
+          printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Download pinned manager persistence and incarnation evidence
+        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify exact manager native prerequisites before fleet mutation
+        if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
+        env:
+          VALIDATION_CASE: ${{ inputs.validation_case }}
+        run: |
+          set -euo pipefail
+          MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \
+            --directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json"
+          printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Download pinned C10 shared-reference encoder
+        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ inputs.validation_tools_run_id }}
+          name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
+
+      - name: Verify C10 exact-source codec and original tests
+        if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
+        run: |
+          set -euo pipefail
+          SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json"
+          python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \
+            --directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
+            > "${VALIDATION_ROOT}/shared-fixture-manifest.json"
+          printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
+
+      - name: Run selected canonical validation case
+        if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
+        env:
+          VALIDATION_CASE: ${{ inputs.validation_case }}
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          ./auto-testing/scripts/run_validation_case.sh \
+            "${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \
+            "${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run"
+
+      - name: Inspect current Heal fleet resources without mutation
+        if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }}
+        env:
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          python3 -B auto-testing/scripts/validation_heal/environment.py \
+            --candidate "${VALIDATION_ROOT}/candidate.json" \
+            --testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment"
+
+      - name: Audit and finalize retained validation environment
+        if: ${{ inputs.validation_cleanup_run_id != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
+          AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
+            --json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json"
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+          run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text())
+          if (run["status"] != "completed" or run["conclusion"] != "failure"
+              or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]):
+              raise ValueError("cleanup source must be a completed failed Functional Chain run")
+          PY
+          gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
+            --pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact"
+          python3 -B auto-testing/scripts/validation_heal/recover_environment.py \
+            --failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \
+            --candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery"
+
+      - name: Show validation report
+        if: ${{ always() }}
+        run: |
+          if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then
+            cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}"
+          fi
+
+      - name: Upload validation evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
+          if-no-files-found: error
+
+  validation-tools:
+    needs: [validation]
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }}
+    uses: ./.github/workflows/rustfs-validation-tools.yml
+    with:
+      candidate_sha: ${{ needs.validation.outputs.source_sha }}
+      auto_testing_sha: ${{ inputs.auto_testing_sha }}
+      build_native_heal: ${{ inputs.validation_native_tools }}
+      build_shared_heal: ${{ inputs.validation_shared_tools }}
+      build_manager_native: ${{ inputs.validation_manager_tools }}
+    secrets: inherit
+
  upgrade:
    needs: [prepare]
    if: ${{ always() && needs.prepare.result == 'success' }}
@@ -0,0 +1,187 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+name: RustFS Validation Package
+
+on:
+  workflow_call:
+    inputs:
+      source_sha:
+        description: Exact RustFS commit SHA to build
+        required: true
+        type: string
+    secrets:
+      R2_ACCESS_KEY_ID:
+        required: true
+      R2_SECRET_ACCESS_KEY:
+        required: true
+      R2_ENDPOINT:
+        required: true
+      R2_BUCKET:
+        required: true
+  workflow_dispatch:
+    inputs:
+      source_sha:
+        description: Exact RustFS commit SHA to build
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rustfs-validation-package-${{ inputs.source_sha }}
+  cancel-in-progress: false
+
+jobs:
+  resolve-source:
+    if: github.repository == 'rustfs/rustfs'
+    name: Resolve exact validation source
+    runs-on: sm-standard-2
+    timeout-minutes: 10
+    outputs:
+      source_sha: ${{ steps.source.outputs.sha }}
+      source_ref: ${{ steps.source.outputs.ref }}
+    steps:
+      - name: Validate requested source
+        id: source
+        env:
+          REQUESTED_SHA: ${{ inputs.source_sha }}
+        run: |
+          set -euo pipefail
+          [[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")"
+          trap 'rm -rf "${source_directory}"' EXIT
+          git init -q "${source_directory}"
+          git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}"
+          test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"
+          echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
+          echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
+
+  build:
+    needs: resolve-source
+    name: Build immutable validation DEB
+    runs-on: sm-standard-4
+    timeout-minutes: 150
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
+      VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}
+    steps:
+      - name: Checkout exact source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          ref: ${{ needs.resolve-source.outputs.source_sha }}
+
+      - name: Confirm checked out source
+        env:
+          EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }}
+        run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          cache-shared-key: validation-package-x86_64-unknown-linux-gnu
+          cache-save-if: "false"
+          install-build-packaging-tools: "false"
+          install-test-tools: "false"
+
+      - name: Build RustFS
+        run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
+
+      - name: Build validation DEB
+        id: package
+        shell: bash
+        run: |
+          set -euo pipefail
+          command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot
+          package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}"
+          package_file="rustfs-validation-${VALIDATION_SOURCE_SHA}.deb"
+          binary="target/x86_64-unknown-linux-gnu/release/rustfs"
+          test -x "${binary}"
+          mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \
+            "${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs"
+          cp "${binary}" "${package_dir}/usr/bin/rustfs"
+          chmod 755 "${package_dir}/usr/bin/rustfs"
+          cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/"
+          cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF'
+          # RustFS Environment Configuration
+          # RUSTFS_ACCESS_KEY=<changeme>
+          # RUSTFS_SECRET_KEY=<changeme>
+          # RUSTFS_VOLUMES=""
+          # RUSTFS_ADDRESS=":9000"
+          ENVEOF
+          cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF'
+          Package: rustfs
+          Version: 0.0.0~validation
+          Section: utils
+          Priority: optional
+          Architecture: amd64
+          Depends: libc6 (>= 2.31)
+          Maintainer: RustFS Team <support@rustfs.com>
+          Description: RustFS immutable validation package
+           Candidate package for RustFS validation only.
+          CONTROLEOF
+          printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles"
+          ./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst"
+          ./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm"
+          ./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm"
+          chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm"
+          cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/"
+          fakeroot dpkg-deb --build "${package_dir}" "${package_file}"
+          test -s "${package_file}"
+          echo "package_file=${package_file}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload validation package to immutable R2 path
+        id: publish
+        env:
+          PACKAGE_FILE: ${{ steps.package.outputs.package_file }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+          R2_BUCKET: ${{ secrets.R2_BUCKET }}
+          AWS_EC2_METADATA_DISABLED: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then
+            echo "Validation package publication requires complete R2 configuration" >&2
+            exit 1
+          fi
+          test "$(git rev-parse HEAD)" = "${VALIDATION_SOURCE_SHA}"
+          export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto
+          package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)"
+          key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb"
+          url="https://dl.rustfs.com/${key}"
+          AWS_CLI=aws
+          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
+            sudo apt-get update
+            sudo apt-get install -y -qq python3-venv
+            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")"
+            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
+            python3 -m venv "${AWS_CLI_DIR}"
+            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
+            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
+          fi
+          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
+          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \
+            --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
+          downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)"
+          test "${downloaded_sha256}" = "${package_sha256}"
+          manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
+          jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \
+            --arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \
+            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
+            --arg package_url "${url}" --arg package_sha256 "${package_sha256}" \
+            '{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}"
+          echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}"
+
+      - name: Upload validation candidate manifest
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ steps.publish.outputs.candidate_file }}
+          if-no-files-found: error
@@ -0,0 +1,300 @@
+# Build source-bound helpers separately so live validation needs only artifacts.
+name: RustFS Validation Tools
+
+on:
+  workflow_call:
+    inputs:
+      candidate_sha:
+        type: string
+        required: true
+      auto_testing_sha:
+        type: string
+        required: true
+      build_native_heal:
+        type: boolean
+        required: false
+        default: false
+      build_shared_heal:
+        type: boolean
+        required: false
+        default: false
+      build_manager_native:
+        type: boolean
+        required: false
+        default: false
+    secrets:
+      PF_TESTING_GH_TOKEN:
+        required: true
+  workflow_dispatch:
+    inputs:
+      candidate_sha:
+        description: Exact RustFS source SHA from the validation candidate manifest
+        type: string
+        required: true
+      auto_testing_sha:
+        description: Exact auto-testing SHA containing the Heal metadata inspector
+        type: string
+        required: true
+      build_native_heal:
+        description: Also compile and run separate exact-source native Heal regressions
+        type: boolean
+        required: false
+        default: false
+      build_shared_heal:
+        description: Build the controlled C10 shared-reference fixture encoder
+        type: boolean
+        required: false
+        default: false
+      build_manager_native:
+        description: Build and run exact candidate checkpoint/MRF/incarnation regressions
+        type: boolean
+        required: false
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  manager-native-regressions:
+    if: ${{ inputs.build_manager_native }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 180
+    env:
+      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+    steps:
+      - name: Validate immutable manager native inputs
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
+
+      - name: Checkout exact manager candidate source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ inputs.candidate_sha }}
+          path: candidate
+          persist-credentials: false
+
+      - name: Checkout exact manager collector source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Install manager native build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
+          df -h .
+
+      - name: Build exact unmodified candidate manager test executable
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          python3 -B auto-testing/scripts/validation_heal/manager_native.py build \
+            --source candidate --commit "${CANDIDATE_SHA}" --out manager-tools
+
+      - name: Run exact manager regressions independently
+        run: |
+          set -euo pipefail
+          python3 -B auto-testing/scripts/validation_heal/manager_native.py run \
+            --directory manager-tools --out manager-evidence
+
+      - name: Upload manager executable identities and original diagnostics
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            manager-tools/heal-manager-native-tests
+            manager-tools/*.json
+            manager-tools/*.jsonl
+            manager-tools/*.stderr
+            manager-tools/*.log
+            manager-evidence/
+          if-no-files-found: warn
+
+  native-heal-regressions:
+    if: ${{ inputs.build_native_heal }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 180
+    env:
+      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+    steps:
+      - name: Validate immutable native source inputs
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
+
+      - name: Checkout exact native candidate source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ inputs.candidate_sha }}
+          path: candidate
+          persist-credentials: false
+
+      - name: Checkout exact native collector source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Install native build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
+          df -h .
+
+      - name: Build and enumerate the unmodified candidate test executable
+        run: |
+          set -euo pipefail
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \
+            --source candidate --commit "${CANDIDATE_SHA}" --out native-tools
+
+      - name: Run exact native regressions separately from fleet validation
+        run: |
+          set -euo pipefail
+          python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \
+            --directory native-tools --out native-evidence
+
+      - name: Upload native executable identities and original diagnostics
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            native-tools/ecstore-native-tests
+            native-tools/*.json
+            native-tools/*.jsonl
+            native-tools/*.stderr
+            native-tools/*.log
+            native-evidence/
+          if-no-files-found: warn
+
+  heal-metadata-inspector:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      CANDIDATE_SHA: ${{ inputs.candidate_sha }}
+      AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
+    steps:
+      - name: Validate immutable source inputs
+        run: |
+          set -euo pipefail
+          [[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
+          [[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
+
+      - name: Checkout exact candidate source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          ref: ${{ inputs.candidate_sha }}
+          path: candidate
+          persist-credentials: false
+
+      - name: Checkout exact inspector source
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ inputs.auto_testing_sha }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
+
+      - name: Build the read-only candidate metadata decoder
+        run: |
+          set -euo pipefail
+          test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}"
+          test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
+          df -h .
+          python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \
+            --source candidate --commit "${CANDIDATE_SHA}" --out inspector-build
+          # Seed the helper's separate graph from the candidate's frozen graph.
+          # Cargo adds the helper package; both lock identities are retained.
+          cp candidate/Cargo.lock inspector-build/Cargo.lock
+          cargo build --release --manifest-path inspector-build/Cargo.toml
+          python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \
+            --binary inspector-build/target/release/heal-meta-inspect --source candidate
+          mkdir validation-tools
+          cp inspector-build/target/release/heal-meta-inspect validation-tools/
+          cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock
+          python3 - <<'PY'
+          import hashlib
+          import json
+          import os
+          from pathlib import Path
+          import subprocess
+
+          source = json.loads(Path("inspector-build/source.json").read_text())
+          binary = Path("validation-tools/heal-meta-inspect")
+          source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"],
+                        binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
+                        inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(),
+                        toolchain=subprocess.check_output(["rustc", "-vV"], text=True),
+                        target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"],
+                        runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
+          Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n")
+          PY
+          git -C candidate diff --exit-code
+
+      - name: Build and test the controlled C10 metadata encoder
+        if: ${{ inputs.build_shared_heal }}
+        run: |
+          set -euo pipefail
+          python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \
+            --source candidate --commit "${CANDIDATE_SHA}" --out shared-build
+          cp candidate/Cargo.lock shared-build/Cargo.lock
+          export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target"
+          cargo build --release --manifest-path shared-build/Cargo.toml
+          mkdir shared-tools
+          if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \
+            > shared-tools/shared-fixture-tests.log 2>&1; then
+            cat shared-tools/shared-fixture-tests.log
+            exit 1
+          fi
+          cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/
+          cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock
+          python3 - <<'PY'
+          import hashlib
+          import json
+          import os
+          from pathlib import Path
+          import subprocess
+
+          source = json.loads(Path("shared-build/source.json").read_text())
+          source.update(schemaVersion=1, built=True, sourceUnmodified=True,
+              candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(),
+              testingCommit=os.environ["AUTO_TESTING_SHA"],
+              binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(),
+              sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(),
+              toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu",
+              runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
+          Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n")
+          PY
+          git -C candidate diff --exit-code
+
+      - name: Upload controlled C10 fixture encoder evidence
+        if: ${{ always() && inputs.build_shared_heal }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
+          path: shared-tools/
+          if-no-files-found: warn
+
+      - name: Upload immutable validation tools
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
+          path: validation-tools/
+          if-no-files-found: error
@@ -740,7 +740,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
        .version_id(version)
        .send()
        .await?;
-    opa.expect_bucket_tags("s3:GetObject", source, Some("finance")).await?;
+    opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
    client.list_objects_v2().bucket(source).send().await?;
    opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
    client.list_object_versions().bucket(source).send().await?;
@@ -9221,7 +9221,7 @@ mod test {
        };
        let snapshot = PersistentListMetadataObject::from_object_info(&object);
        assert!(snapshot.to_object_info("bucket").etag.is_none());
-        let mut old_snapshot = snapshot.clone();
+        let mut old_snapshot = snapshot;
        old_snapshot.etag = object.etag.clone();
        let old_contents = format!(
            "# rustfs-listobjects-key-only-v2\n{}\n",
@@ -274,9 +274,9 @@ impl HealTask {
                    let mut progress = self.progress.write().await;
                    progress.update_stage(3, 3);
                }
-                Err(Error::TaskExecutionFailed {
-                    message: format!("Failed to heal bucket {bucket}: {e}"),
-                })
+                // Keep storage/quorum identity for the cluster retry classifier.
+                // The structured log above already carries the bucket context.
+                Err(e)
            }
        }
    }
@@ -727,6 +727,85 @@ mod canonical_outcome {
        );
    }

+    #[tokio::test(start_paused = true)]
+    async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() {
+        for (failures, succeeds) in [(1, true), (4, false)] {
+            let storage = Arc::new(MockStorage {
+                listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
+                typed_bucket_heal_errors: Mutex::new(HashMap::from([(
+                    "bucket-a".to_string(),
+                    (0..failures)
+                        .map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum))
+                        .collect(),
+                )])),
+                ..Default::default()
+            });
+            let task = HealTask::from_request(
+                HealRequest::new(
+                    HealType::Cluster,
+                    HealOptions {
+                        recursive: true,
+                        timeout: None,
+                        ..Default::default()
+                    },
+                    HealPriority::Normal,
+                ),
+                storage.clone(),
+            );
+            let result = task.execute().await;
+            assert_eq!(
+                result.is_ok(),
+                succeeds,
+                "original cluster task must honor its bounded bucket retry budget"
+            );
+            let outcome = task.get_outcome().await;
+            if succeeds {
+                assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
+                assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
+            } else {
+                assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
+                assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
+                let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure");
+                assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
+            }
+            let calls = storage.bucket_heal_calls.lock().expect("original bucket calls");
+            assert_eq!(
+                calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(),
+                if succeeds { 2 } else { 4 }
+            );
+            assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited");
+        }
+    }
+
+    #[tokio::test]
+    async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() {
+        for error in [
+            Error::Storage(EcstoreError::ErasureWriteQuorum),
+            Error::Disk(DiskError::FileAccessDenied),
+        ] {
+            let retryable = error.is_recoverable_heal();
+            let storage = Arc::new(MockStorage {
+                typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])),
+                ..Default::default()
+            });
+            let task = bucket_task(storage.clone());
+            let observed = task
+                .heal_bucket("bucket-a")
+                .await
+                .expect_err("bucket prepass must retain the storage error");
+            assert!(matches!(
+                observed,
+                Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied)
+            ));
+            assert_eq!(observed.is_recoverable_heal(), retryable);
+            assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
+            assert!(
+                storage.heal_object_calls.lock().expect("object calls").is_empty(),
+                "failed prepass must not heal objects"
+            );
+        }
+    }
+
    #[tokio::test(start_paused = true)]
    async fn cancellation_and_deadline_leave_partial_coverage() {
        for cancel in [false, true] {
@@ -1482,6 +1561,7 @@ struct MockStorage {
    include_object_dir_candidate: Mutex<bool>,
    listed_buckets: Mutex<Option<Vec<String>>>,
    bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
+    typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
    bucket_heal_calls: Mutex<Vec<String>>,
    block_heal_object: Mutex<bool>,
    resume_disk: Mutex<Option<DiskStore>>,
@@ -2247,6 +2327,15 @@ impl HealStorageAPI for MockStorage {
    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
        self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
        self.bucket_heal_opts.lock().unwrap().push(*opts);
+        if let Some(error) = self
+            .typed_bucket_heal_errors
+            .lock()
+            .unwrap()
+            .get_mut(bucket)
+            .and_then(VecDeque::pop_front)
+        {
+            return Err(error);
+        }
        if let Some(message) = self
            .bucket_heal_errors
            .lock()
@@ -563,9 +563,31 @@ fn admin_root_completes_after_unversioned_delete() {
    run_delete_case(DeleteCase::Unversioned);
 }

-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[test]
 #[serial]
-async fn admin_root_rejects_recreated_bucket_before_successor_repair() {
+fn admin_root_rejects_recreated_bucket_before_successor_repair() {
+    // Incarnation-bound repairs use a spawned owner; retain the server's stack
+    // budget for the whole scenario. block_on alone would poll that future on
+    // libtest's smaller calling-thread stack.
+    let stack_size = 8 * 1024 * 1024;
+    std::thread::Builder::new()
+        .name("heal-concurrent-delete".to_owned())
+        .stack_size(stack_size)
+        .spawn(move || {
+            let runtime = tokio::runtime::Builder::new_multi_thread()
+                .worker_threads(4)
+                .thread_stack_size(stack_size)
+                .enable_all()
+                .build()
+                .expect("concurrent delete runtime");
+            runtime.block_on(recreated_bucket_repair());
+        })
+        .expect("concurrent delete scenario thread starts")
+        .join()
+        .expect("concurrent delete scenario must not panic");
+}
+
+async fn recreated_bucket_repair() {
    let directory = tempfile::tempdir().expect("recreated bucket fixture");
    let env = TestECStoreEnv::builder().base_dir(directory.path()).build().await;
    env.make_bucket(BUCKET, false).await;
@@ -29,6 +29,8 @@ The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree

 The [Pool layout compatibility reference](pool-layout-compatibility.md) defines the topology and EC regression matrix for single-drive, single-node multi-drive, and multi-node expansion pools.

+The [immutable validation candidate reference](validation-candidates.md) describes isolated package builds and exact candidate resolution for the current auto-testing fleet.
+
 ## Naming conventions

 ### Reserved test-name substrings (migration gate)
@@ -0,0 +1,52 @@
+# Immutable validation candidates
+
+Canonical validation on the current auto-testing fleet consumes one exact
+package, source commit and candidate-bound tool build. A successful package
+build is provenance evidence, not a canonical test PASS.
+
+The scheduled Functional Chain continues to resolve successful Nightly GNU
+Build attempts from `main`. Optional manual validation can also resolve a
+validation package built without publishing nightly aliases or release assets.
+
+## Build a validation package
+
+Dispatch **RustFS Functional Chain** from the desired driver revision with
+only `validation_package_sha` set to the full 40-character RustFS source SHA.
+Keep candidate, case, helper, tools and cleanup inputs empty or false. The
+request validator rejects mixed package/fleet execution before building.
+
+The package job calls `rustfs-validation-package.yml`, checks out the requested
+source, confirms its Git SHA and builds the ordinary release binary and DEB.
+This job uses build runners and does not acquire or mutate the Heal fleet.
+The same workflow can be dispatched directly with its `source_sha` input when
+it is available on the repository's default branch.
+
+Publication requires complete R2 configuration, conditional `PutObject` with
+`If-None-Match: *`, and a public download whose SHA-256 matches the local DEB.
+Failure emits no candidate manifest. An existing object is never overwritten.
+The only object path is:
+
+```text
+artifacts/rustfs/packages/validation/runs/<run>/<attempt>/<package-sha256>/rustfs.deb
+```
+
+The schema-3 artifact `validation-candidate-<run>-<attempt>` records the
+workflow SHA independently of the exact source SHA and source ref, the
+`validation` channel, build run/attempt, immutable URL and package checksum.
+The resolver verifies the successful manual run, repository/workflow identity,
+exact artifact name and digest, archive members, schema and immutable URL.
+The ordinary scheduled chain does not enable validation package resolution.
+
+## Validate the candidate
+
+After the package run succeeds, dispatch Functional Chain with its exact
+`build_run_id` and `build_run_attempt`, a canonical `validation_case`, and an
+exact `auto_testing_sha`. Heal also requires a new tool build verified against
+that source and helper revision. Reassess source-bound contracts and native
+prerequisites when the source changes; an older tool artifact does not certify
+a new candidate.
+
+The fleet lane retains the shared environment lock, per-case evidence and
+cleanup gate. Record new results separately. A package build, native test,
+diagnostic replay, clean recovery or partial topology result cannot replace
+complete original source assertions or rewrite a retained canonical FAIL.
@@ -964,7 +964,11 @@ async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explic
        .await
        .expect_err("explicit CopyObject overwrite of a locked version should be blocked");

-    assert_eq!(err.code(), &s3s::S3ErrorCode::AccessDenied);
+    assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
+    assert_eq!(
+        err.message(),
+        Some("Invalid argument: A destination versionId requires replication authorization")
+    );
    assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), dst_object).await, source_payload);
    assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), dst_object).await, 2);
 }
@@ -3002,7 +3002,7 @@ impl DefaultObjectUsecase {
            // resolver; only the response headers, authorization and audit
            // summary are derived here, without a second KMS unwrap.
            Some(headers) => (
-                Some(headers.server_side_encryption),
+                Some(headers.server_side_encryption).filter(|_| headers.sse_customer_algorithm.is_none()),
                headers.sse_customer_algorithm,
                headers.sse_customer_key_md5,
                headers.ssekms_key_id,
@@ -492,12 +492,13 @@ impl DefaultObjectUsecase {
        })?;

        let metadata_map = info.user_defined.clone();
-        let server_side_encryption = metadata_map
-            .get("x-amz-server-side-encryption")
-            .map(|v| ServerSideEncryption::from(v.clone()));
        let sse_customer_algorithm = metadata_map
            .get("x-amz-server-side-encryption-customer-algorithm")
            .map(|v| SSECustomerAlgorithm::from(v.clone()));
+        let server_side_encryption = metadata_map
+            .get("x-amz-server-side-encryption")
+            .filter(|_| sse_customer_algorithm.is_none())
+            .map(|v| ServerSideEncryption::from(v.clone()));
        let sse_customer_key_md5 = metadata_map.get("x-amz-server-side-encryption-customer-key-md5").cloned();
        let sse_kms_key_id = metadata_map.get("x-amz-server-side-encryption-aws-kms-key-id").cloned();
        let storage_class = response_storage_class(&info, &metadata_map);
@@ -1408,7 +1408,10 @@ impl DefaultObjectUsecase {

        let output = PutObjectOutput {
            e_tag,
-            server_side_encryption: committed.effective_sse.clone(),
+            server_side_encryption: committed
+                .effective_sse
+                .clone()
+                .filter(|_| committed.sse_customer_algorithm.is_none()),
            sse_customer_algorithm: committed.sse_customer_algorithm.clone(),
            sse_customer_key_md5: committed.sse_customer_key_md5.clone(),
            ssekms_key_id: committed.effective_kms_key_id.clone(),
@@ -3688,6 +3691,12 @@ mod tests {
    async fn sse_c_put_is_accepted_on_a_bucket_with_default_encryption() {
        use md5::{Digest as _, Md5};

+        let context = crate::storage::storage_api::bootstrap_instance_ctx();
+        assert!(
+            context.object_encryption_resolver().is_some(),
+            "real SSE-C reads need the production encryption resolver"
+        );
+
        for (algorithm, kms_key_id, prefix) in [
            (ServerSideEncryption::AES256, None, "ssec-over-aes256-default"),
            (ServerSideEncryption::AWS_KMS, Some("bucket-key"), "ssec-over-kms-default"),
@@ -3709,14 +3718,69 @@ mod tests {
                .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64")))
                .sse_customer_algorithm(Some("AES256".to_string()))
                .sse_customer_key(Some(base64_simd::STANDARD.encode_to_string(customer_key)))
-                .sse_customer_key_md5(Some(key_md5))
+                .sse_customer_key_md5(Some(key_md5.clone()))
                .build()
                .expect("SSE-C PUT input must build");

-            DefaultObjectUsecase::from_global()
+            let usecase = DefaultObjectUsecase::from_global();
+            let response = usecase
                .execute_put_object(&FS::new(), build_request(input, Method::PUT))
                .await
                .unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}"));
+            assert_eq!(response.output.server_side_encryption, None);
+            assert_eq!(response.output.sse_customer_algorithm.as_deref(), Some("AES256"));
+            assert_eq!(response.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
+
+            let encoded_key = base64_simd::STANDARD.encode_to_string(customer_key);
+            let mut read_headers = HeaderMap::new();
+            read_headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
+            read_headers.insert(
+                "x-amz-server-side-encryption-customer-key",
+                HeaderValue::from_str(&encoded_key).expect("encoded customer key is a valid header"),
+            );
+            read_headers.insert(
+                "x-amz-server-side-encryption-customer-key-md5",
+                HeaderValue::from_str(&key_md5).expect("customer key MD5 is a valid header"),
+            );
+            let head_input = HeadObjectInput::builder()
+                .bucket(bucket.clone())
+                .key("ledger.csv".to_string())
+                .sse_customer_algorithm(Some("AES256".to_string()))
+                .sse_customer_key(Some(encoded_key.clone()))
+                .sse_customer_key_md5(Some(key_md5.clone()))
+                .build()
+                .expect("SSE-C HEAD input must build");
+            let mut head_request = build_request(head_input, Method::HEAD);
+            head_request.headers = read_headers.clone();
+            let head = Box::pin(usecase.execute_head_object(head_request))
+                .await
+                .expect("matching SSE-C key must authorize HEAD");
+            assert_eq!(head.output.server_side_encryption, None);
+            assert_eq!(head.output.sse_customer_algorithm.as_deref(), Some("AES256"));
+            assert_eq!(head.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
+
+            let get_input = GetObjectInput::builder()
+                .bucket(bucket.clone())
+                .key("ledger.csv".to_string())
+                .sse_customer_algorithm(Some("AES256".to_string()))
+                .sse_customer_key(Some(encoded_key))
+                .sse_customer_key_md5(Some(key_md5.clone()))
+                .build()
+                .expect("SSE-C GET input must build");
+            let mut get_request = build_request(get_input, Method::GET);
+            get_request.headers = read_headers;
+            let get = Box::pin(usecase.execute_get_object(get_request))
+                .await
+                .expect("matching SSE-C key must authorize GET");
+            assert_eq!(get.output.server_side_encryption, None);
+            assert_eq!(get.output.sse_customer_algorithm.as_deref(), Some("AES256"));
+            assert_eq!(get.output.sse_customer_key_md5.as_deref(), Some(key_md5.as_str()));
+            let mut body = get.output.body.expect("SSE-C GET must include the decrypted body");
+            let mut plaintext = Vec::new();
+            while let Some(chunk) = body.next().await {
+                plaintext.extend_from_slice(&chunk.expect("SSE-C body must decrypt completely"));
+            }
+            assert_eq!(plaintext, payload, "header classification must preserve the complete decrypted payload");

            let stored = store
                .get_object_info(&bucket, "ledger.csv", &ObjectOptions::default())
@@ -4343,15 +4343,8 @@ mod tests {
            (S3Action::DeleteObjectTaggingAction, S3Action::DeleteObjectVersionTaggingAction),
        ] {
            for version in [None, Some("null"), Some("0194e0f1-0000-7000-8000-000000000000")] {
-                let action = versioned_tagging_action(current.clone(), version);
-                assert_eq!(
-                    action,
-                    Action::S3Action(if version.is_some() {
-                        versioned.clone()
-                    } else {
-                        current.clone()
-                    })
-                );
+                let action = versioned_tagging_action(current, version);
+                assert_eq!(action, Action::S3Action(if version.is_some() { versioned } else { current }));
                let args = Args {
                    account: "writer",
                    groups: &groups,
@@ -415,7 +415,12 @@ pub(crate) fn object_lock_retention_from_metadata(metadata: &HashMap<String, Str
            date.checked_to_offset(time::UtcOffset::UTC)
                .filter(|date| (0..=9999).contains(&date.year()))
                .map(Timestamp::from)
-                .ok_or_else(|| s3_error!(InternalError, "Stored retention date is outside the supported UTC range"))
+                .ok_or_else(|| {
+                    S3Error::with_message(
+                        S3ErrorCode::InternalError,
+                        "Stored retention date is outside the supported UTC range".to_string(),
+                    )
+                })
        })
        .transpose()?;
    Ok(ObjectLockRetention { mode, retain_until_date })
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Resolve a nightly manifest from its exact GitHub build attempt and artifact."""
+"""Resolve a candidate manifest from its exact GitHub build attempt and artifact."""
 from __future__ import annotations

 import hashlib
@@ -15,6 +15,7 @@ import zipfile
 REPOSITORY = "rustfs/rustfs"
 ROOT = Path(__file__).resolve().parents[1]
 MAX_ARCHIVE = 1024 * 1024
+VALIDATION_BUILD_WORKFLOWS = {".github/workflows/rustfs-validation-package.yml", ".github/workflows/rustfs-functional-chain.yml"}


 def require(condition, message):
@@ -39,8 +40,11 @@ def validate_manifest(manifest, run):
    require(isinstance(manifest, dict), "manifest must be an object")
    common = {"schema", "source_sha", "build_run_id", "build_run_attempt", "package_url", "package_sha256"}
    version = manifest.get("schema")
-    require(type(version) is int and version in (1, 2), "unsupported candidate schema")
-    require(set(manifest) == (common if version == 1 else common | {"workflow_sha", "source_ref"}), "unexpected candidate fields")
+    require(type(version) is int and version in (1, 2, 3), "unsupported candidate schema")
+    expected_fields = common if version == 1 else common | {"workflow_sha", "source_ref"}
+    if version == 3:
+        expected_fields |= {"channel"}
+    require(set(manifest) == expected_fields, "unexpected candidate fields")
    require(positive(manifest["build_run_id"]) and positive(manifest["build_run_attempt"]), "invalid build identity")
    require((manifest["build_run_id"], manifest["build_run_attempt"]) == (run["id"], run["run_attempt"]), "candidate belongs to another build attempt")
    require(sha(manifest["source_sha"]) and sha(manifest["package_sha256"], 64), "invalid candidate hash")
@@ -49,7 +53,11 @@ def validate_manifest(manifest, run):
    else:
        require(manifest["workflow_sha"] == run["head_sha"] and sha(manifest["workflow_sha"]), "candidate workflow SHA differs from artifact provenance")
        require(isinstance(manifest["source_ref"], str) and bool(re.fullmatch(r"[A-Za-z0-9_./-]{1,200}", manifest["source_ref"])), "invalid build source ref")
-    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/{run['id']}/"
+        if version == 3:
+            require(manifest["channel"] == "validation", "validation candidate has the wrong publication channel")
+            require(manifest["source_ref"] == manifest["source_sha"], "validation candidate must bind an exact source SHA")
+    channel = "validation" if version == 3 else "nightly"
+    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/{channel}/runs/{run['id']}/"
                f"{run['run_attempt']}/{manifest['package_sha256']}/rustfs.deb")
    require(manifest["package_url"] == expected, "package URL does not bind the run, attempt and checksum")
    return manifest
@@ -72,19 +80,31 @@ def read_json_artifact(archive, artifact, run, expected_name, member, max_json=1


 def read_manifest(archive, artifact, run):
-    name = f"nightly-candidate-{run['id']}-{run['run_attempt']}"
-    return validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
+    prefix = "validation-candidate" if run.get("path") in VALIDATION_BUILD_WORKFLOWS else "nightly-candidate"
+    name = f"{prefix}-{run['id']}-{run['run_attempt']}"
+    manifest = validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
+    if prefix == "validation-candidate":
+        require(manifest["schema"] == 3 and manifest["channel"] == "validation",
+                "validation workflow requires a schema-3 validation candidate")
+    else:
+        require(manifest["schema"] != 3, "nightly workflow cannot publish a validation candidate")
+    return manifest


-def resolve(run_id, attempt):
+def resolve(run_id, attempt, *, allow_validation=False):
    require(positive(run_id) and positive(attempt), "build run and attempt are required positive integers")
    endpoint = f"repos/{REPOSITORY}/actions/runs/{run_id}"
    run = api(f"{endpoint}/attempts/{attempt}")
    require(run.get("id") == run_id and run.get("run_attempt") == attempt, "GitHub returned a different build attempt")
-    require(run.get("path") == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main", "candidate must come from nightly-gnu on main")
+    path = run.get("path")
+    validation = allow_validation and path in VALIDATION_BUILD_WORKFLOWS
+    require(path == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main" or validation,
+            "candidate must come from nightly-gnu on main or an explicitly allowed validation build")
    require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "candidate came from another repository")
-    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "nightly attempt has not completed successfully")
-    name = f"nightly-candidate-{run_id}-{attempt}"
+    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "candidate attempt has not completed successfully")
+    if validation:
+        require(run.get("event") == "workflow_dispatch", "validation candidate must be manually dispatched")
+    name = f"{'validation' if validation else 'nightly'}-candidate-{run_id}-{attempt}"
    artifacts = []
    for page in range(1, 11):
        batch = api(f"{endpoint}/artifacts?per_page=100&page={page}")["artifacts"]
@@ -76,6 +76,98 @@ class CandidateTests(unittest.TestCase):
        self.assertEqual(result["artifact_id"], 789)
        self.assertEqual(result["manifest"]["source_sha"], "b" * 40)

+    def test_validation_manifest_binds_requested_source_and_private_channel(self):
+        manifest = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"],
+                    "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")}
+        run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"}
+        payload = io.BytesIO()
+        with zipfile.ZipFile(payload, "w") as archive:
+            archive.writestr("validation-candidate-123-2.json", json.dumps(manifest))
+        raw = payload.getvalue()
+        artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False,
+                    "size_in_bytes": len(raw), "digest": "sha256:" + hashlib.sha256(raw).hexdigest(),
+                    "workflow_run": {"id": 123, "head_sha": "a" * 40}}
+        result = candidate.read_manifest(raw, artifact, run)
+        self.assertEqual(result["channel"], "validation")
+        self.assertIn("/packages/validation/runs/123/2/", result["package_url"])
+
+    def test_validation_resolver_requires_manual_workflow_dispatch(self):
+        payload, artifact = self.archive()
+        run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40,
+               "path": ".github/workflows/rustfs-validation-package.yml", "head_branch": "main",
+               "head_repository": {"full_name": "rustfs/rustfs"}, "event": "schedule",
+               "status": "completed", "conclusion": "success", "run_started_at": "2026-10-12T00:00:00Z"}
+        with mock.patch.object(candidate, "api", side_effect=[run]):
+            with self.assertRaisesRegex(ValueError, "manually dispatched"):
+                candidate.resolve(123, 2, allow_validation=True)
+
+    def test_validation_workflow_rejects_legacy_manifest_schema(self):
+        manifest = {**self.manifest}
+        payload_buffer = io.BytesIO()
+        with zipfile.ZipFile(payload_buffer, "w") as archive:
+            archive.writestr("validation-candidate-123-2.json", json.dumps(manifest))
+        payload = payload_buffer.getvalue()
+        artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False,
+                    "size_in_bytes": len(payload), "digest": "sha256:" + hashlib.sha256(payload).hexdigest(),
+                    "workflow_run": {"id": 123, "head_sha": "a" * 40}}
+        run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"}
+        with self.assertRaisesRegex(ValueError, "schema-3"):
+            candidate.read_manifest(payload, artifact, run)
+
+    def test_private_candidate_resolution_is_explicit_and_attempt_bound(self):
+        self.manifest.update(schema=3, channel="validation", source_ref=self.manifest["source_sha"],
+                             package_url=self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/"))
+        payload, artifact = self.archive(["validation-candidate-123-2.json"])
+        artifact["name"] = "validation-candidate-123-2"
+        base = {**self.run, "head_branch": "cxymds/validation-runner", "head_repository": {"full_name": "rustfs/rustfs"},
+                "event": "workflow_dispatch", "status": "completed", "conclusion": "success",
+                "run_started_at": "2026-10-12T00:00:00Z"}
+        for path in candidate.VALIDATION_BUILD_WORKFLOWS:
+            run = {**base, "path": path}
+            with self.subTest(path=path), mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]):
+                resolved = candidate.resolve(123, 2, allow_validation=True)
+                self.assertEqual(resolved["manifest"]["source_sha"], "b" * 40)
+                self.assertEqual(resolved["workflow_sha"], "a" * 40)
+            with mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError):
+                candidate.resolve(123, 2)
+        for field, value in (("id", 124), ("run_attempt", 1), ("event", "push"), ("status", "in_progress"),
+                             ("conclusion", "failure"), ("path", ".github/workflows/other.yml"),
+                             ("head_repository", {"full_name": "untrusted/rustfs"})):
+            run = {**base, "path": ".github/workflows/rustfs-functional-chain.yml", field: value}
+            with self.subTest(field=field), mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError):
+                candidate.resolve(123, 2, allow_validation=True)
+
+    def test_private_manifest_rejects_alias_channel_source_and_schema_substitution(self):
+        valid = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"],
+                 "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")}
+        for key, value in (("channel", "nightly"), ("source_ref", "main"), ("source_sha", "e" * 40),
+                           ("package_url", self.manifest["package_url"]), ("workflow_sha", "e" * 40),
+                           ("build_run_attempt", 3), ("schema", True)):
+            with self.subTest(key=key), self.assertRaises(ValueError):
+                candidate.validate_manifest({**valid, key: value}, self.run)
+
+    def test_package_only_mode_rejects_conflicting_fleet_and_tool_requests(self):
+        from check_test_wiring import yaml_block
+        workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml"
+        job = yaml_block(workflow.read_text().splitlines(), "validation-package-request", 2)
+        begin = job.index("        run: |") + 1
+        script = "\n".join(line[10:] for line in job[begin:] if not line.strip() or line.startswith("          "))
+        import os
+        inputs = {"validation_package_sha": "a" * 40}
+        env = {**os.environ, "PACKAGE_INPUTS": json.dumps(inputs)}
+        result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        for field in ("build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha", "validation_tools_only",
+                      "validation_native_tools", "validation_shared_tools", "validation_manager_tools",
+                      "validation_cleanup_run_id", "validation_tools_run_id"):
+            with self.subTest(field=field):
+                env["PACKAGE_INPUTS"] = json.dumps({**inputs, field: "not-empty"})
+                result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True)
+                self.assertNotEqual(result.returncode, 0)
+        for value in ("main", "a" * 39, "a" * 40 + "\nFORGED=value"):
+            env["PACKAGE_INPUTS"] = json.dumps({"validation_package_sha": value})
+            self.assertNotEqual(subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True).returncode, 0)
+

 class EvidenceTests(unittest.TestCase):
    def setUp(self):
@@ -221,5 +221,82 @@ SH
        self.assertNotIn("          overwrite: true", upload)


+class ValidationPackageTests(unittest.TestCase):
+    def setUp(self):
+        # Exercise the real upload shell with the same conditional object-store
+        # fixture used by the nightly publication tests.
+        self.fixture = NightlyCandidateTests("test_success_binds_actual_package_checkout_and_attempt")
+        self.fixture.setUp()
+        self.addCleanup(self.fixture.temp.cleanup)
+        self.workflow = ROOT / ".github/workflows/rustfs-validation-package.yml"
+        job = yaml_block(self.workflow.read_text().splitlines(), "build", 2)
+        start = job.index("      - name: Upload validation package to immutable R2 path")
+        end = job.index("      - name: Upload validation candidate manifest")
+        upload = job[start:end]
+        begin = upload.index("        run: |") + 1
+        self.fixture.shell = "\n".join(line[10:] for line in upload[begin:] if not line.strip() or line.startswith("          "))
+        self.fixture.shims.write_text(self.fixture.shims.read_text().replace("/packages/nightly/runs/*", "/packages/validation/runs/*"))
+        self.fixture.env.update(PACKAGE_FILE=self.fixture.package.name, VALIDATION_SOURCE_SHA=self.fixture.sha,
+                                VALIDATION_SOURCE_REF=self.fixture.sha, GITHUB_SHA="f" * 40)
+
+    def test_success_uploads_only_one_immutable_object_and_exact_source(self):
+        result = self.fixture.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        manifest = self.fixture.manifest()
+        self.assertEqual(manifest["schema"], 3)
+        self.assertEqual(manifest["channel"], "validation")
+        self.assertEqual(manifest["source_sha"], self.fixture.sha)
+        self.assertEqual(manifest["source_ref"], self.fixture.sha)
+        self.assertEqual(manifest["workflow_sha"], "f" * 40)
+        validate_manifest(manifest, {"id": 12345, "run_attempt": 1, "head_sha": "f" * 40})
+        objects = [path for path in self.fixture.store.rglob("*") if path.is_file()]
+        self.assertEqual(len(objects), 1)
+        self.assertEqual(objects[0].read_bytes(), self.fixture.package.read_bytes())
+        self.assertEqual(str(objects[0].relative_to(self.fixture.store)),
+                         f"artifacts/rustfs/packages/validation/runs/12345/1/{self.fixture.digest}/rustfs.deb")
+        self.assertNotIn("s3 cp", (self.fixture.root / "aws.log").read_text())
+
+    def test_failure_or_missing_credentials_cannot_certify_a_candidate(self):
+        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download",
+                                      "bootstrap-failure", "install-failure", "broken-install")):
+            with self.subTest(mode=mode):
+                result = self.fixture.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
+                self.assertNotEqual(result.returncode, 0)
+                self.assertFalse(self.fixture.output.exists())
+                self.assertEqual(list(self.fixture.root.glob("validation-candidate-*.json")), [])
+        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
+            with self.subTest(key=key):
+                self.assertNotEqual(self.fixture.run_publish(**{key: ""}).returncode, 0)
+                self.assertFalse(self.fixture.output.exists())
+
+    def test_stale_source_and_duplicate_objects_are_rejected(self):
+        result = self.fixture.run_publish(VALIDATION_SOURCE_SHA="e" * 40)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertFalse(self.fixture.store.exists())
+        result = self.fixture.run_publish()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        manifest = self.fixture.manifest()
+        key = manifest["package_url"].removeprefix("https://dl.rustfs.com/")
+        stored = self.fixture.store / key
+        stored.write_bytes(b"preexisting conflicting object")
+        (self.fixture.root / "validation-candidate-12345-1.json").unlink()
+        result = self.fixture.run_publish()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
+        self.assertFalse(self.fixture.output.exists())
+
+    def test_exact_source_and_workflow_identity_are_preserved_separately(self):
+        lines = self.workflow.read_text().splitlines()
+        build = "\n".join(yaml_block(lines, "build", 2))
+        source = "\n".join(yaml_block(lines, "resolve-source", 2))
+        self.assertIn('[[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]', source)
+        self.assertIn('test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"', source)
+        self.assertIn("ref: ${{ needs.resolve-source.outputs.source_sha }}", build)
+        self.assertIn('--arg source_sha "${VALIDATION_SOURCE_SHA}"', build)
+        self.assertIn('--arg workflow_sha "${GITHUB_SHA}"', build)
+        self.assertNotIn("PF_TESTING_GH_TOKEN", build)
+        self.assertNotIn("publish_nightly_assets", build)
+
+
 if __name__ == "__main__":
    unittest.main()
Author	SHA1	Message	Date
cxymds	16cf19817e	ci(validation): dispatch isolated packages through the driver	2026-10-12 12:29:11 +08:00
cxymds	32fd1e704a	test(validation): verify isolated package publication failures	2026-10-12 12:22:53 +08:00
cxymds	ac197ea655	test(validation): enforce private candidate manifest schema	2026-10-12 12:18:15 +08:00
cxymds	e18ed73942	ci(validation): publish isolated immutable candidate packages	2026-10-12 12:06:59 +08:00
cxymds	3c131f2b2e	fix(heal): preserve typed errors and SSE-C response headers	2026-10-12 11:36:29 +08:00
cxymds	c5f4b5a4f7	fix(heal): preserve typed bucket errors for bounded retries	2026-10-12 10:45:27 +08:00
cxymds	03fb3d78a0	ci(validation): consume pinned manager native prerequisites	2026-10-11 11:55:36 +08:00
cxymds	b8126e50f7	test(ci): expose manager regressions through tools-only dispatch	2026-10-11 11:21:35 +08:00
cxymds	59fb72aa82	test(ci): add isolated manager native validation builds	2026-10-11 11:20:46 +08:00
cxymds	9de01c9170	docs(ci): describe supported validation recovery	2026-10-11 11:07:20 +08:00
cxymds	2ec26fa4b9	feat(ci): expose read-only validation fleet inventory	2026-10-11 10:42:48 +08:00
cxymds	f4507178a3	test(ci): bind C10 execution to the verified shared codec	2026-10-11 10:09:26 +08:00
cxymds	9b4cc40abd	test(validation): build optional C10 fixture codec on hosted Linux	2026-10-11 09:44:09 +08:00
cxymds	b14ff4eeb1	test(validation): bind C07 to pinned native authority evidence	2026-10-11 09:27:59 +08:00
cxymds	ae9b5cbabf	Run validation tool resolution on hosted Linux without acquiring the fleet lock	2026-10-11 07:49:26 +08:00
cxymds	6c78b0d996	test(validation): route native tools through the existing driver	2026-10-11 06:06:17 +08:00
cxymds	0066e34764	test(validation): add optional native Heal regression tooling	2026-10-11 06:03:43 +08:00
cxymds	fbddcc2965	ci: audit retained validation environments before finalization	2026-10-10 23:28:39 +08:00
cxymds	b1793b55a0	test(validation): verify Heal tools against released shard captures	2026-10-10 21:11:43 +08:00
cxymds	835bd3f52d	test(validation): verify pinned Heal tools before fleet execution	2026-10-10 20:46:40 +08:00
cxymds	cb32d55878	test(validation): expose tool artifacts through the existing manual lane	2026-10-10 20:35:14 +08:00
cxymds	864e58592c	test(validation): build the candidate-bound Heal inspector artifact	2026-10-10 20:32:34 +08:00
cxymds	3016209330	test(validation): provide the Python client for Heal cases	2026-10-10 20:30:10 +08:00
cxymds	64e0eeba91	Install validation client for replication cases	2026-10-10 18:52:42 +08:00
cxymds	35fa7488a5	Prepare Python client for pool validation cases	2026-10-10 18:41:48 +08:00
cxymds	1b77846f8e	Install pinned ILM client for canonical validation lane	2026-10-10 18:26:49 +08:00
cxymds	b0c861841e	test(validation): add pinned auto-testing manual lane	2026-10-10 17:34:37 +08:00
Chris	072b57b375	fix: auto-repair breaking changes from `446479791` (#8451 ) * fix: repair s3s footprint ratchet violation, clippy warnings, and stack overflow - Replace s3_error! macro with S3Error::with_message in ecfs.rs to keep s3s footprint ratchet at baseline (1586) after retention validation refactor - Remove redundant clone on Copy type in list_objects.rs (clippy::redundant_clone) - Remove clone_on_copy calls in access.rs test (clippy::clone_on_copy) - Use manual thread with 8MB stack for heal_concurrent_delete_test to fix stack overflow on macOS (consistent with sibling tests in same file) * test: align CI assertions with version authorization rules	2026-10-09 17:01:48 -07:00