fix(ci): verify complete functional chain evidence (#7690)

* fix(ci): bind nightly lanes to one resolved source

* fix(ci): verify complete functional chain evidence

.config/functional-script-revision.txt

@@ -0,0 +1 @@
+8bb710ef325d4b3f891e70deb671348a1f1d850a

.config/make/tests.mak

@@ -55,6 +55,8 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh

.github/workflows/functional-chain-health.yml

@@ -0,0 +1,30 @@
+name: Functional chain health
+on:
+  schedule:
+    - cron: '23 * * * *'
+  workflow_dispatch:
+permissions:
+  contents: read
+  actions: read
+concurrency:
+  group: functional-chain-health
+  cancel-in-progress: false
+jobs:
+  collect:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Collect and publish verified chain health
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: python3 scripts/functional_chain_health.py --publish --output "${RUNNER_TEMP}/chain-health.json"
+      - name: Retain health observation
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-health-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-health.json
+          if-no-files-found: error

.github/workflows/rustfs-functional-chain.yml

@@ -12,50 +12,165 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Functional chain driver: runs the ten functional suites in a fixed order
-# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
-# when its tests fail.
-#
-# Each suite workflow can still be dispatched standalone (workflow_dispatch);
-# only chain-triggered runs forward to the next suite via repository_dispatch,
-# so a standalone run never drags the rest of the chain behind it.
-#
-# Why not workflow_run chaining: GitHub does not guarantee delivery of
-# workflow_run events (they are fire-and-forget), and the head-SHA filter made
-# newly added suites (storage) unable to trigger at all. Explicit
-# repository_dispatch handoffs are verifiable and re-drivable.
-
+# Reusable workflows run from this driver's commit in one Actions run. Each
+# suite still runs after an earlier suite fails; the final job requires all ten.
 name: RustFS Functional Chain
 
 on:
   workflow_dispatch:
+    inputs:
+      build_run_id:
+        description: Successful nightly build run on main
+        type: string
+        required: true
+      build_run_attempt:
+        description: Exact successful build attempt
+        type: string
+        required: true
   workflow_run:
-    # Entry point: start the chain after the nightly build completes. The
-    # build's own conclusion does not gate the chain; each suite reports its
-    # own result to rustfs/backlog and the dashboard.
     workflows: ["Nightly GNU Build"]
     types: [completed]
 
 permissions:
   contents: read
+  actions: read
+
+concurrency:
+  group: rustfs-functional-chain-runs
+  cancel-in-progress: false
 
 jobs:
-  start-chain:
-    name: Start functional chain (upgrade first)
+  prepare:
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+    outputs:
+      manifest: ${{ steps.candidate.outputs.manifest }}
     steps:
-      - name: Dispatch first suite (upgrade)
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Resolve published candidate
+        id: candidate
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-upgrade' \
-            -F 'client_payload[from_suite]=nightly-build'
+          GH_TOKEN: ${{ github.token }}
+          BUILD_RUN_ID: ${{ inputs.build_run_id }}
+          BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
+          CHAIN_OUTPUT: ${{ runner.temp }}/chain-candidate.json
+        run: python3 scripts/resolve_functional_candidate.py
+      - name: Retain candidate identity
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-candidate-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-candidate.json
+          if-no-files-found: error
+
+  upgrade:
+    needs: [prepare]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-upgrade-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  s3:
+    needs: [prepare, upgrade]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-s3-compat-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  kms:
+    needs: [prepare, s3]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-kms-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  tier:
+    needs: [prepare, kms]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-tier-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  storage:
+    needs: [prepare, tier]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-storage-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  heal:
+    needs: [prepare, storage]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-heal-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  pool:
+    needs: [prepare, heal]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-pool-expand-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  security:
+    needs: [prepare, pool]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-security-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  replication:
+    needs: [prepare, security]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-replication-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  performance:
+    needs: [prepare, replication]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    uses: ./.github/workflows/rustfs-performance-test.yml
+    with:
+      chain_manifest: ${{ needs.prepare.outputs.manifest }}
+    secrets: inherit
+
+  complete-chain:
+    needs: [prepare, upgrade, s3, kms, tier, storage, heal, pool, security, replication, performance]
+    if: ${{ always() && needs.prepare.result == 'success' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Download suite evidence
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+        with:
+          pattern: functional-chain-*-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-evidence
+          merge-multiple: true
+      - name: Verify every required suite
+        env:
+          CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
+          CHAIN_NEEDS: ${{ toJSON(needs) }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py aggregate
+          --directory "${RUNNER_TEMP}/chain-evidence"
+          --output "${RUNNER_TEMP}/chain-complete.json"
+      - name: Upload complete-chain evidence
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-complete-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-complete.json
+          if-no-files-found: error

.github/workflows/rustfs-heal-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Heal Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       package_url:
@@ -57,8 +63,13 @@ jobs:
     timeout-minutes: 480
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize functional evidence
         id: evidence
         run: |
@@ -74,25 +85,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -152,6 +159,7 @@ jobs:
             --log-file "${LOG_FILE}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -426,3 +434,24 @@ jobs:
           echo "RustFS heal test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite heal
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
+          --output "${RUNNER_TEMP}/chain-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/heal.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
+          if-no-files-found: error

.github/workflows/rustfs-kms-test.yml

@@ -1,6 +1,12 @@
 name: RustFS KMS Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -50,7 +56,7 @@ jobs:
   kms-test:
     runs-on: smoke-testing
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -71,25 +77,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -156,6 +158,7 @@ jobs:
           ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -375,3 +378,24 @@ jobs:
         run: |
           echo "RustFS KMS suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite kms
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/kms.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
+          if-no-files-found: error

.github/workflows/rustfs-performance-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Performance Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       package_url:
@@ -90,8 +96,13 @@ jobs:
       RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize functional evidence
         id: evidence
         run: |
@@ -108,25 +119,21 @@ jobs:
             printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -192,6 +199,7 @@ jobs:
           } > "${VERSION_FILE}"
 
       - name: Upload report to dashboard (reports/YYYY-MM-DD.md)
+        id: chain_report
         if: ${{ steps.benchmark.conclusion == 'success' }}
         env:
           GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
@@ -325,3 +333,24 @@ jobs:
           echo "RustFS performance test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.benchmark.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite performance
+          --report "${RUSTFS_RESULT_DIR}/summary.tsv"
+          --output "${RUNNER_TEMP}/chain-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/performance.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
+          if-no-files-found: error

.github/workflows/rustfs-pool-expand-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Pool Expansion Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -77,7 +83,7 @@ jobs:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -85,27 +91,29 @@ jobs:
       RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
       RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
     steps:
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Initialize pool test artifacts
+        id: evidence
         run: |
           set -euo pipefail
           ARTIFACT_DIR="${RUNNER_TEMP}/rustfs-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
@@ -363,6 +371,7 @@ jobs:
           fi
 
       - name: Generate report
+        id: chain_report
         if: always()
         run: |
           set -euo pipefail
@@ -707,3 +716,24 @@ jobs:
           echo "RustFS pool expansion test failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded log artifact for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.pool_test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite pool
+          --report "${POOL_ARTIFACT_DIR}/pool-steps.md"
+          --output "${RUNNER_TEMP}/chain-pool-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/pool.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
+          if-no-files-found: error

.github/workflows/rustfs-replication-test.yml

@@ -15,6 +15,12 @@
 name: RustFS Replication Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -62,7 +68,7 @@ jobs:
   replication-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -83,25 +89,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -153,6 +155,7 @@ jobs:
           ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -380,3 +383,24 @@ jobs:
           echo "RustFS replication suite failed"
           echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite replication
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/replication.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
+          if-no-files-found: error

.github/workflows/rustfs-s3-compat-test.yml

@@ -1,6 +1,12 @@
 name: RustFS S3 Compatibility Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -38,7 +44,7 @@ jobs:
   s3-compat-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -59,25 +65,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -122,6 +124,7 @@ jobs:
           ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -352,3 +355,24 @@ jobs:
         run: |
           echo "RustFS S3 compatibility suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite s3
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-s3-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/s3.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
+          if-no-files-found: error

.github/workflows/rustfs-security-test.yml

@@ -15,6 +15,12 @@
 name: RustFS Security Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -75,7 +81,7 @@ jobs:
   security-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       # Checkout the repository into its own subdirectory. Checking out at
       # the workspace root would wipe the auto-testing clone above (that is
@@ -95,25 +101,21 @@ jobs:
           mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
           printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -355,3 +357,24 @@ jobs:
           echo "RustFS security test failed"
           echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and logs for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite security
+          --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
+          --output "${RUNNER_TEMP}/chain-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/security.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
+          if-no-files-found: error

.github/workflows/rustfs-storage-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Storage Engine Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -47,7 +53,7 @@ jobs:
   storage-test:
     runs-on: smoke-testing
     timeout-minutes: 360
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -68,25 +74,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -137,6 +139,7 @@ jobs:
           ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -367,3 +370,24 @@ jobs:
         run: |
           echo "RustFS storage engine suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite storage
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/storage.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
+          if-no-files-found: error

.github/workflows/rustfs-tier-test.yml

@@ -1,6 +1,12 @@
 name: RustFS Tier Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       rustfs_version:
@@ -62,8 +68,13 @@ jobs:
   tier-test:
     runs-on: smoke-testing
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
+      - name: Checkout chain tooling
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
       - name: Initialize run evidence directory
         id: evidence
         run: |
@@ -76,25 +87,21 @@ jobs:
           test -d "${TIER_ARTIFACTS_DIR}"
           test ! -L "${TIER_ARTIFACTS_DIR}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Prepare pinned RustFS CLI
         id: rc
@@ -299,6 +306,7 @@ jobs:
           mv "${TMP_FILE}" "${RESULT_FILE}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -619,3 +627,24 @@ jobs:
         run: |
           echo "RustFS tier suite failed"
           echo "See the uploaded report and log artifacts for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite tier
+          --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"
+          --output "${RUNNER_TEMP}/chain-tier-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/tier.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
+          if-no-files-found: error

.github/workflows/rustfs-upgrade-test.yml

@@ -15,6 +15,12 @@
 name: RustFS Upgrade Test
 
 on:
+  workflow_call:
+    inputs:
+      chain_manifest:
+        description: Verified candidate and chain attempt from the chain driver
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       from_version:
@@ -80,7 +86,7 @@ jobs:
   upgrade-test:
     runs-on: smoke-testing
     timeout-minutes: 420
-    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -101,25 +107,21 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
-      # auto-testing is private: clone it with the dedicated PF token (not
-      # GITHUB_TOKEN) and retry transient GitHub/network failures.
-      - name: Checkout auto-testing scripts (with retry)
+      - name: Bind functional candidate
+        id: chain
+        if: ${{ inputs.chain_manifest != '' }}
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          rm -rf auto-testing
-          for attempt in 1 2 3 4 5; do
-            if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
-              echo "auto-testing cloned (attempt ${attempt})"
-              exit 0
-            fi
-            rm -rf auto-testing
-            echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
-            sleep $((attempt * 15))
-          done
-          echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
-          exit 1
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+        run: python3 scripts/functional_chain_evidence.py consume
+
+      - name: Checkout auto-testing scripts
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          repository: rustfs/auto-testing
+          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          path: auto-testing
+          persist-credentials: false
 
       - name: Show environment
         run: |
@@ -218,6 +220,7 @@ jobs:
           ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
 
       - name: Generate report
+        id: chain_report
         if: ${{ always() && steps.evidence.outcome == 'success' }}
         run: |
           set -euo pipefail
@@ -454,3 +457,24 @@ jobs:
           echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
           echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
           echo "See the uploaded report and logs for details."
+
+      - name: Record chain evidence
+        id: chain_record
+        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}
+        env:
+          CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
+          CHAIN_JOB_STATUS: ${{ job.status }}
+          CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}
+          CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: >-
+          python3 scripts/functional_chain_evidence.py record --suite upgrade
+          --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
+          --output "${RUNNER_TEMP}/chain-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/upgrade.json"
+
+      - name: Upload chain evidence
+        if: ${{ always() && steps.chain_record.outcome == 'success' }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
+          if-no-files-found: error

docs/testing/functional-chain.md

@@ -0,0 +1,25 @@
+# Functional chain evidence
+
+The functional-chain driver calls ten reusable suite workflows sequentially in one Actions run. A suite failure does not suppress later suites. `complete-chain` requires every suite job and its evidence artifact to succeed. The root uses its own concurrency group so an active chain can finish; the suites retain the shared VM lock used by standalone tests.
+
+## Candidate identity
+
+The driver resolves one successful `nightly-gnu.yml` attempt from main. Automatic runs select the triggering scheduled attempt. Manual runs require an explicit build run ID and attempt. Resolution verifies the artifact name, run association, ZIP size and digest, single JSON member, and immutable package URL containing run, attempt and package checksum.
+
+Schema 2 distinguishes the workflow SHA from the actual build SHA and source ref. A release build triggered by a main workflow remains release evidence. Legacy schema 1 is accepted only when its source SHA equals the producer workflow SHA. The consumer does not resolve the source branch again, so branch movement cannot silently select a different package.
+
+All suite installers receive the same package URL and checksum (`PACKAGE_SHA256`, or `TO_SHA256` for upgrade). Their existing package checks run before installation. The private test repository is checked out at `.config/functional-script-revision.txt`; change that pin only to a reviewed, merged revision.
+
+## Completion and reruns
+
+Every suite records its chain run/attempt, workflow SHA, private pin, candidate identity, report hash and execution counts. A missing/empty report, zero passing executions, failed or unfinished case, failed test/report step, cancelled job, or mismatched private checkout invalidates the evidence. Uploading the suite proof requires successful proof generation.
+
+The final job checks all ten expected suite results and all ten proof files against the same envelope. It emits `functional-chain-complete-<run>-<attempt>` only after those checks pass. Failed partial reruns cannot combine an old successful lane's proof with a new attempt. Use **Re-run all jobs** for a new complete acceptance attempt.
+
+## Health publication
+
+`functional-chain-health.yml` inspects recent main-branch chain runs hourly. It validates complete evidence against the exact producer artifact again and checks the private pin from the chain's workflow commit. Its JSON separates the latest attempt from the last complete success for each source. A later failure preserves historical success without turning the new failure green.
+
+Evidence expires 36 hours after the producer attempt started. The dashboard also treats collection older than two hours as stale. Workflow enablement, owner, source identities, evidence version and expiry are visible. An invalid legacy chain-driver success is not complete evidence, and release success cannot authorize moving main PR coverage to nightly.
+
+The dashboard's `src/chain-health.json` must exist before enabling publication. Updates use the read blob SHA and reject unsupported, null or newer existing state. The companion dashboard view is required to display this data. Until a real same-candidate chain completes, these workflow and script checks do not satisfy backlog #2481 or unblock coverage migration in #2483.

scripts/functional_chain_evidence.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Bind functional-suite evidence to one candidate and one chain attempt."""
+from __future__ import annotations
+
+import argparse
+import csv
+from datetime import datetime, timezone
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+
+from resolve_functional_candidate import ROOT, positive, require, sha, validate_manifest
+
+SUITES = ("upgrade", "s3", "kms", "tier", "storage", "heal", "pool", "security", "replication", "performance")
+MAX_REPORT = 8 * 1024 * 1024
+
+
+def current_chain():
+    chain = json.loads(os.environ["CHAIN_MANIFEST"])
+    require(isinstance(chain, dict) and set(chain) == {"schema", "run_id", "attempt", "workflow_sha", "testing_sha", "candidate"}, "invalid chain envelope")
+    require(type(chain["schema"]) is int and chain["schema"] == 1, "unsupported chain schema")
+    require(positive(chain["run_id"]) and positive(chain["attempt"]), "invalid chain run identity")
+    require(chain["run_id"] == int(os.environ["GITHUB_RUN_ID"]) and chain["attempt"] == int(os.environ["GITHUB_RUN_ATTEMPT"]), "chain belongs to another run attempt; rerun all jobs")
+    require(sha(chain["workflow_sha"]) and chain["workflow_sha"] == os.environ["GITHUB_SHA"], "chain workflow source mismatch")
+    head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
+    require(head == chain["workflow_sha"], "lane checkout differs from chain workflow source")
+    require(chain["testing_sha"] == (ROOT / ".config/functional-script-revision.txt").read_text().strip() and sha(chain["testing_sha"]), "private script pin differs from chain")
+    candidate = chain["candidate"]
+    require(isinstance(candidate, dict) and set(candidate) == {"manifest", "artifact_id", "artifact_digest", "workflow_sha", "workflow_ref", "build_started_at"}, "invalid candidate envelope")
+    manifest = candidate["manifest"]
+    require(positive(candidate["artifact_id"]) and isinstance(candidate["artifact_digest"], str) and bool(re.fullmatch(r"sha256:[0-9a-f]{64}", candidate["artifact_digest"])), "invalid candidate artifact identity")
+    require(sha(candidate["workflow_sha"]) and candidate["workflow_ref"] == "main", "candidate workflow source is invalid")
+    validate_manifest(manifest, {"id": manifest["build_run_id"], "run_attempt": manifest["build_run_attempt"], "head_sha": candidate["workflow_sha"]})
+    return chain
+
+
+def consume(chain):
+    manifest = chain["candidate"]["manifest"]
+    with open(os.environ["GITHUB_ENV"], "a") as output:
+        # Every pinned installer already verifies these hashes before dpkg.
+        for key, value in (("RUSTFS_NIGHTLY_PACKAGE_URL", manifest["package_url"]),
+                           ("PACKAGE_SHA256", manifest["package_sha256"]), ("TO_SHA256", manifest["package_sha256"])):
+            output.write(key + "=" + value + "\n")
+    with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+        output.write("testing_sha=" + chain["testing_sha"] + "\n")
+
+
+def report_counts(text, performance=False):
+    counts = {"PASS": 0, "FAIL": 0, "SKIP": 0, "UNSUPPORTED": 0, "RUNNING": 0}
+    if performance:
+        rows = list(csv.DictReader(io.StringIO(text), delimiter="\t"))
+        seen = set()
+        for row in rows:
+            key = (row.get("method"), row.get("size"))
+            require(key[0] in ("get", "put", "mixed") and key[1] and key not in seen, "invalid or duplicate performance round")
+            seen.add(key)
+            fields = ("throughput", "obj_per_s", "req_avg", "req_p50")
+            if key[0] != "mixed":
+                fields += ("req_p90", "req_p99")
+            require(all(isinstance(row.get(field), str) and row[field].strip() for field in fields), "missing benchmark metrics")
+        counts["PASS"] = len(rows)
+        return counts
+    column = None
+    for line in text.splitlines():
+        if not line.startswith("|"):
+            column = None
+            continue
+        cells = [cell.strip().strip("*") for cell in line.strip().strip("|").split("|")]
+        for label in ("Status", "Result"):
+            if cells[0] in ("ID", "Case", "Topology", "Step") and label in cells:
+                column = cells.index(label)
+                break
+        else:
+            if column is not None:
+                require(len(cells) > column, "incomplete report row")
+                status = cells[column]
+                if re.fullmatch(r":?-+:?", status):
+                    continue
+                require(status in counts, "unknown case result")
+                counts[status] += 1
+    return counts
+
+
+def record(chain, suite, report, output):
+    require(suite in SUITES, "unknown suite")
+    result = {"schema": 1, "suite": suite, "chain": chain, "valid": False, "counts": {}, "report_sha256": None}
+    error = None
+    try:
+        private_head = subprocess.check_output(["git", "-C", "auto-testing", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
+        require(private_head == chain["testing_sha"], "suite used a different private script revision")
+        require(report.is_file() and 0 < report.stat().st_size <= MAX_REPORT, "missing, empty or oversized report")
+        data = report.read_bytes()
+        result["report_sha256"] = hashlib.sha256(data).hexdigest()
+        result["counts"] = report_counts(data.decode("utf-8"), suite == "performance")
+        require(result["counts"]["PASS"] > 0 and not result["counts"]["FAIL"] and not result["counts"]["RUNNING"], "no passing executions or incomplete/failed cases")
+        require(all(os.environ[key] == "success" for key in ("CHAIN_JOB_STATUS", "CHAIN_TEST_OUTCOME", "CHAIN_REPORT_OUTCOME")), "suite, report or job did not succeed")
+        result["valid"] = True
+    except (OSError, ValueError, subprocess.SubprocessError) as exc:
+        error = exc
+    output.parent.mkdir(parents=True, exist_ok=False)
+    output.write_text(json.dumps(result, sort_keys=True) + "\n")
+    if error:
+        raise error
+
+
+def aggregate(chain, directory, needs):
+    require(set(needs) == set(SUITES), "aggregate is missing a required lane")
+    require(all(value.get("result") == "success" for value in needs.values()), "a required suite did not succeed")
+    require({path.name for path in directory.iterdir()} == {suite + ".json" for suite in SUITES}, "missing or unexpected suite evidence")
+    records = [json.loads((directory / (suite + ".json")).read_text()) for suite in SUITES]
+    validate_records(chain, records)
+    return {"schema": 1, "chain": chain, "suites": records, "complete": True, "completed_at": datetime.now(timezone.utc).isoformat()}
+
+
+def validate_records(chain, records):
+    require(isinstance(records, list) and len(records) == len(SUITES), "missing suite evidence")
+    require([record.get("suite") for record in records] == list(SUITES), "missing, duplicate or reordered suite evidence")
+    for suite, result in zip(SUITES, records):
+        require(type(result.get("schema")) is int and result["schema"] == 1 and result.get("suite") == suite and result.get("chain") == chain, "suite evidence identity mismatch")
+        require(result.get("valid") is True and sha(result.get("report_sha256"), 64), "suite evidence is invalid")
+        counts = result.get("counts", {})
+        require(set(counts) == {"PASS", "FAIL", "SKIP", "UNSUPPORTED", "RUNNING"}, "missing suite counts")
+        require(all(type(value) is int and value >= 0 for value in counts.values()) and counts["PASS"] > 0 and counts["FAIL"] == counts["RUNNING"] == 0, "suite has no complete passing evidence")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("mode", choices=("consume", "record", "aggregate"))
+    parser.add_argument("--suite", choices=SUITES)
+    parser.add_argument("--report", type=Path)
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--directory", type=Path)
+    args = parser.parse_args()
+    chain = current_chain()
+    if args.mode == "consume":
+        consume(chain)
+    elif args.mode == "record":
+        record(chain, args.suite, args.report, args.output)
+    else:
+        needs = json.loads(os.environ["CHAIN_NEEDS"])
+        needs.pop("prepare", None)
+        result = aggregate(chain, args.directory, needs)
+        args.output.write_text(json.dumps(result, sort_keys=True) + "\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/functional_chain_health.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Report the latest chain attempt separately from verified complete successes."""
+from __future__ import annotations
+
+import argparse
+import base64
+from datetime import datetime, timedelta, timezone
+import json
+from pathlib import Path
+import subprocess
+
+from functional_chain_evidence import validate_records
+from resolve_functional_candidate import REPOSITORY, api, read_json_artifact, require, resolve, sha
+
+WORKFLOW = "rustfs-functional-chain.yml"
+MAX_AGE = timedelta(hours=36)
+
+
+def timestamp(value):
+    require(isinstance(value, str), "missing evidence timestamp")
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    require(parsed.tzinfo is not None, "evidence timestamp has no timezone")
+    return parsed
+
+
+def validate_summary(summary, run):
+    require(isinstance(summary, dict) and type(summary.get("schema")) is int and summary["schema"] == 1 and summary.get("complete") is True, "unsupported complete-chain evidence")
+    chain = summary["chain"]
+    require(chain["run_id"] == run["id"] and chain["attempt"] == run["run_attempt"] and chain["workflow_sha"] == run["head_sha"], "complete evidence belongs to another run attempt")
+    require(sha(chain["testing_sha"]), "missing test-script pin")
+    validate_records(chain, summary["suites"])
+    candidate = chain["candidate"]
+    manifest = candidate["manifest"]
+    require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed")
+    config = api(f"repos/{REPOSITORY}/contents/.config/functional-script-revision.txt?ref={run['head_sha']}")
+    require(base64.b64decode(config["content"]).decode().strip() == chain["testing_sha"], "private pin differs from workflow source")
+    completed = timestamp(summary["completed_at"])
+    require(timestamp(run["run_started_at"]) <= completed <= datetime.now(timezone.utc) + timedelta(minutes=5), "invalid completion timestamp")
+    source_ref = manifest.get("source_ref", candidate["workflow_ref"])
+    return {"run_id": run["id"], "attempt": run["run_attempt"], "url": run["html_url"],
+            "workflow_sha": chain["workflow_sha"], "testing_sha": chain["testing_sha"],
+            "candidate": candidate, "source_ref": source_ref, "source_sha": manifest["source_sha"],
+            "completed_at": completed.isoformat(), "expires_at": (timestamp(candidate["build_started_at"]) + MAX_AGE).isoformat(),
+            "verified_at": datetime.now(timezone.utc).isoformat(), "evidence_schema": 1}
+
+
+def complete_success(run):
+    require(run["path"] == ".github/workflows/" + WORKFLOW and run["head_branch"] == "main", "unexpected chain workflow source")
+    require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "chain came from another repository")
+    require(run.get("status") == "completed" and run.get("conclusion") == "success", "chain has not completed successfully")
+    name = f"functional-chain-complete-{run['id']}-{run['run_attempt']}"
+    payload = api(f"repos/{REPOSITORY}/actions/runs/{run['id']}/artifacts?per_page=100")
+    require(payload["total_count"] <= 100, "chain artifact listing is incomplete")
+    artifacts = [item for item in payload["artifacts"] if item.get("name") == name]
+    require(len(artifacts) == 1, "missing or ambiguous complete-chain artifact")
+    artifact = artifacts[0]
+    require(type(artifact.get("size_in_bytes")) is int and 0 < artifact["size_in_bytes"] <= 1024 * 1024, "complete evidence size is invalid")
+    archive = api(f"repos/{REPOSITORY}/actions/artifacts/{artifact['id']}/zip", binary=True)
+    summary = read_json_artifact(archive, artifact, run, name, "chain-complete.json", max_json=128 * 1024)
+    result = validate_summary(summary, run)
+    result["artifact_id"] = artifact["id"]
+    result["artifact_digest"] = artifact["digest"]
+    return result
+
+
+def collect(limit=20):
+    observed = datetime.now(timezone.utc)
+    workflow = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}")
+    runs = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={limit}")["workflow_runs"]
+    runs.sort(key=lambda run: timestamp(run["run_started_at"]), reverse=True)
+    result = {"schema": 1, "observed_at": observed.isoformat(), "workflow_state": workflow["state"],
+              "owner": "@overtrue", "scan_limit": limit, "inspection_complete": True,
+              "latest_attempt": None, "last_complete_success": {}, "healthy": False}
+    for index, listed in enumerate(runs):
+        run = api(f"repos/{REPOSITORY}/actions/runs/{listed['id']}/attempts/{listed['run_attempt']}")
+        if index == 0:
+            result["latest_attempt"] = {"run_id": run["id"], "attempt": run["run_attempt"], "url": run["html_url"],
+                                        "status": run["status"], "conclusion": run["conclusion"], "verification": "not_complete"}
+        if run.get("conclusion") != "success":
+            continue
+        try:
+            complete = complete_success(run)
+        except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:
+            if index == 0:
+                result["latest_attempt"]["verification"] = "invalid"
+            result["inspection_complete"] = False
+            continue
+        source = complete["source_ref"]
+        if source.startswith("refs/heads/"):
+            source = source[len("refs/heads/"):]
+        if index == 0:
+            result["latest_attempt"]["verification"] = "complete"
+            result["latest_attempt"]["source_ref"] = source
+            result["latest_attempt"]["source_sha"] = complete["source_sha"]
+        previous = result["last_complete_success"].get(source)
+        if previous is None or timestamp(complete["completed_at"]) > timestamp(previous["completed_at"]):
+            result["last_complete_success"][source] = complete
+    # Reject a snapshot if a new attempt started while artifacts were checked.
+    refreshed = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={limit}")["workflow_runs"]
+    fields = ("id", "run_attempt", "status", "conclusion", "head_sha", "run_started_at")
+    snapshot = lambda values: sorted(tuple(item.get(key) for key in fields) for item in values)
+    require(snapshot(runs) == snapshot(refreshed), "chain attempts changed during inspection; retry collection")
+    for complete in result["last_complete_success"].values():
+        complete["fresh"] = observed <= timestamp(complete["expires_at"])
+    latest = result["latest_attempt"] or {}
+    result["healthy"] = (result["workflow_state"] == "active" and latest.get("verification") == "complete"
+                         and latest.get("source_ref") == "main" and result["last_complete_success"].get("main", {}).get("fresh") is True)
+    return result
+
+
+def merge_history(current, previous):
+    require(isinstance(previous, dict) and type(previous.get("schema")) is int and previous["schema"] == 1, "unsupported existing dashboard state")
+    require(timestamp(previous["observed_at"]) <= timestamp(current["observed_at"]), "refusing an older dashboard observation")
+    for source, value in previous.get("last_complete_success", {}).items():
+        if source not in current["last_complete_success"] or timestamp(value["completed_at"]) > timestamp(current["last_complete_success"][source]["completed_at"]):
+            value = dict(value)
+            value["fresh"] = timestamp(current["observed_at"]) <= timestamp(value["expires_at"])
+            value["retained_history"] = True
+            current["last_complete_success"][source] = value
+    return current
+
+
+def publish(result):
+    endpoint = "repos/rustfs/dashboard/contents/src/chain-health.json"
+    existing = api(endpoint)
+    previous = json.loads(base64.b64decode(existing["content"]))
+    merge_history(result, previous)
+    body = {"message": "chore(ci): update functional chain health", "sha": existing["sha"],
+            "content": base64.b64encode((json.dumps(result, indent=2) + "\n").encode()).decode()}
+    subprocess.run(["gh", "api", "--method", "PUT", endpoint, "--input", "-"], input=json.dumps(body), text=True, check=True, capture_output=True, timeout=60)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--publish", action="store_true")
+    args = parser.parse_args()
+    result = collect()
+    if args.publish:
+        publish(result)
+    args.output.write_text(json.dumps(result, indent=2) + "\n")
+    print(json.dumps({key: result[key] for key in ("workflow_state", "inspection_complete", "healthy", "latest_attempt")}))
+    return 0 if result["healthy"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/resolve_functional_candidate.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Resolve a nightly manifest from its exact GitHub build attempt and artifact."""
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import zipfile
+
+REPOSITORY = "rustfs/rustfs"
+ROOT = Path(__file__).resolve().parents[1]
+MAX_ARCHIVE = 1024 * 1024
+
+
+def require(condition, message):
+    if not condition:
+        raise ValueError(message)
+
+
+def sha(value, length=40):
+    return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{%d}" % length, value) is not None
+
+
+def positive(value):
+    return type(value) is int and value > 0
+
+
+def api(path, binary=False):
+    result = subprocess.run(["gh", "api", path], check=True, capture_output=True, timeout=60)
+    return result.stdout if binary else json.loads(result.stdout)
+
+
+def validate_manifest(manifest, run):
+    require(isinstance(manifest, dict), "manifest must be an object")
+    common = {"schema", "source_sha", "build_run_id", "build_run_attempt", "package_url", "package_sha256"}
+    version = manifest.get("schema")
+    require(type(version) is int and version in (1, 2), "unsupported candidate schema")
+    require(set(manifest) == (common if version == 1 else common | {"workflow_sha", "source_ref"}), "unexpected candidate fields")
+    require(positive(manifest["build_run_id"]) and positive(manifest["build_run_attempt"]), "invalid build identity")
+    require((manifest["build_run_id"], manifest["build_run_attempt"]) == (run["id"], run["run_attempt"]), "candidate belongs to another build attempt")
+    require(sha(manifest["source_sha"]) and sha(manifest["package_sha256"], 64), "invalid candidate hash")
+    if version == 1:
+        require(manifest["source_sha"] == run["head_sha"], "legacy manifest cannot identify a different build source")
+    else:
+        require(manifest["workflow_sha"] == run["head_sha"] and sha(manifest["workflow_sha"]), "candidate workflow SHA differs from artifact provenance")
+        require(isinstance(manifest["source_ref"], str) and bool(re.fullmatch(r"[A-Za-z0-9_./-]{1,200}", manifest["source_ref"])), "invalid build source ref")
+    expected = (f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/{run['id']}/"
+                f"{run['run_attempt']}/{manifest['package_sha256']}/rustfs.deb")
+    require(manifest["package_url"] == expected, "package URL does not bind the run, attempt and checksum")
+    return manifest
+
+
+def read_json_artifact(archive, artifact, run, expected_name, member, max_json=16384):
+    require(artifact.get("expired") is False, "candidate artifact expired")
+    require(positive(artifact.get("id")), "invalid artifact id")
+    require(artifact.get("name") == expected_name, "candidate artifact belongs to another attempt")
+    provenance = artifact.get("workflow_run") or {}
+    require(provenance.get("id") == run["id"] and provenance.get("head_sha") == run["head_sha"], "candidate artifact belongs to another workflow run")
+    require(0 < len(archive) <= MAX_ARCHIVE and artifact.get("size_in_bytes") == len(archive), "candidate artifact size mismatch")
+    require(artifact.get("digest") == "sha256:" + hashlib.sha256(archive).hexdigest(), "candidate artifact checksum mismatch")
+    with zipfile.ZipFile(io.BytesIO(archive)) as source:
+        files = source.infolist()
+        require(len(files) == 1 and files[0].filename == member, "unexpected candidate archive members")
+        require(0 < files[0].file_size <= max_json and not files[0].is_dir(), "candidate manifest too large or empty")
+        manifest = json.loads(source.read(files[0]))
+    return manifest
+
+
+def read_manifest(archive, artifact, run):
+    name = f"nightly-candidate-{run['id']}-{run['run_attempt']}"
+    return validate_manifest(read_json_artifact(archive, artifact, run, name, name + ".json"), run)
+
+
+def resolve(run_id, attempt):
+    require(positive(run_id) and positive(attempt), "build run and attempt are required positive integers")
+    endpoint = f"repos/{REPOSITORY}/actions/runs/{run_id}"
+    run = api(f"{endpoint}/attempts/{attempt}")
+    require(run.get("id") == run_id and run.get("run_attempt") == attempt, "GitHub returned a different build attempt")
+    require(run.get("path") == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main", "candidate must come from nightly-gnu on main")
+    require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "candidate came from another repository")
+    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "nightly attempt has not completed successfully")
+    name = f"nightly-candidate-{run_id}-{attempt}"
+    artifacts = []
+    for page in range(1, 11):
+        batch = api(f"{endpoint}/artifacts?per_page=100&page={page}")["artifacts"]
+        artifacts.extend(item for item in batch if item.get("name") == name)
+        if len(batch) < 100:
+            break
+    else:
+        raise ValueError("too many build artifacts to resolve safely")
+    require(len(artifacts) == 1, "missing or ambiguous candidate artifact")
+    artifact = artifacts[0]
+    require(type(artifact.get("size_in_bytes")) is int and 0 < artifact["size_in_bytes"] <= MAX_ARCHIVE, "candidate artifact size is invalid")
+    archive = api(f"repos/{REPOSITORY}/actions/artifacts/{artifact['id']}/zip", binary=True)
+    manifest = read_manifest(archive, artifact, run)
+    return {"manifest": manifest, "artifact_id": artifact["id"], "artifact_digest": artifact["digest"],
+            "workflow_sha": run["head_sha"], "workflow_ref": run["head_branch"], "build_started_at": run["run_started_at"]}
+
+
+def prepare():
+    event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
+    if os.environ["GITHUB_EVENT_NAME"] == "workflow_run":
+        build = event["workflow_run"]
+        require(build.get("event") == "schedule", "automatic chain requires a scheduled build")
+        run_id, attempt = build["id"], build["run_attempt"]
+    else:
+        run_id, attempt = int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"])
+    candidate = resolve(run_id, attempt)
+    revision = (ROOT / ".config/functional-script-revision.txt").read_text().strip()
+    require(sha(revision), "private test script revision must be pinned")
+    chain = {"schema": 1, "run_id": int(os.environ["GITHUB_RUN_ID"]), "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]),
+             "workflow_sha": os.environ["GITHUB_SHA"], "testing_sha": revision, "candidate": candidate}
+    encoded = json.dumps(chain, sort_keys=True, separators=(",", ":"))
+    with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+        output.write("manifest=" + encoded + "\n")
+    Path(os.environ["CHAIN_OUTPUT"]).write_text(encoded + "\n")
+
+
+if __name__ == "__main__":
+    prepare()

scripts/test_functional_chain.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Exercise candidate substitution and complete-chain acceptance boundaries."""
+import copy
+import hashlib
+import io
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest import mock
+import zipfile
+
+import functional_chain_evidence as evidence
+import resolve_functional_candidate as candidate
+
+
+class CandidateTests(unittest.TestCase):
+    def setUp(self):
+        self.run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40}
+        self.manifest = {"schema": 2, "workflow_sha": "a" * 40, "source_sha": "b" * 40, "source_ref": "release",
+                         "build_run_id": 123, "build_run_attempt": 2, "package_sha256": "c" * 64,
+                         "package_url": "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/123/2/" + "c" * 64 + "/rustfs.deb"}
+
+    def archive(self, names=None):
+        output = io.BytesIO()
+        with zipfile.ZipFile(output, "w") as archive:
+            for name in names or ["nightly-candidate-123-2.json"]:
+                archive.writestr(name, json.dumps(self.manifest))
+        payload = output.getvalue()
+        artifact = {"id": 789, "name": "nightly-candidate-123-2", "expired": False, "size_in_bytes": len(payload),
+                    "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), "workflow_run": {"id": 123, "head_sha": "a" * 40}}
+        return payload, artifact
+
+    def test_distinct_build_source_preserves_both_identities(self):
+        payload, artifact = self.archive()
+        result = candidate.read_manifest(payload, artifact, self.run)
+        self.assertEqual(result["source_sha"], "b" * 40)
+        self.assertEqual(result["workflow_sha"], "a" * 40)
+        self.assertEqual(result["source_ref"], "release")
+
+    def test_legacy_requires_the_build_and_workflow_sha_to_agree(self):
+        self.manifest["schema"] = 1
+        del self.manifest["workflow_sha"], self.manifest["source_ref"]
+        with self.assertRaisesRegex(ValueError, "legacy"):
+            candidate.validate_manifest(self.manifest, self.run)
+        self.manifest["source_sha"] = self.run["head_sha"]
+        candidate.validate_manifest(self.manifest, self.run)
+
+    def test_manifest_substitutions_fail(self):
+        for key, value in (("workflow_sha", "d" * 40), ("build_run_id", 124), ("build_run_attempt", 1),
+                           ("package_sha256", "d" * 64), ("package_url", "https://example.com/package.deb"),
+                           ("schema", True), ("source_ref", "release\nFORGED=value")):
+            with self.subTest(key=key), self.assertRaises(ValueError):
+                candidate.validate_manifest({**self.manifest, key: value}, self.run)
+
+    def test_artifact_substitutions_and_archive_members_fail(self):
+        payload, artifact = self.archive()
+        for key, value in (("expired", True), ("name", "nightly-candidate-123-1"), ("size_in_bytes", 1),
+                           ("digest", "sha256:" + "d" * 64), ("workflow_run", {"id": 124, "head_sha": "a" * 40})):
+            with self.subTest(key=key), self.assertRaises(ValueError):
+                candidate.read_manifest(payload, {**artifact, key: value}, self.run)
+        for names in (["../nightly-candidate-123-2.json"], ["nightly-candidate-123-2.json", "extra.json"]):
+            payload, artifact = self.archive(names)
+            with self.assertRaises(ValueError):
+                candidate.read_manifest(payload, artifact, self.run)
+
+    def test_resolver_uses_attempt_metadata_and_does_not_resolve_moving_branch(self):
+        payload, artifact = self.archive()
+        run = {**self.run, "path": ".github/workflows/nightly-gnu.yml", "head_branch": "main", "head_repository": {"full_name": "rustfs/rustfs"},
+               "event": "schedule", "status": "completed", "conclusion": "success", "run_started_at": "2026-09-12T00:00:00Z"}
+        with mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]) as api:
+            result = candidate.resolve(123, 2)
+        self.assertTrue(api.call_args_list[0].args[0].endswith("/attempts/2"))
+        self.assertFalse(any("branches/" in call.args[0] for call in api.call_args_list))
+        self.assertEqual(result["artifact_id"], 789)
+        self.assertEqual(result["manifest"]["source_sha"], "b" * 40)
+
+
+class EvidenceTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.directory = Path(self.temp.name)
+        self.chain = {"run_id": 456, "attempt": 1, "candidate": {"source_sha": "a" * 40, "workflow_sha": "b" * 40}}
+        self.needs = {suite: {"result": "success"} for suite in evidence.SUITES}
+        for suite in evidence.SUITES:
+            value = {"schema": 1, "suite": suite, "chain": self.chain, "valid": True, "report_sha256": "c" * 64,
+                     "counts": {"PASS": 1, "FAIL": 0, "SKIP": 0, "UNSUPPORTED": 0, "RUNNING": 0}}
+            (self.directory / (suite + ".json")).write_text(json.dumps(value))
+
+    def test_complete_chain_retains_source_identity(self):
+        result = evidence.aggregate(self.chain, self.directory, self.needs)
+        self.assertTrue(result["complete"])
+        self.assertEqual(result["chain"], self.chain)
+        self.assertEqual(len(result["suites"]), 10)
+
+    def test_missing_failed_cancelled_or_skipped_lane_never_passes(self):
+        for state in ("failure", "cancelled", "skipped", "pending"):
+            with self.subTest(state=state), self.assertRaises(ValueError):
+                evidence.aggregate(self.chain, self.directory, {**self.needs, "s3": {"result": state}})
+        del self.needs["s3"]
+        with self.assertRaises(ValueError):
+            evidence.aggregate(self.chain, self.directory, self.needs)
+
+    def test_partial_rerun_missing_artifact_and_zero_test_fail(self):
+        path = self.directory / "s3.json"
+        original = json.loads(path.read_text())
+        values = [{**original, "chain": {**self.chain, "attempt": 2}},
+                  {**original, "counts": {**original["counts"], "PASS": 0}},
+                  {**original, "counts": {**original["counts"], "FAIL": 1}},
+                  {**original, "valid": False}, {**original, "report_sha256": ""}]
+        for value in values:
+            path.write_text(json.dumps(value))
+            with self.assertRaises(ValueError):
+                evidence.aggregate(self.chain, self.directory, self.needs)
+        path.unlink()
+        with self.assertRaises(ValueError):
+            evidence.aggregate(self.chain, self.directory, self.needs)
+
+    def test_reports_count_case_status_not_cleanup_status(self):
+        text = "| Topology | Case | Name | Status | Cleanup |\n| --- | --- | --- | --- | --- |\n| sns | TIER-1 | test | UNSUPPORTED | PASS |\n| sns | TIER-2 | test | FAIL | PASS |\n"
+        counts = evidence.report_counts(text)
+        self.assertEqual(counts["PASS"], 0)
+        self.assertEqual(counts["FAIL"], 1)
+        self.assertEqual(counts["UNSUPPORTED"], 1)
+        self.assertEqual(evidence.report_counts("")["PASS"], 0)
+
+    def test_performance_needs_real_complete_metrics(self):
+        header = "method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\n"
+        row = "put\t1MiB\t100MiB/s\t100\t1ms\t1ms\t2ms\t3ms\n"
+        self.assertEqual(evidence.report_counts(header + row, True)["PASS"], 1)
+        self.assertEqual(evidence.report_counts(header, True)["PASS"], 0)
+        with self.assertRaises(ValueError):
+            evidence.report_counts(header + row + row, True)
+        with self.assertRaises(ValueError):
+            evidence.report_counts(header + "put\t1MiB\t\t\t\t\t\t\n", True)
+
+
+class EnvelopeTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        (self.root / ".config").mkdir()
+        (self.root / ".config/functional-script-revision.txt").write_text("d" * 40)
+        manifest = {"schema": 2, "workflow_sha": "a" * 40, "source_sha": "b" * 40, "source_ref": "release",
+                    "build_run_id": 123, "build_run_attempt": 2, "package_sha256": "c" * 64,
+                    "package_url": "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/123/2/" + "c" * 64 + "/rustfs.deb"}
+        self.chain = {"schema": 1, "run_id": 456, "attempt": 3, "workflow_sha": "e" * 40, "testing_sha": "d" * 40,
+                      "candidate": {"manifest": manifest, "artifact_id": 789, "artifact_digest": "sha256:" + "f" * 64,
+                                    "workflow_sha": "a" * 40, "workflow_ref": "main", "build_started_at": "2026-09-12T00:00:00Z"}}
+        self.env = {"CHAIN_MANIFEST": json.dumps(self.chain), "GITHUB_RUN_ID": "456", "GITHUB_RUN_ATTEMPT": "3", "GITHUB_SHA": "e" * 40,
+                    "GITHUB_ENV": str(self.root / "env"), "GITHUB_OUTPUT": str(self.root / "output"),
+                    "CHAIN_JOB_STATUS": "success", "CHAIN_TEST_OUTCOME": "success", "CHAIN_REPORT_OUTCOME": "success"}
+
+    def test_consumer_exports_the_same_package_and_checksum_to_installers(self):
+        with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40):
+            evidence.consume(evidence.current_chain())
+        variables = dict(line.split("=", 1) for line in (self.root / "env").read_text().splitlines())
+        self.assertEqual(variables["PACKAGE_SHA256"], "c" * 64)
+        self.assertEqual(variables["TO_SHA256"], "c" * 64)
+        self.assertEqual(variables["RUSTFS_NIGHTLY_PACKAGE_URL"], self.chain["candidate"]["manifest"]["package_url"])
+
+    def test_partial_rerun_and_wrong_lane_checkout_fail_before_install(self):
+        for changes, head in (({"GITHUB_RUN_ATTEMPT": "4"}, "e" * 40), ({}, "f" * 40), ({"GITHUB_RUN_ID": "457"}, "e" * 40)):
+            with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, {**self.env, **changes}), mock.patch.object(evidence.subprocess, "check_output", return_value=head), self.assertRaises(ValueError):
+                evidence.current_chain()
+        self.assertFalse((self.root / "env").exists())
+
+    def test_report_or_swallowed_test_failure_cannot_produce_valid_evidence(self):
+        report = self.root / "cases.md"
+        report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n")
+        for index, (key, status) in enumerate((("CHAIN_TEST_OUTCOME", "failure"), ("CHAIN_REPORT_OUTCOME", "failure"), ("CHAIN_JOB_STATUS", "cancelled"))):
+            output = self.root / str(index) / "kms.json"
+            with mock.patch.dict(evidence.os.environ, {**self.env, key: status}), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40), self.assertRaises(ValueError):
+                evidence.record(self.chain, "kms", report, output)
+            self.assertFalse(json.loads(output.read_text())["valid"])
+        output = self.root / "success" / "kms.json"
+        with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40):
+            evidence.record(self.chain, "kms", report, output)
+        self.assertTrue(json.loads(output.read_text())["valid"])
+
+    def test_unknown_status_cannot_hide_among_passing_cases(self):
+        text = "| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n| KMS-2 | fixture | NOT RUN |\n"
+        with self.assertRaises(ValueError):
+            evidence.report_counts(text)
+
+    def test_driver_passes_one_manifest_and_runs_every_lane_after_failure(self):
+        from check_test_wiring import yaml_block
+        lines = (candidate.ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text().splitlines()
+        previous = None
+        for suite in evidence.SUITES:
+            job = "\n".join(yaml_block(lines, suite, 2))
+            self.assertIn("needs: [prepare" + (", " + previous if previous else "") + "]", job)
+            self.assertIn("if: ${{ always() && needs.prepare.result == 'success' }}", job)
+            self.assertIn("chain_manifest: ${{ needs.prepare.outputs.manifest }}", job)
+            previous = suite
+        complete = "\n".join(yaml_block(lines, "complete-chain", 2))
+        self.assertIn("needs: [prepare, " + ", ".join(evidence.SUITES) + "]", complete)
+        self.assertIn("functional_chain_evidence.py aggregate", complete)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_functional_chain_health.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Historical successes must not hide incomplete evidence or newer attempts."""
+import base64
+import copy
+from datetime import datetime, timedelta, timezone
+import json
+import unittest
+from unittest import mock
+
+import functional_chain_health as health
+from functional_chain_evidence import SUITES
+
+
+class HealthTests(unittest.TestCase):
+    def setUp(self):
+        now = datetime.now(timezone.utc)
+        self.now = now.isoformat()
+        self.run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40, "html_url": "https://github.com/rustfs/rustfs/actions/runs/123",
+                    "run_started_at": (now - timedelta(hours=1)).isoformat()}
+        self.candidate = {"manifest": {"build_run_id": 456, "build_run_attempt": 3, "source_ref": "release", "source_sha": "b" * 40},
+                          "workflow_sha": "c" * 40, "workflow_ref": "main", "build_started_at": (now - timedelta(hours=2)).isoformat()}
+        self.chain = {"run_id": 123, "attempt": 2, "workflow_sha": "a" * 40, "testing_sha": "d" * 40, "candidate": self.candidate}
+        self.summary = {"schema": 1, "complete": True, "chain": self.chain, "completed_at": (now - timedelta(minutes=1)).isoformat(),
+                        "suites": [{"schema": 1, "suite": suite, "chain": self.chain, "valid": True, "report_sha256": "f" * 64,
+                                    "counts": {"PASS": 1, "FAIL": 0, "SKIP": 0, "UNSUPPORTED": 0, "RUNNING": 0}} for suite in SUITES]}
+        self.config = {"content": base64.b64encode(("d" * 40 + "\n").encode()).decode()}
+
+    def validate(self, summary=None, candidate=None, config=None):
+        with mock.patch.object(health, "resolve", return_value=candidate or self.candidate), mock.patch.object(health, "api", return_value=config or self.config):
+            return health.validate_summary(summary or self.summary, self.run)
+
+    def test_release_success_preserves_both_sources(self):
+        result = self.validate()
+        self.assertEqual(result["source_ref"], "release")
+        self.assertEqual(result["source_sha"], "b" * 40)
+        self.assertEqual(result["workflow_sha"], "a" * 40)
+        self.assertEqual(result["candidate"]["workflow_sha"], "c" * 40)
+
+    def test_substituted_producer_pin_attempt_or_empty_suite_fails(self):
+        with self.assertRaises(ValueError):
+            self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40})
+        with self.assertRaises(ValueError):
+            self.validate(config={"content": base64.b64encode(b"wrong pin").decode()})
+        wrong = copy.deepcopy(self.summary)
+        wrong["chain"]["attempt"] = 1
+        with self.assertRaises(ValueError):
+            self.validate(wrong)
+        wrong = copy.deepcopy(self.summary)
+        wrong["suites"][0]["counts"]["PASS"] = 0
+        with self.assertRaises(ValueError):
+            self.validate(wrong)
+
+    def test_new_failure_retains_last_complete_success_without_becoming_healthy(self):
+        complete = self.validate()
+        previous = {"schema": 1, "observed_at": self.now, "last_complete_success": {"release": complete}}
+        current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False,
+                   "latest_attempt": {"conclusion": "failure"}}
+        result = health.merge_history(current, previous)
+        self.assertFalse(result["healthy"])
+        self.assertEqual(result["latest_attempt"]["conclusion"], "failure")
+        self.assertEqual(result["last_complete_success"]["release"]["source_sha"], "b" * 40)
+        self.assertTrue(result["last_complete_success"]["release"]["retained_history"])
+
+    def test_expired_history_is_not_fresh_and_null_or_stale_state_cannot_publish(self):
+        complete = self.validate()
+        complete["expires_at"] = "2000-01-01T00:00:00Z"
+        previous = {"schema": 1, "observed_at": self.now, "last_complete_success": {"release": complete}}
+        current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
+        self.assertFalse(health.merge_history(current, previous)["last_complete_success"]["release"]["fresh"])
+        with self.assertRaises(ValueError):
+            health.merge_history(current, None)
+        with self.assertRaises(ValueError):
+            health.merge_history({**current, "observed_at": "2000-01-01T00:00:00Z"}, previous)
+        existing = {"sha": "old-blob", "content": base64.b64encode(b"null").decode()}
+        with mock.patch.object(health, "api", return_value=existing), mock.patch.object(health.subprocess, "run") as write:
+            with self.assertRaises(ValueError):
+                health.publish(current)
+            write.assert_not_called()
+
+    def test_collection_rejects_a_concurrent_rerun(self):
+        run = {**self.run, "status": "completed", "conclusion": "failure"}
+        responses = [{"state": "active"}, {"workflow_runs": [run]}, run,
+                     {"workflow_runs": [{**run, "run_attempt": 3, "status": "queued"}]}]
+        with mock.patch.object(health, "api", side_effect=responses):
+            with self.assertRaisesRegex(ValueError, "changed during inspection"):
+                health.collect()
+
+    def test_publication_uses_the_read_blob_sha(self):
+        current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
+        existing = {"sha": "reviewed-blob", "content": base64.b64encode(json.dumps(current).encode()).decode()}
+        with mock.patch.object(health, "api", return_value=existing), mock.patch.object(health.subprocess, "run") as write:
+            health.publish(current)
+        body = json.loads(write.call_args.kwargs["input"])
+        self.assertEqual(body["sha"], "reviewed-blob")
+        self.assertFalse(json.loads(base64.b64decode(body["content"]))["healthy"])
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_nightly_candidate.py

@@ -10,6 +10,7 @@ import tempfile
 import unittest
 
 from check_test_wiring import yaml_block
+from resolve_functional_candidate import validate_manifest
 
 
 ROOT = Path(__file__).resolve().parents[1]
@@ -172,6 +173,7 @@ SH
         self.assertEqual(self.manifest()["source_sha"], self.sha)
         self.assertEqual(self.manifest()["workflow_sha"], "f" * 40)
         self.assertEqual(self.manifest()["source_ref"], "release")
+        validate_manifest(self.manifest(), {"id": 12345, "run_attempt": 1, "head_sha": "f" * 40})
 
     def test_every_lane_uses_the_same_resolved_source(self):
         lines = WORKFLOW.read_text().splitlines()

scripts/test_security_workflow.py

@@ -133,7 +133,7 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
 
     def test_workflow_wiring(self) -> None:
         names = list(self.steps)
-        self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
+        self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts"))
         self.assertNotIn("    continue-on-error: true", self.job)
         self.assertIn("        continue-on-error: true", self.steps["Run security suite"])
         for name in ("Initialize security evidence", "Generate report"):
@@ -291,7 +291,7 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                 cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
                 self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
 
-    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
+    def test_legacy_replication_hands_off_after_failure(self) -> None:
         for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
             with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
                 self.setUp()
@@ -337,16 +337,6 @@ fi
                     RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
                 )
                 self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
-                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
-                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
-                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
-                started = self.run_step("Dispatch first suite (upgrade)")
-                self.assertEqual(started.returncode, 0, started.stderr)
-                self.assertEqual(dispatches.read_text().splitlines(), [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
-                ])
-                dispatches.unlink()
-
                 replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
                 job = yaml_block(replication.splitlines(), "replication-test", 2)
                 self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
@@ -570,10 +560,15 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
                 self.prepare(suite)
                 self.assertNotIn("/tmp/rustfs-", self.source)
                 names = list(self.steps)
-                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
+                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts"))
                 if suite in FunctionalWorkflowTests.DIRECT_TESTS:
-                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
+                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts"))
                 for name, lines in self.steps.items():
+                    if name == "Upload chain evidence":
+                        self.assertIn("        if: ${{ always() && steps.chain_record.outcome == 'success' }}", lines)
+                        self.assertIn("        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}", self.steps["Record chain evidence"])
+                        self.assertIn("          if-no-files-found: error", lines)
+                        continue
                     if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
                         self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
                     if any("uses: actions/upload-artifact@" in line for line in lines):