fix(ci): distinguish nextest setup scripts from test results

.config/ci-plan.json

@@ -0,0 +1,16 @@
+{
+  "schema_version": 1,
+  "rollout": "shadow",
+  "core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
+  "connect_packages": [
+    "rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",
+    "rustfs-iam", "rustfs-policy", "rustfs-kms", "rustfs-license", "rustfs-security-governance",
+    "rustfs-utils", "rustfs-tls-runtime", "rustfs-trusted-proxies", "rustfs-filemeta",
+    "rustfs-lock", "rustfs-obs", "rustfs-signer", "rustfs-io-metrics", "rustfs-s3-ops"
+  ],
+  "full_paths": [
+    "Cargo.toml", "Cargo.lock", "*/Cargo.toml", "rust-toolchain*", "rustfmt.toml",
+    "clippy.toml", ".cargo/*", ".config/*", ".github/*", "Makefile",
+    "scripts/*", "build.rs", "*.proto"
+  ]
+}

.config/make/lint-fmt.mak

@@ -90,6 +90,7 @@ test-wiring-check: ## Check tests stay registered and selected by their intended
 	@echo "🧪 Checking test wiring..."
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
+	$(RUSTFS_PYTHON_BIN) ./scripts/validation_registry.py check
 
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source

.config/make/tests.mak

@@ -59,12 +59,18 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
+	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/validation_registry.py check
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py

.config/nextest.toml

@@ -969,3 +969,17 @@ test-group = 'e2e-vault'
 [[profile.e2e-full.overrides]]
 filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
 threads-required = "num-test-threads"
+
+# Fixed, small contract for every executable server change. These existing
+# scenarios exercise write/read, anonymous denial and retained bytes on restart.
+[profile.core-contract]
+default-filter = '''
+    package(e2e_test) & test(/^(anonymous_access_test::test_anonymous_access_denied_when_restrict_public_buckets_enabled|head_object_consistency_test::head_object_consistency_after_write_and_multipart_and_presigned_head|storage_class_capability_test::tests::historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier)$/)
+'''
+retries = 0
+fail-fast = false
+test-threads = 1
+slow-timeout = { period = "60s", terminate-after = 3 }
+
+[profile.core-contract.junit]
+path = "junit.xml"

.github/actions/quick-checks/action.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: Quick Checks
-description: Run the shared compile-free RustFS quality checks.
+description: Run the shared RustFS quality checks.
 
 runs:
   using: composite
@@ -24,6 +24,7 @@ runs:
         tool: |
           ripgrep@15.2.0
           shellcheck@0.11.0
+          cargo-nextest@0.9.138
 
     - name: Install actionlint
       shell: bash
@@ -37,6 +38,15 @@ runs:
         rm "$actionlint_dir/actionlint.tar.gz"
         echo "$actionlint_dir" >> "$GITHUB_PATH"
 
+    - name: Install Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      with:
+        python-version: "3.12"
+
+    - name: Install workflow parser
+      shell: bash
+      run: python3 -m pip install --disable-pip-version-check PyYAML==6.0.3
+
     - name: Install Rust toolchain
       uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       with:
@@ -104,7 +114,10 @@ runs:
 
     - name: Check test wiring
       shell: bash
-      run: python3 ./scripts/check_test_wiring.py
+      run: |
+        python3 ./scripts/check_test_wiring.py
+        python3 ./scripts/ci_gate.py --check-workflow
+        python3 ./scripts/validation_registry.py check
 
     - name: Check no planning docs committed
       shell: bash

.github/actions/runner-pilot/action.yml

@@ -0,0 +1,107 @@
+name: Runner capability and build pilot
+description: Inspect an existing runner and measure isolated fixed-source cold/warm builds
+inputs:
+  workload:
+    description: probe, server or workspace-tests
+    required: true
+  source-sha:
+    description: Exact merged main commit for builds
+    default: ""
+  toolchain:
+    description: Exact Rust version for builds
+    default: ""
+  scratch-root:
+    description: Existing absolute scratch root
+    default: ""
+  build-jobs:
+    description: Cargo concurrency
+    default: "2"
+  local-ssd:
+    description: Require operator storage attestation
+    default: "false"
+runs:
+  using: composite
+  steps:
+    - name: Validate pilot input and inspect existing capabilities
+      shell: bash
+      env:
+        PILOT_WORKLOAD: ${{ inputs.workload }}
+        PILOT_SHA: ${{ inputs.source-sha }}
+        PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
+        PILOT_SCRATCH: ${{ inputs.scratch-root }}
+        PILOT_JOBS: ${{ inputs.build-jobs }}
+        PILOT_SSD: ${{ inputs.local-ssd }}
+      run: |
+        [[ "$PILOT_WORKLOAD" =~ ^(probe|server|workspace-tests)$ ]]
+        [[ "$PILOT_JOBS" =~ ^([1-9]|1[0-6])$ ]]
+        if [[ "$PILOT_WORKLOAD" != probe ]]; then
+          [[ "$PILOT_SHA" =~ ^[a-f0-9]{40}$ ]]
+          [[ "$PILOT_TOOLCHAIN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
+        fi
+        scratch="${PILOT_SCRATCH:-$RUNNER_TEMP}"
+        [[ "$scratch" == /* && -d "$scratch" ]]
+        minimum_disk=1
+        minimum_memory=0
+        if [[ "$PILOT_WORKLOAD" == server ]]; then minimum_disk=24; minimum_memory=8; fi
+        if [[ "$PILOT_WORKLOAD" == workspace-tests ]]; then minimum_disk=90; minimum_memory=16; fi
+        extra=()
+        if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
+        python3 scripts/ci/runner_pilot.py --root "$scratch" \
+          --output "$RUNNER_TEMP/runner-pilot-evidence" \
+          --min-free-gib "$minimum_disk" --min-memory-gib "$minimum_memory" --architecture x86_64 "${extra[@]}"
+    - name: Checkout the trusted baseline history
+      if: inputs.workload != 'probe'
+      uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+      with:
+        ref: main
+        path: .runner-pilot-source
+        fetch-depth: 0
+        persist-credentials: false
+    - name: Bind the build to a merged main commit
+      if: inputs.workload != 'probe'
+      shell: bash
+      env:
+        PILOT_SHA: ${{ inputs.source-sha }}
+      run: |
+        git -C .runner-pilot-source merge-base --is-ancestor "$PILOT_SHA" HEAD
+        git -C .runner-pilot-source checkout --detach "$PILOT_SHA"
+    - name: Install build dependencies
+      if: inputs.workload != 'probe'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential pkg-config libssl-dev cmake clang time
+    - name: Install the fixed protobuf compiler
+      if: inputs.workload != 'probe'
+      uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
+      with:
+        version: "35.1"
+        repo-token: ${{ github.token }}
+    - name: Install the fixed flatbuffer compiler
+      if: inputs.workload != 'probe'
+      uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
+      with:
+        version: "25.12.19"
+    - name: Install the requested exact Rust toolchain
+      if: inputs.workload != 'probe'
+      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+      with:
+        toolchain: ${{ inputs.toolchain }}
+    - name: Measure cold and warm builds without shared target caches
+      if: inputs.workload != 'probe'
+      shell: bash
+      env:
+        PILOT_WORKLOAD: ${{ inputs.workload }}
+        PILOT_SHA: ${{ inputs.source-sha }}
+        PILOT_TOOLCHAIN: ${{ inputs.toolchain }}
+        PILOT_SCRATCH: ${{ inputs.scratch-root }}
+        PILOT_JOBS: ${{ inputs.build-jobs }}
+        PILOT_SSD: ${{ inputs.local-ssd }}
+      run: |
+        extra=()
+        if [[ "$PILOT_SSD" == true ]]; then extra+=(--local-ssd-attestation /etc/rustfs-runner/storage.json); fi
+        python3 scripts/ci/runner_pilot.py --root "${PILOT_SCRATCH:-$RUNNER_TEMP}" \
+          --output "$RUNNER_TEMP/runner-pilot-evidence" \
+          --source .runner-pilot-source --source-sha "$PILOT_SHA" \
+          --toolchain "$PILOT_TOOLCHAIN" --benchmark "$PILOT_WORKLOAD" \
+          --jobs "$PILOT_JOBS" --timeout-seconds 3600 "${extra[@]}"

.github/actions/test-runtime/action.yml

@@ -0,0 +1,16 @@
+name: Verification runtime
+description: Install the runtime for prebuilt test archives without a target cache
+runs:
+  using: composite
+  steps:
+    - name: Install runtime tools
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libssl-dev ripgrep
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+    - name: Install cargo-nextest
+      uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
+      with:
+        tool: cargo-nextest@0.9.138

.github/workflows/README.md

@@ -32,7 +32,7 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
 
 Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
 
-The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection stays on the operator-managed `smoke-testing` runner as part of the automation environment. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
+The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection uses `ubuntu-latest` so a lab outage cannot also stop its independent monitor. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
 
 Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
 

.github/workflows/ci-resource-accounting.yml

@@ -0,0 +1,60 @@
+name: CI Resource Accounting
+
+on:
+  workflow_dispatch:
+    inputs:
+      since:
+        description: "Inclusive run creation timestamp, for example 2026-10-01T00:00:00Z"
+        required: true
+        type: string
+      until:
+        description: "Inclusive end timestamp; keep the window below 1000 runs"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: read
+
+concurrency:
+  group: ci-resource-accounting
+  cancel-in-progress: false
+
+jobs:
+  collect:
+    if: github.repository == 'rustfs/rustfs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - name: Collect all workflow attempts in the window
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ACCOUNTING_SINCE: ${{ inputs.since }}
+          ACCOUNTING_UNTIL: ${{ inputs.until }}
+          ACCOUNTING_REPOSITORY: ${{ github.repository }}
+        run: |
+          python3 scripts/ci/actions_cost_report.py \
+            --repository "$ACCOUNTING_REPOSITORY" \
+            --since "$ACCOUNTING_SINCE" --until "$ACCOUNTING_UNTIL" \
+            --output "$RUNNER_TEMP/ci-accounting.json" \
+            --summary "$RUNNER_TEMP/ci-accounting.md"
+      - name: Show available accounting summary
+        if: always()
+        run: |
+          if [[ -f "$RUNNER_TEMP/ci-accounting.md" ]]; then
+            cat "$RUNNER_TEMP/ci-accounting.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Retain recomputable accounting evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: ci-accounting-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            ${{ runner.temp }}/ci-accounting.json
+            ${{ runner.temp }}/ci-accounting.md
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/ci-runner-pilot.yml

@@ -0,0 +1,92 @@
+name: CI Runner Pilot
+
+on:
+  workflow_dispatch:
+    inputs:
+      pool:
+        description: "Existing runner pool; no infrastructure is created"
+        required: true
+        type: choice
+        options: [hosted, arc]
+        default: hosted
+      workload:
+        description: "Probe only, or perform cold/warm fixed-source builds"
+        required: true
+        type: choice
+        options: [probe, server, workspace-tests]
+        default: probe
+      source_sha:
+        description: "Exact 40-character merged main SHA; required for builds"
+        type: string
+      toolchain:
+        description: "Exact Rust version such as 1.92.0; required for builds"
+        type: string
+      scratch_root:
+        description: "Existing absolute scratch directory; empty uses runner.temp"
+        type: string
+      build_jobs:
+        description: "Fixed Cargo concurrency for both comparisons (1-16)"
+        type: string
+        default: "2"
+      local_ssd:
+        description: "Require operator /etc/rustfs-runner/storage.json attestation"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-runner-pilot
+  cancel-in-progress: false
+
+jobs:
+  hosted:
+    if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'hosted'
+    runs-on: ubuntu-latest
+    timeout-minutes: 150
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/runner-pilot
+        with:
+          workload: ${{ inputs.workload }}
+          source-sha: ${{ inputs.source_sha }}
+          toolchain: ${{ inputs.toolchain }}
+          scratch-root: ${{ inputs.scratch_root }}
+          build-jobs: ${{ inputs.build_jobs }}
+          local-ssd: ${{ inputs.local_ssd }}
+      - name: Retain hosted pilot evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runner-pilot-hosted-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/runner-pilot-evidence
+          if-no-files-found: error
+          retention-days: 30
+  arc:
+    if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main' && inputs.pool == 'arc'
+    # Disk and hardware: measure the existing workspace build pool's storage.
+    runs-on: sm-standard-4
+    timeout-minutes: 150
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/runner-pilot
+        with:
+          workload: ${{ inputs.workload }}
+          source-sha: ${{ inputs.source_sha }}
+          toolchain: ${{ inputs.toolchain }}
+          scratch-root: ${{ inputs.scratch_root }}
+          build-jobs: ${{ inputs.build_jobs }}
+          local-ssd: ${{ inputs.local_ssd }}
+      - name: Retain ARC pilot evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: runner-pilot-arc-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/runner-pilot-evidence
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/ci.yml

@@ -83,6 +83,10 @@ jobs:
     timeout-minutes: 10
     outputs:
       mode: ${{ steps.scope.outputs.mode }}
+      plan: ${{ steps.scope.outputs.plan }}
+      connect: ${{ steps.scope.outputs.connect || 'true' }}
+      packages: ${{ steps.scope.outputs.packages }}
+      full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
@@ -91,16 +95,36 @@ jobs:
       - name: Select scope using the base revision's policy
         id: scope
         env:
-          CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
+          CI_EVENT_ACTION: ${{ github.event.action }}
         run: |
           if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
-            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
+            python3 -I scripts/ci_gate.py select
           elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
             python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
           else
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT"; then
+            printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' >> "$GITHUB_OUTPUT"
+            echo "Legacy policy has no verification plan; retaining the full matrix."
+          fi
+      - name: Retain verification plan
+        if: steps.scope.outputs.plan != ''
+        env:
+          CI_PLAN: ${{ steps.scope.outputs.plan }}
+        run: |
+          mkdir -p artifacts/ci-plan
+          printf '%s\n' "$CI_PLAN" > artifacts/ci-plan/plan.json
+      - name: Upload verification plan
+        if: steps.scope.outputs.plan != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: verification-plan-${{ github.run_attempt }}
+          path: artifacts/ci-plan/plan.json
+          if-no-files-found: error
+          retention-days: 7
 
   typos:
     name: Typos
@@ -130,13 +154,14 @@ jobs:
         uses: ./.github/actions/quick-checks
 
   test-and-lint:
-    name: Workspace Test and Lint
+    name: Workspace Lint and Build
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -179,72 +204,31 @@ jobs:
         run: |
           ./scripts/ci/resource_sampler.sh start clippy
           trap './scripts/ci/resource_sampler.sh stop' EXIT
-          cargo clippy --all-targets -- -D warnings
+          python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
 
-      - name: Run nextest tests
+      - name: Archive workspace test binaries
         env:
-          # #5394 mitigation, settled by the backlog#1601 experiment.
-          #
-          # 2 was chosen when three concurrent workspace test links were believed
-          # to saturate the runner's overlay I/O and wedge Cargo until the 75m
-          # timeout. cgroup v2 readings from the sampler show the pod actually
-          # has 14 CPUs and 28GB (peak use 2.1GB): the label name "sm-standard-4"
-          # had led everyone, including the original mitigation, to assume 4
-          # cores, and memory was never the constraint.
-          #
-          # Measured on the same ci-dev cache, the test-profile compile took
-          # 28m21s at 2 on a pull request and 16-23 min at 3 across three
-          # successful main runs. The first pull-request run at 3 that met a
-          # busy node (run 39193827281) then hit the 75m timeout before any
-          # test started: the sampler shows cargo spinning for 70 minutes with
-          # the pod's io.pressure "full" at 86-88% and CPU pressure at zero,
-          # while node-level IO pressure was already 36% from co-tenants when
-          # the step began. The constraint is the pod's shared storage, not
-          # CPU or memory, and the limit that storage sustains on a contended
-          # node is 2. Main pushes and dispatches keep 3 (rarer, and three
-          # successes); pull requests stay at 2 until the pods get local disk
-          # for target/, which is the lever that would unlock the 14 cores.
-          #
-          # Must stay step-level: rust-cache hashes CARGO/CC/CFLAGS/CXX/CMAKE/RUST
-          # prefixed variables from process.env into the cache key, so promoting
-          # this to job level would rotate every key on this lane.
           CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
         run: |
-          mkdir -p artifacts/test-and-lint
-          rm -f target/nextest/ci/junit.xml
-          ./scripts/ci/resource_sampler.sh start nextest
+          ./scripts/ci/resource_sampler.sh start workspace-archive
           trap './scripts/ci/resource_sampler.sh stop' EXIT
-          set +e
-          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
-            cargo nextest run --profile ci --all --exclude e2e_test \
-            --status-level all --final-status-level all \
-            2>&1 | tee artifacts/test-and-lint/nextest.log
-          status=${PIPESTATUS[0]}
-          if [[ "${status}" -eq 0 ]]; then
-            cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
-              > artifacts/test-and-lint/core-test-listing.json \
-              && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
-              && test -s target/nextest/ci/junit.xml || status=$?
-          fi
-          {
-            echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
-            echo "exit_status=${status}"
-            echo "finished_at=$(date --utc --iso-8601=seconds)"
-            echo
-            echo "Remaining test-related processes:"
-            pgrep -af 'cargo|nextest|target/.*/deps/' || true
-            echo
-            echo "Kernel OOM / kill events:"
-            dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
-          } > artifacts/test-and-lint/nextest-diagnostics.txt
-          exit "${status}"
+          python3 scripts/ci_test_artifact.py build --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
+
+      - name: Upload workspace test binaries
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: workspace-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/workspace-tests.tar.zst*
+          compression-level: 0
+          if-no-files-found: error
+          retention-days: 1
 
       - name: Run documentation tests
         run: |
           mkdir -p artifacts/test-and-lint
           set +e
           timeout --verbose --signal=TERM --kill-after=30s 15m \
-            cargo test --all --exclude e2e_test --doc \
+            python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
             2>&1 | tee artifacts/test-and-lint/doctest.log
           status=${PIPESTATUS[0]}
           {
@@ -261,9 +245,8 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: junit-test-and-lint-${{ github.run_number }}
+          name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
-            target/nextest/ci/junit.xml
             artifacts/test-and-lint
           retention-days: 3
           if-no-files-found: error
@@ -275,33 +258,73 @@ jobs:
       - name: Check log-analyzer rule anchors
         run: ./scripts/check_log_analyzer_rules.sh
 
-      # Preserve the migration floor and require successful execution evidence
-      # from the workspace run, without compiling or executing its subset again.
-      - name: Verify rebalance/decommission migration proofs
-        run: ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+  workspace-tests:
+    name: Workspace tests
+    needs: [ test-and-lint, classify-changes ]
+    # The archive retains compile-time fixture paths and the large test set
+    # requires the same runner image until the hosted-runtime pilot passes.
+    runs-on: sm-standard-4
+    timeout-minutes: 90
+    env:
+      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/test-runtime
+      - name: Protect Connect test home
+        run: chmod go-w "$(realpath "$HOME")"
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: workspace-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}
+      - name: Verify workspace archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
+
+      - name: Run nextest tests
+        run: |
+          mkdir -p artifacts/test-and-lint
+          rm -f target/nextest/ci/junit.xml
+          ./scripts/ci/resource_sampler.sh start nextest
+          trap './scripts/ci/resource_sampler.sh stop' EXIT
+          set +e
+          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
+            cargo nextest run --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite \
+            --status-level all --final-status-level all \
+            2>&1 | tee artifacts/test-and-lint/nextest.log
+          status=${PIPESTATUS[0]}
+          if [[ "${status}" -eq 0 ]]; then
+            cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --message-format json \
+              > artifacts/test-and-lint/core-test-listing.json \
+              && python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
+              && test -s target/nextest/ci/junit.xml || status=$?
+          fi
+          {
+            echo "command=workspace archive execution"
+            echo "exit_status=${status}"
+            echo "finished_at=$(date --utc --iso-8601=seconds)"
+            echo
+            echo "Remaining test-related processes:"
+            pgrep -af 'cargo|nextest|target/.*/deps/' || true
+            echo
+            echo "Kernel OOM / kill events:"
+            dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
+          } > artifacts/test-and-lint/nextest-diagnostics.txt
+          exit "${status}"
+
+      - name: Verify workspace execution completeness
+        run: |
+          python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
+          ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
 
-      # The workspace already compiled these ignored tests. Keep the same Cargo
-      # package selection to reuse their binaries, then run them serially in
-      # separate processes because they share fixed ports and global state.
-      # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
-      # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
-      # disk_index 0), not an EC metadata-distribution issue.
-      # restore_object_usecase_reports_ongoing_conflict_and_completion was
-      # re-enabled by backlog#1304 (restore accepts serialize on a short CAS
-      # guard; the copy-back no longer holds the #4877 whole-copy-back lock,
-      # so the mid-restore ongoing read and fast 409 rejection it asserts are
-      # the implemented contract).
       - name: Run ignored ILM integration tests serially
         if: needs.classify-changes.outputs.mode == 'full'
-        env:
-          # Match the preceding workspace build and reuse its feature graph.
-          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
         run: |
           mkdir -p artifacts/ilm-integration
           set +e
           NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
             cargo nextest run --profile default -j1 --run-ignored ignored-only \
-            --all --exclude e2e_test \
+            --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite \
             -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
             --status-level all --final-status-level all \
             2>&1 | tee artifacts/ilm-integration/nextest.log
@@ -315,21 +338,21 @@ jobs:
           } > artifacts/ilm-integration/diagnostics.txt
           exit "${status}"
 
-      - name: Upload ILM test diagnostics
-        if: always() && needs.classify-changes.outputs.mode == 'full'
+      - name: Upload workspace test evidence
+        if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: ilm-integration-${{ github.run_number }}-${{ github.run_attempt }}
-          path: artifacts/ilm-integration
+          name: workspace-test-evidence-${{ github.run_attempt }}
+          path: |
+            target/nextest/ci/junit.xml
+            artifacts/test-and-lint
+            artifacts/ilm-integration
           retention-days: 3
+          if-no-files-found: error
 
-  # Both Connect boundaries rebuild only the rustfs crate on top of the restored
-  # ci-dev cache: the two e2e features are empty, so every dependency keeps its
-  # cached dev-profile artifact and each feature set recompiles rustfs once.
-  # The budget covers a ci-dev miss, where the first step builds the whole graph.
   connect-feature-boundaries:
     name: Connect Feature Boundaries
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -374,26 +397,7 @@ jobs:
       # link-concurrency limit; it is cold because ci-dev holds no release
       # artifacts.
       - name: Reject short credentials in release builds
-        run: |
-          log="$(mktemp)"
-          set +e
-          CARGO_TERM_COLOR=never cargo check -p rustfs --release \
-            --features connect-e2e-short-credentials >"$log" 2>&1
-          status=$?
-          set -e
-          cat "$log"
-          expected='error: connect-e2e-short-credentials is restricted to debug builds'
-          summary="error: could not compile \`rustfs\` (lib) due to 1 previous error"
-          expected_count="$(grep -Fxc "$expected" "$log" || true)"
-          summary_count="$(grep -Fc "$summary" "$log" || true)"
-          error_count="$(grep -Ec '^error(:|\[)' "$log" || true)"
-          if [ "$status" -ne 101 ] || [ "$expected_count" -ne 1 ] \
-            || [ "$summary_count" -ne 1 ] || [ "$error_count" -ne 2 ]; then
-            echo "feature gate did not fail solely at the expected compile_error" >&2
-            rm -f "$log"
-            exit 1
-          fi
-          rm -f "$log"
+        run: python3 scripts/check_connect_release_boundary.py
 
       - name: Check offline enrollment E2E root boundary
         run: ./scripts/check_offline_enrollment_e2e.sh
@@ -453,7 +457,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: junit-test-and-lint-rio-v2-${{ github.run_number }}
+          name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/ci/junit.xml
             artifacts/test-and-lint
@@ -541,7 +545,7 @@ jobs:
     if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
-    timeout-minutes: 30
+    timeout-minutes: 60
     env:
       FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
     steps:
@@ -608,7 +612,7 @@ jobs:
       - name: Upload debug binary
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: |
             target/debug/rustfs
             target/debug/rustfs.e2e.json
@@ -616,6 +620,23 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
+      - name: Archive E2E test binaries
+        env:
+          CARGO_BUILD_JOBS: "2"
+        run: python3 scripts/ci_test_artifact.py build --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
+
+      - name: Run E2E documentation tests
+        run: cargo test --locked -p e2e_test --doc
+
+      - name: Upload E2E test binaries
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/e2e-tests.tar.zst*
+          compression-level: 0
+          if-no-files-found: error
+          retention-days: 1
+
   build-rustfs-debug-binary-rio-v2:
     name: Build RustFS Debug Binary (rio-v2)
     # Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
@@ -649,7 +670,7 @@ jobs:
       - name: Upload debug binary
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: rustfs-debug-binary-rio-v2
+          name: rustfs-debug-binary-rio-v2-${{ github.run_attempt }}
           path: |
             target/debug/rustfs
             target/debug/rustfs.e2e.json
@@ -732,10 +753,51 @@ jobs:
         # scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
         run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture
 
+  core-contract:
+    name: Core contract
+    needs: [ build-rustfs-debug-binary ]
+    # Compile-time fixture paths must match the producer until the pilot passes.
+    runs-on: sm-standard-4
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/test-runtime
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: rustfs-debug-binary-${{ github.run_attempt }}
+          path: target/debug
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: e2e-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}
+      - name: Run core storage and authorization contract
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/core-contract-logs
+        run: |
+          chmod +x target/debug/rustfs
+          python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
+          cargo nextest list --profile core-contract --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --message-format json > "$RUNNER_TEMP/core-contract-list.json"
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile core-contract --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --no-tests fail
+          python3 scripts/ci_test_artifact.py evidence --core "$RUNNER_TEMP/core-contract-list.json" target/nextest/core-contract/junit.xml
+      - name: Upload core contract evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: core-contract-${{ github.run_attempt }}
+          path: |
+            target/nextest/core-contract/junit.xml
+            ${{ runner.temp }}/core-contract-list.json
+            ${{ runner.temp }}/core-contract-logs
+          if-no-files-found: error
+          retention-days: 3
+
   e2e-tests:
     name: End-to-End Tests
     needs: [ build-rustfs-debug-binary ]
-    runs-on: sm-standard-2
+    # Retain the producer image and fixture paths until the runtime pilot passes.
+    runs-on: sm-standard-4
     timeout-minutes: 30
     steps:
       - name: Checkout repository
@@ -743,16 +805,17 @@ jobs:
         with:
           persist-credentials: false
 
-      # Full setup with dependency caching: the smoke-suite step below
-      # compiles the e2e_test crate, which pulls in most of the workspace.
-      # Without a restored cache this was a cold multi-GB build on every run.
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
+      - name: Install verification runtime
+        uses: ./.github/actions/test-runtime
+
+      - name: Download E2E test binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          name: e2e-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}
+
+      - name: Verify E2E archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
 
       - name: Set up Python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
@@ -772,7 +835,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -781,20 +844,15 @@ jobs:
       # Build the e2e test graph once. The archive is reused by the smoke
       # selection guard, security exact-count check, and run below, avoiding a
       # second compile of the same e2e_test target on cold runners (backlog#1645).
-      - name: Archive e2e smoke test binaries
+      - name: Verify e2e smoke test membership
         env:
-          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
           NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
         run: |
-          cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
-          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
+          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
           python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
           ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
 
-      # Keep E2E documentation coverage with its already-built package graph.
-      - name: Run E2E documentation tests
-        run: cargo test -p e2e_test --doc
-
       # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
       # profile.e2e-smoke default-filter in .config/nextest.toml is the single
       # wiring mechanism for e2e tests in CI — extend that filter instead of
@@ -802,20 +860,23 @@ jobs:
       # random port and reuses the downloaded debug binary above.
       - name: Run e2e smoke suite
         env:
-          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
+          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
           RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
           SAMPLER_INTERVAL_SECS: "15"
         run: |
           ./scripts/ci/resource_sampler.sh start e2e-smoke
           trap './scripts/ci/resource_sampler.sh stop' EXIT
-          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
+          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite \
             --status-level all --final-status-level all --failure-output final
 
+      - name: Verify e2e smoke execution completeness
+        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
+
       - name: Upload e2e smoke diagnostics
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-smoke-diagnostics-${{ github.run_number }}
+          name: e2e-smoke-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/rustfs-e2e-smoke-logs/
             ${{ runner.temp }}/rustfs-e2e-smoke-list.json
@@ -826,10 +887,13 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-smoke-junit-${{ github.run_number }}
+          name: e2e-smoke-junit-${{ github.run_number }}-${{ github.run_attempt }}
           path: target/nextest/e2e-smoke/junit.xml
           if-no-files-found: warn
 
+      - name: Install tool build dependencies
+        run: sudo apt-get install -y build-essential cmake pkg-config
+
       - name: Install s3s-e2e test tool
         uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
         with:
@@ -851,7 +915,7 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-test-logs-${{ github.run_number }}
+          name: e2e-test-logs-${{ github.run_number }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
           retention-days: 3
 
@@ -868,7 +932,8 @@ jobs:
       github.event_name == 'merge_group' ||
       (github.event_name == 'push' && github.ref == 'refs/heads/main')
     needs: [ build-rustfs-debug-binary ]
-    runs-on: sm-standard-2
+    # Retain the producer image and fixture paths until the runtime pilot passes.
+    runs-on: sm-standard-4
     timeout-minutes: 55
     steps:
       - name: Checkout repository
@@ -876,13 +941,17 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Setup Rust environment
-        uses: ./.github/actions/setup
+      - name: Install verification runtime
+        uses: ./.github/actions/test-runtime
+
+      - name: Download E2E test binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          rust-version: stable
-          cache-shared-key: ci-dev
-          cache-save-if: 'false'
-          install-build-packaging-tools: 'false'
+          name: e2e-test-binaries-${{ github.run_attempt }}
+          path: ${{ runner.temp }}
+
+      - name: Verify E2E archive identity
+        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
 
       - name: Install network fault-injection tools
         run: |
@@ -940,7 +1009,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -981,7 +1050,7 @@ jobs:
         env:
           NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
         run: |
-          cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          cargo nextest list --profile e2e-full --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
           python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
 
       # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
@@ -995,13 +1064,16 @@ jobs:
           RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
           RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
           RUSTFS_HEAL_CHAOS_LOG_ROOT: ${{ runner.temp }}/rustfs-heal-chaos-logs
-        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
+        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite
+
+      - name: Verify e2e full execution completeness
+        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-full-list.json" target/nextest/e2e-full/junit.xml
 
       - name: Upload junit
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-full-junit-${{ github.run_number }}
+          name: e2e-full-junit-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             target/nextest/e2e-full/junit.xml
             ${{ runner.temp }}/rustfs-e2e-full-list.json
@@ -1011,7 +1083,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: fresh-startup-cas-evidence-${{ github.run_number }}
+          name: fresh-startup-cas-evidence-${{ github.run_number }}-${{ github.run_attempt }}
           path: |
             ${{ runner.temp }}/rustfs-startup-cas-evidence
             ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
@@ -1022,7 +1094,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-full-heal-logs-${{ github.run_number }}
+          name: e2e-full-heal-logs-${{ github.run_number }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/rustfs-heal-chaos-logs
           if-no-files-found: warn
           retention-days: 7
@@ -1049,7 +1121,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary-rio-v2
+          name: rustfs-debug-binary-rio-v2-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1082,13 +1154,13 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-test-logs-rio-v2-${{ github.run_number }}
+          name: e2e-test-logs-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
           path: /tmp/rustfs.log
           retention-days: 3
 
   s3-implemented-tests:
     name: S3 Implemented Tests
-    needs: [ build-rustfs-debug-binary, e2e-tests ]
+    needs: [ build-rustfs-debug-binary ]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
@@ -1100,7 +1172,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1125,7 +1197,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: s3tests-implemented-${{ github.run_number }}
+          name: s3tests-implemented-${{ github.run_number }}-${{ github.run_attempt }}
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
@@ -1162,7 +1234,7 @@ jobs:
     # prebuilt debug binary (no cargo build), and s3-implemented-tests — which
     # already waits on e2e-tests — finishes later anyway, so a green PR's total
     # wall clock is unchanged.
-    needs: [ build-rustfs-debug-binary, e2e-tests ]
+    needs: [ build-rustfs-debug-binary ]
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -1174,7 +1246,7 @@ jobs:
       - name: Download debug binary
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: rustfs-debug-binary
+          name: rustfs-debug-binary-${{ github.run_attempt }}
           path: target/debug
 
       - name: Make binary executable
@@ -1207,7 +1279,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: s3tests-lifecycle-behavior-${{ github.run_number }}
+          name: s3tests-lifecycle-behavior-${{ github.run_number }}-${{ github.run_attempt }}
           path: artifacts/s3tests-single/**
           if-no-files-found: ignore
           retention-days: 3
@@ -1220,6 +1292,8 @@ jobs:
       - typos
       - quick-checks
       - test-and-lint
+      - workspace-tests
+      - core-contract
       - connect-feature-boundaries
       - test-and-lint-rio-v2
       - test-and-lint-protocols
@@ -1252,6 +1326,8 @@ jobs:
       - typos
       - quick-checks
       - test-and-lint
+      - workspace-tests
+      - core-contract
       - test-and-lint-rio-v2
       - test-and-lint-protocols
       - build-rustfs-debug-binary

.github/workflows/functional-chain-health.yml

@@ -12,44 +12,13 @@ concurrency:
 jobs:
   collect:
     if: github.repository == 'rustfs/rustfs'
-    runs-on: smoke-testing
+    # Health remains available when either laboratory fleet is offline.
+    runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
-      - name: Ensure GitHub CLI
-        shell: bash
-        run: |
-          if command -v gh >/dev/null 2>&1; then
-            gh --version
-            exit 0
-          fi
-
-          gh_version="2.101.0"
-          case "$(uname -m)" in
-            x86_64) gh_arch="amd64" ;;
-            aarch64|arm64) gh_arch="arm64" ;;
-            *)
-              echo "Unsupported Linux architecture for GitHub CLI: $(uname -m)" >&2
-              exit 1
-              ;;
-          esac
-
-          gh_install_root="${RUNNER_TEMP:-$PWD}/gh-cli"
-          rm -rf "$gh_install_root"
-          mkdir -p "$gh_install_root/bin"
-          archive="$gh_install_root/gh.tar.gz"
-          curl --fail --location --retry 3 \
-            "https://github.com/cli/cli/releases/download/v${gh_version}/gh_${gh_version}_linux_${gh_arch}.tar.gz" \
-            --output "$archive"
-          tar -xzf "$archive" -C "$gh_install_root"
-          install -m 0755 \
-            "$gh_install_root/gh_${gh_version}_linux_${gh_arch}/bin/gh" \
-            "$gh_install_root/bin/gh"
-          echo "$gh_install_root/bin" >> "$GITHUB_PATH"
-          export PATH="$gh_install_root/bin:$PATH"
-          gh --version
       - name: Collect and publish verified chain health
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}

.github/workflows/rustfs-fault-tolerance-matrix.yml

@@ -93,6 +93,12 @@ jobs:
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
 
+      - name: Checkout environment guard
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+          path: rustfs-tools
+
       - name: Checkout auto-testing scripts
         timeout-minutes: 5
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -118,6 +124,13 @@ jobs:
           echo "auto-testing: ${AUTO_TESTING_SHA}"
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
+        run: python3 rustfs-tools/scripts/functional_environment.py enter --suite fault-tolerance
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -215,7 +228,24 @@ jobs:
           if-no-files-found: warn
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
-          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
+
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-matrix.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 rustfs-tools/scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-matrix.json
+          if-no-files-found: error

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -50,10 +50,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes, ahead of
-    # the performance suite.
-    types: [rustfs-chain-fault-tolerance]
 
 permissions:
   contents: read
@@ -80,7 +76,7 @@ jobs:
   fault-tolerance-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -136,6 +132,13 @@ jobs:
           echo "auto-testing: ${AUTO_TESTING_SHA}"
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
+        run: python3 scripts/functional_environment.py enter --suite fault-tolerance
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -305,58 +308,27 @@ jobs:
           if-no-files-found: warn
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
-          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
+          ./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
 
-      - name: "Continue functional chain (next: Performance)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff retries, then files an alert issue in rustfs/backlog.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
-              -F 'client_payload[from_suite]=fault-tolerance'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-fault-tolerance.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite fault-tolerance --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-fault-tolerance.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -391,3 +363,13 @@ jobs:
           name: functional-chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}/fault-tolerance.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite fault-tolerance --report "${FUNCTIONAL_ARTIFACTS_DIR}/suite.log"

.github/workflows/rustfs-functional-chain.yml

@@ -13,12 +13,23 @@
 # limitations under the License.
 
 # Reusable workflows run from this driver's commit in one Actions run. Each
-# suite still runs after an earlier suite fails; the final job requires all twelve.
+# selected suite still runs after an earlier failure; the final job checks its plan.
 name: RustFS Functional Chain
 
 on:
   workflow_dispatch:
     inputs:
+      validation_plan:
+        description: Registered cadence plan (existing suite frequencies are preserved)
+        type: choice
+        options: [nightly, weekly, monthly]
+        required: false
+        default: nightly
+      suites:
+        description: Comma-separated suite IDs, or empty for the complete registered plan
+        type: string
+        required: false
+        default: ''
       build_run_id:
         description: Successful nightly or validation package run; empty for package-only build
         type: string
@@ -216,10 +227,25 @@ jobs:
     outputs:
       manifest: ${{ steps.candidate.outputs.manifest }}
       performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}
+      plan: ${{ steps.selection.outputs.plan }}
+      suites: ${{ steps.selection.outputs.suites }}
+      optional_suites: ${{ steps.selection.outputs.optional_suites }}
     steps:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
+      - name: Select registered suites
+        id: selection
+        env:
+          VALIDATION_PLAN: ${{ inputs.validation_plan || 'nightly' }}
+          VALIDATION_SUITES: ${{ inputs.suites }}
+        run: python3 scripts/validation_registry.py select --output "${RUNNER_TEMP}/suite-plan.json"
+      - name: Retain selected responsibilities
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-plan-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/suite-plan.json
+          if-no-files-found: error
       - name: Resolve published candidate
         id: candidate
         env:
@@ -239,6 +265,7 @@ jobs:
           if-no-files-found: error
 
       - name: Check shared functional fleet runner before scheduling suites
+        if: ${{ contains(fromJSON(steps.selection.outputs.environments), 'functional-fleet') }}
         # The eleven functional suites run on the smoke-testing fleet; a hard
         # requirement. The performance lane runs on pf-testing, which is a
         # separate fleet - probe it without failing the whole chain so an
@@ -248,14 +275,17 @@ jobs:
         run: python3 scripts/check_functional_runners.py smoke-testing
 
       - name: Probe performance fleet runner
+        if: ${{ contains(fromJSON(steps.selection.outputs.suites), 'performance') }}
         id: perf_probe
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+          PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
         run: |
           if python3 scripts/check_functional_runners.py pf-testing; then
             echo "performance_ready=online" >> "$GITHUB_OUTPUT"
           else
             echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
+            test "$PERFORMANCE_OPTIONAL" = true
           fi
 
   validation:
@@ -580,6 +610,10 @@ jobs:
             > "${VALIDATION_ROOT}/shared-fixture-manifest.json"
           printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
 
+      - name: Reject quarantined suite environment before canonical validation
+        if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
+        run: python3 scripts/functional_environment.py check --suite heal
+
       - name: Run selected canonical validation case
         if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
         env:
@@ -665,7 +699,7 @@ jobs:
 
   upgrade:
     needs: [prepare]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'upgrade') }}
     uses: ./.github/workflows/rustfs-upgrade-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -673,7 +707,7 @@ jobs:
 
   s3:
     needs: [prepare, upgrade]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 's3') }}
     uses: ./.github/workflows/rustfs-s3-compat-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -681,7 +715,7 @@ jobs:
 
   kms:
     needs: [prepare, s3]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'kms') }}
     uses: ./.github/workflows/rustfs-kms-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -689,7 +723,7 @@ jobs:
 
   tier:
     needs: [prepare, kms]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'tier') }}
     uses: ./.github/workflows/rustfs-tier-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -697,7 +731,7 @@ jobs:
 
   storage:
     needs: [prepare, tier]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'storage') }}
     uses: ./.github/workflows/rustfs-storage-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -705,7 +739,7 @@ jobs:
 
   heal:
     needs: [prepare, storage]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'heal') }}
     uses: ./.github/workflows/rustfs-heal-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -713,7 +747,7 @@ jobs:
 
   pool:
     needs: [prepare, heal]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'pool') }}
     uses: ./.github/workflows/rustfs-pool-expand-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -721,7 +755,7 @@ jobs:
 
   security:
     needs: [prepare, pool]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'security') }}
     uses: ./.github/workflows/rustfs-security-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -729,7 +763,7 @@ jobs:
 
   replication:
     needs: [prepare, security]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'replication') }}
     uses: ./.github/workflows/rustfs-replication-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -737,7 +771,7 @@ jobs:
 
   fault-tolerance:
     needs: [prepare, replication]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'fault-tolerance') }}
     uses: ./.github/workflows/rustfs-fault-tolerance-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -745,15 +779,15 @@ jobs:
 
   table:
     needs: [prepare, fault-tolerance]
-    if: ${{ always() && needs.prepare.result == 'success' }}
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'table') }}
     uses: ./.github/workflows/rustfs-table-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
     secrets: inherit
 
   performance:
-    needs: [prepare, table]
-    if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}
+    needs: [prepare]
+    if: ${{ always() && needs.prepare.result == 'success' && contains(fromJSON(needs.prepare.outputs.suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready == 'online' }}
     uses: ./.github/workflows/rustfs-performance-test.yml
     with:
       chain_manifest: ${{ needs.prepare.outputs.manifest }}
@@ -781,8 +815,10 @@ jobs:
         env:
           CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
           CHAIN_NEEDS: ${{ toJSON(needs) }}
+          CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
         run: >-
           python3 scripts/functional_chain_evidence.py summarize
+          --allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
           --directory "${RUNNER_TEMP}/chain-evidence"
           --output "${RUNNER_TEMP}/chain-report.json"
       - name: Retain chain report regardless of test verdict
@@ -799,9 +835,10 @@ jobs:
         env:
           CHAIN_MANIFEST: ${{ needs.prepare.outputs.manifest }}
           CHAIN_NEEDS: ${{ toJSON(needs) }}
+          CHAIN_PLAN: ${{ needs.prepare.outputs.plan }}
         run: >-
           python3 scripts/functional_chain_evidence.py aggregate
-          --allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
+          --allow-skipped "${{ contains(fromJSON(needs.prepare.outputs.optional_suites || '[]'), 'performance') && needs.prepare.outputs.performance_ready != 'online' && 'performance' || '' }}"
           --directory "${RUNNER_TEMP}/chain-evidence"
           --output "${RUNNER_TEMP}/chain-complete.json"
       - name: Upload complete-chain evidence

.github/workflows/rustfs-heal-test.yml

@@ -29,11 +29,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the storage suite finishes. Heal runs
-    # exactly once per chain; the pool expansion workflow no longer embeds
-    # its own heal pass.
-    types: [rustfs-chain-heal]
 
 permissions:
   contents: read
@@ -63,7 +58,7 @@ jobs:
     timeout-minutes: 60
     # Standalone manual run, or one link of the nightly functional chain
     # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -109,6 +104,13 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
+        run: python3 scripts/functional_environment.py enter --suite heal
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -154,8 +156,7 @@ jobs:
       - name: Run heal test (write -> outage -> heal -> verify)
         timeout-minutes: 45
         id: test
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           ./auto-testing/rustfs_heal_test.sh \
@@ -241,9 +242,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing steps keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns turn the workflow red.
+          # Preserve harness completion separately; the final verdict checks
+          # every case after retaining reports and cleanup evidence.
           STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)"
           STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)"
           STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 ))
@@ -281,7 +281,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; step failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -377,8 +377,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -396,54 +397,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Pool expansion)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-pool' \
-              -F 'client_payload[from_suite]=heal'; then
-              echo "dispatched next suite Pool expansion (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after heal (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-pool'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-heal.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-heal-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-heal.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -472,3 +440,13 @@ jobs:
           name: functional-chain-heal-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-heal-${{ github.run_id }}-${{ github.run_attempt }}/heal.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite heal --report "${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"

.github/workflows/rustfs-kms-test.yml

@@ -29,9 +29,6 @@ on:
         description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
         required: false
         type: string
-  repository_dispatch:
-    # Chain handoff: dispatched when the S3 compatibility suite finishes.
-    types: [rustfs-chain-kms]
 
 permissions:
   contents: read
@@ -56,7 +53,7 @@ jobs:
   kms-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -100,6 +97,13 @@ jobs:
           openssl version
           docker --version || true
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
+        run: python3 scripts/functional_environment.py enter --suite kms
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -131,8 +135,7 @@ jobs:
       - name: Run KMS suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -185,10 +188,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -226,7 +227,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -325,8 +326,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -344,54 +346,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Tier)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-tier' \
-              -F 'client_payload[from_suite]=kms'; then
-              echo "dispatched next suite Tier (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after kms (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-tier'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-kms.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite kms --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-kms-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-kms.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -419,3 +388,13 @@ jobs:
           name: functional-chain-kms-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-kms-${{ github.run_id }}-${{ github.run_attempt }}/kms.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite kms --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-performance-test.yml

@@ -54,9 +54,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes.
-    types: [rustfs-chain-performance]
 
 permissions:
   contents: read
@@ -89,7 +86,7 @@ env:
 
 jobs:
   runner-check:
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -113,7 +110,7 @@ jobs:
       RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
     # Run on manual dispatch, or when the nightly build completed successfully.
     # Skipped when nightly failed.
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -159,6 +156,13 @@ jobs:
           warp --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
+        run: python3 scripts/functional_environment.py enter --suite performance
+
       - name: Reset test environment (before)
         if: ${{ inputs.cleanup_before != 'false' }}
         run: |
@@ -323,10 +327,27 @@ jobs:
           if-no-files-found: error
 
       - name: Reset test environment (after)
-        if: ${{ always() && inputs.cleanup_after != 'false' }}
+        id: environment_cleanup
+        if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
           ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
 
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-performance.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite performance --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-performance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-performance.json
+          if-no-files-found: error
+
       - name: Notify on failure
         if: failure()
         run: |
@@ -354,3 +375,13 @@ jobs:
           name: functional-chain-performance-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-performance-${{ github.run_id }}-${{ github.run_attempt }}/performance.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.benchmark.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite performance --report "${RUSTFS_RESULT_DIR}/summary.tsv"

.github/workflows/rustfs-pool-expand-test.yml

@@ -56,9 +56,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the heal suite finishes.
-    types: [rustfs-chain-pool]
 
 permissions:
   contents: read
@@ -86,14 +83,13 @@ env:
   RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
 
 jobs:
-  # Pool expansion: dispatched by the heal suite's chain handoff. Heal
-  # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
+  # Pool expansion owns an independent reusable suite.
   pool-expansion-test:
     name: Pool expansion / decommission test
     runs-on: smoke-testing
     # Leave setup and finalizer headroom beyond the default 240-minute test.
     timeout-minutes: 360
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     env:
       RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
       RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -149,6 +145,13 @@ jobs:
             exit 1
           fi
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
+        run: python3 scripts/functional_environment.py enter --suite pool
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' }}
@@ -270,8 +273,7 @@ jobs:
         # timeout-minutes requires; the bare || form yielded the string '240'
         # and the runner rejected the template (Unexpected value '240').
         timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -o pipefail
@@ -698,8 +700,9 @@ jobs:
             ./auto-testing/rustfs_pool_nginx_stage.sh cleanup
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}
+        if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -717,54 +720,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Security)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-security' \
-              -F 'client_payload[from_suite]=pool'; then
-              echo "dispatched next suite Security (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after pool (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-security'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-pool.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite pool --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-pool-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-pool.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -793,3 +763,13 @@ jobs:
           name: functional-chain-pool-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-pool-${{ github.run_id }}-${{ github.run_attempt }}/pool.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.pool_test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite pool --report "${POOL_ARTIFACT_DIR}/pool-steps.md"

.github/workflows/rustfs-replication-test.yml

@@ -39,9 +39,6 @@ on:
           - bucket
           - site
         default: all
-  repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes.
-    types: [rustfs-chain-replication]
 
 permissions:
   contents: read
@@ -68,7 +65,7 @@ jobs:
   replication-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -113,6 +110,13 @@ jobs:
           aws --version
           df -h /data | tail -1 || true
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
+        run: python3 scripts/functional_environment.py enter --suite replication
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -135,8 +139,7 @@ jobs:
       - name: Run replication suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -192,10 +195,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -234,7 +235,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -333,8 +334,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -352,50 +354,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Fault tolerance)"
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-fault-tolerance' \
-              -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Fault tolerance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            trap 'rm -f "${BODY_FILE}"' EXIT
-            {
-              echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-replication.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite replication --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-replication.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -424,3 +397,13 @@ jobs:
           name: functional-chain-replication-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-replication-${{ github.run_id }}-${{ github.run_attempt }}/replication.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite replication --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-s3-compat-test.yml

@@ -17,9 +17,6 @@ on:
         description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
         required: false
         type: string
-  repository_dispatch:
-    # Chain handoff: dispatched when the upgrade suite finishes.
-    types: [rustfs-chain-s3]
 
 permissions:
   contents: read
@@ -44,7 +41,7 @@ jobs:
   s3-compat-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -88,6 +85,13 @@ jobs:
           openssl version
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
+        run: python3 scripts/functional_environment.py enter --suite s3
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -110,8 +114,7 @@ jobs:
       - name: Run S3 compatibility suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -161,10 +164,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -203,7 +204,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -302,8 +303,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -321,54 +323,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: KMS)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-kms' \
-              -F 'client_payload[from_suite]=s3'; then
-              echo "dispatched next suite KMS (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after s3 (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-kms'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-s3.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite s3 --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-s3-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-s3.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -396,3 +365,13 @@ jobs:
           name: functional-chain-s3-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-s3-${{ github.run_id }}-${{ github.run_attempt }}/s3.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite s3 --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-security-test.yml

@@ -52,9 +52,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Chain handoff: dispatched when the pool expansion suite finishes.
-    types: [rustfs-chain-security]
 
 permissions:
   contents: read
@@ -81,7 +78,7 @@ jobs:
   security-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       # Checkout the repository into its own subdirectory. Checking out at
       # the workspace root would wipe the auto-testing clone above (that is
@@ -126,6 +123,13 @@ jobs:
           docker --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
+        run: python3 rustfs-repo/scripts/functional_environment.py enter --suite security
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
@@ -149,8 +153,7 @@ jobs:
       - name: Run security suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         env:
           REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
@@ -194,9 +197,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns turn the workflow red. The suite log lines carry ANSI
+          # The final verdict checks cases after diagnostics and cleanup.
+          # The suite log lines carry ANSI
           # color escapes both before and after the verdict tag
           # (e.g. "\e[1;31m[FAIL]\e[0m STS-105 ..."), so the pattern allows
           # any number of escapes on each side of it.
@@ -236,7 +238,7 @@ jobs:
             fi
           } > "${SECURITY_ARTIFACTS_DIR}/report.md"
           cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -336,8 +338,9 @@ jobs:
           retention-days: 3
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
+        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -355,23 +358,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Replication)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          echo "Dispatching next functional suite: Replication"
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-replication' \
-            -F 'client_payload[from_suite]=security'
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-security.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 rustfs-repo/scripts/functional_environment.py finish --suite security --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-security-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-security.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -400,3 +401,13 @@ jobs:
           name: functional-chain-security-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-security-${{ github.run_id }}-${{ github.run_attempt }}/security.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 rustfs-repo/scripts/functional_chain_evidence.py verify-report --suite security --report "${SECURITY_ARTIFACTS_DIR}/suite-report.md"

.github/workflows/rustfs-storage-test.yml

@@ -26,9 +26,6 @@ on:
           - single-multi
           - multi-multi
         default: all
-  repository_dispatch:
-    # Chain handoff: dispatched when the tier suite finishes.
-    types: [rustfs-chain-storage]
 
 permissions:
   contents: read
@@ -53,7 +50,7 @@ jobs:
   storage-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -97,6 +94,13 @@ jobs:
           openssl version
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
+        run: python3 scripts/functional_environment.py enter --suite storage
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -119,8 +123,7 @@ jobs:
       - name: Run storage engine suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         run: |
           set -euo pipefail
@@ -176,10 +179,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -218,7 +219,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -317,8 +318,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -336,54 +338,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Heal)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-heal' \
-              -F 'client_payload[from_suite]=storage'; then
-              echo "dispatched next suite Heal (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after storage (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-heal'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-storage.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite storage --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-storage.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -411,3 +380,13 @@ jobs:
           name: functional-chain-storage-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-storage-${{ github.run_id }}-${{ github.run_attempt }}/storage.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite storage --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-table-test.yml

@@ -89,6 +89,13 @@ jobs:
           aws --version
           python3 --version
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
+        run: python3 scripts/functional_environment.py enter --suite table
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         continue-on-error: true
@@ -162,10 +169,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -203,7 +208,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -292,11 +297,12 @@ jobs:
           retention-days: 30
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
-        continue-on-error: true
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
-          set -uo pipefail
+          set -euo pipefail
+          cleanup_status=0
           for node in ${RUSTFS_NODES}; do
             ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
               set -euo pipefail
@@ -307,26 +313,25 @@ jobs:
               fi
               for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
               ${SUDO} rm -f /etc/default/rustfs
-            ' || echo "cleanup failed on ${node} (continuing)"
+            ' || cleanup_status=$?
           done
+          exit "${cleanup_status}"
 
-      - name: "Continue functional chain (next: Performance)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          echo "Dispatching next functional suite: Performance"
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=table'
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-table.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite table --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-table-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-table.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -358,3 +363,13 @@ jobs:
           name: functional-chain-table-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-table-${{ github.run_id }}-${{ github.run_attempt }}/table.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite table --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/rustfs-tier-test.yml

@@ -37,9 +37,6 @@ on:
         required: false
         default: false
         type: boolean
-  repository_dispatch:
-    # Chain handoff: dispatched when the KMS suite finishes.
-    types: [rustfs-chain-tier]
 
 permissions:
   contents: read
@@ -68,7 +65,7 @@ jobs:
   tier-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -216,6 +213,13 @@ jobs:
           openssl version
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
+        run: python3 scripts/functional_environment.py enter --suite tier
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         run: |
@@ -269,8 +273,7 @@ jobs:
       - name: Run tier suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         env:
           PACKAGE_URL_INPUT: ${{ inputs.package_url }}
@@ -467,8 +470,9 @@ jobs:
           if-no-files-found: error
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: always()
+        if: ${{ always() && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
@@ -488,6 +492,22 @@ jobs:
             '
           done
 
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-tier.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite tier --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-tier-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-tier.json
+          if-no-files-found: error
+
       - name: Cleanup pinned RustFS CLI
         timeout-minutes: 2
         if: always()
@@ -664,55 +684,6 @@ jobs:
             --package-source "${PACKAGE_SOURCE}" \
             --date "$(date -u +%Y-%m-%d)"
 
-      - name: "Continue functional chain (next: Storage engine)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-storage' \
-              -F 'client_payload[from_suite]=tier'; then
-              echo "dispatched next suite Storage engine (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after tier (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-storage'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
-
       - name: Notify on failure
         if: failure()
         run: |
@@ -739,3 +710,13 @@ jobs:
           name: functional-chain-tier-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-tier-${{ github.run_id }}-${{ github.run_attempt }}/tier.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite tier --report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md"

.github/workflows/rustfs-upgrade-test.yml

@@ -59,9 +59,6 @@ on:
         description: 'Reset the nodes after the test (DESTROYS test data/config)'
         type: boolean
         default: true
-  repository_dispatch:
-    # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
-    types: [rustfs-chain-upgrade]
 
 permissions:
   contents: read
@@ -86,7 +83,7 @@ jobs:
   upgrade-test:
     runs-on: smoke-testing
     timeout-minutes: 60
-    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
+    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
     steps:
       - name: Checkout repository (for report parser)
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -132,6 +129,13 @@ jobs:
           docker --version || true
           df -h /data | tail -1
 
+      - name: Admit clean shared environment
+        id: environment
+        timeout-minutes: 5
+        env:
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
+        run: python3 scripts/functional_environment.py enter --suite upgrade
+
       - name: Cleanup environment (before)
         timeout-minutes: 5
         if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
@@ -165,8 +169,7 @@ jobs:
       - name: Run upgrade compatibility suite
         id: test
         timeout-minutes: 45
-        # Case failures keep the run green: the report and the backlog
-        # issue manager carry the product signal.
+        # Collect reports and backlog evidence before the final suite verdict.
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -258,10 +261,8 @@ jobs:
             RESULT=success
           fi
 
-          # Product gate: failing cases keep the run green — they are reported
-          # below and tracked in rustfs/backlog. Only harness/environment
-          # breakdowns (the suite never reached case level, or a wholesale
-          # failure with zero passes) turn the workflow red.
+          # Report harness completion here; the final verdict checks every
+          # case result after diagnostics and cleanup have been retained.
           CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)"
           CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 ))
@@ -304,7 +305,7 @@ jobs:
             fi
           } | tee "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Red only for harness/environment breakdowns; case failures stay green.
+          # Preserve the harness verdict separately from the final case verdict.
           [ "${HARNESS_OK}" = "1" ]
 
       - name: Upload functional report to dashboard
@@ -412,8 +413,9 @@ jobs:
           retention-days: 3
 
       - name: Cleanup environment (after)
+        id: environment_cleanup
         timeout-minutes: 5
-        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
+        if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}
         run: |
           set -euo pipefail
           read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
@@ -431,54 +433,21 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: S3 compatibility)"
-        # Only chain-triggered runs forward to the next suite; standalone
-        # workflow_dispatch runs stop after their own cleanup. A failed
-        # handoff must never pass silently: it retries, then files an alert
-        # issue in rustfs/backlog so a stalled chain is visible.
-        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        continue-on-error: true
+      - name: Release environment only after successful cleanup
+        if: ${{ always() && steps.environment.outcome == 'success' }}
+        timeout-minutes: 5
         env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-s3' \
-              -F 'client_payload[from_suite]=upgrade'; then
-              echo "dispatched next suite S3 compatibility (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after upgrade (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            {
-              echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-s3'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-upgrade.json
+          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
+        run: python3 scripts/functional_environment.py finish --suite upgrade --cleanup-outcome "$CLEANUP_OUTCOME"
+
+      - name: Retain environment ownership and cleanup receipt
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: functional-environment-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/environment-upgrade.json
+          if-no-files-found: error
 
       - name: Notify on failure
         if: failure()
@@ -508,3 +477,13 @@ jobs:
           name: functional-chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
           path: ${{ runner.temp }}/chain-upgrade-${{ github.run_id }}-${{ github.run_attempt }}/upgrade.json
           if-no-files-found: error
+
+      - name: Enforce suite and report result
+        if: ${{ always() }}
+        env:
+          SUITE_OUTCOME: ${{ steps.test.outcome }}
+          REPORT_OUTCOME: ${{ steps.chain_report.outcome }}
+        run: |
+          test "$SUITE_OUTCOME" = success
+          test "$REPORT_OUTCOME" = success
+          python3 scripts/functional_chain_evidence.py verify-report --suite upgrade --report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"

.github/workflows/scheduled-validation-freshness.yml

@@ -34,7 +34,7 @@ jobs:
     # self-hosted sm-standard-* pool is down, which is exactly when the
     # scheduled validations it watches stall.
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
     permissions:
       actions: read
       contents: read
@@ -47,6 +47,7 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          NIGHTLY_SOURCE_REF: ${{ vars.NIGHTLY_BRANCH || 'main' }}
         run: |
           set +e
           python3 scripts/check_scheduled_validation_freshness.py \

.github/workflows/scheduled-validation-watchdog.yml

@@ -16,19 +16,8 @@ name: Scheduled Validation Watchdog
 
 on:
   workflow_run:
-    workflows:
-      - "Security Audit"
-      - "Build and Release"
-      - "Continuous Integration"
-      - "coverage"
-      - "e2e-nightly"
-      - "e2e-distributed"
-      - "e2e-s3tests"
-      - "Fuzz"
-      - "mint"
-      - "minio-interop"
-      - "Nightly GNU Build"
-      - "Runner Hygiene"
+    # The registry filters this broad subscription; names are not duplicated here.
+    workflows: ["*"]
     types: [completed]
 
 permissions:
@@ -37,11 +26,7 @@ permissions:
 jobs:
   alert-on-incomplete-run:
     name: Alert on incomplete scheduled run
-    if: >-
-      github.repository == 'rustfs/rustfs' &&
-      github.event.workflow_run.event == 'schedule' &&
-      github.event.workflow_run.conclusion != 'success' &&
-      github.event.workflow_run.conclusion != 'failure'
+    if: github.repository == 'rustfs/rustfs'
     # Stay on GitHub-hosted runners: this monitor must keep running when the
     # self-hosted sm-standard-* pool is down, which is exactly when the
     # scheduled validations it watches stall.
@@ -55,7 +40,11 @@ jobs:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
+      - name: Select expected completion from registry
+        id: registry
+        run: python3 scripts/validation_registry.py watchdog
       - name: Open or update incomplete-run issue
+        if: steps.registry.outputs.selected == 'true'
         uses: ./.github/actions/schedule-failure-issue
         with:
           github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}

docs/testing/README.md

@@ -31,6 +31,8 @@ The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree
 
 The [scanner memory acceptance runner](scanner-memory-acceptance.md) verifies owned S3 datasets, scanner restart progress, and Linux process/cgroup memory evidence under a fixed container limit.
 
+The [Actions resource accounting guide](ci-resource-accounting.md) covers all attempts, cross-workflow attribution and unknown costs. The [runner capability and storage pilot](ci-runner-pilot.md) provides manual preflight and fixed-source build comparisons without changing required job placement.
+
 The [Pool layout compatibility reference](pool-layout-compatibility.md) defines the topology and EC regression matrix for single-drive, single-node multi-drive, and multi-node expansion pools.
 
 The [Scanner/Heal topology protection gates](scanner-heal-topology-gates.md) map G-EXP, G-REB and G-DEC and their fault windows to the `scanner_heal_topology_` cases, their recorded inventory, and the basis of their timeouts.

docs/testing/ci-gates.md

@@ -16,15 +16,15 @@ The `main` ruleset (`6436880`) requires exactly these contexts, with `strict_req
 
 Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `scripts/ci_gate.py` to select a conservative documentation-only path: root Markdown/licenses, `AGENTS.md`, Markdown under `docs/` or `.agents/skills/`, and documentation images. Unknown paths, unavailable Git history, an empty diff, or a missing base policy select the full scope for that event. Renames include their deleted source path. Documentation-only PRs still run Quick Checks and Typos; the aggregate requires the expensive jobs to be skipped exactly as selected.
 
-PRs changing only Rust sources under `crates/e2e_test/src/` and known E2E selection digests (optionally with those documentation paths) use the `e2e` scope. They keep workspace validation, the server build, E2E smoke, and both S3 lanes; independent workflows retain their path selection. Connect boundary, serial ILM, optional protocol/rio-v2, and real io_uring jobs are skipped because their sources and test inputs are unchanged. Manifests, fixtures, shared nextest configuration, scripts, production sources, and unknown paths still select `full`. A dependency on `e2e_test` from another workspace member or reachable local dependency disables this shortcut. All non-PR events retain their full selection.
+PRs changing only Rust sources under `crates/e2e_test/src/` and known E2E selection digests (optionally with those documentation paths) use the `e2e` scope. They keep workspace validation, the server build, E2E smoke, and both S3 lanes; independent workflows retain their path selection. Connect boundary, serial ILM, optional protocol/rio-v2, and real io_uring jobs are skipped because their sources and test inputs are unchanged. Manifests, fixtures, shared nextest configuration, scripts, production sources, and unknown paths still select `full`. A dependency on `e2e_test` from another workspace member or reachable local dependency disables this shortcut. All non-PR events retain their full selection. The structured planner also records proposed affected-package and Connect selections in shadow mode; effective responsibility remains unchanged until a reviewed promotion. See [verification plans and artifacts](ci-verification-pipeline.md) for identity, rollout and queue-switch contracts.
 
 Alternate-feature unit tests and Clippy for rio-v2, Swift, SFTP, and HTTP/3 run on merge groups and on scheduled (nightly) or manual validation. PRs and main pushes run the default workspace and retain ILM, S3, real io_uring, and Connect security boundaries in their applicable scopes. The aggregate requires the alternate-feature jobs to be skipped on PRs and main pushes and successful on merge-group, scheduled, and manual events. This avoids repeating four feature builds on every PR revision and on every merge; feature-specific regressions are detected by the nightly run when the merge queue is not used. For changes to these features, dispatch `ci.yml` against the candidate branch before merging when that signal is needed, including PRs targeting `gateway/integration`, which has no automatic push trigger in this workflow. Scheduled results supplement the main baseline and do not replace it.
 
-Workspace Clippy runs on every event with a selected workspace Rust job, pull requests included, ahead of the workspace tests in the same job. `cache-warm.yml` warms Clippy's check artifacts under `ci-dev` with the same invocation to reuse compatible dependency artifacts. `make pre-commit` does not run Clippy. Select local Clippy targets, features, and documentation exemptions from the root `AGENTS.md` verification tier; the workspace command below reproduces the CI lane and is not a universal pre-push requirement.
+Workspace Clippy runs on every event with a selected workspace Rust job, pull requests included, in `Workspace Lint and Build`. That job also creates the exact test archive and runs doctests; `Workspace tests` consumes the archive without rebuilding it. `cache-warm.yml` warms Clippy's check artifacts under `ci-dev` with the same invocation to reuse compatible dependency artifacts. `make pre-commit` does not run Clippy. Select local Clippy targets, features, and documentation exemptions from the root `AGENTS.md` verification tier; the workspace command below reproduces the CI lane and is not a universal pre-push requirement.
 
-Serial ILM tests run at the end of `test-and-lint` for the `full` scope, reusing the workspace's Cargo package and feature selection. The existing ignored-test filter and `-j1` remain; `--profile default` keeps the no-retry execution and preserves the separate core CI JUnit report. Any ILM failure fails the workspace job and required aggregate. Logs remain in the `ilm-integration` artifact. There is no separate ILM runner or cache restore.
+Serial ILM tests run at the end of `workspace-tests` for the `full` scope, consuming the same verified archive as ordinary workspace tests. The existing ignored-test filter and `-j1` remain; `--profile default` keeps the no-retry execution and preserves the separate core CI JUnit report. Any ILM failure fails the consumer and required aggregate. Logs remain in the workspace test evidence artifact. There is no separate ILM compilation or cache restore.
 
-`required-checks` runs even after failed or skipped dependencies. `scripts/ci_gate.py verify` rejects missing jobs, unexpected jobs, failure, cancellation, and unexpected skips; optional lanes are required only on their declared events. `Workspace Test and Lint` is the ordinary Rust job, while `Test and Lint` uniquely names the aggregate. New validation jobs must update both its direct dependencies and the script contract. Test this wiring and its failure cases with `python3 scripts/ci_gate.py --self-test`.
+`required-checks` runs even after failed or skipped dependencies. `scripts/ci_gate.py verify` rejects missing jobs, unexpected jobs, failure, cancellation, and unexpected skips; optional lanes are required only on their declared events. `Workspace Lint and Build` and `Workspace tests` are the ordinary Rust producer and consumer, while `Test and Lint` uniquely names the aggregate. New validation jobs must update both its direct dependencies and the script contract. Test this wiring and its failure cases with `python3 scripts/ci_gate.py --self-test`.
 
 The `gateway/integration` ruleset (`24663286`) requires the same three contexts with the same review rules for the s3s to gateway integration branch (rustfs/backlog#2734). It omits `required_linear_history` because `gateway-integration-sync.yml` lands `--no-ff` merge commits from `main` on that branch every weekday, and it names the `rustfs-cla-assistant` app as the bypass actor that pushes them. `ci.yml` runs on pull requests targeting either branch.
 
@@ -50,14 +50,15 @@ The four site-replication state-writer concurrency proofs reserve the available
 | Event | Check name | Workflow / job | Merge status | Reproduce |
 |---|---|---|---|---|
 | PR, non-doc change | `Quick Checks` | `ci.yml` `quick-checks` | Required | `make pre-commit` |
-| PR, non-doc change, and every non-PR event | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --exclude e2e_test --doc`; migration evidence check described below |
+| PR, non-doc change, and every non-PR event | `Workspace Lint and Build`, `Workspace tests` | `ci.yml` `test-and-lint` / `workspace-tests` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --exclude e2e_test --doc`; migration evidence check described below |
 | PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
-| Full selection | `Workspace Test and Lint` (serial ILM step) | `ci.yml` `test-and-lint` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
+| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
 | Merge group, nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs and main pushes | `cargo nextest run` with the job's `--features` |
 | PR, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
 | PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
 | PR, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
 | PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Via aggregate | `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` under the same wrapper; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
+| PR, non-doc change | `Core contract` | `ci.yml` `core-contract` | Via aggregate | existing write/read, anonymous denial and restart-byte tests via `core-contract` profile and the E2E archive |
 | PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Via aggregate | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env |
 | PR, non-doc change | `S3 Lifecycle Behavior Tests` | `ci.yml` `s3-lifecycle-behavior-tests` | Via aggregate | `scripts/s3-tests/run.sh` with the job's accelerated-scanner env |
 | PR touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
@@ -77,39 +78,42 @@ Architecture migration rules run once in the required Quick Checks job. Migratio
 
 Scanner usage and heal rebuild coverage are intentionally split by risk and cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect authoritative scanner usage publication, quota-visible usage, or admin usage snapshots get an end-to-end signal before merge review. `heal_erasure_disk_rebuild_test` runs in `e2e-full` so core erasure heal rebuild regressions are caught no later than the merge queue or `main` push lane; it also remains in `e2e-nightly` with the serialized cluster fault-domain suites for scheduled soak signal.
 
+The E2E producer builds the hooks binary and harness archive once. Smoke, full and core consumers verify their identity and execution membership. The two S3 lanes depend directly on that producer and run alongside E2E. Runtime consumers retain the producer checkout path because existing fixtures embed compile-time paths.
+
 ## Scheduled validation
 
-Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has no recent attempt or completed successful scheduled run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`.
+Scheduled lanes never block a PR. Their workflow-local gate fails the run. The versioned `.github/scheduled-validations.json` registry maps workflow responsibility, cadence, failure ownership, live-state policy and suite requirements. Hosted watchdog and freshness monitors derive their expected set from it; disabled or paused entries remain visible without claiming success or enabling them. Attempts and completed successes have separate freshness checks. See [periodic validation orchestration](validation-orchestration.md) for single-suite selection, complete-chain evidence, shared-fleet quarantine and recovery.
 
-| Workflow (cadence) | Jobs | Verdict and artifacts | In freshness list | Reproduce |
+| Workflow (cadence) | Jobs | Verdict and artifacts | Registry monitor | Reproduce |
 |---|---|---|---|---|
 | `ci.yml` (nightly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | strict aggregate; the full E2E lane runs on dispatch, merge groups, and main pushes | yes | dispatch `ci.yml` |
-| `cache-warm.yml` (every six hours, plus main pushes that change `Cargo.lock`, `rust-toolchain.toml` or the setup action) | `warm-ci-dev`, `warm-ci-feat-rio`, `warm-ci-feat-proto`, `warm-ci-uring` | no verdict; sole writer of the `rust-cache` keys that `ci.yml` restores | no | dispatch `cache-warm.yml` |
+| `cache-warm.yml` (every six hours, plus main pushes that change `Cargo.lock`, `rust-toolchain.toml` or the setup action) | `warm-ci-dev`, `warm-ci-feat-rio`, `warm-ci-feat-proto`, `warm-ci-uring` | no verdict; sole writer of the `rust-cache` keys that `ci.yml` restores | yes | dispatch `cache-warm.yml` |
 | `build.yml` (nightly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--features e2e-test-hooks` on both steps with `--profile e2e-nightly`; `--features ftps,webdav,sftp` on both steps with `-j 1 --profile e2e-protocols` |
 | `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, run `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
-| `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
+| `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes (disabled) | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
-| `on-demand-migration-interop.yml` (nightly) | `minio-source`, `cloud-source` (`aws`, `r2`, `gcs`) | report-only provider interop; one JSON report per provider naming cases, timings and source request counts, plus JUnit and MinIO logs. A cloud provider whose `ODM_INTEROP_*` secrets are absent is skipped with a summary note, not failed | no | start the pinned MinIO container as in the job, export the `RUSTFS_ODM_INTEROP_*` variables, run `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test` |
+| `on-demand-migration-interop.yml` (nightly) | `minio-source`, `cloud-source` (`aws`, `r2`, `gcs`) | report-only provider interop; one JSON report per provider naming cases, timings and source request counts, plus JUnit and MinIO logs. A cloud provider whose `ODM_INTEROP_*` secrets are absent is skipped with a summary note, not failed | yes | start the pinned MinIO container as in the job, export the `RUSTFS_ODM_INTEROP_*` variables, run `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test` |
 | `nightly-gnu.yml` (nightly) | `build`, `kms-vault-lane`, `kms-vault-ha-failover` | build, live Vault, and HA failover gates | yes | commands and pinned Vault images in the workflow |
 | `audit.yml` (nightly) | `cargo-deny`, `workflow-pin-report` | dependency and workflow-pin gates | yes | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
 | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
-| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
+| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
-| `e2e-upgrade.yml` (weekly) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | no | see the PR row |
-| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
-| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
-| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | no | dispatch, optionally with `source_ref` |
+| `e2e-upgrade.yml` (weekly) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | see the PR row |
+| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
+| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
+| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |
 | `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on missing or stale attempts or completed successes | n/a | dispatch |
 
 Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues.
 
 ## Packaged functional acceptance
 
-`rustfs-functional-chain.yml` dispatches the packaged-build suites in `rustfs-*-test.yml` on the shared lab runners. A failing suite step or job must fail its workflow. Report collection, cleanup, and dispatch of the next suite can still run with `always()`; continuing diagnostics does not make the failed suite successful.
+`rustfs-functional-chain.yml` calls selected packaged-build suites in one Actions run, using the registry's plan and the existing immutable candidate manifest. Shared-fleet suites remain serial and retain their lock through cleanup; the isolated performance fleet can run alongside them. A failed suite does not prevent later selected suites from gathering evidence, but required failures, cancellation, missing or foreign evidence fail the aggregate. The old repository-dispatch relay is removed; manual single-suite workflows end after their own suite.
+
+Functional-chain health runs on a hosted runner independently of the lab fleet. It verifies the full plan, exact candidate/attempt, current expected source SHA and completion freshness. A subset or stale candidate cannot satisfy full-chain health. Cleanup failure or interruption leaves a persistent node marker and blocks subsequent mutation until the documented recovery procedure completes. These code paths require live acceptance runs before the operational rollout can be declared complete.
 
-Workflow status preserves errors that the test scripts report. It does not establish complete execution or a common package identity across the chain: inspect the current run's case results, package identity, and test-script revision as well. A script that returns zero after a failed tool invocation needs its own result check.
 
 ## Release validation
 

docs/testing/ci-resource-accounting.md

@@ -0,0 +1,61 @@
+# Actions resource accounting
+
+**Use this when:** comparing CI changes across workflows and attempts, allocating observed work to PRs, or distinguishing runner occupancy from a provider bill.
+
+The read-only collector is `scripts/ci/actions_cost_report.py`. The independent manual entry is `.github/workflows/ci-resource-accounting.yml`; it does not run on PRs. `scripts/ci_timing_report.py` continues to own the successful ordinary-code sample definition, including both the historical combined lane and the workspace/core archive consumers.
+
+## Collect and reproduce
+
+Use an authenticated GitHub CLI with Actions and pull-request read access. Start with a small, closed creation window. The collector deliberately rejects an inventory above GitHub's filtered-run limit rather than silently truncating it; split such a window and retain each report. Do not sum overlapping windows without deduplicating repository, run ID, attempt and job ID.
+
+```bash
+python3 scripts/ci/actions_cost_report.py \
+  --repository rustfs/rustfs \
+  --since 2026-10-01T00:00:00Z \
+  --until 2026-10-01T12:00:00Z \
+  --output /tmp/ci-accounting.json \
+  --summary /tmp/ci-accounting.md
+
+python3 scripts/ci/actions_cost_report.py \
+  --input /tmp/ci-accounting.json \
+  --output /tmp/ci-accounting-recomputed.json \
+  --summary /tmp/ci-accounting-recomputed.md
+```
+
+The snapshot retains selected raw run, attempt, job and step fields alongside the summary. Every attempt from 1 to the observed latest attempt has its own Jobs API pagination. Failed and cancelled jobs remain in the sum. A missing attempt or a run that changes during collection marks the report incomplete and returns a failing exit status while retaining available evidence. Partial reports are useful for diagnosis; they are not a complete baseline.
+
+GitHub's [run inventory](https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository) is filtered by original creation time. A rerun of an older run can occur inside the wall-clock interval without entering this inventory. Widen the creation window for lifecycle studies; API retention and unidentified downstream events still prevent a completeness claim. The [attempt-specific Jobs API](https://docs.github.com/en/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run-attempt) provides the job evidence; logs are not downloaded, so expired or mixed build/test logs are never interpreted as zero compiler time.
+
+## Allocation and unknowns
+
+Direct PR events use their recorded PR members. A push at a known merged PR's exact merge commit is allocated to that PR. The collector never guesses all merge-group members from the first PR in a synthetic branch, or assigns a downstream workflow solely because its SHA matches. Unresolved jobs stay in the unattributed bucket.
+
+For reviewed merge-group membership or downstream provenance, supply an attribution file bound to the observed run ID and head SHA:
+
+```json
+{
+  "schema": 1,
+  "repository": "rustfs/rustfs",
+  "runs": [
+    {
+      "run_id": 123456,
+      "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+      "pull_requests": [123, 124],
+      "evidence": "https://github.com/rustfs/rustfs/actions/runs/123456"
+    }
+  ],
+  "pull_requests": {}
+}
+```
+
+Pass it with `--attributions` when recomputing the snapshot. Its evidence URL is an operator-reviewed assertion, not an API proof of membership. Include the original event payload or another retained membership receipt in that review. Each job's execution and scheduler-wait minutes are divided equally among recorded members. Periodic work remains separate and cannot be reassigned with the manifest. The unallocated original total remains available to check conservation. PR creation and merge timestamps are observations, while `lifecycle_complete` stays false because retained Actions history cannot prove all historical work exists.
+
+Known runner occupancy is the sum of completed-job start-to-end durations. The queue field measures creation-to-start wait; the API timestamps do not separately identify scheduler and dependency delay. Missing or inverted timestamps remain unknown. Explicitly skipped jobs consume zero execution minutes; queued, running or cancelled jobs with missing end times do not. The successful-code feedback sample uses the rerun attempt's start, excluding time before that attempt. That sample is separate from the accumulated PR cost across revisions and workflows.
+
+Phase totals are available only for dedicated steps named `CI phase: setup`, `CI phase: compile`, `CI phase: test`, `CI phase: archive` or `CI phase: transfer`. Other step names and timestamps remain visible without inferring a compiler/test split. Labels classify standard hosted, fixed-capacity, metered and unknown runners according to the repository's existing placement policy. Per-job rounded minutes are only a comparison model. Unit prices, capacity invoices and actual billing remain unknown; reduced demand does not establish reduced spend.
+
+## Comparisons and validation
+
+Preserve the baseline snapshot and source revision. Compare the same event, source policy, runner image, toolchain, build configuration and cache conditions. Image and cache identity require the runner pilot or build receipts; they are not recoverable from a label. Report known totals and unknown coverage together, including failed and cancelled work. A faster isolated successful run cannot replace the lifecycle report or establish equivalent test coverage.
+
+Run the accounting fixtures with `python3 -m unittest discover -s scripts/ci -p 'test_actions_cost_report.py'` and the shared timing tests with `python3 -m unittest discover -s scripts -p 'test_ci_timing_report.py'`. The fixtures cover pagination, partial retention, reruns, cancelled consumption, unknown timestamps, independent workflows, shared groups and allocation conservation. Disable the manual workflow to stop collection; no product gate or runner placement depends on it.

docs/testing/ci-runner-pilot.md

@@ -0,0 +1,50 @@
+# Runner capability and storage pilot
+
+**Use this when:** checking whether a runner can consume a build artifact, comparing workspace build storage, or preparing a limited local-SSD trial in the existing runner infrastructure.
+
+`scripts/ci/runner_pilot.py` owns capability checks and isolated cold/warm build measurements. `.github/workflows/ci-runner-pilot.yml` is a manual entry using the existing hosted and ARC labels. It creates no cloud resources, changes no pool capacity and moves no required CI jobs. The ARC controller configuration, node images and storage provisioning live outside this repository, as documented by the existing runner hygiene workflow. A successful probe does not certify local physical storage or worker destruction.
+
+## Probe and benchmark
+
+Start with the `probe` workload. The JSON receipt records architecture, kernel/libc, effective CPU and memory limits, free disk and mount, Docker daemon availability, an actual minimal io_uring setup attempt, installed tool versions and available image identity. Required capabilities fail closed through the CLI options. Unrequested unsupported capabilities remain observations. Network reachability, dynamic-library compatibility of a specific artifact and physical storage locality are not established by this probe.
+
+```bash
+python3 scripts/ci/runner_pilot.py \
+  --root /mnt/build-scratch \
+  --output /tmp/runner-pilot-evidence \
+  --architecture x86_64 --min-free-gib 90 --min-memory-gib 8
+```
+
+For a build, select `server` or `workspace-tests` in the manual workflow and supply an exact merged-main SHA and an exact Rust version. The workflow checks ancestry before executing the source, installs the requested compiler and the fixed protobuf/flatbuffer tool versions from its composite action, then invokes the tool. Run both workloads separately with the same source, toolchain and concurrency when comparing storage. Hosted runners may fail the disk preflight; this is an explicit capacity result, not a build failure to bypass.
+
+The CLI's `--benchmark`, `--source-sha`, `--toolchain` and `--jobs` options provide the same entry locally on a prepared Linux worker. Build commands are defined by `build_commands`; the workspace workload compiles tests without executing them. This pilot does not replace functional acceptance or the dedicated compile-attribution work.
+
+The cold build uses a new task-owned Cargo target directory with compiler wrappers and incremental compilation disabled. The warm build reuses only that target. Cargo registry downloads may already be warm, and the operating-system page cache is not dropped. GNU time records wall time, maximum RSS and filesystem I/O counters; these are not disk bandwidth or a simultaneous memory total across every process. Each attempt keeps logs and a JSON receipt. A failed or timed-out cold build stops the pair instead of producing a warm success. Ordinary termination kills the child process group and removes the task-owned target; a hard job timeout or lost node still requires infrastructure destruction. Queue, setup and evidence-upload costs come from the [resource accounting report](ci-resource-accounting.md).
+
+## Controlled existing-infrastructure trial
+
+An operator must make the actual ARC configuration change in its owning infrastructure repository. Use that deployment mechanism and retain its reviewed revision; do not install a second scheduler from this repository. Keep the current production pool available. Bound the candidate to one worker and one manual job at a time, isolate its scratch mount and credentials, and ensure the selected existing label is routed exclusively to the intended candidate during the trial. If that cannot be guaranteed without affecting ordinary jobs, use an operator-provisioned dedicated label and review its workflow/actionlint declaration before running a trial.
+
+The image must have a pinned digest and the required tools or a recorded setup phase. A locally attached SSD/NVMe mount must be a task-local filesystem, never a persistent shared target between untrusted jobs. Do not infer physical locality from a disk's nonrotational flag: virtual network-backed disks can report the same property. For `local_ssd: true`, the operator provides a root-owned, non-group/world-writable `/etc/rustfs-runner/storage.json` and sets `RUSTFS_RUNNER_IMAGE_DIGEST` to the actual image digest:
+
+```json
+{
+  "schema": 1,
+  "storage": "local-ssd",
+  "mount_source": "/dev/nvme0n1",
+  "mount_target": "/mnt/build-scratch",
+  "image_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+}
+```
+
+The tool requires that assertion to match the observed filesystem, mount and image digest. It reports the source as an operator attestation and keeps `storage_locality_verified` false. The operator must retain infrastructure evidence of device locality separately. Likewise, `worker_destruction_verified` stays false: directory cleanup is not a proof that a worker or credentials were destroyed.
+
+Before admitting PR work, the infrastructure owner must execute two separate jobs and demonstrate that the second cannot read a marker or credential fixture left by the first. Repeat for cancellation and abnormal termination, retaining controller events and worker identities. Never put publication credentials in this pool. The monthly `scripts/ci/check_runner_ephemerality.sh` audit remains independent. No deployment or isolation result is implied by the fixtures in this repository.
+
+## Artifact consumers and rollout decision
+
+Run the existing artifact identity/ABI verifier before a real consumer trial. A matching architecture and glibc version alone is insufficient: execute the exact selected test set, verify required dynamic libraries and Docker/privilege requirements, and retain the archive's result contract. Rust fixtures can embed absolute build-time paths. Preserve the producer workspace path and required fixture files, or explicitly establish a supported relocation strategy before using a different runner layout. Required consumers keep their current runner until that acceptance is complete.
+
+Repeat the same benchmark pair enough times to expose variability, including an empty target-cache case and a failed or cancelled run. Keep source, compiler, features, profile, concurrency and image fixed while changing storage. Compare queue, compilation, consumer execution, transfer and total occupied minutes; do not attribute simultaneous compiler or cache-policy changes to the SSD. Keep fixed-capacity charges separate from demand. Promotion needs measured total benefit and unchanged test responsibility; this tooling does not claim deployment, savings or a performance target.
+
+Rollback is to stop the manual workflow and restore the infrastructure owner's previous image, scratch mount and capacity revision. No main CI placement changes need reversal. Keep receipts outside the disposable target. Local validation is `python3 -m unittest discover -s scripts/ci -p 'test_runner_pilot.py'`; deployment, real cloud storage, queueing, node destruction and artifact migration remain live acceptance work.

docs/testing/ci-timing.md

@@ -1,5 +1,7 @@
 # CI timing samples
 
+**Use this when:** sampling the ordinary successful PR feedback path. For all attempts, independent workflows and PR allocation, use [Actions resource accounting](ci-resource-accounting.md); for controlled storage comparisons, use the [runner pilot](ci-runner-pilot.md).
+
 Collect a bounded sample of completed PR CI runs created in the last seven days with `python3 scripts/ci_timing_report.py --limit 30 --output /tmp/ci-timing.json`. This requires an authenticated `gh` CLI with Actions read access. The JSON retains run SHA, attempt, job and step timestamps so another reviewer can reproduce the summary with `--input /tmp/ci-timing.json --output /tmp/ci-timing-summary.json` without GitHub access.
 
 The report separates job creation-to-start wait, job execution, and individual step durations. The successful-code sample excludes documentation-only runs and successful PR-closure cancellation handlers. It uses the selected attempt's start for reruns, includes parallel jobs in the runner-minute sum, and leaves missing timestamps unknown. It does not estimate compiler time from a combined build-and-test step or treat runner minutes as wall time or a bill.

docs/testing/ci-verification-pipeline.md

@@ -0,0 +1,63 @@
+# Verification plans and reusable CI artifacts
+
+Use this guide when changing validation selection, test execution, or promoting a measured CI optimization. The workflow is `.github/workflows/ci.yml`; the selection policy is `.config/ci-plan.json`; the periodic responsibility registry is `.github/scheduled-validations.json`.
+
+## Selection and rollout
+
+The classifier reads the pull request base revision's planner and policy. It computes the changed packages and their reverse dependency closure from both revisions, including renamed dependencies, build dependencies, target dependencies and proc macros. Core package responsibilities are unioned with the committed test inventory. Unknown paths, changed build inputs, unavailable history or unresolved inventory expand validation. The proposed plan explains each selection and records candidate commit/tree, policy revision, event, configurations and expected job outcomes.
+
+The initial policy is `shadow`: proposed selection is retained for comparison, while effective execution retains the full existing workspace and Connect responsibility for ordinary code changes. Existing documentation and isolated E2E scopes remain conservative. A base policy predating structured plans selects the full matrix. Promotion to `enforced` is a separate reviewed policy change after representative shadow comparisons, changed-test leak checks and failed-selection fixtures pass. Changing a PR's own policy cannot authorize skipping its base revision's requirements.
+
+`Test and Lint` remains the required aggregate name. Every verification producer and consumer is a direct dependency. The verifier accepts only the effective plan's exact expected success/skipped outcomes; failure, cancellation, missing results, extra jobs or a plan for a different candidate fail. Quick Checks and CLA retain their separate required names.
+
+## Build and execute
+
+The workspace producer performs Clippy including the E2E harness with warnings denied, builds the selected nextest archive, executes doctests and checks log-rule anchors. The workspace consumer verifies and executes the archive, validates actual JUnit membership against its nextest listing, retains the core inventory and migration count floor, then executes the existing ignored ILM selection serially without a second compilation. The producer retains the measured Cargo build budget; execution retains nextest test groups and retry policy.
+
+The E2E producer builds the hooks server and its existing provenance sidecars, builds the harness archive once, and runs harness doctests. E2E smoke, full E2E and the short core contract consume that archive. S3 implemented and lifecycle tests depend directly on the server producer and can run alongside E2E. The core profile uses three existing tests for object write/read, anonymous authorization denial and bytes surviving restart; its execution report must contain exactly those tests.
+
+`scripts/ci_test_artifact.py` binds each archive to source content, commit/tree, lockfile, Rust and nextest versions, configuration, build flags, repository and run/attempt, then verifies its digest and size before use. A consumer with an older glibc, different platform, different selection or different identity fails. Re-running only a failed consumer does not borrow a previous attempt's archive: re-run the producer and consumers together. Archives expire after one day and are never used as release certification or uploaded into a privileged publishing context.
+
+Consumer setup does not restore a Cargo target cache. It installs the runtime, then downloads and validates the current artifact. Several tests embed absolute manifest paths and binary paths at compile time. Until those fixtures are made relocatable and tested, consumers require the same absolute checkout path as the producer, use the same runner family, and extract into that checkout's `target` directory. `--workspace-remap` alone does not rewrite strings compiled into a test executable. Source fixtures remain in the exact checkout, while non-test binaries and build outputs travel in the archive. The external s3s tool retains its independent pinned installation.
+
+Workspace execution initially has one shard. There is no measured evidence yet that distributing this workspace archive improves total occupancy after compression, transfer and extraction. This also preserves every cross-test resource group. A future multi-shard promotion must use observed durations, keep every shared-resource component in one shard, cap concurrency, and prove the union of successful shard executions equals the selected listing. Merely applying nextest hash partitions would split process-local resource groups and is not an acceptable promotion.
+
+## Measurements and operational switches
+
+Use [resource accounting](ci-resource-accounting.md) to compare the same candidate and include all workflow attempts, cancellation, queue delay, shared merge-group allocation, and unknown costs. Use the independent [runner pilot](ci-runner-pilot.md) to establish storage, runtime and cleanup capabilities before modifying placement. Neither a generated archive nor a successful static check proves the target latency or cost reduction.
+
+Periodic and manual suites use the responsibility registry and retain the existing immutable candidate/evidence contract. A selective manual run carries an explicit suite plan and cannot substitute for a full periodic chain. Shared environments remain locked through preparation, execution and cleanup. Health runs separately on hosted runners and checks the expected candidate, attempt and completed evidence; see [periodic validation orchestration](validation-orchestration.md).
+
+## Merge-queue preparation and rollback
+
+The merge-group event plans and validates its actual combined SHA/tree with the full Linux and feature configuration matrix. Main pushes keep complete validation; no reuse is inferred from tree equality because commit identity and build inputs can matter. CLA remains an independent queue-entry requirement.
+
+`scripts/ci_merge_queue.py` prepares a bounded queue proposal from a live ruleset snapshot without calling a mutation API. It preserves all existing checks, reviews, bypass actors and branch conditions, adds an `ALLGREEN` squash queue with at most two concurrent builds/two merged entries, and produces the original rollback payload. Its timeout accommodates the existing full gate while measurements are collected. API fields follow [GitHub's ruleset contract](https://docs.github.com/en/rest/repos/rules).
+
+```bash
+mkdir -p artifacts/queue-switch
+gh api repos/rustfs/rustfs/rulesets/6436880 > artifacts/queue-switch/before.json
+python3 scripts/ci_merge_queue.py artifacts/queue-switch/before.json > artifacts/queue-switch/prepared.json
+python3 scripts/ci_merge_queue.py artifacts/queue-switch/before.json \
+  --prepared artifacts/queue-switch/prepared.json --phase before
+```
+
+Enabling the queue is an explicit maintainer operation after the proposal is accepted and real merge-group validation succeeds. Before that operation, refresh the live snapshot and compare `--phase before`; drift requires a new proposal. Keep the unchanged required check names throughout the switch. After application, read the same ruleset again and compare `--phase after`; record the operation, resulting ruleset and actual merge-group run. Rollback restores the saved `before` payload and verifies it with `--phase before`, retaining PR checks and full main validation. The helper does not apply either payload.
+
+The deployment rehearsal must include two individually passing changes whose combination fails, queue regrouping after a base update, cancelled/missing results and a mismatched artifact. The combined candidate must remain blocked until its own full verification succeeds. Static fixture tests exercise the selection, identity and result contracts; they are not evidence that the live queue is enabled. The current rollout retains the pre-existing repository protection until the separate switch is performed.
+
+## Reproduction and regression gates
+
+```bash
+python3 -m pip install PyYAML==6.0.3
+python3 scripts/ci_gate.py --self-test
+python3 scripts/ci_gate.py --check-workflow
+python3 scripts/test_ci_test_artifact.py
+python3 scripts/check_connect_release_boundary.py --self-test
+python3 scripts/test_ci_merge_queue.py
+python3 scripts/check_test_wiring.py --self-test
+python3 scripts/check_test_wiring.py
+actionlint
+```
+
+The full `make script-tests` includes these contracts plus the periodic registry, environment cleanup and accounting tests. The semantic workflow checks accept equivalent YAML block scalars while rejecting conditions or shell constructs that suppress required execution or its failure.

docs/testing/validation-orchestration.md

@@ -0,0 +1,49 @@
+# Periodic validation orchestration
+
+Use this reference to select packaged suites, interpret a health observation, or recover a quarantined lab environment. The machine-readable responsibility map is [`.github/scheduled-validations.json`](../../.github/scheduled-validations.json). Its `workflows` entries preserve every baseline workflow and distinguish new entries, independent maintenance, operator-disabled policies, alert writers, and duties without a periodic obligation. Its `suites` entries own selection, dependencies, environments, resource locks, requiredness, and evidence exits.
+
+## Selection and scheduling
+
+[The functional driver](../../.github/workflows/rustfs-functional-chain.yml) consumes the existing immutable nightly candidate and exact build attempt. Automatic nightly completion and manual runs select through the same registry function. The `validation_plan` input accepts `nightly`, `weekly`, or `monthly`; larger cadence plans include the shorter cadence duties. Existing workflow crons remain their execution entrypoints, and the registry check rejects event or cron drift. Listing an independent periodic duty in a plan does not dispatch it or change its budget.
+
+The twelve existing packaged suites retain their nightly cadence. An empty `suites` input selects the complete cadence plan. A comma-separated list selects only those suites and declared dependencies. For example, `suites=s3` runs S3 compatibility alone. Explicit selection makes every selected suite required, including performance. A full plan may omit performance only when its registered optional policy and the preflight result both permit it; the complete evidence records the reason.
+
+Inspect selection without fleet access:
+
+```bash
+python3 scripts/validation_registry.py select --cadence nightly --suites s3
+python3 scripts/validation_registry.py select --cadence weekly --event schedule
+python3 scripts/validation_registry.py check
+```
+
+Shared functional suites run serially, and an earlier test failure does not stop later selected suites from gathering results. Performance uses an isolated fleet and can run concurrently. The final aggregate rejects required failures, cancellation, unexpected skips, missing or foreign evidence, and execution outside the selected plan. A successful subset remains subset evidence; it cannot satisfy full-chain health.
+
+The old `repository_dispatch` relay is retired. Each individual workflow remains a manual debugging entrypoint and stops after its own suite. Package-only builds, tools, read-only runner/fleet inspection, and interrupted evidence recovery remain separate driver modes. Their success does not represent suite completion. No workflow is enabled automatically by registration or monitoring.
+
+## Shared environment ownership
+
+Workflow concurrency holds the registered fleet lock across admission, setup, tests, cleanup, and evidence collection. Before any fleet mutation, the suite atomically records its run, attempt, suite, and lock identity in `~/.rustfs-functional-environment/active.json` on every node. This requires Python 3 on the lab nodes and a persistent SSH-user home directory. The receipt is retained as a `functional-environment-<suite>-<run>-<attempt>` artifact.
+
+Only successful cleanup releases the markers. Cancellation, runner loss, partial admission, skipped cleanup, or cleanup failure leaves the fleet quarantined. A later suite fails before changing it. The fault-tolerance matrix observes the same policy, and canonical validation cases check for quarantine before executing. Read-only inspection and the existing evidence-based recovery entrypoints remain available.
+
+For recovery, inspect the receipt and retained test evidence, ensure the original attempt has stopped, and clean every listed node while holding the same workflow resource lock. Then release only the original identity. For example, in an operator-controlled recovery step under that lock:
+
+```bash
+GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=2 \
+RUSTFS_NODES='node1 node2 node3' RUSTFS_SSH_USER=azureuser \
+python3 scripts/functional_environment.py recover --suite s3 --cleanup-outcome success
+```
+
+The `success` argument is an operator assertion that cleanup already completed; it does not perform cleanup. Recovery permits missing markers from partial admission but refuses to remove another attempt's marker. Keep the original evidence and recovery receipt. Do not delete quarantine files as a substitute for cleanup or run a second test against retained failed state.
+
+## Independent health
+
+The completion watchdog subscribes broadly and filters through the registry, so adding a monitored duty does not require a second workflow-name list. Existing in-workflow failure alerts remain their own writers; the watchdog handles cancellations and other incomplete endings, plus failures assigned to it. The freshness scan displays declared policy separately from live Actions state and recent attempts separately from successful runs. Disabled or paused entries are visible without a success claim or an enable operation.
+
+[Functional-chain health](../../scripts/functional_chain_health.py) runs on `ubuntu-latest`, independently of both lab fleets. It reuses the candidate and suite evidence verifier. Health requires the expected source ref's current SHA, the correct run and attempt, the full selected responsibility set, and fresh completion evidence. A newly completed real attempt on an unchanged source SHA refreshes completion freshness even when its original package is older. Reusing old artifacts, verifying an older source SHA, or running only one suite does not.
+
+A newer failed or cancelled full-plan attempt remains the latest attempt even when an earlier complete success is still fresh. Pagination gaps, API errors, identity mismatches, and source movement during inspection produce an incomplete observation. Previously published successes remain historical facts and cannot make that observation healthy. The dashboard keeps its existing publication and history format; this orchestration does not create a release certification path.
+
+## Verification boundaries
+
+The local registry, aggregation, freshness, and quarantine tests exercise selection and failure transitions with fixed fixtures; they do not claim a successful fleet run. Before treating deployment as accepted, retain one fixed-candidate full-plan run, one single-suite run, a cleanup-failure isolation exercise, and an independent-monitor alert/recovery exercise. Existing workflow disablement policy and private test-script revisions require their own authorized changes.

scripts/check_connect_release_boundary.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Prove the normal release compiles and the debug-only feature is rejected."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import subprocess
+import sys
+import unittest
+
+MESSAGE = "connect-e2e-short-credentials is restricted to debug builds"
+SOURCE = "rustfs/src/connect/registration.rs"
+
+
+def validate_rejection(status: int, output: str, root: Path) -> list[str]:
+    errors, diagnostics, completed = [], [], []
+    for line in output.splitlines():
+        try:
+            value = json.loads(line)
+        except ValueError:
+            errors.append("Cargo emitted non-JSON output on its diagnostic stream")
+            continue
+        if not isinstance(value, dict):
+            errors.append("Cargo emitted an invalid diagnostic object")
+            continue
+        if value.get("reason") == "compiler-message" and value.get("message", {}).get("level") == "error":
+            diagnostics.append(value)
+        elif value.get("reason") == "build-finished":
+            completed.append(value.get("success"))
+    if status != 101 or completed != [False]:
+        errors.append("negative release build did not end with a Cargo compilation rejection")
+    if len(diagnostics) != 1:
+        errors.append("negative release build must contain only the expected compiler error")
+        return errors
+    diagnostic = diagnostics[0]
+    message = diagnostic["message"]
+    if message.get("message") != MESSAGE or message.get("code") is not None:
+        errors.append("release failed for an unrelated compiler error")
+    target = diagnostic.get("target", {})
+    if target.get("name") != "rustfs" or "lib" not in target.get("kind", []):
+        errors.append("release rejection did not come from the RustFS library")
+    primary = [span for span in message.get("spans", []) if span.get("is_primary") is True]
+    valid = False
+    for span in primary:
+        path = Path(span.get("file_name", ""))
+        path = (root / path).resolve() if not path.is_absolute() else path.resolve()
+        if path != (root / SOURCE).resolve():
+            continue
+        try:
+            source = path.read_text().splitlines()
+            start, end = span["line_start"], span["line_end"]
+            if type(start) is int and type(end) is int and 1 <= start <= end <= len(source):
+                excerpt = "\n".join(source[start - 1:end])
+                valid = "compile_error!" in source[start - 1] and MESSAGE in excerpt
+        except (OSError, KeyError, TypeError):
+            pass
+    if not valid:
+        errors.append("release rejection is not located at the credential lifetime guard")
+    return errors
+
+
+def check(root: Path) -> int:
+    command = ["cargo", "check", "--locked", "-p", "rustfs", "--lib", "--release", "--message-format=json"]
+    positive = subprocess.run(command, cwd=root)
+    if positive.returncode:
+        print("ERROR: normal release configuration must compile before the negative check", file=sys.stderr)
+        return 1
+    negative = subprocess.run(command + ["--features", "connect-e2e-short-credentials"], cwd=root, capture_output=True, text=True)
+    sys.stderr.write(negative.stderr)
+    errors = validate_rejection(negative.returncode, negative.stdout, root)
+    for error in errors:
+        print(f"ERROR: {error}", file=sys.stderr)
+    if errors:
+        # Cargo JSON may contain useful source diagnostics; retain it on failure.
+        sys.stdout.write(negative.stdout)
+    else:
+        print("Connect release boundary passed: normal configuration compiles; short credentials are rejected at their guard")
+    return bool(errors)
+
+
+if __name__ == "__main__":
+    if sys.argv[1:] == ["--self-test"]:
+        suite = unittest.defaultTestLoader.discover(str(Path(__file__).resolve().parent), pattern="test_connect_release_boundary.py")
+        raise SystemExit(not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful())
+    if sys.argv[1:]:
+        raise SystemExit("usage: check_connect_release_boundary.py [--self-test]")
+    raise SystemExit(check(Path(__file__).resolve().parent.parent))

scripts/check_scheduled_validation_freshness.py

@@ -17,14 +17,14 @@ from unittest import mock
 from urllib.parse import parse_qs, quote, urlencode, urlsplit
 from urllib.request import Request, urlopen
 
+from validation_registry import load_registry, monitored_workflows
+
 
 ROOT = Path(__file__).resolve().parents[1]
 
 
 def load_validations(path: Path) -> list[tuple[str, int, datetime | None]]:
-    data = json.loads(path.read_text())
-    if not isinstance(data, list) or not data:
-        raise ValueError("scheduled validation config must be a non-empty list")
+    data = [item for item in monitored_workflows(load_registry(path)) if item["monitor"] == "workflow"]
 
     validations: list[tuple[str, int, datetime | None]] = []
     seen: set[str] = set()
@@ -161,7 +161,8 @@ def write_report(path: Path, rows: list[tuple[str, int, str, str, str]], default
         "",
         f"Default branch: `{default_branch}`. Ages use scheduled-run creation time; rerunning an old commit does not refresh its evidence.",
         "Attempt outcomes are shown independently of successful-run freshness.",
-        "Success is the GitHub workflow run conclusion; suite completeness remains the responsibility of each workflow.",
+        "Ordinary duties use workflow conclusions; the functional chain reuses its complete candidate and attempt evidence verifier.",
+        "Disabled and paused responsibilities remain visible and are never enabled by monitoring.",
         "",
         "| Workflow | Limit | Freshness | Last attempt | Last completed success |",
         "| --- | ---: | --- | --- | --- |",
@@ -172,13 +173,38 @@ def write_report(path: Path, rows: list[tuple[str, int, str, str, str]], default
     path.write_text("\n".join(lines) + "\n")
 
 
+def workflow_state(repository, workflow, token, api_url):
+    endpoint = f"{api_url.rstrip('/')}/repos/{repository}/actions/workflows/{Path(workflow).name}"
+    request = Request(endpoint, headers={"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"})
+    with urlopen(request, timeout=15) as response:
+        value = json.load(response)
+    state = value.get("state")
+    if state not in ("active", "disabled_manually", "disabled_inactivity", "disabled_fork", "deleted"):
+        raise ValueError("unknown workflow execution state")
+    return state
+
+
 def check_freshness(
     config: Path, report: Path, repository: str, token: str, api_url: str, default_branch: str
 ) -> int:
     now = datetime.now(timezone.utc)
+    registry = load_registry(config)
+    definitions = {item["workflow"]: item for item in monitored_workflows(registry)}
     rows: list[tuple[str, int, str, str, str]] = []
     failed = False
     for workflow, max_age_hours, never_ran_grace_until in load_validations(config):
+        definition = definitions[workflow]
+        try:
+            state = workflow_state(repository, workflow, token, api_url)
+        except Exception as error:
+            failed = True
+            rows.append((workflow, max_age_hours, f"Observation incomplete: {error}", "Inspection failed", "Inspection failed"))
+            continue
+        if definition["policy"] != "active" or state != "active":
+            failed |= definition["policy"] == "active"
+            rows.append((workflow, max_age_hours, f"Policy: {definition['policy']}; runtime: {state}; no health claim",
+                         "Not scheduled by this monitor", "Historical successes do not override execution policy"))
+            continue
         runs: dict[str, dict[str, object] | None] = {}
         reasons: list[str] = []
         for label, successful in (("Last attempt", False), ("Last completed success", True)):
@@ -188,7 +214,6 @@ def check_freshness(
                 )
             except Exception as error:
                 reasons.append(f"{label}: inspection failed: {error}")
-        # A failed inspection or any recorded attempt ends first-run grace.
         initial_grace = (
             len(runs) == 2
             and all(run is None for run in runs.values())
@@ -200,8 +225,13 @@ def check_freshness(
                 reason = stale_reason(run, now, max_age_hours)
                 if reason is not None:
                     reasons.append(f"{label}: {reason}")
+        attempt = runs.get("Last attempt") or {}
+        if attempt.get("status") == "completed" and attempt.get("conclusion") != "success":
+            reasons.append("Latest attempt: " + str(attempt.get("conclusion")))
         failed |= bool(reasons)
         result = "; ".join(reasons) if reasons else "Fresh"
+        if attempt.get("status") in ("queued", "in_progress", "waiting", "pending"):
+            result += "; latest attempt " + str(attempt["status"]) + ", completion pending"
         if initial_grace:
             result = f"Initial grace until {never_ran_grace_until.isoformat()}"
         evidence = [
@@ -209,6 +239,24 @@ def check_freshness(
             for label in ("Last attempt", "Last completed success")
         ]
         rows.append((workflow, max_age_hours, result, *evidence))
+    for definition in definitions.values():
+        if definition["monitor"] != "complete-chain":
+            continue
+        try:
+            # Use the existing candidate/attempt/evidence verifier. A green
+            # GitHub conclusion alone never satisfies this responsibility.
+            from functional_chain_health import collect
+            health = collect(source_ref=os.environ.get("NIGHTLY_SOURCE_REF") or default_branch)
+            latest = health["latest_attempt"] or {}
+            success = health["last_complete_success"].get(health.get("expected_source_ref"), {})
+            verdict = "Verified complete and current" if health["healthy"] else "Incomplete, stale, disabled or wrong candidate"
+            failed |= not health["healthy"] and definition["policy"] == "active"
+            rows.append((definition["workflow"], definition["max_age_hours"], verdict,
+                         json.dumps(latest, sort_keys=True), json.dumps(success, sort_keys=True)))
+        except Exception as error:
+            failed = True
+            rows.append((definition["workflow"], definition["max_age_hours"], f"Observation incomplete: {error}",
+                         "Inspection failed", "No complete-evidence claim"))
     write_report(report, rows, default_branch)
     return 1 if failed else 0
 
@@ -226,58 +274,16 @@ class SelfTests(unittest.TestCase):
     def test_config_rejects_duplicate_and_invalid_entries(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             path = Path(tmp) / "validations.json"
-            path.write_text(
-                json.dumps(
-                    [
-                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36},
-                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 0},
-                    ]
-                )
-            )
-            with self.assertRaises(ValueError):
-                load_validations(path)
-            path.write_text(
-                json.dumps(
-                    [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}]
-                )
-            )
-            with self.assertRaises(ValueError):
-                load_validations(path)
-            for bad_grace in (36, "not-a-timestamp", "2026-09-01T00:00:00"):
-                path.write_text(
-                    json.dumps(
-                        [
-                            {
-                                "workflow": ".github/workflows/ci.yml",
-                                "max_age_hours": 36,
-                                "never_ran_grace_until": bad_grace,
-                            }
-                        ]
-                    )
-                )
+            original = load_registry()
+            for mutate in (lambda value: value["workflows"].append(value["workflows"][0]),
+                           lambda value: value["workflows"][0].update(max_age_hours=0),
+                           lambda value: value.update(schema=2)):
+                value = json.loads(json.dumps(original))
+                mutate(value)
+                path.write_text(json.dumps(value))
                 with self.assertRaises(ValueError):
                     load_validations(path)
-            path.write_text(
-                json.dumps(
-                    [
-                        {
-                            "workflow": ".github/workflows/ci.yml",
-                            "max_age_hours": 36,
-                            "never_ran_grace_until": "2026-09-02T06:37:00Z",
-                        }
-                    ]
-                )
-            )
-            self.assertEqual(
-                load_validations(path),
-                [
-                    (
-                        ".github/workflows/ci.yml",
-                        36,
-                        datetime(2026, 9, 2, 6, 37, tzinfo=timezone.utc),
-                    )
-                ],
-            )
+            self.assertTrue(load_validations(ROOT / ".github/scheduled-validations.json"))
 
     @staticmethod
     def run_fixture(**overrides: object) -> dict[str, object]:
@@ -299,7 +305,7 @@ class SelfTests(unittest.TestCase):
             config = root / "validations.json"
             report = root / "report.md"
             entries = [
-                {"workflow": f".github/workflows/check-{index}.yml", "max_age_hours": 36}
+                {"workflow": f".github/workflows/check-{index}.yml", "max_age_hours": 36, "monitor": "workflow", "policy": "active"}
                 for index in range(workflows)
             ]
             if grace is not None:
@@ -311,6 +317,8 @@ class SelfTests(unittest.TestCase):
                     payload = {"total_count": len(payload["workflow_runs"]), **payload}
                 responses.append(payload if isinstance(payload, Exception) else io.StringIO(json.dumps(payload)))
             with (
+                mock.patch(__name__ + ".load_registry", return_value={"workflows": entries}),
+                mock.patch(__name__ + ".workflow_state", return_value="active"),
                 mock.patch(__name__ + ".urlopen", side_effect=responses) as request,
                 mock.patch(__name__ + ".datetime", wraps=datetime) as clock,
             ):
@@ -372,9 +380,9 @@ class SelfTests(unittest.TestCase):
                     {"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]},
                     {"workflow_runs": [success]},
                 ])
-                self.assertEqual(status, 0)
+                self.assertEqual(status, int(state == "completed" and conclusion != "success"))
                 self.assertIn(f"[{state}" + (f"/{conclusion}" if conclusion else "") + "]", report)
-                self.assertIn("Fresh", report)
+                self.assertIn("Fresh" if status == 0 else "Latest attempt:", report)
                 self.assertNotIn("All critical scheduled validations", report)
 
     def test_grace_requires_two_successful_queries_with_no_history(self) -> None:
@@ -457,6 +465,21 @@ class SelfTests(unittest.TestCase):
         self.assertIn("No recorded run", report)
         self.assertIn("Inspection failed", report)
 
+    def test_disabled_policy_and_runtime_state_remain_visible_without_enable_calls(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            report = Path(tmp) / "report.md"
+            entry = {"workflow": ".github/workflows/check.yml", "max_age_hours": 36,
+                     "monitor": "workflow", "policy": "disabled"}
+            for policy, expected in (("disabled", 0), ("paused", 0), ("active", 1)):
+                entry["policy"] = policy
+                with mock.patch(__name__ + ".load_registry", return_value={"workflows": [entry]}), \
+                     mock.patch(__name__ + ".workflow_state", return_value="disabled_manually"), \
+                     mock.patch(__name__ + ".urlopen") as request:
+                    status = check_freshness(Path("unused"), report, "rustfs/rustfs", "token", "https://api.github.test", "main")
+                self.assertEqual(status, expected)
+                self.assertIn("runtime: disabled_manually; no health claim", report.read_text())
+                request.assert_not_called()
+
     def test_cli_requires_the_repository_default_branch(self) -> None:
         from check_test_wiring import yaml_block
 

scripts/check_test_wiring.py

@@ -7,6 +7,7 @@ import hashlib
 import json
 import os
 import re
+import shlex
 import shutil
 import subprocess
 import sys
@@ -504,7 +505,8 @@ SCANNER_HEAL_RELEASE_PRESSURE_METRICS = (
 )
 SCHEDULED_ALERT_WORKFLOWS = tuple(
     item["workflow"]
-    for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())
+    for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())["workflows"]
+    if item.get("alert_contract") == "scheduled-validation"
 )
 # Each migrated scheduled-alert workflow requires the new pin; this is not a list of pins that
 # every workflow may choose between during the staged checkout update.
@@ -678,46 +680,42 @@ def check_vault_test_groups(root: Path) -> list[str]:
 
 
 def check_ilm_build_budget(root: Path) -> list[str]:
-    workflow_path = root / ".github/workflows/ci.yml"
-    if not workflow_path.is_file():
-        return [".github/workflows/ci.yml: missing workflow"]
-    workflow = workflow_path.read_text()
-    job = re.search(
-        r"^  test-and-lint:\n(?P<body>.*?)(?=^  [a-z0-9-]+:\n)",
-        workflow,
-        re.MULTILINE | re.DOTALL,
-    )
-    if job is None:
-        return [".github/workflows/ci.yml: ILM integration must share the workspace test job"]
-
-    step = re.search(
-        r"^      - name: Run ignored ILM integration tests serially\n(?P<body>.*?)(?=^      - name:|\Z)",
-        job.group("body"),
-        re.MULTILINE | re.DOTALL,
-    )
-    if step is None:
-        return [".github/workflows/ci.yml: missing ignored ILM integration step"]
-
-    # Measured budget: 3 build jobs on main pushes and dispatches, 2 on pull
-    # requests. A pull-request run at 3 (39193827281) spent 70 minutes with the
-    # pod's io.pressure "full" at 86-88% and timed out before any test started;
-    # the shared pod storage, not CPU, bounds the PR lane.
-    budget = (
-        "CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || "
-        "github.event_name == 'workflow_dispatch') && '3' || '2' }}"
-    )
-    if budget not in step.group("body"):
-        return [".github/workflows/ci.yml: ILM integration must keep the measured 3/2 Cargo build budget"]
-    for required in (
-        "        if: needs.classify-changes.outputs.mode == 'full'",
-        "cargo nextest run --profile default -j1 --run-ignored ignored-only",
-        "--all --exclude e2e_test",
-        "-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'",
-        'status=${PIPESTATUS[0]}',
-        'exit "${status}"',
-    ):
-        if required not in step.group("body"):
-            return [f".github/workflows/ci.yml: ILM integration is missing {required.strip()}"]
+    """Preserve the serial ignored-test contract while allowing archive reuse."""
+    from ci_gate import condition, workflow_document
+    try:
+        jobs = workflow_document((root / ".github/workflows/ci.yml").read_text())["jobs"]
+        matches = [(name, job, step) for name, job in jobs.items() for step in job.get("steps", [])
+                   if "--run-ignored" in step.get("run", "")]
+        if len(matches) != 1:
+            return ["CI must have exactly one ignored ILM execution"]
+        name, job, step = matches[0]
+        command = step.get("run", "").replace("\\\n", " ")
+        tokens = shlex.split(command)
+        for mode in ("docs", "e2e", "full"):
+            if condition(step.get("if", "true"), {"needs.classify-changes.outputs.mode": mode}) != (mode == "full"):
+                return ["ILM integration must run for full validation only"]
+        for required in ("cargo", "nextest", "run", "--profile", "default", "-j1", "--run-ignored", "ignored-only",
+                         "-E", "binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))"):
+            if required not in tokens:
+                return [f"ILM integration is missing {required}"]
+        if "--archive-file" not in tokens or "test-and-lint" not in job.get("needs", []):
+            return ["ILM integration must consume the workspace producer archive"]
+        if any(item.get("continue-on-error", "false") != "false" for item in (job, step)):
+            return ["ILM integration must propagate failures"]
+        if 'status=${PIPESTATUS[0]}' not in command or 'exit "${status}"' not in command or "|| true" in command.split('status=${PIPESTATUS[0]}')[0]:
+            return ["ILM integration must retain the actual test exit status"]
+        producer_steps = jobs.get("test-and-lint", {}).get("steps", [])
+        builds = [item for item in producer_steps if "ci_test_artifact.py build --configuration workspace" in item.get("run", "")]
+        if len(builds) != 1:
+            return ["ILM integration requires one workspace archive producer"]
+        budget = builds[0].get("env", {}).get("CARGO_BUILD_JOBS", "")
+        # This remains the measured build budget; execution has no Cargo build.
+        compact = re.sub(r"\s+", "", budget)
+        expected = "${{(github.event_name=='push'||github.event_name=='workflow_dispatch')&&'3'||'2'}}"
+        if compact != expected:
+            return ["workspace archive must keep the measured 3/2 Cargo build budget"]
+    except (OSError, KeyError, TypeError, ValueError) as error:
+        return [f"invalid ILM workflow contract: {error}"]
     return []
 
 
@@ -1046,63 +1044,52 @@ def yaml_scalar_continues(lines: list[str], index: int, indent: int) -> bool:
     return following is not None and len(following) - len(following.lstrip()) > indent
 
 
-def check_quick_checks(root: Path) -> list[str]:
-    errors: list[str] = []
-    bypass_key = r'''(?:if|continue-on-error|needs|"if"|"continue-on-error"|"needs"|'if'|'continue-on-error'|'needs')\s*:'''
-    for name in ("ci.yml",):
-        relative = f".github/workflows/{name}"
-        path = root / relative
-        job = yaml_block(path.read_text().splitlines(), "quick-checks", 2) if path.is_file() else None
-        if job is None:
-            errors.append(f"{relative}: missing Quick Checks job")
-            continue
-        conditions = [index for index, line in enumerate(job) if re.match(rf"^    {bypass_key}", line)]
-        expected = ["if: github.event_name != 'pull_request' || github.event.action != 'closed'"] if name == "ci.yml" else []
-        if [job[index].strip() for index in conditions] != expected or any(
-            yaml_scalar_continues(job, index, 4) for index in conditions
-        ):
-            errors.append(f"{relative}: Quick Checks job must not add dependencies, bypass failures, or change its event condition")
-        checkout_ref = SCHEDULED_ALERT_CHECKOUT_OVERRIDES.get(
-            relative, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
-        )
-        checkout = workflow_step_block(job, checkout_ref)
-        action = workflow_step_block(job, "./.github/actions/quick-checks")
-        if checkout is None or action is None:
-            errors.append(f"{relative}: Quick Checks requires checkout and the shared quick-checks action")
-            continue
-        if checkout[0] >= action[0]:
-            errors.append(f"{relative}: checkout must run before shared Quick Checks")
-        if "          persist-credentials: false" not in checkout[1]:
-            errors.append(f"{relative}: Quick Checks checkout must disable persisted credentials")
-        for step in (checkout, action):
-            if any(re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]):
-                errors.append(f"{relative}: Quick Checks checkout and shared action must run without bypasses")
+def shell_command_contract(source: str) -> list[str]:
+    """Normalize harmless multiline shell spelling; never accept extra commands."""
+    source = source.replace("\\\n", " ")
+    lines = [line.strip() for line in source.splitlines() if line.strip() and not line.lstrip().startswith("#")]
+    if lines and lines[0] == "set -euo pipefail":
+        lines.pop(0)
+    tokens = shlex.split(" ; ".join(lines), comments=True)
+    return [";" if token == "&&" else token for token in tokens]
 
-    relative = ".github/actions/quick-checks/action.yml"
-    path = root / relative
-    runs = yaml_block(path.read_text().splitlines(), "runs", 0) if path.is_file() else None
-    if runs is None or "  using: composite" not in runs:
-        errors.append(f"{relative}: missing composite action")
-        return errors
-    steps = yaml_block(runs, "steps", 2) or []
-    for command in (
-        "shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests",
-        "python3 ./scripts/test_object_generation_protocol_model.py",
-    ):
-        step = workflow_step_block(steps, command, key="run", indent=4)
-        if step is None:
-            errors.append(f"{relative}: missing direct execution of {command}")
-            continue
-        if "      shell: bash" not in step[1] or any(
-            re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]
-        ):
-            errors.append(f"{relative}: {command} must use bash without a condition or continue-on-error")
-        run_index = next(
-            index for index, line in enumerate(step[1])
-            if line.split("#", 1)[0].rstrip() in (f"      run: {command}", f"    - run: {command}")
-        )
-        if yaml_scalar_continues(step[1], run_index, 6):
-            errors.append(f"{relative}: {command} must remain a single-line run scalar")
+
+def check_quick_checks(root: Path) -> list[str]:
+    from ci_gate import condition, workflow_document
+    errors: list[str] = []
+    try:
+        caller = workflow_document((root / ".github/workflows/ci.yml").read_text())
+        job = caller["jobs"]["quick-checks"]
+        if job.get("needs") or job.get("continue-on-error", "false") != "false":
+            errors.append("Quick Checks must not depend on other jobs or bypass failures")
+        for event in ("pull_request", "push", "merge_group", "schedule", "workflow_dispatch"):
+            for action in ("opened", "closed", "synchronize"):
+                context = {"github.event_name": event, "github.event.action": action}
+                if condition(job.get("if", "true"), context) != (event != "pull_request" or action != "closed"):
+                    errors.append("Quick Checks event condition changes required responsibility")
+        steps = job.get("steps", [])
+        checkout = [(index, step) for index, step in enumerate(steps) if step.get("uses", "").startswith("actions/checkout@")]
+        actions = [(index, step) for index, step in enumerate(steps) if step.get("uses") == "./.github/actions/quick-checks"]
+        if len(checkout) != 1 or len(actions) != 1 or checkout[0][0] >= actions[0][0]:
+            errors.append("Quick Checks requires checkout before its shared action")
+        else:
+            if checkout[0][1].get("with", {}).get("persist-credentials") != "false":
+                errors.append("Quick Checks checkout must disable persisted credentials")
+            for _, step in checkout + actions:
+                if "if" in step or step.get("continue-on-error", "false") != "false":
+                    errors.append("Quick Checks steps must not bypass execution or failure")
+        action = workflow_document((root / ".github/actions/quick-checks/action.yml").read_text())["runs"]
+        if action.get("using") != "composite":
+            errors.append("Quick Checks must use the shared composite action")
+        for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests",
+                        "python3 ./scripts/test_object_generation_protocol_model.py"):
+            matching = [step for step in action.get("steps", []) if shell_command_contract(step.get("run", "")) == shell_command_contract(command)]
+            if len(matching) != 1:
+                errors.append(f"Quick Checks must execute {command} exactly once with failure propagation")
+            elif matching[0].get("shell") != "bash" or "if" in matching[0] or matching[0].get("continue-on-error", "false") != "false":
+                errors.append(f"Quick Checks must execute {command} in fail-fast bash without bypasses")
+    except (OSError, KeyError, TypeError, ValueError) as error:
+        errors.append(f"invalid Quick Checks contract: {error}")
     return errors
 
 
@@ -1238,21 +1225,9 @@ def check_scheduled_alerts(root: Path) -> list[str]:
         return errors
     watchdog_on = yaml_block(watchdog_lines, "on", 0)
     watchdog_run = yaml_block(watchdog_on or [], "workflow_run", 2)
-    watchdog_workflows = yaml_block(watchdog_run or [], "workflows", 4)
-    if watchdog_workflows is None:
-        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing workflow_run workflows")
-        return errors
-    watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_workflows)
-    for relative in SCHEDULED_ALERT_WORKFLOWS:
-        path = root / relative
-        if not path.is_file():
-            continue
-        source = path.read_text()
-        match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE)
-        if not match:
-            errors.append(f"{relative}: missing workflow name")
-        elif f'- "{match.group(1).strip()}"' not in watchdog_sources:
-            errors.append(f"{relative}: missing from scheduled completion watchdog")
+    watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_run or [])
+    if not re.search(r'^    workflows:\s*\["\*"\]\s*$', watchdog_sources, re.MULTILINE):
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing registry-wide completion subscription")
     watchdog_job_lines = yaml_block(watchdog_lines, "alert-on-incomplete-run", 2)
     if watchdog_job_lines is None:
         errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing alert-on-incomplete-run job")
@@ -1263,9 +1238,9 @@ def check_scheduled_alerts(root: Path) -> list[str]:
         "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
     )
     required = (
-        "github.event.workflow_run.event == 'schedule'",
-        "github.event.workflow_run.conclusion != 'success'",
-        "github.event.workflow_run.conclusion != 'failure'",
+        "python3 scripts/validation_registry.py watchdog",
+        "id: registry",
+        "if: steps.registry.outputs.selected == 'true'",
         "actions: read",
         "issues: write",
         f"uses: {watchdog_checkout_ref}",
@@ -1288,7 +1263,7 @@ def check_scheduled_alerts(root: Path) -> list[str]:
             ".github/workflows/scheduled-validation-watchdog.yml: " + error
             for error in alert_step_errors(
                 watchdog_job_lines,
-                None,
+                "if: steps.registry.outputs.selected == 'true'",
                 ("actions: read", "issues: write"),
                 (
                     "github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}",
@@ -3547,6 +3522,11 @@ class SelfTests(unittest.TestCase):
                 path.parent.mkdir(parents=True, exist_ok=True)
                 path.write_text(source)
             self.assertEqual(check_quick_checks(root), [])
+            for command in ("shellcheck --version && actionlint", "make script-tests"):
+                formatted = action.replace(f"run: {command}", f"run: |\n        {command}")
+                (root / ".github/actions/quick-checks/action.yml").write_text(formatted)
+                self.assertEqual(check_quick_checks(root), [])
+            (root / ".github/actions/quick-checks/action.yml").write_text(action)
             for relative in (".github/workflows/ci.yml",):
                 source = sources[relative]
                 mutations = {
@@ -5572,46 +5552,30 @@ class SelfTests(unittest.TestCase):
             listing.write_text("not json")
             self.assertIn("cannot read", check_core_listing(root, listing)[0])
 
-    def test_ilm_lane_keeps_the_measured_cargo_build_budget(self) -> None:
+    def test_ilm_lane_keeps_archive_reuse_and_serial_execution(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
             workflow = root / ".github/workflows/ci.yml"
             workflow.parent.mkdir(parents=True)
-            valid = (
-                "jobs:\n"
-                "  test-and-lint:\n"
-                "    steps:\n"
-                "      - name: Run ignored ILM integration tests serially\n"
-                "        if: needs.classify-changes.outputs.mode == 'full'\n"
-                "        env:\n"
-                "          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}\n"
-                "        run: |\n"
-                "          cargo nextest run --profile default -j1 --run-ignored ignored-only --all --exclude e2e_test \\\n"
-                "            -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'\n"
-                "          status=${PIPESTATUS[0]}\n"
-                '          exit "${status}"\n'
-                "  next-job:\n"
-                "    steps: []\n"
-            )
+            valid = (ROOT / ".github/workflows/ci.yml").read_text()
             workflow.write_text(valid)
             self.assertEqual(check_ilm_build_budget(root), [])
-
-            workflow.write_text(valid.replace("          CARGO_BUILD_JOBS:", "          BUILD_JOBS:"))
-            self.assertEqual(len(check_ilm_build_budget(root)), 1)
             for old, new in (
                 ("&& '3' || '2' }}", "&& '3' || '3' }}"),
-                ("&& '3' || '2' }}", "&& '2' || '2' }}"),
-                ("test-and-lint:", "test-ilm-integration-serial:"),
                 ("outputs.mode == 'full'", "outputs.mode == 'docs'"),
-                ("--profile default", "--profile ci"),
-                ("-j1", "-j2"),
+                ("--profile default -j1", "--profile ci -j1"),
+                ("-j1 --run-ignored", "-j2 --run-ignored"),
                 ("ignored-only", "default"),
-                ("--all --exclude e2e_test", "-p rustfs-scanner -p rustfs"),
                 ("binary(lifecycle_integration_test)", "none()"),
                 ('exit "${status}"', "exit 0"),
+                ("--archive-file", "--not-an-archive"),
+                ("build --configuration workspace", "build --configuration e2e"),
             ):
                 workflow.write_text(valid.replace(old, new))
                 self.assertTrue(check_ilm_build_budget(root), (old, new))
+            # Formatting and the consumer job ID are not execution contracts.
+            workflow.write_text(valid.replace("  workspace-tests:", "  renamed-workspace-tests:"))
+            self.assertEqual(check_ilm_build_budget(root), [])
 
     def test_vault_tests_require_cross_process_serialization(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
@@ -5984,19 +5948,19 @@ class SelfTests(unittest.TestCase):
             freshness_checkout = new_checkout
             watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml"
             watchdog.write_text(
-                "on:\n  workflow_run:\n    workflows:\n"
-                + "\n".join(f'      - "{name}"' for name in names)
-                + "\njobs:\n"
+                'on:\n  workflow_run:\n    workflows: ["*"]\n'
+                + "jobs:\n"
                 + "  alert-on-incomplete-run:\n"
-                + "    github.event.workflow_run.event == 'schedule'\n"
-                + "    github.event.workflow_run.conclusion != 'success'\n"
-                + "    github.event.workflow_run.conclusion != 'failure'\n"
+
                 + "    permissions:\n"
                 + "      actions: read\n"
                 + "      issues: write\n"
                 + "    steps:\n"
                 + f"      - uses: {watchdog_checkout}\n"
+                + "      - run: python3 scripts/validation_registry.py watchdog\n"
+                + "        id: registry\n"
                 + "      - uses: ./.github/actions/schedule-failure-issue\n"
+                + "        if: steps.registry.outputs.selected == 'true'\n"
                 + "        with:\n"
                 + "          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}\n"
                 + "          workflow-name: ${{ github.event.workflow_run.name }}\n"
@@ -6171,14 +6135,14 @@ class SelfTests(unittest.TestCase):
             first.write_text(first_original)
 
             watchdog.write_text(
-                watchdog_original.replace(f'      - "{names[0]}"\n', "")
+                watchdog_original.replace('    workflows: ["*"]\n', '    workflows: ["unrelated"]\n')
                 + f'  decoy:\n    strategy:\n      matrix:\n        workflow:\n          - "{names[0]}"\n'
                 + '    runs-on: ubuntu-latest\n    steps:\n      - run: true\n'
             )
             self.assertEqual(len(check_scheduled_alerts(root)), 1)
             watchdog.write_text(watchdog_original)
 
-            watchdog.write_text(watchdog_original.replace(f'      - "{names[0]}"\n', ""))
+            watchdog.write_text(watchdog_original.replace('    workflows: ["*"]\n', '    workflows: ["unrelated"]\n'))
             self.assertEqual(len(check_scheduled_alerts(root)), 1)
             watchdog.write_text(watchdog_original)
             original = first.read_text()

scripts/ci/actions_cost_report.py

@@ -0,0 +1,317 @@
+#!/usr/bin/env python3
+"""Read all Actions attempts in a creation window and account for observed work."""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter, defaultdict
+from datetime import datetime, timezone
+import json
+import math
+from pathlib import Path
+import re
+import subprocess
+import sys
+from urllib.parse import urlencode
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from ci_timing_report import minutes, summarize as timing_summary, timestamp
+
+RUN_FIELDS = ("id", "run_attempt", "workflow_id", "path", "name", "event", "head_sha", "head_branch",
+              "created_at", "run_started_at", "updated_at", "status", "conclusion", "html_url", "pull_requests")
+JOB_FIELDS = ("id", "run_id", "run_attempt", "name", "status", "conclusion", "created_at", "started_at",
+              "completed_at", "labels", "runner_name", "runner_group_name", "steps", "html_url")
+PHASES = ("setup", "compile", "test", "archive", "transfer")
+
+
+def require(condition, message):
+    if not condition:
+        raise ValueError(message)
+
+
+def api(path):
+    # Every request is explicitly GET; stderr may contain credentials or URLs.
+    result = subprocess.run(["gh", "api", "--method", "GET", path], capture_output=True, text=True, timeout=90)
+    if result.returncode:
+        raise RuntimeError("GitHub read failed")
+    return json.loads(result.stdout)
+
+
+def pages(path, field, *, cap=10000):
+    rows, seen, expected = [], set(), None
+    for page in range(1, math.ceil(cap / 100) + 1):
+        response = api(f"{path}{'&' if '?' in path else '?'}per_page=100&page={page}")
+        batch = response.get(field)
+        total = response.get("total_count")
+        require(isinstance(batch, list) and type(total) is int and 0 <= total <= cap,
+                "missing count or inventory exceeds limit; narrow the collection window")
+        require(expected is None or total == expected, "inventory changed during pagination")
+        expected = total
+        for row in batch:
+            require(type(row.get("id")) is int and row["id"] not in seen, "duplicate or missing inventory identity")
+            seen.add(row["id"])
+        rows.extend(batch)
+        if len(rows) >= total or len(batch) < 100:
+            require(len(rows) == total, "incomplete pagination")
+            return rows
+    raise ValueError("pagination limit reached")
+
+
+def repository_members(run, repository):
+    # Actions can associate a shared SHA with PRs targeting other forks. A PR
+    # number is only meaningful inside its base repository.
+    return sorted({row["number"] for row in (run.get("pull_requests") or [])
+                   if type(row.get("number")) is int and row["number"] > 0
+                   and row.get("url", "").lower() == f"https://api.github.com/repos/{repository}/pulls/{row['number']}".lower()})
+
+
+def collect(repository, since, until):
+    require(re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository), "invalid repository")
+    require(timestamp(since) < timestamp(until), "collection interval must increase")
+    snapshot = {"schema": 1, "repository": repository, "observed_at": datetime.now(timezone.utc).isoformat(),
+                "created_since": since, "created_until": until, "complete": False, "issues": [],
+                "runs": [], "pull_requests": {}, "attributions": {}}
+    prefix = f"repos/{repository}"
+    try:
+        query = urlencode({"created": f"{since}..{until}", "exclude_pull_requests": "false"})
+        inventory = pages(f"{prefix}/actions/runs?{query}", "workflow_runs", cap=1000)
+    except (ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
+        snapshot["issues"].append({"scope": "run_inventory", "error_type": type(error).__name__})
+        return snapshot
+    for run in inventory:
+        run_id = run["id"]
+        try:
+            require(timestamp(since) <= timestamp(run.get("created_at")) <= timestamp(until), "run outside interval")
+            latest = run.get("run_attempt")
+            require(type(latest) is int and 1 <= latest <= 1000, "invalid attempt count")
+            endpoint = f"{prefix}/actions/runs/{run_id}"
+            for attempt in range(1, latest + 1):
+                sample = {"id": run_id, "run_attempt": attempt, "jobs": [], "complete": False}
+                snapshot["runs"].append(sample)
+                try:
+                    detail = api(f"{endpoint}/attempts/{attempt}")
+                    require(detail.get("id") == run_id and detail.get("run_attempt") == attempt
+                            and detail.get("head_sha") == run["head_sha"], "attempt identity mismatch")
+                    sample.update({key: detail.get(key) for key in RUN_FIELDS})
+                    jobs = pages(f"{endpoint}/attempts/{attempt}/jobs", "jobs")
+                    for job in jobs:
+                        require(job.get("run_id") == run_id and job.get("run_attempt", attempt) == attempt,
+                                "job belongs to a different attempt")
+                    sample["jobs"] = [{key: job.get(key) for key in JOB_FIELDS} for job in jobs]
+                    sample["complete"] = True
+                except (ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
+                    snapshot["issues"].append({"scope": "attempt", "run_id": run_id, "attempt": attempt,
+                                               "error_type": type(error).__name__})
+            observed = api(endpoint)
+            require(all(observed.get(key) == run.get(key) for key in ("run_attempt", "head_sha", "status", "conclusion", "updated_at")),
+                    "run changed while collecting")
+        except (ValueError, TypeError, RuntimeError, subprocess.TimeoutExpired) as error:
+            snapshot["issues"].append({"scope": "run", "run_id": run_id, "error_type": type(error).__name__})
+    members = {number for run in snapshot["runs"] for number in repository_members(run, repository)}
+    for number in sorted(members):
+        try:
+            detail = api(f"{prefix}/pulls/{number}")
+            require(detail.get("number") == number, "PR identity mismatch")
+            snapshot["pull_requests"][str(number)] = {key: detail.get(key)
+                for key in ("number", "created_at", "merged_at", "merge_commit_sha", "html_url")}
+        except (ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
+            snapshot["issues"].append({"scope": "pull_request", "number": number, "error_type": type(error).__name__})
+    snapshot["complete"] = not snapshot["issues"]
+    return snapshot
+
+
+def add_attributions(snapshot, manifest):
+    """Bind operator-reviewed missing event provenance to immutable run identities."""
+    require(manifest.get("schema") == 1 and manifest.get("repository") == snapshot["repository"],
+            "attribution manifest repository/schema mismatch")
+    rows = manifest.get("runs", [])
+    require(isinstance(rows, list), "attribution runs must be a list")
+    seen = set()
+    for row in rows:
+        run_id = row.get("run_id")
+        require(type(run_id) is int and run_id not in seen, "duplicate or invalid attribution run")
+        seen.add(run_id)
+        matches = [run for run in snapshot["runs"] if run["id"] == run_id]
+        require(matches and all(run.get("head_sha") == row.get("head_sha") for run in matches), "attribution SHA mismatch")
+        members = row.get("pull_requests")
+        require(isinstance(members, list) and members and all(type(value) is int and value > 0 for value in members)
+                and len(set(members)) == len(members), "attribution requires unique PR members")
+        require(isinstance(row.get("evidence"), str) and row["evidence"].startswith("https://github.com/"),
+                "attribution requires a reviewable evidence URL")
+        require(not any(run.get("event") == "schedule" for run in matches), "periodic work must remain separate")
+        snapshot["attributions"][str(run_id)] = row
+    for number, metadata in manifest.get("pull_requests", {}).items():
+        snapshot["pull_requests"].setdefault(number, metadata)
+
+
+def runner_model(labels):
+    labels = labels or []
+    if any(label.startswith(("sm-", "dind-", "smoke-", "pf-")) for label in labels):
+        return "fixed_capacity"
+    if any(label.startswith("ubicloud-") for label in labels):
+        return "metered"
+    if "self-hosted" not in labels and any(label.startswith(("ubuntu-", "windows-", "macos-")) for label in labels):
+        return "standard_hosted"
+    return "unknown"
+
+
+def safe_minutes(start, end):
+    try:
+        return minutes(start, end)
+    except (ValueError, TypeError, AttributeError):
+        return None
+
+
+def job_measurement(job):
+    skipped = job.get("conclusion") == "skipped"
+    duration = 0 if skipped else (safe_minutes(job.get("started_at"), job.get("completed_at"))
+                                 if job.get("status") == "completed" else None)
+    wait = 0 if skipped else safe_minutes(job.get("created_at"), job.get("started_at"))
+    phases = {phase: [] for phase in PHASES}
+    steps = []
+    for step in job.get("steps") or []:
+        phase = step.get("name", "").removeprefix("CI phase: ")
+        phase = phase if phase in PHASES else "unknown"
+        elapsed = safe_minutes(step.get("started_at"), step.get("completed_at")) if step.get("status") == "completed" else None
+        steps.append({"name": step.get("name"), "phase": phase, "minutes": elapsed})
+        if phase in phases:
+            phases[phase].append(elapsed)
+    return {"runner_minutes": duration, "queue_minutes": wait,
+            "runner_model": runner_model(job.get("labels")),
+            "phases_minutes": {phase: sum(values) if values and all(value is not None for value in values) else None
+                               for phase, values in phases.items()}, "steps": steps}
+
+
+def attribution(run, snapshot):
+    if run.get("event") == "schedule":
+        return [], "periodic"
+    explicit = snapshot.get("attributions", {}).get(str(run["id"]))
+    if explicit:
+        require(explicit["head_sha"] == run.get("head_sha"), "stored attribution SHA mismatch")
+        return explicit["pull_requests"], "reviewed_manifest"
+    # Merge-group membership and workflow_run provenance cannot be recovered
+    # safely from a synthetic branch name or a matching current SHA alone.
+    if run.get("event") in ("pull_request", "pull_request_target"):
+        members = repository_members(run, snapshot["repository"])
+        if members:
+            return members, "event_pull_requests"
+    if run.get("event") == "push":
+        merged = [int(number) for number, pr in snapshot.get("pull_requests", {}).items()
+                  if pr.get("merged_at") and pr.get("merge_commit_sha") == run.get("head_sha")]
+        if merged:
+            return sorted(merged), "exact_merge_commit"
+    return [], "unattributed"
+
+
+def new_bucket():
+    return {"known_runner_minutes": 0.0, "known_queue_minutes": 0.0,
+            "unknown_runner_jobs": 0, "unknown_queue_jobs": 0, "job_share": 0.0,
+            "per_job_rounded_known_minutes": 0.0, "monetary_cost": None}
+
+
+def accumulate(bucket, metric, weight=1):
+    bucket["job_share"] += weight
+    for kind in ("runner", "queue"):
+        duration = metric[f"{kind}_minutes"]
+        if duration is None:
+            bucket[f"unknown_{kind}_jobs"] += weight
+        else:
+            bucket[f"known_{kind}_minutes"] += duration * weight
+    if metric["runner_minutes"] is not None:
+        bucket["per_job_rounded_known_minutes"] += math.ceil(metric["runner_minutes"]) * weight
+
+
+def summarize(snapshot):
+    total, periodic, unattributed = new_bucket(), new_bucket(), new_bucket()
+    pools, prs = defaultdict(new_bucket), defaultdict(new_bucket)
+    statuses, unknowns, observations, identities = Counter(), Counter(), [], set()
+    all_jobs = set()
+    for run in snapshot["runs"]:
+        identity = (run["id"], run["run_attempt"])
+        require(identity not in identities, "duplicate run attempt in snapshot")
+        identities.add(identity)
+        statuses[run.get("conclusion") or run.get("status") or "unknown"] += 1
+        members, source = attribution(run, snapshot)
+        for job in run["jobs"]:
+            require(job["id"] not in all_jobs, "duplicate job across the snapshot")
+            all_jobs.add(job["id"])
+            metric = job_measurement(job)
+            accumulate(total, metric)
+            accumulate(pools[metric["runner_model"]], metric)
+            if members:
+                for member in members:
+                    accumulate(prs[str(member)], metric, 1 / len(members))
+            else:
+                accumulate(periodic if source == "periodic" else unattributed, metric)
+            for key in ("runner_minutes", "queue_minutes"):
+                unknowns[key] += metric[key] is None
+            unknowns["runner_model"] += metric["runner_model"] == "unknown"
+            unknowns["phase_breakdown_jobs"] += any(value is None for value in metric["phases_minutes"].values())
+            observations.append({"run_id": run["id"], "run_attempt": run["run_attempt"], "head_sha": run.get("head_sha"),
+                                 "workflow": run.get("path"), "event": run.get("event"), "job_id": job["id"],
+                                 "job_name": job.get("name"), "conclusion": job.get("conclusion"),
+                                 "pull_requests": members, "attribution_source": source, **metric})
+    for number, bucket in prs.items():
+        metadata = snapshot.get("pull_requests", {}).get(number, {})
+        bucket["merged_at"] = metadata.get("merged_at")
+        bucket["created_at"] = metadata.get("created_at")
+        # A creation-window sample cannot prove retained historical runs or all
+        # externally triggered validation were available. Never call it a bill.
+        bucket["lifecycle_complete"] = False
+    successful = timing_summary([run for run in snapshot["runs"]
+                                 if run.get("path") == ".github/workflows/ci.yml" and run.get("complete")])
+    return {"collection_complete": snapshot.get("complete", False), "attempts": len(identities),
+            "attempt_statuses": dict(statuses), "unknowns": dict(unknowns), "total_unallocated": total,
+            "runner_models": dict(pools), "pull_requests": dict(prs), "periodic_unallocated": periodic,
+            "unattributed": unattributed, "job_observations": observations,
+            "successful_code_pr": {"runner_sum": successful["successful_code_job_sum"], "feedback": successful["successful_code_wall"]},
+            "allocation_rule": "Each job and its queue wait are shared equally by recorded PR members; periodic work stays separate.",
+            "limits": ["Known minutes are partial sums; unknown timestamps and missing attempts are never zero-filled.",
+                       "Creation-to-start wait does not separate scheduler and dependency delay and is not runner occupancy.",
+                       "Creation windows omit older runs rerun inside the window; retained API history may be incomplete.",
+                       "Only explicit CI phase step labels split compile, test and transfer; combined steps remain unknown.",
+                       "Rounded minutes are a comparison model, not a provider invoice. Prices and fixed capacity bills are unknown.",
+                       "Cache state, image identity and compiler attribution require benchmark or build receipts; labels do not prove them."]}
+
+
+def markdown(report):
+    total = report["total_unallocated"]
+    return ("# Actions resource accounting\n\n"
+            f"Collection complete: {report['collection_complete']}. Attempts: {report['attempts']}.\n\n"
+            f"Known runner occupancy: {total['known_runner_minutes']:.2f} minutes; "
+            f"jobs with unknown occupancy: {total['unknown_runner_jobs']}.\n\n"
+            f"Known creation-to-start wait sum: {total['known_queue_minutes']:.2f} minutes; "
+            f"jobs with unknown wait: {total['unknown_queue_jobs']}.\n\n"
+            "| Allocation | Known runner minutes | Unknown runner job shares |\n|---|---:|---:|\n"
+            + "".join(f"| {name} | {bucket['known_runner_minutes']:.2f} | {bucket['unknown_runner_jobs']:.2f} |\n"
+                      for name, bucket in [(f"PR #{number}", value) for number, value in sorted(report["pull_requests"].items())]
+                      + [("Periodic", report["periodic_unallocated"]), ("Unattributed", report["unattributed"])])
+            + "\n" + "\n".join(f"- {limit}" for limit in report["limits"]) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repository", default="rustfs/rustfs")
+    parser.add_argument("--since", help="inclusive run creation timestamp, with timezone")
+    parser.add_argument("--until", help="inclusive run creation timestamp, with timezone")
+    parser.add_argument("--input", type=Path, help="recompute an existing snapshot offline")
+    parser.add_argument("--attributions", type=Path)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    args = parser.parse_args()
+    if not args.input and not (args.since and args.until):
+        parser.error("provide --input or both --since and --until")
+    snapshot = json.loads(args.input.read_text()) if args.input else collect(args.repository, args.since, args.until)
+    require(snapshot.get("schema") == 1 and isinstance(snapshot.get("runs"), list), "unsupported snapshot")
+    if args.attributions:
+        add_attributions(snapshot, json.loads(args.attributions.read_text()))
+    snapshot["summary"] = summarize(snapshot)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(snapshot, indent=2) + "\n")
+    args.summary.write_text(markdown(snapshot["summary"]))
+    print(f"{len(snapshot['runs'])} attempts; collection_complete={snapshot['complete']}; see {args.output}")
+    return 0 if snapshot["complete"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/fixtures/cost/attempts.json

@@ -0,0 +1,29 @@
+{
+  "schema": 1,
+  "repository": "owner/repo",
+  "complete": true,
+  "created_since": "2026-10-01T00:00:00Z",
+  "created_until": "2026-10-02T00:00:00Z",
+  "issues": [],
+  "pull_requests": {
+    "11": {"number": 11, "created_at": "2026-10-01T00:00:00Z", "merged_at": "2026-10-01T01:00:00Z", "merge_commit_sha": "cccccccccccccccccccccccccccccccccccccccc"},
+    "12": {"number": 12, "created_at": "2026-10-01T00:00:00Z", "merged_at": "2026-10-01T01:00:00Z"}
+  },
+  "attributions": {
+    "102": {"run_id": 102, "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "pull_requests": [11, 12], "evidence": "https://github.com/owner/repo/actions/runs/102"}
+  },
+  "runs": [
+    {"id": 101, "run_attempt": 1, "path": ".github/workflows/ci.yml", "event": "pull_request", "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "status": "completed", "conclusion": "failure", "created_at": "2026-10-01T00:00:00Z", "run_started_at": "2026-10-01T00:00:00Z", "pull_requests": [{"number": 11, "url": "https://api.github.com/repos/owner/repo/pulls/11"}], "complete": true,
+      "jobs": [{"id": 1001, "run_id": 101, "run_attempt": 1, "name": "Workspace Test and Lint", "status": "completed", "conclusion": "failure", "labels": ["sm-standard-4"], "created_at": "2026-10-01T00:00:00Z", "started_at": "2026-10-01T00:02:00Z", "completed_at": "2026-10-01T00:12:00Z", "steps": []}]},
+    {"id": 101, "run_attempt": 2, "path": ".github/workflows/ci.yml", "event": "pull_request", "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "status": "completed", "conclusion": "success", "created_at": "2026-10-01T00:00:00Z", "run_started_at": "2026-10-01T00:20:00Z", "pull_requests": [{"number": 11, "url": "https://api.github.com/repos/owner/repo/pulls/11"}], "complete": true,
+      "jobs": [{"id": 1002, "run_id": 101, "run_attempt": 2, "name": "Workspace Test and Lint", "status": "completed", "conclusion": "success", "labels": ["sm-standard-4"], "created_at": "2026-10-01T00:20:00Z", "started_at": "2026-10-01T00:21:00Z", "completed_at": "2026-10-01T00:31:00Z", "steps": [{"name": "Run nextest tests", "status": "completed", "conclusion": "success", "started_at": "2026-10-01T00:22:00Z", "completed_at": "2026-10-01T00:30:00Z"}]}]},
+    {"id": 102, "run_attempt": 1, "path": ".github/workflows/build.yml", "event": "merge_group", "head_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "status": "completed", "conclusion": "cancelled", "complete": true,
+      "jobs": [{"id": 1003, "run_id": 102, "run_attempt": 1, "name": "Build", "status": "completed", "conclusion": "cancelled", "labels": ["ubicloud-standard-16-arm"], "created_at": "2026-10-01T00:30:00Z", "started_at": "2026-10-01T00:34:00Z", "completed_at": "2026-10-01T00:40:00Z", "steps": []}]},
+    {"id": 103, "run_attempt": 1, "path": ".github/workflows/nightly.yml", "event": "schedule", "head_sha": "cccccccccccccccccccccccccccccccccccccccc", "status": "completed", "conclusion": "success", "complete": true,
+      "jobs": [{"id": 1004, "name": "Nightly", "status": "completed", "conclusion": "success", "labels": ["sm-standard-4"], "created_at": null, "started_at": "2026-10-01T00:40:00Z", "completed_at": "2026-10-01T00:45:00Z", "steps": []}]},
+    {"id": 104, "run_attempt": 1, "path": ".github/workflows/downstream.yml", "event": "workflow_run", "head_sha": "cccccccccccccccccccccccccccccccccccccccc", "status": "completed", "conclusion": "cancelled", "complete": true,
+      "jobs": [{"id": 1005, "name": "Expired details", "status": "completed", "conclusion": "cancelled", "labels": ["custom-pool"], "created_at": null, "started_at": null, "completed_at": null, "steps": null}]},
+    {"id": 105, "run_attempt": 1, "path": ".github/workflows/post-merge.yml", "event": "push", "head_sha": "cccccccccccccccccccccccccccccccccccccccc", "status": "completed", "conclusion": "success", "complete": true,
+      "jobs": [{"id": 1006, "name": "Post merge", "status": "completed", "conclusion": "success", "labels": ["ubuntu-latest"], "created_at": "2026-10-01T01:00:00Z", "started_at": "2026-10-01T01:01:00Z", "completed_at": "2026-10-01T01:03:00Z", "steps": []}]}
+  ]
+}

scripts/ci/runner_pilot.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""Inspect runner capabilities and measure isolated, fixed-source Cargo builds."""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+from datetime import datetime, timezone
+import errno
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+
+def command(arguments, *, timeout=15, cwd=None):
+    try:
+        result = subprocess.run(arguments, capture_output=True, text=True, timeout=timeout, cwd=cwd)
+        return result.stdout.strip() if result.returncode == 0 else None
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return None
+
+
+def read(path):
+    try:
+        return Path(path).read_text().strip()
+    except OSError:
+        return None
+
+
+def memory_limit():
+    memory = read("/proc/meminfo") or ""
+    match = re.search(r"^MemTotal:\s+(\d+) kB$", memory, re.MULTILINE)
+    limits = [int(match[1]) * 1024] if match else []
+    for path in ("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"):
+        value = read(path)
+        if value and value.isdigit():
+            limits.append(int(value))
+    return min(limits) if limits else None
+
+
+def cpu_limit():
+    limits = [float(os.cpu_count() or 1)]
+    if hasattr(os, "sched_getaffinity"):
+        limits.append(float(len(os.sched_getaffinity(0))))
+    quota = read("/sys/fs/cgroup/cpu.max")
+    if quota:
+        fields = quota.split()
+        if len(fields) == 2 and all(field.isdigit() for field in fields) and int(fields[1]) > 0:
+            limits.append(int(fields[0]) / int(fields[1]))
+    else:
+        quota = read("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
+        period = read("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
+        if quota and period and quota.isdigit() and period.isdigit() and int(period) > 0:
+            limits.append(int(quota) / int(period))
+    return min(limits)
+
+
+def io_uring():
+    if platform.system() != "Linux" or platform.machine() not in ("x86_64", "aarch64"):
+        return {"available": None, "reason": "unsupported_probe_platform"}
+    libc = ctypes.CDLL(None, use_errno=True)
+    # A zeroed io_uring_params requests no features and one queue entry. The
+    # syscall creates only a descriptor; it does not submit I/O or map memory.
+    parameters = ctypes.create_string_buffer(256)
+    result = libc.syscall(ctypes.c_long(425), ctypes.c_uint(1), ctypes.byref(parameters))
+    if result >= 0:
+        os.close(result)
+        return {"available": True, "reason": "setup_succeeded"}
+    return {"available": False, "reason": errno.errorcode.get(ctypes.get_errno(), "UNKNOWN")}
+
+
+def inspect(root):
+    root = root.resolve(strict=True)
+    usage = shutil.disk_usage(root)
+    mount_raw = command(["findmnt", "--json", "--target", str(root), "--output", "TARGET,SOURCE,FSTYPE,OPTIONS"])
+    mount = json.loads(mount_raw)["filesystems"][0] if mount_raw else None
+    docker = command(["docker", "info", "--format", "{{.ServerVersion}}"])
+    return {"schema": 1, "observed_at": datetime.now(timezone.utc).isoformat(),
+            "architecture": platform.machine(), "system": platform.system(), "kernel": platform.release(),
+            "libc": list(platform.libc_ver()), "effective_cpu_count": cpu_limit(), "memory_limit_bytes": memory_limit(),
+            "disk": {"root": str(root), "free_bytes": usage.free, "total_bytes": usage.total, "mount": mount},
+            "docker": {"available": docker is not None, "server_version": docker}, "io_uring": io_uring(),
+            "tools": {tool: command(arguments) for tool, arguments in {
+                "rustc": ["rustc", "-Vv"], "cargo": ["cargo", "--version"], "cc": ["cc", "--version"],
+                "cmake": ["cmake", "--version"], "protoc": ["protoc", "--version"],
+                "clang": ["clang", "--version"], "pkg-config": ["pkg-config", "--version"]}.items()},
+            "image": {"hosted_image": os.environ.get("ImageOS"), "hosted_version": os.environ.get("ImageVersion"),
+                      "operator_image_digest": os.environ.get("RUSTFS_RUNNER_IMAGE_DIGEST")},
+            "run": {"id": os.environ.get("GITHUB_RUN_ID"), "attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+                    "runner_name": os.environ.get("RUNNER_NAME")},
+            "storage_locality_verified": False, "worker_destruction_verified": False}
+
+
+def preflight(report, *, min_free_gib=0, min_memory_gib=0, architecture=None, docker=False, uring=False):
+    failures = []
+    if report["system"] != "Linux":
+        failures.append("Linux is required for this pilot")
+    if architecture and report["architecture"] != architecture:
+        failures.append("architecture does not match the requested artifact/build target")
+    if report["disk"]["free_bytes"] < min_free_gib * 1024 ** 3:
+        failures.append("insufficient free disk")
+    if min_memory_gib and (report["memory_limit_bytes"] is None or report["memory_limit_bytes"] < min_memory_gib * 1024 ** 3):
+        failures.append("insufficient or unknown memory limit")
+    if docker and not report["docker"]["available"]:
+        failures.append("Docker daemon unavailable")
+    if uring and report["io_uring"]["available"] is not True:
+        failures.append("io_uring setup unavailable")
+    return failures
+
+
+def check_storage_attestation(path, report):
+    """Check an operator assertion without treating nonrotational disks as local."""
+    metadata = path.stat()
+    if metadata.st_uid != 0 or metadata.st_mode & 0o022:
+        raise ValueError("storage attestation must be root-owned and not group/world-writable")
+    attestation = json.loads(path.read_text())
+    mount = report["disk"]["mount"] or {}
+    if not (attestation.get("schema") == 1 and attestation.get("storage") == "local-ssd"
+            and attestation.get("mount_source") == mount.get("source")
+            and attestation.get("mount_target") == mount.get("target")
+            and mount.get("fstype") in ("ext4", "xfs", "btrfs")
+            and re.fullmatch(r"sha256:[a-f0-9]{64}", attestation.get("image_digest", ""))):
+        raise ValueError("local SSD attestation does not match this mount and image identity")
+    if report["image"]["operator_image_digest"] != attestation["image_digest"]:
+        raise ValueError("running image digest must match the operator storage attestation")
+    report["storage_attestation"] = attestation
+    # Local block devices can still be backed by network storage. This is an
+    # operator assertion, never an independent physical-locality measurement.
+    report["storage_locality_source"] = "operator_attestation"
+
+
+def build_commands(workload):
+    if workload == "server":
+        return ["build", "--locked", "-p", "rustfs", "--bin", "rustfs"]
+    if workload == "workspace-tests":
+        return ["test", "--locked", "--workspace", "--all-targets", "--no-run"]
+    raise ValueError("unsupported workload")
+
+
+def measure(arguments, environment, cwd, log, time_file, timeout):
+    start = time.monotonic()
+    # GNU time records peak RSS and I/O for the Cargo process tree. Signal
+    # handling kills that entire group before the task-owned target is removed.
+    with log.open("w") as output:
+        process = subprocess.Popen(["/usr/bin/time", "-v", "-o", str(time_file), *arguments],
+                                   cwd=cwd, env=environment, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
+        timed_out = False
+        try:
+            result = process.wait(timeout=timeout)
+        except (subprocess.TimeoutExpired, KeyboardInterrupt):
+            timed_out = True
+            os.killpg(process.pid, signal.SIGTERM)
+            try:
+                result = process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                os.killpg(process.pid, signal.SIGKILL)
+                result = process.wait(timeout=10)
+            # The leader can exit before a child that ignores TERM. Do not
+            # remove its target directory while that process group survives.
+            try:
+                os.killpg(process.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+    timings = read(time_file) or ""
+    def field(name):
+        match = re.search(r"^\s*" + re.escape(name) + r":\s+(\d+)$", timings, re.MULTILINE)
+        return int(match[1]) if match else None
+    return {"elapsed_seconds": time.monotonic() - start, "exit_code": result, "timed_out": timed_out,
+            "max_rss_kib": field("Maximum resident set size (kbytes)"),
+            "filesystem_inputs": field("File system inputs"), "filesystem_outputs": field("File system outputs")}
+
+
+def benchmark(source, root, output, sha, toolchain, workload, jobs, timeout, *, measure_fn=measure):
+    if not re.fullmatch(r"[a-f0-9]{40}", sha) or not re.fullmatch(r"\d+\.\d+\.\d+", toolchain):
+        raise ValueError("benchmark requires an exact source SHA and Rust toolchain version")
+    if command(["git", "rev-parse", "HEAD"], cwd=source) != sha:
+        raise ValueError("source checkout does not match the requested SHA")
+    if command(["git", "status", "--porcelain", "--untracked-files=no"], cwd=source) != "":
+        raise ValueError("source checkout has tracked modifications")
+    version = command(["rustup", "run", toolchain, "rustc", "-Vv"])
+    if not version or f"release: {toolchain}\n" not in version + "\n":
+        raise ValueError("the exact requested Rust toolchain is not installed")
+    if command(["/usr/bin/time", "--version"]) is None:
+        raise ValueError("GNU time is required for resource measurements")
+    output.mkdir(parents=True, exist_ok=True)
+    report = {"schema": 1, "source_sha": sha, "rustc": version, "workload": workload,
+              "profile": "dev", "features": "default", "build_jobs": jobs, "measurements": [],
+              "cache_policy": "fresh Cargo target for cold; same target for warm; registry downloads may be warm",
+              "passed": False, "task_directory_removed": False}
+    target = Path(tempfile.mkdtemp(prefix="rustfs-runner-pilot-", dir=root))
+    try:
+        # Compare the checked-in build configuration, not ambient tuning left
+        # by a previous job or interactive shell. Registry/network settings may
+        # remain, but all known compiler/profile/target overrides are removed.
+        environment = {key: value for key, value in os.environ.items()
+                       if not key.startswith(("CARGO_PROFILE_", "CARGO_TARGET_", "CARGO_BUILD_"))
+                       and key not in {"RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "RUSTDOCFLAGS", "CARGO_ENCODED_RUSTDOCFLAGS",
+                                       "RUSTC", "RUSTDOC", "CC", "CXX", "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS",
+                                       "RUSTFS_BUILD_VERSION"}}
+        environment.update(CARGO_TARGET_DIR=str(target / "target"), CARGO_BUILD_JOBS=str(jobs),
+                           CARGO_INCREMENTAL="0", RUSTC_WRAPPER="", RUSTC_WORKSPACE_WRAPPER="")
+        arguments = ["rustup", "run", toolchain, "cargo", *build_commands(workload)]
+        report["command"] = arguments
+        for cache in ("cold", "warm"):
+            log, time_file = output / f"{cache}.log", output / f"{cache}.time.txt"
+            metric = measure_fn(arguments, environment, source, log, time_file, timeout)
+            report["measurements"].append({"cache": cache, **metric})
+            (output / "benchmark.json").write_text(json.dumps(report, indent=2) + "\n")
+            if metric["exit_code"] != 0 or metric["timed_out"]:
+                break
+        report["passed"] = len(report["measurements"]) == 2 and all(item["exit_code"] == 0 and not item["timed_out"] for item in report["measurements"])
+    finally:
+        shutil.rmtree(target)
+        report["task_directory_removed"] = not target.exists()
+        (output / "benchmark.json").write_text(json.dumps(report, indent=2) + "\n")
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, required=True, help="existing scratch directory on the disk under test")
+    parser.add_argument("--output", type=Path, required=True, help="durable evidence directory outside scratch")
+    parser.add_argument("--min-free-gib", type=float, default=0)
+    parser.add_argument("--min-memory-gib", type=float, default=0)
+    parser.add_argument("--architecture", choices=("x86_64", "aarch64"))
+    parser.add_argument("--require-docker", action="store_true")
+    parser.add_argument("--require-io-uring", action="store_true")
+    parser.add_argument("--local-ssd-attestation", type=Path)
+    parser.add_argument("--benchmark", choices=("server", "workspace-tests"))
+    parser.add_argument("--source", type=Path, default=Path.cwd())
+    parser.add_argument("--source-sha")
+    parser.add_argument("--toolchain")
+    parser.add_argument("--jobs", type=int, default=2)
+    parser.add_argument("--timeout-seconds", type=int, default=3600)
+    args = parser.parse_args()
+    if not (1 <= args.jobs <= 16 and 60 <= args.timeout_seconds <= 7200
+            and 0 <= args.min_free_gib <= 10000 and 0 <= args.min_memory_gib <= 1000):
+        parser.error("resource requirements are outside the bounded pilot limits")
+    if args.benchmark and not (args.source_sha and args.toolchain):
+        parser.error("--benchmark requires --source-sha and --toolchain")
+    args.output.mkdir(parents=True, exist_ok=True)
+    report = inspect(args.root)
+    failures = preflight(report, min_free_gib=args.min_free_gib, min_memory_gib=args.min_memory_gib,
+                         architecture=args.architecture, docker=args.require_docker, uring=args.require_io_uring)
+    if args.local_ssd_attestation:
+        try:
+            check_storage_attestation(args.local_ssd_attestation, report)
+        except (ValueError, OSError) as error:
+            failures.append(str(error))
+    report["preflight_failures"] = failures
+    (args.output / "capabilities.json").write_text(json.dumps(report, indent=2) + "\n")
+    if failures:
+        print("Runner preflight failed: " + "; ".join(failures), file=sys.stderr)
+        return 1
+    if args.benchmark:
+        def cancel(_signum, _frame):
+            raise KeyboardInterrupt
+        signal.signal(signal.SIGTERM, cancel)
+        result = benchmark(args.source.resolve(), args.root.resolve(), args.output.resolve(), args.source_sha,
+                           args.toolchain, args.benchmark, args.jobs, args.timeout_seconds)
+        return 0 if result["passed"] else 1
+    print("Runner preflight passed; see capabilities.json for observed capabilities and unverified infrastructure properties")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/test_actions_cost_report.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Accounting fixtures preserve failed, cancelled, shared and unknown work."""
+
+import copy
+import json
+from pathlib import Path
+import unittest
+from unittest.mock import patch
+from urllib.parse import parse_qs, urlsplit
+
+import actions_cost_report as report
+
+
+class AccountingTests(unittest.TestCase):
+    def setUp(self):
+        self.snapshot = json.loads((Path(__file__).parent / "fixtures/cost/attempts.json").read_text())
+
+    def test_all_attempts_and_cross_workflow_allocation_conserve_cost(self):
+        summary = report.summarize(self.snapshot)
+        self.assertEqual(summary["attempts"], 6)
+        self.assertEqual(summary["attempt_statuses"], {"failure": 1, "success": 3, "cancelled": 2})
+        self.assertEqual(summary["total_unallocated"]["known_runner_minutes"], 33)
+        self.assertEqual(summary["pull_requests"]["11"]["known_runner_minutes"], 25)
+        self.assertEqual(summary["pull_requests"]["12"]["known_runner_minutes"], 3)
+        self.assertEqual(summary["pull_requests"]["11"]["known_queue_minutes"], 6)
+        allocated = sum(pr["known_runner_minutes"] for pr in summary["pull_requests"].values())
+        self.assertEqual(allocated + summary["periodic_unallocated"]["known_runner_minutes"]
+                         + summary["unattributed"]["known_runner_minutes"], 33)
+        self.assertFalse(summary["pull_requests"]["11"]["lifecycle_complete"])
+        self.assertIsNone(summary["runner_models"]["fixed_capacity"]["monetary_cost"])
+
+    def test_success_sample_uses_attempt_start_and_preserves_unknown_phases(self):
+        summary = report.summarize(self.snapshot)
+        self.assertEqual(summary["successful_code_pr"]["runner_sum"]["median_minutes"], 10)
+        self.assertEqual(summary["successful_code_pr"]["feedback"]["median_minutes"], 11)
+        self.assertEqual(summary["unknowns"]["runner_minutes"], 1)
+        self.assertEqual(summary["unknowns"]["queue_minutes"], 2)
+        self.assertEqual(summary["unknowns"]["phase_breakdown_jobs"], 6)
+        self.assertIsNone(summary["job_observations"][1]["phases_minutes"]["compile"])
+
+    def test_unknown_group_membership_stays_unallocated(self):
+        self.snapshot["attributions"] = {}
+        self.snapshot["runs"][2]["head_branch"] = "gh-readonly-queue/main/pr-11-fixture"
+        summary = report.summarize(self.snapshot)
+        self.assertEqual(summary["unattributed"]["known_runner_minutes"], 6)
+        self.assertNotIn("12", summary["pull_requests"])
+
+    def test_foreign_repository_pr_numbers_cannot_be_attributed_locally(self):
+        run = self.snapshot["runs"][0]
+        run["pull_requests"].append({"number": 12, "url": "https://api.github.com/repos/foreign/repo/pulls/12"})
+        self.assertEqual(report.repository_members(run, "owner/repo"), [11])
+        self.assertEqual(report.attribution(run, self.snapshot), ([11], "event_pull_requests"))
+        run["pull_requests"] = [{"number": 11, "url": "https://api.github.com/repos/foreign/repo/pulls/11"}]
+        self.assertEqual(report.attribution(run, self.snapshot), ([], "unattributed"))
+
+    def test_manifest_is_bound_to_run_sha_and_cannot_allocate_periodic_work(self):
+        row = copy.deepcopy(self.snapshot["attributions"]["102"])
+        manifest = {"schema": 1, "repository": "owner/repo", "runs": [row]}
+        report.add_attributions(self.snapshot, manifest)
+        for changes in ({"head_sha": "d" * 40}, {"pull_requests": [11, 11]},
+                        {"run_id": 103, "head_sha": "c" * 40}):
+            with self.subTest(changes=changes), self.assertRaises(ValueError):
+                report.add_attributions(self.snapshot, {**manifest, "runs": [{**row, **changes}]})
+
+    def test_duplicate_attempt_or_job_is_rejected(self):
+        for duplicate_attempt in (True, False):
+            snapshot = copy.deepcopy(self.snapshot)
+            if duplicate_attempt:
+                snapshot["runs"].append(copy.deepcopy(snapshot["runs"][0]))
+            else:
+                snapshot["runs"][0]["jobs"] *= 2
+            with self.assertRaises(ValueError):
+                report.summarize(snapshot)
+        self.snapshot["runs"][1]["jobs"][0]["id"] = self.snapshot["runs"][0]["jobs"][0]["id"]
+        with self.assertRaisesRegex(ValueError, "duplicate job"):
+            report.summarize(self.snapshot)
+
+    def test_pagination_requires_complete_unique_inventory(self):
+        first = [{"id": number} for number in range(100)]
+        with patch.object(report, "api", side_effect=[{"total_count": 101, "jobs": first}, {"total_count": 101, "jobs": [{"id": 100}]}]) as api:
+            self.assertEqual(len(report.pages("fixture", "jobs")), 101)
+            self.assertIn("page=2", api.call_args.args[0])
+        for second in ({"total_count": 101, "jobs": []}, {"total_count": 101, "jobs": [{"id": 0}]},
+                       {"total_count": 102, "jobs": [{"id": 100}]}):
+            with patch.object(report, "api", side_effect=[{"total_count": 101, "jobs": first}, second]), self.assertRaises(ValueError):
+                report.pages("fixture", "jobs")
+
+    def collect_fixture(self, *, missing_attempt=False, changed=False):
+        attempts = self.snapshot["runs"][:2]
+        latest = {key: value for key, value in attempts[1].items() if key != "jobs"}
+        calls = []
+        def api(path):
+            calls.append(path)
+            clean = urlsplit(path).path
+            query = parse_qs(urlsplit(path).query)
+            if clean.endswith("/actions/runs"):
+                self.assertNotIn("event", query)
+                self.assertNotIn("status", query)
+                return {"workflow_runs": [latest], "total_count": 1}
+            if clean.endswith("/pulls/11"):
+                return self.snapshot["pull_requests"]["11"]
+            if clean.endswith("/attempts/1") and missing_attempt:
+                raise RuntimeError("expired")
+            if "/attempts/" in clean:
+                attempt = int(clean.split("/attempts/")[1].split("/")[0])
+                if clean.endswith("/jobs"):
+                    return {"jobs": attempts[attempt - 1]["jobs"], "total_count": 1}
+                return attempts[attempt - 1]
+            return {**latest, "run_attempt": 3} if changed else latest
+        with patch.object(report, "api", side_effect=api):
+            snapshot = report.collect("owner/repo", "2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z")
+        return snapshot, calls
+
+    def test_collector_fetches_each_attempt_and_retains_api_failures(self):
+        snapshot, calls = self.collect_fixture()
+        self.assertTrue(snapshot["complete"])
+        self.assertEqual([run["run_attempt"] for run in snapshot["runs"]], [1, 2])
+        self.assertTrue(any("/attempts/1/jobs?" in call for call in calls))
+        self.assertTrue(any("/attempts/2/jobs?" in call for call in calls))
+        partial, _ = self.collect_fixture(missing_attempt=True)
+        self.assertFalse(partial["complete"])
+        self.assertEqual(len(partial["runs"]), 2)
+        self.assertFalse(partial["runs"][0]["complete"])
+        self.assertTrue(partial["runs"][1]["complete"])
+        self.assertEqual(partial["issues"][0]["scope"], "attempt")
+
+    def test_changed_latest_attempt_and_capped_inventory_are_incomplete(self):
+        snapshot, _ = self.collect_fixture(changed=True)
+        self.assertFalse(snapshot["complete"])
+        with patch.object(report, "api", return_value={"total_count": 1001, "workflow_runs": []}):
+            snapshot = report.collect("owner/repo", "2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z")
+        self.assertFalse(snapshot["complete"])
+        self.assertEqual(snapshot["issues"][0]["scope"], "run_inventory")
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci/test_runner_pilot.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Runner pilots refuse unsupported capabilities and preserve failed build evidence."""
+
+import copy
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import runner_pilot as pilot
+
+
+class PilotTests(unittest.TestCase):
+    def setUp(self):
+        self.capabilities = {"system": "Linux", "architecture": "x86_64", "memory_limit_bytes": 8 * 1024 ** 3,
+                             "disk": {"free_bytes": 100 * 1024 ** 3, "mount": {"source": "/dev/nvme0n1", "target": "/scratch", "fstype": "xfs"}},
+                             "docker": {"available": True}, "io_uring": {"available": True},
+                             "image": {"operator_image_digest": "sha256:" + "a" * 64}}
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.scratch = self.root / "scratch"
+        self.scratch.mkdir()
+        self.output = self.root / "evidence"
+
+    def test_preflight_refuses_disk_memory_abi_and_required_daemon(self):
+        self.assertEqual(pilot.preflight(self.capabilities, min_free_gib=80, min_memory_gib=8, docker=True, uring=True), [])
+        broken = copy.deepcopy(self.capabilities)
+        broken.update(architecture="aarch64", memory_limit_bytes=None)
+        broken["docker"]["available"] = False
+        broken["io_uring"]["available"] = None
+        failures = pilot.preflight(broken, min_free_gib=101, min_memory_gib=8, architecture="x86_64", docker=True, uring=True)
+        self.assertEqual(len(failures), 5)
+
+    def test_memory_limit_obeys_container_limit(self):
+        values = {"/proc/meminfo": "MemTotal:       67108864 kB",
+                  "/sys/fs/cgroup/memory.max": str(8 * 1024 ** 3)}
+        with patch.object(pilot, "read", side_effect=values.get):
+            self.assertEqual(pilot.memory_limit(), 8 * 1024 ** 3)
+
+    def test_local_ssd_requires_matching_operator_mount_and_image(self):
+        attestation = self.root / "storage.json"
+        row = {"schema": 1, "storage": "local-ssd", "mount_source": "/dev/nvme0n1", "mount_target": "/scratch", "image_digest": "sha256:" + "a" * 64}
+        attestation.write_text(json.dumps(row))
+        metadata = type("Metadata", (), {"st_uid": 0, "st_mode": 0o100644})()
+        with patch.object(Path, "stat", return_value=metadata):
+            pilot.check_storage_attestation(attestation, self.capabilities)
+            self.assertEqual(self.capabilities["storage_locality_source"], "operator_attestation")
+            attestation.write_text(json.dumps({**row, "mount_source": "/dev/other"}))
+            with self.assertRaises(ValueError):
+                pilot.check_storage_attestation(attestation, self.capabilities)
+        metadata.st_mode = 0o100666
+        with patch.object(Path, "stat", return_value=metadata), self.assertRaises(ValueError):
+            pilot.check_storage_attestation(attestation, self.capabilities)
+
+    def invoke_benchmark(self, *, fail=False, raise_error=False):
+        environments = []
+        def command(arguments, **kwargs):
+            if arguments[:2] == ["git", "rev-parse"]:
+                return "a" * 40
+            if arguments[:2] == ["git", "status"]:
+                return ""
+            if arguments[0] == "rustup":
+                return "rustc 1.92.0\nrelease: 1.92.0\nhost: x86_64-unknown-linux-gnu"
+            return "GNU time"
+        def measure(arguments, environment, source, log, time_file, timeout):
+            environments.append(environment)
+            target = Path(environment["CARGO_TARGET_DIR"])
+            target.mkdir(exist_ok=True)
+            (target / "compiled").write_text("fixture")
+            log.write_text("controlled build output")
+            time_file.write_text("fixture resources")
+            if raise_error:
+                raise OSError("controlled build failure")
+            return {"exit_code": 1 if fail else 0, "timed_out": False, "elapsed_seconds": 1,
+                    "max_rss_kib": 1024, "filesystem_inputs": 1, "filesystem_outputs": 2}
+        with patch.object(pilot, "command", side_effect=command):
+            result = pilot.benchmark(self.root, self.scratch, self.output, "a" * 40, "1.92.0", "server", 2, 60, measure_fn=measure)
+        return result, environments
+
+    def test_cold_warm_share_only_task_target_and_cleanup(self):
+        with patch.dict(os.environ, {"CARGO_PROFILE_DEV_OVERFLOW_CHECKS": "false", "RUSTFLAGS": "-C opt-level=3"}):
+            result, environments = self.invoke_benchmark()
+        self.assertTrue(result["passed"])
+        self.assertEqual([row["cache"] for row in result["measurements"]], ["cold", "warm"])
+        self.assertEqual(environments[0]["CARGO_TARGET_DIR"], environments[1]["CARGO_TARGET_DIR"])
+        self.assertEqual(environments[0]["RUSTC_WRAPPER"], "")
+        self.assertEqual(environments[0]["CARGO_INCREMENTAL"], "0")
+        self.assertNotIn("CARGO_PROFILE_DEV_OVERFLOW_CHECKS", environments[0])
+        self.assertNotIn("RUSTFLAGS", environments[0])
+        self.assertTrue(result["task_directory_removed"])
+        self.assertEqual(list(self.scratch.iterdir()), [])
+
+    def test_failed_build_has_no_warm_success_and_keeps_receipt(self):
+        result, _ = self.invoke_benchmark(fail=True)
+        self.assertFalse(result["passed"])
+        self.assertEqual(len(result["measurements"]), 1)
+        self.assertTrue(result["task_directory_removed"])
+        self.assertEqual(json.loads((self.output / "benchmark.json").read_text()), result)
+
+    def test_unexpected_failure_still_removes_only_task_directory(self):
+        unrelated = self.scratch / "unrelated"
+        unrelated.mkdir()
+        with self.assertRaises(OSError):
+            self.invoke_benchmark(raise_error=True)
+        self.assertEqual(list(self.scratch.iterdir()), [unrelated])
+        self.assertFalse(json.loads((self.output / "benchmark.json").read_text())["passed"])
+
+    def test_requires_pinned_source_and_toolchain(self):
+        for sha, version in (("main", "1.92.0"), ("a" * 40, "stable")):
+            with self.assertRaises(ValueError):
+                pilot.benchmark(self.root, self.scratch, self.output, sha, version, "server", 2, 60)
+        with patch.object(pilot, "command", return_value="b" * 40), self.assertRaises(ValueError):
+            pilot.benchmark(self.root, self.scratch, self.output, "a" * 40, "1.92.0", "server", 2, 60)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/ci_merge_queue.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Prepare or compare merge-queue ruleset payloads without changing GitHub."""
+
+import argparse
+import copy
+import hashlib
+import json
+from pathlib import Path
+import sys
+
+REQUIRED_CHECKS = {"Test and Lint", "Quick Checks", "CLA Check"}
+WRITABLE_FIELDS = ("name", "target", "enforcement", "bypass_actors", "conditions", "rules")
+QUEUE_RULE = {
+    "type": "merge_queue",
+    "parameters": {
+        "check_response_timeout_minutes": 180,
+        "grouping_strategy": "ALLGREEN",
+        "max_entries_to_build": 2,
+        "max_entries_to_merge": 2,
+        "merge_method": "SQUASH",
+        "min_entries_to_merge": 1,
+        "min_entries_to_merge_wait_minutes": 5,
+    },
+}
+
+
+def writable(snapshot):
+    if not isinstance(snapshot, dict) or snapshot.get("target") != "branch" or snapshot.get("enforcement") != "active":
+        raise ValueError("an active branch ruleset snapshot is required")
+    result = {key: copy.deepcopy(snapshot[key]) for key in WRITABLE_FIELDS if key in snapshot}
+    rules = result.get("rules", [])
+    required = [rule for rule in rules if rule.get("type") == "required_status_checks"]
+    if len(required) != 1:
+        raise ValueError("exactly one required-check rule is required")
+    contexts = required[0].get("parameters", {}).get("required_status_checks", [])
+    if len(contexts) != len(REQUIRED_CHECKS) or {item.get("context") for item in contexts} != REQUIRED_CHECKS:
+        raise ValueError("required checks changed; reconcile the workflow before preparing a switch")
+    if any(item.get("integration_id") != 15368 for item in contexts):
+        raise ValueError("required checks must be owned by GitHub Actions")
+    reviews = [rule for rule in rules if rule.get("type") == "pull_request"]
+    if len(reviews) != 1 or reviews[0].get("parameters", {}).get("required_approving_review_count", 0) < 1:
+        raise ValueError("queue preparation requires the existing review boundary")
+    if "squash" not in reviews[0]["parameters"].get("allowed_merge_methods", []):
+        raise ValueError("ruleset does not permit the proposed squash method")
+    includes = result.get("conditions", {}).get("ref_name", {}).get("include", [])
+    if includes not in (["~DEFAULT_BRANCH"], ["refs/heads/main"]):
+        raise ValueError("queue preparation is restricted to the main ruleset")
+    return result
+
+
+def fingerprint(value):
+    return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+
+
+def prepare(snapshot):
+    before = writable(snapshot)
+    if any(rule.get("type") == "merge_queue" for rule in before["rules"]):
+        raise ValueError("merge queue already configured; compare the existing policy instead")
+    after = copy.deepcopy(before)
+    after["rules"].append(copy.deepcopy(QUEUE_RULE))
+    return {"schema": 1, "ruleset_id": snapshot.get("id"), "before_sha256": fingerprint(before),
+            "status": "prepared-not-applied", "before": before, "after": after}
+
+
+def compare(prepared, snapshot, phase):
+    if prepared.get("schema") != 1 or phase not in ("before", "after"):
+        raise ValueError("invalid switch record or phase")
+    if prepared.get("ruleset_id") != snapshot.get("id"):
+        raise ValueError("readback belongs to a different ruleset")
+    if fingerprint(prepared["before"]) != prepared.get("before_sha256"):
+        raise ValueError("original snapshot was modified")
+    expected = prepare(dict(prepared["before"], id=prepared["ruleset_id"]))
+    if prepared != expected:
+        raise ValueError("prepared change differs from the bounded queue policy")
+    if writable(snapshot) != prepared[phase]:
+        raise ValueError(f"live ruleset differs from the {phase} snapshot")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("snapshot", type=Path)
+    parser.add_argument("--prepared", type=Path)
+    parser.add_argument("--phase", choices=("before", "after"), default="before")
+    args = parser.parse_args()
+    try:
+        snapshot = json.loads(args.snapshot.read_text())
+        if args.prepared:
+            compare(json.loads(args.prepared.read_text()), snapshot, args.phase)
+            print(f"Ruleset {args.phase} readback matches")
+        else:
+            print(json.dumps(prepare(snapshot), indent=2))
+    except (OSError, ValueError, KeyError, TypeError) as error:
+        print(f"merge queue preflight: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci_test_artifact.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Bind reusable nextest archives to the source, build configuration and run."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import platform
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+
+from e2e_binary import ROOT, file_hash, source_identity
+
+CONFIGURATIONS = {"workspace": ["--workspace", "--exclude", "e2e_test"], "e2e": ["-p", "e2e_test"]}
+BUILD_ENVIRONMENT = (
+    "RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_BUILD_RUSTFLAGS",
+    "RUSTC", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", "CARGO_BUILD_RUSTC",
+    "RUSTFS_BUILD_VERSION", "CC", "CXX", "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS", "AR",
+)
+CORE_TESTS = {
+    "anonymous_access_test::test_anonymous_access_denied_when_restrict_public_buckets_enabled",
+    "head_object_consistency_test::head_object_consistency_after_write_and_multipart_and_presigned_head",
+    "storage_class_capability_test::tests::historical_label_only_metadata_is_standard_without_hiding_a_real_transition_tier",
+}
+
+
+def output(*args):
+    return subprocess.check_output(args, cwd=ROOT, text=True).strip()
+
+
+def build_environment():
+    keys = set(BUILD_ENVIRONMENT) | {key for key in os.environ if key.startswith(("CARGO_PROFILE_", "CARGO_TARGET_"))}
+    return {key: os.environ.get(key, "") for key in sorted(keys)}
+
+
+def identity(configuration):
+    if os.environ.get("CARGO_BUILD_TARGET"):
+        raise ValueError("verification archives require the native target")
+    if Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")).resolve() != (ROOT / "target").resolve():
+        raise ValueError("verification archives require the checkout target directory")
+    return {
+        "source": source_identity(),
+        "tree": output("git", "rev-parse", "HEAD^{tree}"),
+        # Several fixtures use compile-time manifest/bin paths. Until those are
+        # relocatable, consumers must retain the producer checkout location.
+        "workspace_root": str(ROOT),
+        "lock_sha256": file_hash(ROOT / "Cargo.lock"),
+        "rustc": output("rustc", "-vV"),
+        "nextest": output("cargo", "nextest", "--version"),
+        "configuration": configuration,
+        "profile": "test",
+        "features": [],
+        "build_environment": build_environment(),
+        "system": platform.system(),
+        "machine": platform.machine(),
+        "run_id": os.environ.get("GITHUB_RUN_ID", "local"),
+        "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", "local"),
+        "repository": os.environ.get("GITHUB_REPOSITORY", "local"),
+    }
+
+
+def manifest_path(archive):
+    return archive.with_name(archive.name + ".json")
+
+
+def package_args(configuration):
+    if configuration == "workspace" and os.environ.get("CI_PACKAGES"):
+        packages = json.loads(os.environ["CI_PACKAGES"])
+        if not isinstance(packages, list) or not packages or any(
+            not isinstance(name, str) or not name or name.startswith("-") or name == "e2e_test" for name in packages
+        ):
+            raise ValueError("CI_PACKAGES must be a nonempty workspace package list")
+        return [part for name in sorted(set(packages)) for part in ("-p", name)]
+    return CONFIGURATIONS[configuration]
+
+
+def build(archive, configuration):
+    before = identity(configuration)
+    archive.parent.mkdir(parents=True, exist_ok=True)
+    selection = package_args(configuration)
+    command = ["cargo", "nextest", "archive", "--locked", *selection, "--archive-file", str(archive)]
+    subprocess.run(command, cwd=ROOT, check=True)
+    if identity(configuration) != before:
+        raise ValueError("archive source or build identity changed while compiling")
+    record = {
+        "schema": 1, "identity": before, "package_args": selection,
+        "sha256": file_hash(archive), "size_bytes": archive.stat().st_size,
+        "libc": list(platform.libc_ver()),
+    }
+    manifest_path(archive).write_text(json.dumps(record, indent=2) + "\n")
+
+
+def verify_record(record, expected, digest, size, libc):
+    if record.get("schema") != 1 or record.get("identity") != expected:
+        raise ValueError("archive identity mismatch (source/configuration/toolchain/environment/run)")
+    if record.get("sha256") != digest or record.get("size_bytes") != size or size <= 0:
+        raise ValueError("archive is missing, truncated or corrupt")
+    producer_libc = record.get("libc")
+    if not isinstance(producer_libc, list) or len(producer_libc) != 2 or producer_libc[0] != libc[0]:
+        raise ValueError("archive libc family mismatch")
+    if producer_libc[0] == "glibc":
+        def version(value):
+            return tuple(int(part) for part in value.split("."))
+        if version(producer_libc[1]) > version(libc[1]):
+            raise ValueError("consumer libc is older than the producer")
+    elif producer_libc[1] != libc[1]:
+        raise ValueError("archive libc version mismatch")
+
+
+def verify(archive, configuration):
+    record = json.loads(manifest_path(archive).read_text())
+    verify_record(record, identity(configuration), file_hash(archive), archive.stat().st_size, platform.libc_ver())
+    if record.get("package_args") != package_args(configuration):
+        raise ValueError("archive package selection does not match the verification plan")
+    return record
+
+
+def selected_tests(listing):
+    selected = set()
+    for binary, suite in listing["rust-suites"].items():
+        if suite.get("status") != "listed":
+            raise ValueError(f"test binary {binary} could not be listed")
+        for name, case in suite["testcases"].items():
+            if case.get("filter-match", {}).get("status") == "matches" and not case.get("ignored"):
+                selected.add((binary, name))
+    if not selected:
+        raise ValueError("test selection must not be empty")
+    return selected
+
+
+def verify_execution(listing, junit, core=False):
+    inventory = json.loads(Path(listing).read_text())
+    expected = selected_tests(inventory)
+    known = {(binary, name) for binary, suite in inventory["rust-suites"].items() for name in suite["testcases"]}
+    if core and (len(expected) != len(CORE_TESTS) or {name for _, name in expected} != CORE_TESTS):
+        raise ValueError("core contract must select exactly the write/read, denial and restart proofs")
+    report = ET.parse(junit).getroot()
+    if report.tag != "testsuites" or report.get("failures") != "0" or report.get("errors") != "0":
+        raise ValueError("test report must be a successful nextest run")
+    actual = set()
+    seen = set()
+    for suite in report.findall("testsuite"):
+        name = suite.get("name", "")
+        setup_id = name.removeprefix("@setup-script:") if name.startswith("@setup-script:") else None
+        cases = suite.findall("testcase")
+        # Nextest reports each setup script as a one-case suite outside the
+        # Rust test inventory. Validate its result without counting it as a test.
+        if setup_id is not None and (not setup_id or len(cases) != 1 or cases[0].get("name") != setup_id):
+            raise ValueError("invalid setup-script execution")
+        for case in cases:
+            key = (name, case.get("name"))
+            # Ignored tests may be emitted as skipped; selected tests may not.
+            if key in known - expected and case.find("skipped") is not None:
+                continue
+            if key in seen or case.get("classname") != key[0]:
+                raise ValueError("duplicate or misplaced test execution")
+            if (case.get("status", "passed") != "passed" or case.find("skipped") is not None
+                    or case.find("failure") is not None or case.find("error") is not None):
+                raise ValueError("selected test did not pass")
+            seen.add(key)
+            if setup_id is None:
+                actual.add(key)
+    if actual != expected:
+        raise ValueError(f"test execution differs from plan: missing={sorted(expected - actual)}, extra={sorted(actual - expected)}")
+    return len(actual)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="command", required=True)
+    for name in ("build", "verify"):
+        child = sub.add_parser(name)
+        child.add_argument("--configuration", choices=CONFIGURATIONS, required=True)
+        child.add_argument("--archive", type=Path, required=True)
+    child = sub.add_parser("cargo")
+    child.add_argument("operation", choices=("lint", "doc"))
+    child.add_argument("--configuration", choices=CONFIGURATIONS, required=True)
+    child = sub.add_parser("evidence")
+    child.add_argument("listing", type=Path)
+    child.add_argument("junit", type=Path)
+    child.add_argument("--core", action="store_true")
+    args = parser.parse_args()
+    try:
+        if args.command == "build":
+            build(args.archive.resolve(), args.configuration)
+        elif args.command == "verify":
+            verify(args.archive.resolve(), args.configuration)
+        elif args.command == "cargo":
+            selection = package_args(args.configuration)
+            if args.operation == "lint" and args.configuration == "workspace":
+                # The original all-targets lint included the harness. Archive
+                # and doctest exclusions must not remove that responsibility.
+                selection = [*selection, "-p", "e2e_test"] if "CI_PACKAGES" in os.environ and os.environ["CI_PACKAGES"] else ["--workspace"]
+            command = ["cargo", "clippy" if args.operation == "lint" else "test", "--locked", *selection]
+            command += ["--all-targets", "--", "-D", "warnings"] if args.operation == "lint" else ["--doc"]
+            subprocess.run(command, cwd=ROOT, check=True)
+        else:
+            print(f"Verified {verify_execution(args.listing, args.junit, args.core)} test executions")
+    except (OSError, ValueError, KeyError, TypeError, ET.ParseError, subprocess.CalledProcessError) as error:
+        print(f"verification artifact: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci_timing_report.py

@@ -35,6 +35,25 @@ def distribution(values):
             "median_minutes": statistics.median(known) if known else None}
 
 
+def successful_code_run(run):
+    """Recognize both the historical combined lane and the archive consumers."""
+    jobs = run["jobs"]
+    if not (run.get("event", "pull_request") == "pull_request"
+            and run.get("status") == "completed" and run.get("conclusion") == "success"
+            and jobs and all(job.get("status") == "completed" for job in jobs)):
+        return False
+    consumers = [job for job in jobs if job["name"].startswith(("Workspace tests", "Core contract"))]
+    if consumers:
+        return (any(job["name"] == "Workspace Lint and Build" and job.get("conclusion") == "success" for job in jobs)
+                and all(job.get("conclusion") == "success" for job in consumers)
+                and all(any(job["name"].startswith(prefix) for job in consumers)
+                        for prefix in ("Workspace tests", "Core contract")))
+    return any(job["name"] in ("Workspace Test and Lint", "Test and Lint")
+               and job.get("conclusion") == "success"
+               and any(step.get("name") == "Run nextest tests" and step.get("conclusion") == "success"
+                       for step in (job.get("steps") or [])) for job in jobs)
+
+
 def summarize(runs):
     statuses = Counter()
     jobs_by_name = defaultdict(list)
@@ -46,10 +65,7 @@ def summarize(runs):
             if job.get("conclusion") != "skipped":
                 jobs_by_name[job["name"]].append(job)
         # Exclude docs-only and pull_request.closed cancellation-handler greens.
-        if (run.get("status") == "completed" and run.get("conclusion") == "success" and jobs
-                and any(job["name"] in ("Workspace Test and Lint", "Test and Lint") and job.get("conclusion") == "success"
-                        and any(step.get("name") == "Run nextest tests" and step.get("conclusion") == "success" for step in (job.get("steps") or [])) for job in jobs)
-                and all(job.get("status") == "completed" for job in jobs)):
+        if successful_code_run(run):
             successful_code_runs.append(run)
 
     wall, job_time = [], []
@@ -82,7 +98,7 @@ def summarize(runs):
             "successful_code_wall": distribution(wall), "successful_code_job_sum": distribution(job_time),
             "jobs": job_summary,
             "limits": ["Live collection samples completed runs; ongoing queue depth is not measured.",
-                       "Job creation-to-start is scheduler wait; dependency delay is not included.",
+                       "Job creation-to-start wait does not separately identify scheduler and dependency delays.",
                        "Step times can combine setup, compilation and tests; they do not isolate compiler time.",
                        "Job sums are unweighted runner minutes, not billing or wall time.",
                        "Actions timing does not establish functional completeness, escaped regressions or quarantine health."]}

scripts/functional_chain_evidence.py

@@ -14,8 +14,9 @@ import re
 import subprocess
 
 from resolve_functional_candidate import ROOT, positive, require, sha, validate_manifest
+from validation_registry import load_registry, select_plan, suite_ids, validate_plan
 
-SUITES = ("upgrade", "s3", "kms", "tier", "storage", "heal", "pool", "security", "replication", "fault-tolerance", "table", "performance")
+SUITES = suite_ids()
 MAX_REPORT = 8 * 1024 * 1024
 
 
@@ -128,6 +129,24 @@ def auto_testing_dir():
     return candidates[0]
 
 
+def read_report(suite, report):
+    require(suite in SUITES, "unknown suite")
+    require(report.is_file() and 0 < report.stat().st_size <= MAX_REPORT, "missing, empty or oversized report")
+    data = report.read_bytes()
+    text = data.decode("utf-8")
+    counts = fault_tolerance_counts(text) if suite == "fault-tolerance" else report_counts(text, suite == "performance")
+    return counts, hashlib.sha256(data).hexdigest()
+
+
+def require_passing(counts):
+    require(counts["PASS"] > 0 and not counts["FAIL"] and not counts["RUNNING"], "no passing executions or incomplete/failed cases")
+
+
+def verify_report(suite, report):
+    counts, _ = read_report(suite, report)
+    require_passing(counts)
+
+
 def record(chain, suite, report, output):
     require(suite in SUITES, "unknown suite")
     result = {"schema": 1, "suite": suite, "chain": chain, "valid": False, "counts": {}, "report_sha256": None}
@@ -135,12 +154,8 @@ def record(chain, suite, report, output):
     try:
         private_head = subprocess.check_output(["git", "-C", str(auto_testing_dir()), "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
         require(private_head == chain["testing_sha"], "suite used a different private script revision")
-        require(report.is_file() and 0 < report.stat().st_size <= MAX_REPORT, "missing, empty or oversized report")
-        data = report.read_bytes()
-        result["report_sha256"] = hashlib.sha256(data).hexdigest()
-        text = data.decode("utf-8")
-        result["counts"] = fault_tolerance_counts(text) if suite == "fault-tolerance" else report_counts(text, suite == "performance")
-        require(result["counts"]["PASS"] > 0 and not result["counts"]["FAIL"] and not result["counts"]["RUNNING"], "no passing executions or incomplete/failed cases")
+        result["counts"], result["report_sha256"] = read_report(suite, report)
+        require_passing(result["counts"])
         require(all(os.environ[key] == "success" for key in ("CHAIN_JOB_STATUS", "CHAIN_TEST_OUTCOME", "CHAIN_REPORT_OUTCOME")), "suite, report or job did not succeed")
         result["valid"] = True
     except (OSError, ValueError, subprocess.SubprocessError) as exc:
@@ -156,10 +171,11 @@ def record(chain, suite, report, output):
         raise error
 
 
-def summarize(chain, directory, needs):
+def summarize(chain, directory, needs, plan=None, allow_skipped=()):
     """Retain failed/missing lanes without granting complete-success evidence."""
     lanes = []
-    for suite in SUITES:
+    plan = validate_plan(plan) if plan is not None else select_plan(load_registry())
+    for suite in plan["suites"]:
         lane = {"suite": suite, "result": needs.get(suite, {}).get("result", "missing"),
                 "evidence": None, "error": None}
         try:
@@ -170,11 +186,11 @@ def summarize(chain, directory, needs):
         except (OSError, ValueError, AttributeError) as error:
             lane["error"] = str(error)
         lanes.append(lane)
-    result = {"schema": 1, "chain": chain, "needs": needs, "lanes": lanes,
+    result = {"schema": 1, "chain": chain, "plan": plan, "needs": needs, "lanes": lanes,
               "complete": False, "error": None, "completed_at": datetime.now(timezone.utc).isoformat()}
     try:
         require(chain is not None, "candidate preparation did not complete")
-        aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"})
+        aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"}, allow_skipped, plan)
         result["complete"] = True
     except (OSError, ValueError, KeyError, TypeError, AttributeError) as error:
         result["error"] = str(error)
@@ -206,25 +222,31 @@ def render_summary(result):
     return "\n".join(lines) + "\n"
 
 
-def aggregate(chain, directory, needs, allow_skipped=()):
+def aggregate(chain, directory, needs, allow_skipped=(), plan=None):
+    plan = validate_plan(plan) if plan is not None else select_plan(load_registry())
+    selected = plan["suites"]
     allowed = {name for name in allow_skipped if name}
-    require(allowed <= set(SUITES), "aggregate allow-list names an unknown lane")
+    require(allowed <= set(plan["optional_suites"]), "aggregate allow-list names a required or unknown lane")
     require(set(needs) == set(SUITES), "aggregate is missing a required lane")
-    skipped = {name for name, value in needs.items() if value.get("result") == "skipped"}
+    require(all(value.get("result") == "skipped" for name, value in needs.items() if name not in selected),
+            "an unselected suite unexpectedly ran")
+    skipped = {name for name in selected if needs[name].get("result") == "skipped"}
     require(skipped <= allowed, "a lane was skipped without preflight permission: " + ", ".join(sorted(skipped - allowed)))
-    require(all(value.get("result") == "success" for name, value in needs.items() if name not in skipped), "a required suite did not succeed")
-    expected = [suite for suite in SUITES if suite not in skipped]
+    require(all(needs[name].get("result") == "success" for name in selected if name not in skipped), "a required suite did not succeed")
+    expected = [suite for suite in selected if suite not in skipped]
     require({path.name for path in directory.iterdir()} == {suite + ".json" for suite in expected}, "missing or unexpected suite evidence")
     records = [json.loads((directory / (suite + ".json")).read_text()) for suite in expected]
     validate_records(chain, records, expected)
-    return {"schema": 1, "chain": chain, "suites": records, "complete": True,
-            "skipped_lanes": sorted(skipped), "completed_at": datetime.now(timezone.utc).isoformat()}
+    return {"schema": 1, "chain": chain, "plan": plan, "suites": records, "complete": True,
+            "skipped_lanes": sorted(skipped),
+            "skip_reasons": {name: "optional fleet unavailable during preflight" for name in sorted(skipped)},
+            "completed_at": datetime.now(timezone.utc).isoformat()}
 
 
 def validate_records(chain, records, expected_suites=SUITES):
     require(isinstance(records, list) and len(records) == len(expected_suites), "missing suite evidence")
     require([record.get("suite") for record in records] == list(expected_suites), "missing, duplicate or reordered suite evidence")
-    for suite, result in zip(SUITES, records):
+    for suite, result in zip(expected_suites, records):
         require(type(result.get("schema")) is int and result["schema"] == 1 and result.get("suite") == suite and result.get("chain") == chain, "suite evidence identity mismatch")
         require(result.get("valid") is True and sha(result.get("report_sha256"), 64), "suite evidence is invalid")
         counts = result.get("counts", {})
@@ -234,7 +256,7 @@ def validate_records(chain, records, expected_suites=SUITES):
 
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("mode", choices=("consume", "record", "aggregate", "summarize"))
+    parser.add_argument("mode", choices=("consume", "record", "aggregate", "summarize", "verify-report"))
     parser.add_argument("--suite", choices=SUITES)
     parser.add_argument("--report", type=Path)
     parser.add_argument("--output", type=Path)
@@ -242,9 +264,13 @@ def main():
     parser.add_argument("--allow-skipped", default="",
                         help="comma-separated lanes the preflight deliberately skipped (e.g. performance)")
     args = parser.parse_args()
+    if args.mode == "verify-report":
+        verify_report(args.suite, args.report)
+        return
+    plan = json.loads(os.environ["CHAIN_PLAN"]) if os.environ.get("CHAIN_PLAN") else None
     if args.mode == "summarize":
         chain = current_chain() if os.environ.get("CHAIN_MANIFEST") else None
-        result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]))
+        result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]), plan, args.allow_skipped.split(","))
         args.output.write_text(json.dumps(result, sort_keys=True) + "\n")
         args.output.with_suffix(".md").write_text(render_summary(result))
         if os.environ.get("GITHUB_STEP_SUMMARY"):
@@ -259,7 +285,7 @@ def main():
     else:
         needs = json.loads(os.environ["CHAIN_NEEDS"])
         needs.pop("prepare", None)
-        result = aggregate(chain, args.directory, needs, args.allow_skipped.split(","))
+        result = aggregate(chain, args.directory, needs, args.allow_skipped.split(","), plan)
         args.output.write_text(json.dumps(result, sort_keys=True) + "\n")
 
 

scripts/functional_chain_health.py

@@ -8,12 +8,15 @@ from datetime import datetime, timedelta, timezone
 import json
 from pathlib import Path
 import subprocess
+from urllib.parse import quote
 
 from functional_chain_evidence import validate_records
 from resolve_functional_candidate import REPOSITORY, api, read_json_artifact, require, resolve, sha
+from validation_registry import load_registry, select_plan, suite_ids, validate_plan, workflow_entry
 
 WORKFLOW = "rustfs-functional-chain.yml"
-MAX_AGE = timedelta(hours=36)
+ENTRY = workflow_entry(".github/workflows/" + WORKFLOW)
+MAX_AGE = timedelta(hours=ENTRY["max_age_hours"])
 
 
 def timestamp(value):
@@ -28,7 +31,14 @@ def validate_summary(summary, run):
     chain = summary["chain"]
     require(chain["run_id"] == run["id"] and chain["attempt"] == run["run_attempt"] and chain["workflow_sha"] == run["head_sha"], "complete evidence belongs to another run attempt")
     require(sha(chain["testing_sha"]), "missing test-script pin")
-    validate_records(chain, summary["suites"])
+    plan = validate_plan(summary["plan"]) if "plan" in summary else select_plan(load_registry())
+    require(plan["suites"] == list(suite_ids()), "partial suite execution cannot satisfy complete-chain health")
+    skipped = summary.get("skipped_lanes", [])
+    require(isinstance(skipped, list) and len(skipped) == len(set(skipped))
+            and set(skipped) <= set(plan["optional_suites"]), "required suite was skipped")
+    require(not skipped or all(summary.get("skip_reasons", {}).get(name) for name in skipped),
+            "optional suite omission has no evidence")
+    validate_records(chain, summary["suites"], [name for name in plan["suites"] if name not in skipped])
     candidate = chain["candidate"]
     manifest = candidate["manifest"]
     require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed")
@@ -41,18 +51,18 @@ def validate_summary(summary, run):
     return {"run_id": run["id"], "attempt": run["run_attempt"], "url": run["html_url"],
             "workflow_sha": chain["workflow_sha"], "testing_sha": chain["testing_sha"],
             "candidate": candidate, "source_ref": source_ref, "source_sha": manifest["source_sha"],
-            "completed_at": completed.isoformat(), "expires_at": (timestamp(candidate["build_started_at"]) + MAX_AGE).isoformat(),
+            "completed_at": completed.isoformat(), "expires_at": (completed + MAX_AGE).isoformat(),
+            "skipped_lanes": skipped,
             "verified_at": datetime.now(timezone.utc).isoformat(), "evidence_schema": 1}
 
 
 def complete_success(run):
     require(run["path"] == ".github/workflows/" + WORKFLOW and run["head_branch"] == "main", "unexpected chain workflow source")
     require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "chain came from another repository")
+    require(run.get("event") in ("workflow_run", "workflow_dispatch"), "unexpected chain event")
     require(run.get("status") == "completed" and run.get("conclusion") == "success", "chain has not completed successfully")
     name = f"functional-chain-complete-{run['id']}-{run['run_attempt']}"
-    payload = api(f"repos/{REPOSITORY}/actions/runs/{run['id']}/artifacts?per_page=100")
-    require(payload["total_count"] <= 100, "chain artifact listing is incomplete")
-    artifacts = [item for item in payload["artifacts"] if item.get("name") == name]
+    artifacts = [item for item in run_artifacts(run["id"]) if item.get("name") == name]
     require(len(artifacts) == 1, "missing or ambiguous complete-chain artifact")
     artifact = artifacts[0]
     require(type(artifact.get("size_in_bytes")) is int and 0 < artifact["size_in_bytes"] <= 1024 * 1024, "complete evidence size is invalid")
@@ -64,50 +74,125 @@ def complete_success(run):
     return result
 
 
-def collect(limit=20, source_ref="main"):
+def run_artifacts(run_id):
+    artifacts = []
+    total = None
+    for page in range(1, 21):
+        payload = api(f"repos/{REPOSITORY}/actions/runs/{run_id}/artifacts?per_page=100&page={page}")
+        count, values = payload.get("total_count"), payload.get("artifacts")
+        require(type(count) is int and count >= 0 and isinstance(values, list), "incomplete chain artifact listing")
+        require(total is None or total == count, "chain artifacts changed during pagination")
+        total = count
+        require(len(values) <= 100 and (values or len(artifacts) == total), "incomplete chain artifact page")
+        artifacts.extend(values)
+        require(len(artifacts) <= total and len({item["id"] for item in artifacts}) == len(artifacts),
+                "duplicated chain artifact page")
+        if len(artifacts) == total:
+            return artifacts
+    raise ValueError("chain artifact pagination budget exhausted")
+
+
+def expected_attempt(run):
+    if run.get("event") == "workflow_run":
+        return True
+    require(run.get("event") == "workflow_dispatch", "unexpected chain attempt event")
+    name = f"functional-plan-{run['id']}-{run['run_attempt']}"
+    artifacts = [item for item in run_artifacts(run["id"]) if item.get("name") == name]
+    if not artifacts:
+        # Missing evidence cannot distinguish a non-suite dispatch from a
+        # full-plan attempt that failed before uploading its selection.
+        return True
+    require(len(artifacts) == 1, "ambiguous suite plan artifact")
+    artifact = artifacts[0]
+    require(type(artifact.get("size_in_bytes")) is int and 0 < artifact["size_in_bytes"] <= 1024 * 1024,
+            "selection plan size is invalid")
+    archive = api(f"repos/{REPOSITORY}/actions/artifacts/{artifact['id']}/zip", binary=True)
+    plan = validate_plan(read_json_artifact(archive, artifact, run, name, "suite-plan.json"))
+    return plan["suites"] == list(suite_ids())
+
+
+def list_runs(page_size):
+    require(type(page_size) is int and 1 <= page_size <= 100, "invalid chain page size")
+    runs = []
+    total = None
+    for page in range(1, 101):
+        payload = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={page_size}&page={page}")
+        count, values = payload.get("total_count"), payload.get("workflow_runs")
+        require(type(count) is int and count >= 0 and isinstance(values, list), "incomplete chain run listing")
+        require(total is None or total == count, "chain run count changed during pagination")
+        total = count
+        require(len(values) <= page_size and (values or len(runs) == total), "incomplete chain run page")
+        runs.extend(values)
+        require(len(runs) <= total and len({run["id"] for run in runs}) == len(runs), "duplicated chain run page")
+        if len(runs) == total:
+            return runs
+    raise ValueError("chain pagination budget exhausted; observation is incomplete")
+
+
+def expected_source_sha(source_ref):
+    value = api(f"repos/{REPOSITORY}/git/ref/heads/{quote(source_ref, safe='')}")["object"]["sha"]
+    require(sha(value), "expected source ref has no valid SHA")
+    return value
+
+
+def collect(limit=100, source_ref="main"):
     source_ref = source_ref.removeprefix("refs/heads/")
     require(bool(source_ref), "expected nightly source ref is empty")
     observed = datetime.now(timezone.utc)
     workflow = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}")
-    runs = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={limit}")["workflow_runs"]
+    expected_sha = expected_source_sha(source_ref)
+    runs = list_runs(limit)
+    # Only verified partial plans exclude manual attempts. Missing evidence
+    # must not hide an early failure behind an older complete success.
     runs.sort(key=lambda run: timestamp(run["run_started_at"]), reverse=True)
     result = {"schema": 1, "observed_at": observed.isoformat(), "workflow_state": workflow["state"],
-              "owner": "@overtrue", "scan_limit": limit, "inspection_complete": True, "expected_source_ref": source_ref,
+              "owner": ENTRY["owner"], "policy": ENTRY["policy"], "scan_limit": limit,
+              "inspection_complete": True, "expected_source_ref": source_ref, "expected_source_sha": expected_sha,
               "latest_attempt": None, "last_complete_success": {}, "healthy": False}
-    for index, listed in enumerate(runs):
+    for listed in runs:
         run = api(f"repos/{REPOSITORY}/actions/runs/{listed['id']}/attempts/{listed['run_attempt']}")
-        if index == 0:
+        require((run["id"], run["run_attempt"], run["head_sha"]) ==
+                (listed["id"], listed["run_attempt"], listed["head_sha"]), "attempt API returned a different run identity")
+        if not expected_attempt(run):
+            continue
+        latest = result["latest_attempt"] is None
+        if latest:
             result["latest_attempt"] = {"run_id": run["id"], "attempt": run["run_attempt"], "url": run["html_url"],
                                         "status": run["status"], "conclusion": run["conclusion"], "verification": "not_complete"}
         if run.get("conclusion") != "success":
             continue
         try:
             complete = complete_success(run)
-        except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:
-            if index == 0:
+        except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
+            if latest:
                 result["latest_attempt"]["verification"] = "invalid"
-            result["inspection_complete"] = False
+                result["inspection_complete"] = False
             continue
-        source = complete["source_ref"]
-        if source.startswith("refs/heads/"):
-            source = source[len("refs/heads/"):]
-        if index == 0:
-            result["latest_attempt"]["verification"] = "complete"
-            result["latest_attempt"]["source_ref"] = source
-            result["latest_attempt"]["source_sha"] = complete["source_sha"]
+        source = complete["source_ref"].removeprefix("refs/heads/")
+        if latest:
+            result["latest_attempt"] = {"run_id": run["id"], "attempt": run["run_attempt"], "url": run["html_url"],
+                                        "status": run["status"], "conclusion": run["conclusion"], "verification": "complete",
+                                        "source_ref": source, "source_sha": complete["source_sha"]}
         previous = result["last_complete_success"].get(source)
         if previous is None or timestamp(complete["completed_at"]) > timestamp(previous["completed_at"]):
             result["last_complete_success"][source] = complete
-    # Reject a snapshot if a new attempt started while artifacts were checked.
-    refreshed = api(f"repos/{REPOSITORY}/actions/workflows/{WORKFLOW}/runs?branch=main&per_page={limit}")["workflow_runs"]
+        # Older immutable successes add no facts once this source has current
+        # evidence. Historical observations survive separately in the dashboard.
+        if source == source_ref:
+            break
+    refreshed = list_runs(limit)
     fields = ("id", "run_attempt", "status", "conclusion", "head_sha", "run_started_at")
     snapshot = lambda values: sorted(tuple(item.get(key) for key in fields) for item in values)
     require(snapshot(runs) == snapshot(refreshed), "chain attempts changed during inspection; retry collection")
+    require(expected_source_sha(source_ref) == expected_sha, "expected source changed during inspection")
     for complete in result["last_complete_success"].values():
-        complete["fresh"] = observed <= timestamp(complete["expires_at"])
+        complete["fresh"] = (observed <= timestamp(complete["expires_at"])
+                             and complete["source_sha"] == expected_sha)
     latest = result["latest_attempt"] or {}
-    result["healthy"] = (result["workflow_state"] == "active" and latest.get("verification") == "complete"
-                         and latest.get("source_ref") == source_ref and result["last_complete_success"].get(source_ref, {}).get("fresh") is True)
+    result["healthy"] = (result["inspection_complete"] and result["policy"] == "active"
+                         and result["workflow_state"] == "active" and latest.get("verification") == "complete"
+                         and latest.get("source_ref") == source_ref and latest.get("source_sha") == expected_sha
+                         and result["last_complete_success"].get(source_ref, {}).get("fresh") is True)
     return result
 
 
@@ -117,7 +202,8 @@ def merge_history(current, previous):
     for source, value in previous.get("last_complete_success", {}).items():
         if source not in current["last_complete_success"] or timestamp(value["completed_at"]) > timestamp(current["last_complete_success"][source]["completed_at"]):
             value = dict(value)
-            value["fresh"] = timestamp(current["observed_at"]) <= timestamp(value["expires_at"])
+            value["fresh"] = (timestamp(current["observed_at"]) <= timestamp(value["expires_at"])
+                              and value["source_sha"] == current.get("expected_source_sha"))
             value["retained_history"] = True
             current["last_complete_success"][source] = value
     return current
@@ -139,10 +225,15 @@ def main():
     parser.add_argument("--publish", action="store_true")
     parser.add_argument("--source-ref", default="main", help="Expected nightly build source, not the workflow branch")
     args = parser.parse_args()
-    result = collect(source_ref=args.source_ref)
+    try:
+        result = collect(source_ref=args.source_ref)
+    except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error:
+        result = {"schema": 1, "observed_at": datetime.now(timezone.utc).isoformat(),
+                  "workflow_state": "unknown", "inspection_complete": False, "healthy": False,
+                  "latest_attempt": None, "last_complete_success": {}, "error": str(error)}
+    args.output.write_text(json.dumps(result, indent=2) + "\n")
     if args.publish:
         publish(result)
-    args.output.write_text(json.dumps(result, indent=2) + "\n")
     print(json.dumps({key: result[key] for key in ("workflow_state", "inspection_complete", "healthy", "latest_attempt")}))
     return 0 if result["healthy"] else 1
 

scripts/functional_environment.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""Quarantine shared fleet nodes until the owning attempt completes cleanup.
+
+GitHub workflow concurrency owns mutual exclusion. Durable node markers keep
+cancellation, runner loss and failed cleanup from making that environment
+available to the next lock holder. Recovery must inspect the retained receipt,
+clean the nodes under the same resource lock, then recover using the original
+run/attempt/suite identity. Recovery also handles partially admitted fleets.
+"""
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import subprocess
+
+from validation_registry import load_registry, require
+
+# Execute the same small state transition on every node, without relying on
+# runner-local storage surviving a job or being shared by different runners.
+REMOTE = r'''
+import base64,json,os,pathlib,stat,sys
+mode,encoded=sys.argv[1:]
+identity=json.loads(base64.b64decode(encoded))
+root=pathlib.Path.home()/'.rustfs-functional-environment'
+root.mkdir(mode=0o700,exist_ok=True)
+info=root.lstat()
+if not stat.S_ISDIR(info.st_mode) or info.st_uid!=os.getuid() or info.st_mode&0o077:
+    raise ValueError('unsafe environment quarantine directory')
+path=root/'active.json'
+if mode=='check':
+    if path.exists() or path.is_symlink():
+        raise ValueError('environment quarantined by an unfinished attempt: '+path.read_text())
+elif mode=='enter':
+    fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
+    with os.fdopen(fd,'w') as out:
+        out.write(json.dumps(identity,sort_keys=True)+'\n');out.flush();os.fsync(out.fileno())
+elif mode in ('recover-verify','recover-finish') and not path.exists() and not path.is_symlink():
+    pass
+elif mode in ('verify','finish','recover-verify','recover-finish'):
+    if path.is_symlink() or json.loads(path.read_text())!=identity:
+        raise ValueError('quarantine belongs to another attempt')
+    if mode in ('finish','recover-finish'):
+        path.unlink()
+else:
+    raise ValueError('unknown environment operation')
+fd=os.open(root,os.O_RDONLY|os.O_DIRECTORY);os.fsync(fd);os.close(fd)
+'''
+
+
+def identity(suite, env):
+    suites = {item["id"]: item for item in load_registry()["suites"]}
+    require(suite in suites, "unknown environment suite")
+    run_id, attempt = env.get("GITHUB_RUN_ID", ""), env.get("GITHUB_RUN_ATTEMPT", "")
+    require(re.fullmatch(r"[1-9][0-9]*", run_id) and re.fullmatch(r"[1-9][0-9]*", attempt),
+            "invalid quarantine run identity")
+    return {"schema": 1, "run_id": int(run_id), "attempt": int(attempt), "suite": suite,
+            "environment": suites[suite]["environment"], "resource_lock": suites[suite]["resource_lock"]}
+
+
+def remote(node, user, mode, value):
+    encoded = base64.b64encode(json.dumps(value, sort_keys=True).encode()).decode()
+    command = shlex.join(["python3", "-c", REMOTE, mode, encoded])
+    subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o",
+                    "StrictHostKeyChecking=accept-new", user + "@" + node, command],
+                   check=True, capture_output=True, text=True, timeout=30)
+
+
+def transition(mode, suite, env, cleanup_outcome="", execute=remote):
+    value = identity(suite, env)
+    nodes = (env.get("RUSTFS_NODES") or "vm000 vm001 vm002").split()
+    user = env.get("RUSTFS_SSH_USER") or "azureuser"
+    require(nodes and len(nodes) == len(set(nodes)) and all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", node) for node in nodes)
+            and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", user), "invalid fleet SSH identity")
+    receipt = {**value, "nodes": nodes, "mode": mode, "cleanup_outcome": cleanup_outcome,
+               "state": "quarantined", "completed_nodes": []}
+    try:
+        require(mode in ("check", "enter", "finish", "recover"), "unknown fleet transition")
+        if mode in ("finish", "recover"):
+            require(cleanup_outcome == "success", "cleanup incomplete; fleet remains quarantined")
+        # Preflight every node before the first state mutation. Any partial
+        # failure deliberately leaves admission blocked until operator recovery.
+        for node in nodes:
+            execute(node, user, "check" if mode in ("check", "enter") else "recover-verify" if mode == "recover" else "verify", value)
+        for node in ([] if mode == "check" else nodes):
+            execute(node, user, "recover-finish" if mode == "recover" else mode, value)
+            receipt["completed_nodes"].append(node)
+        receipt["state"] = "owned" if mode == "enter" else "clean"
+        return receipt
+    finally:
+        target = env.get("FUNCTIONAL_ENVIRONMENT_RECEIPT")
+        if target:
+            Path(target).write_text(json.dumps(receipt, sort_keys=True) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("mode", choices=("check", "enter", "finish", "recover"))
+    parser.add_argument("--suite", required=True)
+    parser.add_argument("--cleanup-outcome", default="")
+    args = parser.parse_args()
+    print(json.dumps(transition(args.mode, args.suite, os.environ, args.cleanup_outcome), sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

scripts/test_ci_merge_queue.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import copy
+import unittest
+
+from ci_merge_queue import REQUIRED_CHECKS, compare, prepare
+
+
+class QueueTests(unittest.TestCase):
+    def snapshot(self):
+        return {"id": 12, "name": "main", "target": "branch", "enforcement": "active", "bypass_actors": [],
+                "conditions": {"ref_name": {"include": ["~DEFAULT_BRANCH"], "exclude": []}},
+                "rules": [{"type": "required_status_checks", "parameters": {
+                    "required_status_checks": [{"context": name, "integration_id": 15368} for name in sorted(REQUIRED_CHECKS)]}},
+                    {"type": "pull_request", "parameters": {"required_approving_review_count": 1, "allowed_merge_methods": ["squash"]}},
+                    {"type": "required_linear_history"}]}
+
+    def test_prepares_only_additive_queue_rule_and_preserves_rollback(self):
+        snapshot = self.snapshot()
+        original = copy.deepcopy(snapshot)
+        record = prepare(snapshot)
+        self.assertEqual(snapshot, original)
+        self.assertEqual(record["after"]["rules"][:-1], record["before"]["rules"])
+        compare(record, snapshot, "before")
+        compare(record, dict(record["after"], id=snapshot["id"]), "after")
+
+    def test_changed_or_missing_checks_cannot_be_hidden(self):
+        for key, value in (("context", "Other"), ("integration_id", 42)):
+            snapshot = self.snapshot()
+            snapshot["rules"][0]["parameters"]["required_status_checks"][0][key] = value
+            with self.assertRaises(ValueError):
+                prepare(snapshot)
+
+    def test_rule_drift_wrong_id_and_weakened_proposal_fail_readback(self):
+        snapshot = self.snapshot()
+        record = prepare(snapshot)
+        for change in (dict(snapshot, id=99), dict(snapshot, bypass_actors=[{"actor_id": 42}])):
+            with self.assertRaises(ValueError):
+                compare(record, change, "before")
+        record["after"]["rules"][-1]["parameters"]["grouping_strategy"] = "HEADGREEN"
+        with self.assertRaises(ValueError):
+            compare(record, snapshot, "before")
+
+    def test_disabled_ruleset_missing_reviews_or_duplicate_activation_rejected(self):
+        for update in ({"enforcement": "disabled"}, {"target": "tag"}, {"rules": self.snapshot()["rules"][:1]}):
+            with self.assertRaises(ValueError):
+                prepare(dict(self.snapshot(), **update))
+        record = prepare(self.snapshot())
+        with self.assertRaises(ValueError):
+            prepare(dict(record["after"], id=12))
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_ci_plan.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""Regression cases for trusted planning and fail-closed aggregation."""
+import copy
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import ci_gate as gate
+
+
+class Repository:
+    def __init__(self, root):
+        self.root = root
+        self.git('init', '-q')
+        self.write('Cargo.toml', '[workspace]\nmembers = ["rustfs", "core", "leaf", "consumer", "macro", "crates/e2e_test"]\n[workspace.dependencies]\nrenamed = { package="leaf", path="leaf" }\n')
+        for name, directory in [('rustfs', 'rustfs'), ('core', 'core'), ('leaf', 'leaf'), ('consumer', 'consumer'), ('macro', 'macro'), ('e2e_test', 'crates/e2e_test')]:
+            self.write(f'{directory}/Cargo.toml', f'[package]\nname = "{name}"\nversion = "0.1.0"\n')
+            self.write(f'{directory}/src/lib.rs', '// source\n')
+        self.write('consumer/Cargo.toml', '[package]\nname="consumer"\nversion="0.1.0"\n[target.\'cfg(unix)\'.dev-dependencies]\nrenamed = { workspace=true }\n[build-dependencies]\nmacro={path="../macro"}\n')
+        self.write('macro/Cargo.toml', '[package]\nname="macro"\nversion="0.1.0"\n[lib]\nproc-macro=true\n')
+        self.write('rustfs/Cargo.toml', '[package]\nname="rustfs"\nversion="0.1.0"\n[dependencies]\nconsumer={path="../consumer"}\n')
+        self.policy = {'schema_version': 1, 'rollout': 'shadow', 'core_packages': ['core'], 'connect_packages': ['core'],
+                       'full_paths': ['Cargo.toml', 'Cargo.lock', '*/Cargo.toml', '.config/*', '.github/*', 'scripts/*', 'rust-toolchain*']}
+        self.write(gate.POLICY_PATH, json.dumps(self.policy))
+        self.write('.config/ecstore-required-tests.json', json.dumps({'tests': [{'suite': 'core::integration', 'name': 'required'}]}))
+        self.base = self.commit()
+
+    def git(self, *args):
+        return subprocess.check_output(['git', '-c', 'core.hooksPath=/dev/null', '-c', 'user.name=CI Test', '-c', 'user.email=ci@example.invalid', *args], cwd=self.root, stderr=subprocess.PIPE).decode().strip()
+
+    def write(self, path, content):
+        target = self.root / path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(content)
+
+    def commit(self):
+        self.git('add', '-A')
+        self.git('commit', '-qm', 'fixture')
+        return self.git('rev-parse', 'HEAD')
+
+    def plan(self, head=None, event='pull_request', base=None, action='opened'):
+        return gate.select_plan(event, base or self.base, head or self.git('rev-parse', 'HEAD'), 'refs/heads/main', self.root, action)
+
+
+class PlannerTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.repo = Repository(Path(self.temp.name))
+
+    def test_dependency_graph_includes_renamed_target_dev_build_and_proc_macro_edges(self):
+        packages, graph = gate.cargo_graph(self.repo.root, self.repo.base)
+        self.assertEqual(graph['consumer'], {'leaf', 'macro'})
+        self.assertEqual(gate.closure({'leaf'}, graph, reverse=True), {'leaf', 'consumer', 'rustfs'})
+        self.assertEqual(gate.closure({'macro'}, graph, reverse=True), {'macro', 'consumer', 'rustfs'})
+        self.repo.write('leaf/src/lib.rs', '// changed\n')
+        plan = self.repo.plan(self.repo.commit())
+        self.assertEqual(plan['proposed']['packages'], ['consumer', 'core', 'leaf', 'rustfs'])
+        self.assertFalse(plan['proposed']['connect'])
+        self.assertTrue(plan['connect'])
+        self.assertEqual(plan['packages'], sorted(set(packages) - {'e2e_test'}))
+        self.assertEqual(plan['policy']['rollout'], 'shadow')
+
+    def test_deleted_or_moved_source_keeps_original_dependents(self):
+        (self.repo.root / 'leaf/src/lib.rs').rename(self.repo.root / 'core/src/moved.rs')
+        plan = self.repo.plan(self.repo.commit())
+        self.assertEqual(plan['proposed']['packages'], ['consumer', 'core', 'leaf', 'rustfs'])
+        self.assertTrue(plan['proposed']['connect'])
+        self.assertFalse(plan['proposed']['full_validation'])
+
+    def test_shared_unknown_lock_and_policy_changes_are_full(self):
+        for path in ('Cargo.lock', 'rust-toolchain.toml', 'shared/fixture.json', 'scripts/run.sh', '.config/ci-plan.json', 'crates/new/Cargo.toml'):
+            with self.subTest(path=path):
+                if path == gate.POLICY_PATH:
+                    modified = dict(self.repo.policy, rollout='enforced', core_packages=[])
+                    self.repo.write(path, json.dumps(modified))
+                else:
+                    self.repo.write(path, 'changed\n')
+                plan = self.repo.plan(self.repo.commit())
+                self.assertTrue(plan['proposed']['full_validation'])
+                self.assertTrue(plan['connect'])
+                self.assertEqual(plan['policy']['rollout'], 'shadow')
+
+    def test_core_inventory_is_unioned_and_cannot_be_removed_by_candidate(self):
+        self.repo.write('leaf/src/lib.rs', '// change\n')
+        with patch.object(gate, 'read_policy', return_value=dict(self.repo.policy, rollout='enforced', core_packages=[])):
+            plan = self.repo.plan(self.repo.commit())
+            self.assertIn('core', plan['packages'])
+        self.repo.write('.config/ecstore-required-tests.json', json.dumps({'tests': []}))
+        plan = self.repo.plan(self.repo.commit())
+        self.assertTrue(plan['full_validation'])
+        self.assertTrue(plan['connect'])
+
+    def test_docs_and_e2e_shortcuts_do_not_hide_production_or_fixture_inputs(self):
+        self.repo.write('README.md', '# docs\n')
+        docs = self.repo.plan(self.repo.commit())
+        self.assertEqual(docs['mode'], 'docs')
+        self.assertEqual(docs['packages'], [])
+        self.repo.write('crates/e2e_test/src/lib.rs', '// harness change\n')
+        e2e = self.repo.plan(self.repo.commit())
+        self.assertEqual(e2e['mode'], 'e2e')
+        self.assertFalse(e2e['connect'])
+        self.repo.write('leaf/tests/fixtures/README.md', 'fixture input\n')
+        code = self.repo.plan(self.repo.commit())
+        self.assertEqual(code['mode'], 'full')
+        self.assertIn('leaf', code['proposed']['packages'])
+
+    def test_shadow_retains_full_mode_for_harness_build_and_fixture_changes(self):
+        for path in ('crates/e2e_test/build.rs', 'crates/e2e_test/src/fixture.json'):
+            self.repo.write(path, 'changed\n')
+            plan = self.repo.plan(self.repo.commit())
+            self.assertEqual(plan['mode'], 'full')
+            self.assertTrue(plan['connect'])
+
+    def test_new_harness_dependency_disables_e2e_shortcut(self):
+        self.repo.write('consumer/Cargo.toml', '[package]\nname="consumer"\nversion="0.1.0"\n[dependencies]\nharness={path="../crates/e2e_test"}\n')
+        baseline = self.repo.commit()
+        self.repo.write('crates/e2e_test/src/lib.rs', '// change\n')
+        plan = self.repo.plan(self.repo.commit(), base=baseline)
+        self.assertEqual(plan['mode'], 'full')
+        self.assertIn('consumer', plan['proposed']['packages'])
+
+    def test_non_pr_events_validate_complete_combined_candidate_and_closed_cancels(self):
+        self.repo.write('leaf/src/lib.rs', '// PR one\n')
+        self.repo.write('macro/src/lib.rs', '// PR two\n')
+        head = self.repo.commit()
+        for event in ('push', 'merge_group', 'schedule', 'workflow_dispatch'):
+            plan = self.repo.plan(head, event=event)
+            self.assertTrue(plan['full_validation'])
+            self.assertEqual(plan['mode'], 'full')
+            self.assertEqual(plan['head'], head)
+            self.assertEqual(plan['tree'], self.repo.git('rev-parse', head + '^{tree}'))
+            self.assertEqual(gate.validate_plan(plan, event, 'refs/heads/main', head, plan['tree']), plan['jobs'])
+        closed = self.repo.plan(head, action='closed')
+        self.assertEqual(set(closed['jobs'].values()), {'skipped'})
+
+    def test_unavailable_diff_policy_or_base_falls_back_and_invalid_head_fails(self):
+        self.repo.write('leaf/src/lib.rs', '// changed\n')
+        head = self.repo.commit()
+        for base in ('0' * 40, '', '--output=bad'):
+            plan = gate.select_plan('pull_request', base, head, 'refs/heads/main', self.repo.root)
+            self.assertTrue(plan['full_validation'])
+            self.assertTrue(plan['connect'])
+            self.assertTrue(plan['packages'])
+        with patch.object(gate, 'read_policy', side_effect=ValueError('broken policy')):
+            self.assertTrue(self.repo.plan(head)['full_validation'])
+        with self.assertRaises(ValueError):
+            self.repo.plan('--malicious')
+
+    def test_actual_workflow_bootstrap_never_executes_pr_policy_and_upgrades_legacy_output(self):
+        workflow = gate.workflow_document((Path(gate.__file__).resolve().parent.parent / '.github/workflows/ci.yml').read_text())
+        body = next(step['run'] for step in workflow['jobs']['classify-changes']['steps'] if step.get('id') == 'scope')
+        self.repo.write('scripts/ci_gate.py', "import os\nwith open(os.environ['GITHUB_OUTPUT'], 'a') as output: output.write('mode=docs\\n')\n")
+        base = self.repo.commit()
+        self.repo.write('scripts/ci_gate.py', 'raise SystemExit(71)\n')
+        head = self.repo.commit()
+        for revision in (base, '0' * 40, ''):
+            output = self.repo.root / 'outputs'
+            output.write_text('')
+            env = dict(os.environ, GITHUB_EVENT_NAME='pull_request', GITHUB_REF='refs/heads/main', GITHUB_SHA=head,
+                       CI_BASE_SHA=revision, RUNNER_TEMP=str(self.repo.root), GITHUB_OUTPUT=str(output))
+            result = subprocess.run(['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', body],
+                                    cwd=self.repo.root, env=env, capture_output=True, text=True)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            values = dict(line.split('=', 1) for line in output.read_text().splitlines())
+            self.assertEqual(values['mode'], 'full')
+            self.assertEqual(values['connect'], 'true')
+        self.repo.write('scripts/ci_gate.py', 'raise SystemExit(29)\n')
+        broken = self.repo.commit()
+        env['CI_BASE_SHA'] = broken
+        result = subprocess.run(['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', body],
+                                cwd=self.repo.root, env=env, capture_output=True, text=True)
+        self.assertEqual(result.returncode, 29)
+
+    def test_bootstrap_script_is_standalone_and_reads_policy_from_base_objects(self):
+        self.repo.write('leaf/src/lib.rs', '// changed\n')
+        self.repo.write(gate.POLICY_PATH, json.dumps(dict(self.repo.policy, rollout='enforced', core_packages=[])))
+        head = self.repo.commit()
+        script = self.repo.root / 'trusted.py'
+        script.write_text(Path(gate.__file__).read_text())
+        output = self.repo.root / 'outputs'
+        env = dict(os.environ, GITHUB_EVENT_NAME='pull_request', GITHUB_REF='refs/heads/main', GITHUB_SHA=head,
+                   CI_BASE_SHA=self.repo.base, GITHUB_OUTPUT=str(output))
+        result = subprocess.run([sys.executable, '-I', str(script), 'select'], cwd=self.repo.root, env=env, capture_output=True, text=True)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        values = dict(line.split('=', 1) for line in output.read_text().splitlines())
+        self.assertEqual(set(values), {'mode', 'plan', 'connect', 'packages', 'full_validation'})
+        plan = json.loads(values['plan'])
+        self.assertEqual(plan['policy']['revision'], self.repo.base)
+        self.assertEqual(plan['policy']['rollout'], 'shadow')
+        self.assertEqual(values['connect'], 'true')
+
+
+class GateTests(unittest.TestCase):
+    def plan(self, event='pull_request', mode='full', connect=True):
+        jobs = gate.expected_results(mode, event, 'refs/heads/main', connect)
+        return {'schema_version': 1, 'event': event, 'ref': 'refs/heads/main', 'head': 'a' * 40, 'tree': 'b' * 40,
+                'base': 'c' * 40, 'action': '', 'mode': mode, 'connect': connect, 'full_validation': mode == 'full', 'packages': ['rustfs'] if mode != 'docs' else [],
+                'policy': {'revision': 'c' * 40, 'rollout': 'enforced'}, 'jobs': jobs, 'configurations': gate.configurations(jobs),
+                'reasons': ['fixture'], 'not_applicable': {job: 'fixture' for job, state in jobs.items() if state == 'skipped'}}
+
+    def needs(self, plan):
+        needs = {job: {'result': state} for job, state in plan['jobs'].items()}
+        needs['classify-changes']['outputs'] = {'mode': plan['mode'], 'plan': json.dumps(plan)}
+        return needs
+
+    def test_failure_cancel_missing_extra_or_unexpected_skip_is_rejected(self):
+        for event, mode in [('pull_request', 'docs'), ('pull_request', 'e2e'), ('pull_request', 'full'), ('push', 'full'), ('merge_group', 'full'), ('schedule', 'full'), ('workflow_dispatch', 'full')]:
+            plan = self.plan(event, mode)
+            good = self.needs(plan)
+            self.assertEqual(gate.verify_results(good, event, plan['ref'], plan['head'], plan['tree']), [])
+            for job, state in plan['jobs'].items():
+                for bad in (None, 'failure', 'cancelled', 'skipped', 'success'):
+                    if bad == state:
+                        continue
+                    needs = copy.deepcopy(good)
+                    needs[job]['result'] = bad
+                    self.assertTrue(gate.verify_results(needs, event, plan['ref']), (event, mode, job, bad))
+                needs = copy.deepcopy(good)
+                del needs[job]
+                self.assertTrue(gate.verify_results(needs, event, plan['ref']))
+            good['new-unwired-job'] = {'result': 'success'}
+            self.assertTrue(gate.verify_results(good, event, plan['ref']))
+
+    def test_invalid_plan_matrix_and_candidate_identity_fail_closed(self):
+        plan = self.plan('merge_group')
+        for field, value in [('schema_version', 2), ('event', 'push'), ('ref', 'wrong'), ('base', '../main'), ('packages', []), ('connect', 'false'), ('configurations', []), ('jobs', {}), ('not_applicable', {}), ('policy', {})]:
+            bad = dict(plan, **{field: value})
+            needs = self.needs(plan)
+            needs['classify-changes']['outputs']['plan'] = json.dumps(bad)
+            self.assertTrue(gate.verify_results(needs, 'merge_group', plan['ref']), field)
+        good = self.needs(plan)
+        self.assertTrue(gate.verify_results(good, plan['event'], plan['ref'], 'c' * 40, plan['tree']))
+        self.assertTrue(gate.verify_results(good, plan['event'], plan['ref'], plan['head'], 'c' * 40))
+        self.assertTrue(gate.verify_results(good, plan['event'], plan['ref'], base='d' * 40))
+        self.assertTrue(gate.verify_results(good, plan['event'], plan['ref'], action='closed'))
+
+    def test_legacy_base_cannot_authorize_skipped_new_jobs(self):
+        expected = gate.expected_results('full', 'pull_request', 'refs/heads/main')
+        needs = {job: {'result': result} for job, result in expected.items()}
+        needs['classify-changes']['outputs'] = {'mode': 'full'}
+        self.assertEqual(gate.verify_results(needs, 'pull_request', 'refs/heads/main'), [])
+        needs['classify-changes']['outputs']['mode'] = 'docs'
+        self.assertTrue(gate.verify_results(needs, 'pull_request', 'refs/heads/main'))
+        needs['classify-changes']['outputs'] = {'mode': 'full', 'plan': '{'}
+        self.assertTrue(gate.verify_results(needs, 'pull_request', 'refs/heads/main'))
+
+    def workflow(self):
+        jobs = {}
+        for job in gate.ALWAYS_JOBS + gate.CODE_JOBS + gate.OPTIONAL_JOBS:
+            # A complete plan lookup is represented by equivalent explicit expressions here.
+            if job in gate.ALWAYS_JOBS:
+                expression = "true"
+            elif job in gate.EXTENDED_FEATURE_JOBS:
+                expression = "needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('[\"merge_group\",\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
+            elif job in gate.OPTIONAL_JOBS[:2]:
+                expression = "contains(fromJSON('[\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
+            elif job == 'e2e-full':
+                expression = "github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')"
+            elif job == 'connect-feature-boundaries':
+                expression = "needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect == 'true'"
+            elif job in gate.E2E_JOBS:
+                expression = "needs.classify-changes.outputs.mode != 'docs'"
+            else:
+                expression = "needs.classify-changes.outputs.mode == 'full'"
+            expression = f"({expression}) && (github.event_name != 'pull_request' || github.event.action != 'closed')"
+            jobs[job] = {'name': job, 'if': expression, 'steps': []}
+        jobs['test-and-lint']['steps'] = [{'run': 'cargo clippy --all-targets -- -D warnings'}]
+        jobs['test-and-lint-protocols']['strategy'] = {'matrix': {'features': [
+            {'name': 'swift', 'flags': '--features swift'}, {'name': 'sftp', 'flags': '--features sftp'}, {'name': 'http3', 'flags': '--features http3,swift'}]}}
+        jobs['required-checks'] = {'name': 'Test and Lint', 'if': "always() && (github.event_name != 'pull_request' || github.event.action != 'closed')",
+                                   'needs': list(gate.ALWAYS_JOBS + gate.CODE_JOBS + gate.OPTIONAL_JOBS),
+                                   'steps': [{'shell': 'bash', 'env': {'CI_NEEDS': '${{ toJSON(needs) }}'}, 'run': 'python3 scripts/ci_gate.py verify'}]}
+        return {'on': {'pull_request': {}}, 'jobs': jobs}
+
+    def check(self, workflow):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            path = root / '.github/workflows/ci.yml'
+            path.parent.mkdir(parents=True)
+            path.write_text(json.dumps(workflow))
+            return gate.check_workflow(root)
+
+    def test_ignored_ilm_archive_execution_preserves_failure_and_core_report(self):
+        workflow = gate.workflow_document((Path(gate.__file__).resolve().parent.parent / '.github/workflows/ci.yml').read_text())
+        steps = workflow['jobs']['workspace-tests']['steps']
+        bodies = [step['run'] for step in steps if '--run-ignored ignored-only' in step.get('run', '')]
+        self.assertEqual(len(bodies), 1)
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            commands = root / 'bin'
+            commands.mkdir()
+            timeout = commands / 'timeout'
+            timeout.write_text('#!/bin/sh\nshift 4\nexec "$@"\n')
+            timeout.chmod(0o755)
+            cargo = commands / 'cargo'
+            cargo.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$ILM_ARGS"\nexit "$ILM_STATUS"\n')
+            cargo.chmod(0o755)
+            report = root / 'target/nextest/ci/junit.xml'
+            report.parent.mkdir(parents=True)
+            report.write_text('existing core report')
+            args = root / 'args'
+            for status in (0, 23, 124):
+                env = dict(os.environ, PATH=f"{commands}{os.pathsep}{os.environ['PATH']}", ILM_ARGS=str(args),
+                           ILM_STATUS=str(status), RUNNER_TEMP=str(root), GITHUB_WORKSPACE=str(root))
+                result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', bodies[0]], cwd=root, env=env, capture_output=True, text=True)
+                self.assertEqual(result.returncode, status, result.stderr)
+                self.assertEqual(report.read_text(), 'existing core report')
+                arguments = args.read_text().splitlines()
+                self.assertEqual(arguments[:7], ['nextest', 'run', '--profile', 'default', '-j1', '--run-ignored', 'ignored-only'])
+                self.assertIn('--archive-file', arguments)
+                self.assertIn(f'exit_status={status}', (root / 'artifacts/ilm-integration/diagnostics.txt').read_text())
+
+    def test_semantic_yaml_equivalence_and_wiring_boundaries(self):
+        workflow = self.workflow()
+        self.assertEqual(self.check(workflow), [])
+        workflow['jobs']['required-checks']['if'] = "${{ (github.event.action != 'closed' || github.event_name != 'pull_request') && always() }}"
+        workflow['jobs']['required-checks']['needs'].reverse()
+        for job in ('workspace-tests', 'core-contract', 'e2e-tests', 's3-implemented-tests', 's3-lifecycle-behavior-tests'):
+            del workflow['jobs'][job]['if']
+            workflow['jobs'][job]['needs'] = ['test-and-lint' if job == 'workspace-tests' else 'build-rustfs-debug-binary']
+        self.assertEqual(self.check(workflow), [])
+        for mutate in (
+            lambda w: w['jobs']['required-checks']['needs'].pop(),
+            lambda w: w['jobs']['required-checks'].update({'if': 'success()'}),
+            lambda w: w['jobs']['required-checks'].update({'if': "'always()' == 'always()'"}),
+            lambda w: w['jobs']['required-checks']['steps'][0].update({'run': 'python3 scripts/ci_gate.py verify || true'}),
+            lambda w: w['jobs']['typos'].update({'continue-on-error': 'true'}),
+            lambda w: w['jobs']['test-and-lint']['steps'][0].update({'if': 'false'}),
+            lambda w: w['jobs']['test-and-lint-protocols']['strategy']['matrix']['features'].pop(),
+            lambda w: w['jobs'].update({'unaccounted': {'name': 'New'}}),
+            lambda w: w['on']['pull_request'].update({'paths': ['README.md']}),
+            lambda w: w['jobs']['core-contract'].update({'if': 'false'}),
+        ):
+            changed = copy.deepcopy(workflow)
+            mutate(changed)
+            self.assertTrue(self.check(changed))
+        with self.assertRaises(ValueError):
+            gate.workflow_document('jobs: { test: {}, test: {} }')
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/test_ci_test_artifact.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+"""Archive consumers require exact provenance and complete selected executions."""
+
+import copy
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import tempfile
+import subprocess
+import sys
+import unittest
+from unittest.mock import patch
+import xml.etree.ElementTree as ET
+
+import ci_test_artifact as artifact
+
+
+def listing(cases):
+    suites = {}
+    for binary, name, ignored in cases:
+        suite = suites.setdefault(binary, {"status": "listed", "testcases": {}})
+        suite["testcases"][name] = {"ignored": ignored,
+                                   "filter-match": {"status": "mismatch" if ignored else "matches"}}
+    return {"rust-suites": suites}
+
+
+def junit(cases):
+    root = ET.Element("testsuites", tests=str(len(cases)), failures="0", errors="0")
+    suites = {}
+    for binary, name, outcome in cases:
+        if binary not in suites:
+            suites[binary] = ET.SubElement(root, "testsuite", name=binary, failures="0", errors="0")
+        case = ET.SubElement(suites[binary], "testcase", name=name, classname=binary)
+        if outcome:
+            ET.SubElement(case, outcome)
+    return root
+
+
+class ArtifactIdentityTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.archive = self.root / "tests.tar.zst"
+        self.archive.write_bytes(b"controlled archive bytes")
+        self.identity = {"source": {"head": "a" * 40, "sha256": "b" * 64}, "tree": "c" * 40,
+                         "workspace_root": "/workspace/rustfs", "lock_sha256": "d" * 64,
+                         "rustc": "rustc 1.92.0", "nextest": "cargo-nextest 0.9.138",
+                         "configuration": "workspace", "profile": "test", "features": [],
+                         "build_environment": {key: "" for key in artifact.BUILD_ENVIRONMENT},
+                         "system": "Linux", "machine": "x86_64", "repository": "owner/repo",
+                         "run_id": "123", "run_attempt": "2"}
+        self.digest = hashlib.sha256(self.archive.read_bytes()).hexdigest()
+        self.size = self.archive.stat().st_size
+        self.record = {"schema": 1, "identity": copy.deepcopy(self.identity),
+                       "package_args": ["--workspace", "--exclude", "e2e_test"],
+                       "sha256": self.digest, "size_bytes": self.size, "libc": ["glibc", "2.39"]}
+
+    def verify(self, record=None, expected=None, digest=None, size=None, libc=("glibc", "2.39")):
+        artifact.verify_record(record or self.record, expected or self.identity,
+                               digest or self.digest, self.size if size is None else size, libc)
+
+    def test_exact_record_accepts_same_or_newer_glibc(self):
+        self.verify()
+        self.verify(libc=("glibc", "2.40"))
+
+    def test_every_provenance_dimension_rejects_mismatch(self):
+        mutations = {"source": {"head": "e" * 40, "sha256": "f" * 64}, "tree": "e" * 40,
+                     "workspace_root": "/different/checkout", "lock_sha256": "e" * 64,
+                     "rustc": "rustc 1.93.0", "nextest": "cargo-nextest 0.9.140",
+                     "configuration": "e2e", "profile": "release", "features": ["rio-v2"],
+                     "build_environment": {**self.identity["build_environment"], "RUSTFLAGS": "-C opt-level=3"},
+                     "system": "Darwin", "machine": "aarch64", "repository": "other/repo",
+                     "run_id": "124", "run_attempt": "3"}
+        for key, value in mutations.items():
+            with self.subTest(key=key), self.assertRaisesRegex(ValueError, "identity mismatch"):
+                self.verify(expected={**self.identity, key: value})
+
+    def test_modified_truncated_and_empty_archives_are_rejected(self):
+        for key, value in (("schema", 2), ("sha256", "f" * 64), ("size_bytes", self.size + 1)):
+            with self.subTest(key=key), self.assertRaises(ValueError):
+                self.verify(record={**self.record, key: value})
+        with self.assertRaises(ValueError):
+            self.verify(size=0)
+        with self.assertRaises(ValueError):
+            self.verify(digest="0" * 64)
+
+    def test_abi_refuses_older_or_different_or_unparseable_libc(self):
+        for libc in (("glibc", "2.38"), ("musl", "1.2.5"), ("glibc", "unknown")):
+            with self.subTest(libc=libc), self.assertRaises(ValueError):
+                self.verify(libc=libc)
+        for producer in ([], ["glibc"], ["glibc", "not-a-version"]):
+            with self.subTest(producer=producer), self.assertRaises(ValueError):
+                self.verify(record={**self.record, "libc": producer})
+        musl = {**self.record, "libc": ["musl", "1.2.5"]}
+        self.verify(record=musl, libc=("musl", "1.2.5"))
+        with self.assertRaises(ValueError):
+            self.verify(record=musl, libc=("musl", "1.2.6"))
+
+    def test_verify_binds_actual_bytes_and_package_plan(self):
+        artifact.manifest_path(self.archive).write_text(json.dumps(self.record))
+        with patch.object(artifact, "identity", return_value=self.identity), \
+                patch.object(artifact.platform, "libc_ver", return_value=("glibc", "2.39")), \
+                patch.dict(os.environ, {}, clear=True):
+            self.assertEqual(artifact.verify(self.archive, "workspace"), self.record)
+            with patch.dict(os.environ, {"CI_PACKAGES": '["rustfs"]'}), self.assertRaisesRegex(ValueError, "package selection"):
+                artifact.verify(self.archive, "workspace")
+            self.archive.write_bytes(b"corrupt")
+            with self.assertRaisesRegex(ValueError, "corrupt"):
+                artifact.verify(self.archive, "workspace")
+
+    def test_package_selection_rejects_invalid_and_keeps_e2e_separate(self):
+        with patch.dict(os.environ, {"CI_PACKAGES": '["rustfs-utils", "rustfs", "rustfs"]'}):
+            self.assertEqual(artifact.package_args("workspace"), ["-p", "rustfs", "-p", "rustfs-utils"])
+            self.assertEqual(artifact.package_args("e2e"), ["-p", "e2e_test"])
+        for value in ('[]', '{}', '[null]', '["--all-features"]', '["e2e_test"]', '[""]'):
+            with self.subTest(value=value), patch.dict(os.environ, {"CI_PACKAGES": value}), self.assertRaises(ValueError):
+                artifact.package_args("workspace")
+
+    def test_source_drift_during_archive_build_cannot_write_receipt(self):
+        after = {**self.identity, "source": {"head": "e" * 40, "sha256": "f" * 64}}
+        with patch.object(artifact, "identity", side_effect=[self.identity, after]), \
+                patch.object(artifact.subprocess, "run"), patch.dict(os.environ, {}, clear=True), \
+                self.assertRaisesRegex(ValueError, "changed while compiling"):
+            artifact.build(self.archive, "workspace")
+        self.assertFalse(artifact.manifest_path(self.archive).exists())
+
+    def test_codegen_overrides_are_bound_but_build_concurrency_is_not(self):
+        overrides = {"CARGO_PROFILE_TEST_OVERFLOW_CHECKS": "false", "CARGO_PROFILE_TEST_DEBUG_ASSERTIONS": "false",
+                     "CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS": "-C opt-level=3",
+                     "CARGO_BUILD_RUSTFLAGS": "-C debuginfo=0", "RUSTFS_BUILD_VERSION": "fixture",
+                     "CFLAGS": "-O3", "CARGO_BUILD_JOBS": "2"}
+        with patch.dict(os.environ, overrides, clear=True):
+            bound = artifact.build_environment()
+            for key, value in overrides.items():
+                if key != "CARGO_BUILD_JOBS":
+                    self.assertEqual(bound[key], value)
+            with patch.dict(os.environ, {"CARGO_BUILD_JOBS": "4"}):
+                self.assertEqual(artifact.build_environment(), bound)
+            with patch.dict(os.environ, {"CARGO_PROFILE_TEST_OVERFLOW_CHECKS": "true"}):
+                self.assertNotEqual(artifact.build_environment(), bound)
+
+    def test_native_target_and_original_target_directory_are_required(self):
+        for overrides in ({"CARGO_BUILD_TARGET": "aarch64-unknown-linux-gnu"},
+                          {"CARGO_TARGET_DIR": str(self.root / "different-target")}):
+            with self.subTest(overrides=overrides), patch.dict(os.environ, overrides, clear=True), self.assertRaises(ValueError):
+                artifact.identity("workspace")
+
+    def test_lint_command_keeps_all_targets_deny_warnings_and_failure(self):
+        arguments = ["ci_test_artifact.py", "cargo", "lint", "--configuration", "workspace"]
+        expected = ["cargo", "clippy", "--locked", "-p", "rustfs", "-p", "e2e_test", "--all-targets", "--", "-D", "warnings"]
+        with patch.object(sys, "argv", arguments), patch.dict(os.environ, {"CI_PACKAGES": '["rustfs"]'}):
+            with patch.object(artifact.subprocess, "run") as run:
+                self.assertEqual(artifact.main(), 0)
+                self.assertEqual(run.call_args.args[0], expected)
+                self.assertTrue(run.call_args.kwargs["check"])
+            with patch.object(artifact.subprocess, "run", side_effect=subprocess.CalledProcessError(1, expected)), \
+                    patch.object(sys, "stderr", io.StringIO()):
+                self.assertEqual(artifact.main(), 1)
+        for environment in ({}, {"CI_PACKAGES": ""}):
+            with self.subTest(environment=environment), patch.object(sys, "argv", arguments), \
+                    patch.dict(os.environ, environment, clear=True), patch.object(artifact.subprocess, "run") as run:
+                self.assertEqual(artifact.main(), 0)
+                self.assertEqual(run.call_args.args[0], ["cargo", "clippy", "--locked", "--workspace", "--all-targets", "--", "-D", "warnings"])
+
+    def test_doctests_are_an_independent_command(self):
+        with patch.object(sys, "argv", ["ci_test_artifact.py", "cargo", "doc", "--configuration", "workspace"]), \
+                patch.dict(os.environ, {}, clear=True), patch.object(artifact.subprocess, "run") as run:
+            self.assertEqual(artifact.main(), 0)
+        self.assertEqual(run.call_args.args[0], ["cargo", "test", "--locked", "--workspace", "--exclude", "e2e_test", "--doc"])
+
+
+class ExecutionEvidenceTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.listing_file = self.root / "listing.json"
+        self.junit_file = self.root / "junit.xml"
+        self.listing = listing([("crate-a", "one", False), ("crate-b", "two", False), ("crate-b", "ignored", True)])
+        self.report = junit([("crate-a", "one", None), ("crate-b", "two", None)])
+
+    def verify(self, listing_value=None, report=None, *, core=False):
+        self.listing_file.write_text(json.dumps(self.listing if listing_value is None else listing_value))
+        ET.ElementTree(self.report if report is None else report).write(self.junit_file)
+        return artifact.verify_execution(self.listing_file, self.junit_file, core=core)
+
+    def test_exact_selected_set_and_known_ignored_cases(self):
+        self.assertEqual(self.verify(), 2)
+        self.assertEqual(self.verify(report=junit([("crate-a", "one", None), ("crate-b", "two", None),
+                                                   ("crate-b", "ignored", "skipped")])), 2)
+
+    def test_missing_extra_or_misnamed_execution_is_rejected(self):
+        cases = [("crate-a", "one", None), ("crate-b", "two", None)]
+        for altered in (cases[:1], cases + [("crate-a", "extra", None)],
+                        [("wrong-binary", "one", None), cases[1]]):
+            with self.subTest(cases=altered), self.assertRaises(ValueError):
+                self.verify(report=junit(altered))
+
+    def test_selected_skip_failure_error_or_duplicate_is_rejected(self):
+        for outcome in ("skipped", "failure", "error"):
+            with self.subTest(outcome=outcome), self.assertRaises(ValueError):
+                self.verify(report=junit([("crate-a", "one", outcome), ("crate-b", "two", None)]))
+        with self.assertRaisesRegex(ValueError, "duplicate"):
+            self.verify(report=junit([("crate-a", "one", None), ("crate-a", "one", None), ("crate-b", "two", None)]))
+        self.report.find("testsuite/testcase").set("classname", "different")
+        with self.assertRaisesRegex(ValueError, "misplaced"):
+            self.verify()
+
+    def test_extra_skipped_case_must_exist_in_listing(self):
+        with self.assertRaises(ValueError):
+            self.verify(report=junit([("crate-a", "one", None), ("crate-b", "two", None),
+                                      ("unknown-binary", "unknown-case", "skipped")]))
+
+    def test_setup_scripts_must_pass_without_counting_as_selected_tests(self):
+        cases = [("crate-a", "one", None), ("crate-b", "two", None)]
+        script = ("@setup-script:prepare", "prepare", None)
+        self.assertEqual(self.verify(report=junit([*cases, script])), 2)
+        for outcome in ("skipped", "failure", "error"):
+            with self.subTest(outcome=outcome), self.assertRaises(ValueError):
+                self.verify(report=junit([*cases, (*script[:2], outcome)]))
+        with self.assertRaisesRegex(ValueError, "missing"):
+            self.verify(report=junit([cases[0], script]))
+
+    def test_malformed_or_duplicate_setup_script_evidence_is_rejected(self):
+        cases = [("crate-a", "one", None), ("crate-b", "two", None)]
+        script = ("@setup-script:prepare", "prepare", None)
+        for extra in ([script, script], [("@setup-script:", "", None)],
+                      [(script[0], "wrong-name", None)]):
+            with self.subTest(extra=extra), self.assertRaises(ValueError):
+                self.verify(report=junit([*cases, *extra]))
+        report = junit([*cases, script])
+        report.append(copy.deepcopy(report[-1]))
+        with self.assertRaises(ValueError):
+            self.verify(report=report)
+        report = junit(cases)
+        ET.SubElement(report, "testsuite", name=script[0], failures="0", errors="0")
+        with self.assertRaises(ValueError):
+            self.verify(report=report)
+        report = junit([*cases, script])
+        report[-1].find("testcase").set("classname", "wrong-suite")
+        with self.assertRaisesRegex(ValueError, "misplaced"):
+            self.verify(report=report)
+
+    def test_empty_unlisted_or_failed_top_level_report_is_rejected(self):
+        with self.assertRaises(ValueError):
+            self.verify(listing_value={"rust-suites": {}})
+        with self.assertRaises(ValueError):
+            self.verify(listing_value=listing([("crate-a", "ignored", True)]))
+        self.listing["rust-suites"]["crate-a"]["status"] = "not-listed"
+        with self.assertRaises(ValueError):
+            self.verify()
+        self.listing["rust-suites"]["crate-a"]["status"] = "listed"
+        for attribute in ("failures", "errors"):
+            report = copy.deepcopy(self.report)
+            report.set(attribute, "1")
+            with self.subTest(attribute=attribute), self.assertRaises(ValueError):
+                self.verify(report=report)
+
+    def test_core_requires_exactly_three_selected_proofs(self):
+        names = sorted(artifact.CORE_TESTS)
+        core_listing = listing([("e2e", name, False) for name in names])
+        core_report = junit([("e2e", name, None) for name in names])
+        self.assertEqual(self.verify(core_listing, core_report, core=True), 3)
+        for cases in ([name for name in names if name != names[0]], names + ["unrelated"]):
+            with self.subTest(cases=cases), self.assertRaises(ValueError):
+                self.verify(listing([("e2e", name, False) for name in cases]),
+                            junit([("e2e", name, None) for name in cases]), core=True)
+        # Reusing one proof's name in another binary must not expand the suite.
+        duplicate_name = listing([("e2e", name, False) for name in names] + [("other-binary", names[0], False)])
+        duplicate_report = junit([("e2e", name, None) for name in names] + [("other-binary", names[0], None)])
+        with self.assertRaises(ValueError):
+            self.verify(duplicate_name, duplicate_report, core=True)
+
+
+class NextestSetupScriptTests(unittest.TestCase):
+    def test_real_nextest_setup_script_report(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "src").mkdir()
+            (root / ".config").mkdir()
+            (root / "Cargo.toml").write_text(
+                '[package]\nname = "evidence-fixture"\nversion = "0.1.0"\nedition = "2021"\n[workspace]\n'
+            )
+            (root / "rust-toolchain.toml").write_text((artifact.ROOT / "rust-toolchain.toml").read_text())
+            (root / "src/lib.rs").write_text(
+                '#[test]\nfn setup_is_applied() {\n'
+                '    assert_eq!(std::env::var("EVIDENCE_SETUP").unwrap(), "ready");\n}\n'
+            )
+            config = root / ".config/nextest.toml"
+            config.write_text(
+                'experimental = ["setup-scripts"]\n'
+                '[scripts.setup.prepare]\ncommand = ["sh", "prepare.sh"]\n'
+                '[[profile.default.scripts]]\nfilter = "all()"\nsetup = "prepare"\n'
+                '[profile.ci.junit]\npath = "junit.xml"\n'
+            )
+            script = root / "prepare.sh"
+            script.write_text('echo EVIDENCE_SETUP=ready >> "$NEXTEST_ENV"\n')
+            environment = {**os.environ, "CARGO_TARGET_DIR": str(root / "target"), "CARGO_NET_OFFLINE": "true"}
+
+            def nextest(*arguments):
+                return subprocess.run(["cargo", "nextest", *arguments], cwd=root, env=environment,
+                                      capture_output=True, text=True, timeout=120)
+
+            listed = nextest("list", "--profile", "ci", "--message-format", "json")
+            self.assertEqual(listed.returncode, 0, listed.stderr)
+            listing_file = root / "listing.json"
+            listing_file.write_text(listed.stdout)
+            report = root / "target/nextest/ci/junit.xml"
+            executed = nextest("run", "--profile", "ci", "--no-tests", "fail")
+            self.assertEqual(executed.returncode, 0, executed.stderr)
+            suites = ET.parse(report).getroot().findall("testsuite")
+            self.assertIn("@setup-script:prepare", [suite.get("name") for suite in suites])
+            verified = subprocess.run(
+                [sys.executable, str(artifact.ROOT / "scripts/ci_test_artifact.py"), "evidence",
+                 str(listing_file), str(report)], capture_output=True, text=True, timeout=30,
+            )
+            self.assertEqual(verified.returncode, 0, verified.stderr)
+            self.assertIn("Verified 1 test executions", verified.stdout)
+
+            report.unlink()
+            script.write_text('exit 1\n')
+            failed = nextest("run", "--profile", "ci", "--no-tests", "fail")
+            self.assertNotEqual(failed.returncode, 0)
+            with self.assertRaisesRegex(ValueError, "successful nextest run"):
+                artifact.verify_execution(listing_file, report)
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/test_ci_timing_report.py

@@ -45,6 +45,20 @@ class TimingTests(unittest.TestCase):
         self.job["steps"] = []
         self.assertEqual(summarize([self.run])["successful_code_runs"], 0)
 
+    def test_archive_pipeline_requires_workspace_and_core_consumers(self):
+        self.run["jobs"] = [{**self.job, "id": 5, "name": "Workspace Lint and Build"},
+                            {**self.job, "name": "Workspace tests (0)"},
+                            {**self.job, "id": 3, "name": "Workspace tests (1)"},
+                            {**self.job, "id": 4, "name": "Core contract"}]
+        self.assertEqual(summarize([self.run])["successful_code_runs"], 1)
+        self.run["jobs"][0]["conclusion"] = "skipped"
+        self.assertEqual(summarize([self.run])["successful_code_runs"], 0)
+        self.run["jobs"][0]["conclusion"] = "success"
+        self.run["jobs"][-1]["conclusion"] = "skipped"
+        self.assertEqual(summarize([self.run])["successful_code_runs"], 0)
+        self.run["jobs"].pop()
+        self.assertEqual(summarize([self.run])["successful_code_runs"], 0)
+
     def test_collection_rejects_out_of_range_api_results(self):
         with mock.patch("ci_timing_report.api", return_value={"workflow_runs": [self.run]}):
             with self.assertRaisesRegex(ValueError, "outside the requested date range"):

scripts/test_connect_release_boundary.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+import copy
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import check_connect_release_boundary as guard
+
+
+class ReleaseBoundaryTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp.name)
+        source = self.root / guard.SOURCE
+        source.parent.mkdir(parents=True)
+        source.write_text('// header\ncompile_error!("' + guard.MESSAGE + '");\n')
+        self.diagnostic = {"reason": "compiler-message", "target": {"name": "rustfs", "kind": ["lib"]},
+                           "message": {"level": "error", "message": guard.MESSAGE, "code": None,
+                                       "spans": [{"file_name": guard.SOURCE, "is_primary": True, "line_start": 2, "line_end": 2}]}}
+        self.addCleanup(self.temp.cleanup)
+
+    def output(self, *diagnostics):
+        return "\n".join(json.dumps(value) for value in [*diagnostics, {"reason": "build-finished", "success": False}])
+
+    def test_rejection_is_bound_to_diagnostic_kind_and_source_guard(self):
+        self.assertEqual(guard.validate_rejection(101, self.output(self.diagnostic), self.root), [])
+        for status in (0, 1, 137):
+            self.assertTrue(guard.validate_rejection(status, self.output(self.diagnostic), self.root))
+        for field, value in (("message", "unrelated compiler error"), ("code", {"code": "E0599"}), ("level", "warning")):
+            diagnostic = copy.deepcopy(self.diagnostic)
+            diagnostic["message"][field] = value
+            self.assertTrue(guard.validate_rejection(101, self.output(diagnostic), self.root))
+        for field, value in (("file_name", "rustfs/src/main.rs"), ("is_primary", False), ("line_start", 1), ("line_start", 0), ("line_end", 100)):
+            diagnostic = copy.deepcopy(self.diagnostic)
+            diagnostic["message"]["spans"][0][field] = value
+            self.assertTrue(guard.validate_rejection(101, self.output(diagnostic), self.root), (field, value))
+
+    def test_other_errors_or_incomplete_cargo_output_cannot_prove_boundary(self):
+        self.assertTrue(guard.validate_rejection(101, self.output(), self.root))
+        self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic, self.diagnostic), self.root))
+        self.assertTrue(guard.validate_rejection(101, json.dumps(self.diagnostic), self.root))
+        self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic) + '\nerror: network failure', self.root))
+        (self.root / guard.SOURCE).write_text('fn removed_guard() {}\n')
+        self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic), self.root))
+
+    def test_normal_configuration_failure_stops_negative_build(self):
+        with patch.object(guard.subprocess, 'run') as run:
+            run.return_value.returncode = 101
+            self.assertEqual(guard.check(self.root), 1)
+            self.assertEqual(run.call_count, 1)
+
+    def test_real_cargo_release_boundary(self):
+        (self.root / 'Cargo.toml').write_text(
+            '[package]\nname = "rustfs"\nversion = "0.1.0"\nedition = "2021"\n'
+            '[lib]\npath = "' + guard.SOURCE + '"\n'
+            '[features]\nconnect-e2e-short-credentials = []\n[workspace]\n'
+        )
+        (self.root / 'Cargo.lock').write_text(
+            'version = 3\n\n[[package]]\nname = "rustfs"\nversion = "0.1.0"\n'
+        )
+        # The fixture is outside the checkout, so retain its selected toolchain
+        # while keeping compilation offline and out of the workspace target.
+        toolchain = Path(__file__).resolve().parents[1] / 'rust-toolchain.toml'
+        (self.root / toolchain.name).write_text(toolchain.read_text())
+        condition = '#[cfg(all(feature = "connect-e2e-short-credentials", not(debug_assertions)))]\n'
+        cases = (
+            ('expected rejection', condition + 'compile_error!("' + guard.MESSAGE + '");\n', 0),
+            ('missing rejection', '', 1),
+            ('unrelated rejection', condition + 'compile_error!("unrelated compiler error");\n', 1),
+        )
+        with patch.dict(os.environ, {'CARGO_TARGET_DIR': str(self.root / 'target'), 'CARGO_NET_OFFLINE': 'true'}):
+            for name, source, expected in cases:
+                with self.subTest(name=name):
+                    (self.root / guard.SOURCE).write_text(source)
+                    self.assertEqual(guard.check(self.root), expected)
+
+
+if __name__ == '__main__':
+    unittest.main()

scripts/test_functional_chain.py

@@ -817,12 +817,13 @@ class EnvelopeTests(unittest.TestCase):
         previous = None
         for suite in evidence.SUITES:
             job = "\n".join(yaml_block(lines, suite, 2))
-            self.assertIn("needs: [prepare" + (", " + previous if previous else "") + "]", job)
+            self.assertIn("needs: [prepare" + (", " + previous if previous and suite != "performance" else "") + "]", job)
+            selected = " && contains(fromJSON(needs.prepare.outputs.suites || '[]'), '" + suite + "')"
             if suite == "performance":
                 # gated on the preflight probe: runs only when its fleet is online
-                self.assertIn("if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}", job)
+                self.assertIn("if: ${{ always() && needs.prepare.result == 'success'" + selected + " && needs.prepare.outputs.performance_ready == 'online' }}", job)
             else:
-                self.assertIn("if: ${{ always() && needs.prepare.result == 'success' }}", job)
+                self.assertIn("if: ${{ always() && needs.prepare.result == 'success'" + selected + " }}", job)
             self.assertIn("chain_manifest: ${{ needs.prepare.outputs.manifest }}", job)
             previous = suite
         complete = "\n".join(yaml_block(lines, "complete-chain", 2))
@@ -952,7 +953,8 @@ class WorkflowTimeoutTests(unittest.TestCase):
         source = (candidate.ROOT / ".github/workflows/rustfs-performance-test.yml").read_text()
         job = "\n".join(yaml_block(source.splitlines(), "performance-test", 2))
         self.assertIn("    timeout-minutes: 900", job)
-        self.assertNotIn("        timeout-minutes:", job)
+        benchmark = job.split("id: benchmark", 1)[1].split("      - name:", 1)[0]
+        self.assertNotIn("        timeout-minutes:", benchmark)
         self.assertIn("RUSTFS_WARP_DURATION: ${{ inputs.warp_duration || '5m' }}", job)
         self.assertNotIn("RUSTFS_WARP_SLEEP:", job)
         self.assertIn("RUSTFS_WARP_METHODS: ${{ inputs.test_method }}", job)

scripts/test_functional_chain_health.py

@@ -16,7 +16,7 @@ class HealthTests(unittest.TestCase):
         now = datetime.now(timezone.utc)
         self.now = now.isoformat()
         self.run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40, "html_url": "https://github.com/rustfs/rustfs/actions/runs/123",
-                    "run_started_at": (now - timedelta(hours=1)).isoformat()}
+                    "run_started_at": (now - timedelta(hours=1)).isoformat(), "event": "workflow_run"}
         self.candidate = {"manifest": {"build_run_id": 456, "build_run_attempt": 3, "source_ref": "release", "source_sha": "b" * 40},
                           "workflow_sha": "c" * 40, "workflow_ref": "main", "build_started_at": (now - timedelta(hours=2)).isoformat()}
         self.chain = {"run_id": 123, "attempt": 2, "workflow_sha": "a" * 40, "testing_sha": "d" * 40, "candidate": self.candidate}
@@ -85,8 +85,8 @@ class HealthTests(unittest.TestCase):
 
     def test_collection_rejects_a_concurrent_rerun(self):
         run = {**self.run, "status": "completed", "conclusion": "failure"}
-        responses = [{"state": "active"}, {"workflow_runs": [run]}, run,
-                     {"workflow_runs": [{**run, "run_attempt": 3, "status": "queued"}]}]
+        responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, {"workflow_runs": [run], "total_count": 1}, run,
+                     {"workflow_runs": [{**run, "run_attempt": 3, "status": "queued"}], "total_count": 1}]
         with mock.patch.object(health, "api", side_effect=responses):
             with self.assertRaisesRegex(ValueError, "changed during inspection"):
                 health.collect()
@@ -95,11 +95,141 @@ class HealthTests(unittest.TestCase):
         run = {**self.run, "status": "completed", "conclusion": "success"}
         complete = self.validate()
         for expected, healthy in (("release", True), ("refs/heads/release", True), ("main", False)):
-            responses = [{"state": "active"}, {"workflow_runs": [run]}, run, {"workflow_runs": [run]}]
+            responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, {"workflow_runs": [run], "total_count": 1}, run,
+                         {"workflow_runs": [run], "total_count": 1}, {"object": {"sha": "b" * 40}}]
             with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
                 result = health.collect(source_ref=expected)
             self.assertEqual(result["healthy"], healthy)
 
+    def test_same_source_new_execution_refreshes_completion_even_for_old_build(self):
+        self.candidate["build_started_at"] = "2000-01-01T00:00:00Z"
+        complete = self.validate()
+        self.assertGreater(health.timestamp(complete["expires_at"]), datetime.now(timezone.utc))
+        run = {**self.run, "status": "completed", "conclusion": "success"}
+        payload = {"workflow_runs": [run], "total_count": 1}
+        responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, run, payload,
+                     {"object": {"sha": "b" * 40}}]
+        with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
+            self.assertTrue(health.collect(source_ref="release")["healthy"])
+        wrong = copy.deepcopy(self.summary)
+        wrong["chain"]["attempt"] = 1
+        with self.assertRaises(ValueError):
+            self.validate(wrong)
+
+    def test_success_on_stale_sha_or_incomplete_listing_never_becomes_healthy(self):
+        run = {**self.run, "status": "completed", "conclusion": "success"}
+        payload = {"workflow_runs": [run], "total_count": 1}
+        complete = self.validate()
+        responses = [{"state": "active"}, {"object": {"sha": "c" * 40}}, payload, run, payload,
+                     {"object": {"sha": "c" * 40}}]
+        with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
+            result = health.collect(source_ref="release")
+        self.assertFalse(result["healthy"])
+        self.assertFalse(result["last_complete_success"]["release"]["fresh"])
+        with mock.patch.object(health, "api", side_effect=[{"workflow_runs": [run], "total_count": 2},
+                                                          {"workflow_runs": [], "total_count": 2}]):
+            with self.assertRaisesRegex(ValueError, "incomplete chain run page"):
+                health.list_runs(1)
+
+    def test_optional_omission_is_explicit_and_partial_manual_is_not_full_health(self):
+        from validation_registry import load_registry, select_plan
+        value = copy.deepcopy(self.summary)
+        value["plan"] = select_plan(load_registry())
+        value["suites"] = value["suites"][:-1]
+        value["skipped_lanes"] = ["performance"]
+        value["skip_reasons"] = {"performance": "optional fleet unavailable during preflight"}
+        self.assertEqual(self.validate(value)["skipped_lanes"], ["performance"])
+        del value["skip_reasons"]
+        with self.assertRaises(ValueError):
+            self.validate(value)
+        value = copy.deepcopy(self.summary)
+        value["plan"] = select_plan(load_registry(), requested="s3")
+        value["suites"] = [value["suites"][1]]
+        with self.assertRaisesRegex(ValueError, "partial"):
+            self.validate(value)
+
+    def test_new_failed_attempt_does_not_hide_behind_recent_complete_success(self):
+        old = {**self.run, "status": "completed", "conclusion": "success"}
+        new = {**old, "id": 124, "run_started_at": self.now, "conclusion": "cancelled"}
+        payload = {"workflow_runs": [new, old], "total_count": 2}
+        responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, new, old, payload,
+                     {"object": {"sha": "b" * 40}}]
+        with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=self.validate()):
+            result = health.collect(source_ref="release")
+        self.assertFalse(result["healthy"])
+        self.assertEqual(result["latest_attempt"]["conclusion"], "cancelled")
+        self.assertTrue(result["last_complete_success"]["release"]["fresh"])
+
+    def test_manual_attempt_without_plan_cannot_hide_behind_recent_success(self):
+        old = {**self.run, "status": "completed", "conclusion": "success"}
+        complete = self.validate()
+        for status, conclusion in (("completed", "failure"), ("completed", "cancelled"), ("in_progress", None), ("completed", "success")):
+            with self.subTest(status=status, conclusion=conclusion):
+                new = {**old, "id": 124, "run_started_at": self.now, "event": "workflow_dispatch",
+                       "status": status, "conclusion": conclusion}
+                payload = {"workflow_runs": [new, old], "total_count": 2}
+                responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, new,
+                             {"total_count": 0, "artifacts": []}, old, payload, {"object": {"sha": "b" * 40}}]
+
+                def completed(run):
+                    if run["id"] == new["id"]:
+                        raise ValueError("missing complete-chain artifact")
+                    return complete
+
+                with mock.patch.object(health, "api", side_effect=responses), \
+                     mock.patch.object(health, "complete_success", side_effect=completed):
+                    result = health.collect(source_ref="release")
+                self.assertFalse(result["healthy"])
+                self.assertEqual(result["latest_attempt"]["run_id"], 124)
+                self.assertEqual(result["latest_attempt"]["conclusion"], conclusion)
+                self.assertNotEqual(result["latest_attempt"]["verification"], "complete")
+                self.assertTrue(result["last_complete_success"]["release"]["fresh"])
+
+
+    def test_manual_full_plan_failure_is_latest_but_verified_subset_is_not(self):
+        import hashlib
+        import io
+        import zipfile
+        from validation_registry import load_registry, select_plan
+        run = {**self.run, "event": "workflow_dispatch"}
+        name = "functional-plan-123-2"
+        for suites, expected in (("", True), ("s3", False)):
+            buffer = io.BytesIO()
+            with zipfile.ZipFile(buffer, "w") as archive:
+                archive.writestr("suite-plan.json", json.dumps(select_plan(load_registry(), requested=suites)))
+            raw = buffer.getvalue()
+            artifact = {"id": 77, "name": name, "expired": False, "size_in_bytes": len(raw),
+                        "digest": "sha256:" + hashlib.sha256(raw).hexdigest(),
+                        "workflow_run": {"id": 123, "head_sha": "a" * 40}}
+            with mock.patch.object(health, "api", side_effect=[{"total_count": 1, "artifacts": [artifact]}, raw]):
+                self.assertEqual(health.expected_attempt(run), expected)
+        with mock.patch.object(health, "api", return_value={"total_count": 0, "artifacts": []}):
+            self.assertTrue(health.expected_attempt(run))
+        new = {**run, "status": "completed", "conclusion": "failure", "id": 124, "run_started_at": self.now}
+        old = {**self.run, "status": "completed", "conclusion": "success"}
+        payload = {"workflow_runs": [new, old], "total_count": 2}
+        complete = self.validate()
+        responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, new, old, payload,
+                     {"object": {"sha": "b" * 40}}]
+        with mock.patch.object(health, "api", side_effect=responses), \
+             mock.patch.object(health, "expected_attempt", return_value=True), \
+             mock.patch.object(health, "complete_success", return_value=complete):
+            result = health.collect(source_ref="release")
+        self.assertFalse(result["healthy"])
+        self.assertEqual(result["latest_attempt"]["run_id"], 124)
+
+    def test_artifact_pagination_must_be_complete_and_unique(self):
+        first = [{"id": index} for index in range(100)]
+        final = [{"id": 100}]
+        with mock.patch.object(health, "api", side_effect=[{"total_count": 101, "artifacts": first},
+                                                          {"total_count": 101, "artifacts": final}]):
+            self.assertEqual(len(health.run_artifacts(1)), 101)
+        for final in ([], [{"id": 0}]):
+            with mock.patch.object(health, "api", side_effect=[{"total_count": 101, "artifacts": first},
+                                                              {"total_count": 101, "artifacts": final}]):
+                with self.assertRaises(ValueError):
+                    health.run_artifacts(1)
+
     def test_publication_uses_the_read_blob_sha(self):
         current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
         existing = {"sha": "reviewed-blob", "content": base64.b64encode(json.dumps(current).encode()).decode()}

scripts/test_security_workflow.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Exercise functional failures, chain dispatch, and security evidence without remote VMs."""
+"""Exercise functional failures, cleanup isolation, and evidence without remote VMs."""
 
 from __future__ import annotations
 
@@ -298,7 +298,7 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
             self.assertFalse(body.exists())
 
     def test_all_suites_hold_the_expected_lock_for_manual_and_chain_runs(self) -> None:
-        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
+        for suite in (*FunctionalWorkflowTests.JOBS, "security"):
             with self.subTest(suite=suite):
                 source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
                 # Workflow-level concurrency covers every job, including cleanup,
@@ -312,88 +312,41 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                     f"group: {expected_group}", "cancel-in-progress: false",
                 ])
                 self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
-                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
+                self.assertIsNotNone(yaml_block(source, "workflow_call", 2))
+                self.assertIsNone(yaml_block(source, "repository_dispatch", 2))
+                self.assertNotIn("Continue functional chain", "\n".join(source))
                 cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
                 cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
                 self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
 
-    def test_legacy_replication_hands_off_after_failure(self) -> None:
-        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
-            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
-                self.setUp()
-                fake_bin = self.directory / "bin"
-                fake_bin.mkdir()
-                commands = {
-                    "gh": '''#!/usr/bin/env bash
-set -euo pipefail
-if [ "$1" = api ]; then
-  printf '%s\\n' "$*" >> "$DISPATCHES"
-  attempt=$(wc -l < "$DISPATCHES")
-  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
-elif [ "$1 $2" = 'issue create' ]; then
-  printf 'issue\\n' >> "$EXECUTED"
-  while [ "$#" -gt 0 ]; do
-    if [ "$1" = --body-file ]; then
-      cat "$2" > "$CAPTURE_BODY"
-      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
-    fi
-    shift
-  done
-  exit "$ISSUE_EXIT"
-else
-  exit 99
-fi
-''',
-                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
-                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
-                }
-                for name, contents in commands.items():
-                    command = fake_bin / name
-                    command.write_text(contents)
-                    command.chmod(0o755)
-                dispatches = self.directory / "dispatches"
-                executed = self.directory / "executed"
-                body = self.directory / "issue-body.md"
-                body_path = self.directory / "issue-body-path"
-                self.env.update(
-                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
-                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
-                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
-                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
-                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
+    def test_reusable_chain_continues_selected_suites_after_predecessor_failure(self) -> None:
+        source = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text().splitlines()
+        ordered = ("upgrade", "s3", "kms", "tier", "storage", "heal", "pool", "security",
+                   "replication", "fault-tolerance", "table")
+        for index, suite in enumerate((*ordered, "performance")):
+            with self.subTest(suite=suite):
+                job = yaml_block(source, suite, 2)
+                self.assertIsNotNone(job)
+                dependencies = "prepare" if index == 0 or suite == "performance" else f"prepare, {ordered[index - 1]}"
+                self.assertIn(f"    needs: [{dependencies}]", job)
+                condition = (
+                    "    if: ${{ always() && needs.prepare.result == 'success' && "
+                    f"contains(fromJSON(needs.prepare.outputs.suites || '[]'), '{suite}')"
                 )
-                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
-                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
-                job = yaml_block(replication.splitlines(), "replication-test", 2)
-                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
-                self.steps = named_steps(job)
-                handoff = "Continue functional chain (next: Fault tolerance)"
-                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
-                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
-                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
-                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
-                initialized = self.run_step("Initialize functional evidence")
-                self.assertEqual(initialized.returncode, 0, initialized.stderr)
-                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
-                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
-                failed = self.run_step("Run replication suite")
-                self.assertEqual(failed.returncode, 17, failed.stderr)
-                cleaned = self.run_step("Cleanup environment (after)")
-                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
-                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
-                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
-                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
-                forwarded = self.run_step(handoff)
-                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
-                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
-                self.assertEqual(calls, [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-fault-tolerance -F client_payload[from_suite]=replication",
-                ] * (min(failed_attempts + 1, 3) if token else 0))
-                if failed_attempts == 3:
-                    self.assertIn("could not hand off from **replication** to **Fault tolerance**", body.read_text())
-                    self.assertIn("rustfs-chain-fault-tolerance", body.read_text())
-                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
-                    self.assertFalse(Path(body_path.read_text().strip()).exists())
+                if suite == "performance":
+                    condition += " && needs.prepare.outputs.performance_ready == 'online'"
+                self.assertIn(condition + " }}", job)
+                self.assertFalse(any("continue-on-error:" in line for line in job))
+                self.assertIn("      chain_manifest: ${{ needs.prepare.outputs.manifest }}", job)
+        aggregate = yaml_block(source, "complete-chain", 2)
+        self.assertIn("    if: ${{ always() && needs.prepare.result != 'skipped' }}", aggregate)
+        steps = named_steps(aggregate)
+        for name in ("Summarize every suite including failures and missing evidence",
+                     "Retain chain report regardless of test verdict", "Verify every required suite"):
+            self.assertIn("        if: always()", steps[name])
+        names = list(steps)
+        self.assertLess(names.index("Retain chain report regardless of test verdict"),
+                        names.index("Verify every required suite"))
 
 
 class FunctionalWorkflowTests(unittest.TestCase):
@@ -411,7 +364,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
     }
 
     def test_failure_and_always_step_wiring(self) -> None:
-        for suite, job_id in self.JOBS.items():
+        for suite, job_id in {**self.JOBS, "security": "security-test"}.items():
             if suite in getattr(self, "EXCLUDED", ()):
                 continue
             with self.subTest(suite=suite):
@@ -422,57 +375,75 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 steps = named_steps(job)
                 if suite in self.DIRECT_TESTS:
                     test = steps[self.DIRECT_TESTS[suite]]
-                    # Case failures keep the run green; the step records its
-                    # outcome for the report and the backlog issue manager.
+                    # Collect reports after a suite failure; the final verdict
+                    # still rejects failed, cancelled, and skipped suites.
                     self.assertRegex("\n".join(test), r'''(?m)^        continue-on-error: true$''')
                     self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
                 cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                 condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
                 if suite == "pool-expand":
-                    self.assertEqual(condition, "if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' }}")
+                    self.assertEqual(condition, "if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}")
                     self.assertIn("        id: topology", steps["Validate pool topology before destructive cleanup"])
                     self.assertLess(list(steps).index("Validate pool topology before destructive cleanup"),
                                     list(steps).index("Cleanup environment (before)"))
                 else:
                     self.assertIn(condition, (
-                        "if: always()",
-                        "if: ${{ always() && inputs.cleanup_after != 'false' }}",
-                        "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
+                        "if: ${{ always() && steps.environment.outcome == 'success' }}",
+                        "if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}",
+                        "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}",
                     ))
-                if suite != "performance":
-                    handoff = next(
-                        value for name, value in steps.items() if name.startswith("Continue functional chain")
-                    )
-                    self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
+                self.assertIn("        id: environment_cleanup", cleanup)
+                self.assertNotIn("        continue-on-error: true", cleanup)
+                release = steps["Release environment only after successful cleanup"]
+                self.assertIn("        if: ${{ always() && steps.environment.outcome == 'success' }}", release)
+                self.assertIn("          CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}", release)
+                self.assertTrue(any('finish --suite ' in line and '--cleanup-outcome "$CLEANUP_OUTCOME"' in line
+                                    for line in release))
+                names = list(steps)
+                admission = next(name for name, lines in steps.items() if "        id: environment" in lines)
+                self.assertLess(names.index(admission), names.index("Reset test environment (before)" if suite == "performance" else "Cleanup environment (before)"))
+                self.assertLess(names.index("Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"),
+                                names.index("Release environment only after successful cleanup"))
+                verdict = steps["Enforce suite and report result"]
+                self.assertIn("        if: ${{ always() }}", verdict)
+                self.assertNotIn("        continue-on-error: true", verdict)
+                self.assertEqual(names[-1], "Enforce suite and report result")
+                self.assertLess(names.index("Upload chain evidence"), names.index("Enforce suite and report result"))
 
-    def test_failed_suite_preserves_exit_and_cleanup_and_dispatch_execute(self) -> None:
-        for suite, test_name in self.DIRECT_TESTS.items():
-            with self.subTest(suite=suite), tempfile.TemporaryDirectory() as directory:
+    def test_failed_suite_preserves_exit_and_cleanup_failures(self) -> None:
+        for suite, test_name, cleanup_exit in ((suite, name, status)
+                                             for suite, name in self.DIRECT_TESTS.items() for status in (0, 23)):
+            with self.subTest(suite=suite, cleanup_exit=cleanup_exit), tempfile.TemporaryDirectory() as directory:
                 root = Path(directory)
                 (root / "auto-testing").mkdir()
                 script = root / f"auto-testing/rustfs-{suite}-test.sh"
                 if suite == "fault-tolerance":
-                    # FT cleans up through the suite script itself (--cleanup), not ssh;
-                    # emit the marker so the ordering assertion holds
-                    script.write_text('#!/bin/sh\n[ "$1" = "--cleanup" ] && printf "cleanup\\n" >> "$EXECUTED"\nprintf "partial suite diagnostics\\n"\nexit 17\n')
+                    # FT cleanup must propagate its own status independently of
+                    # the preceding suite failure.
+                    script.write_text('#!/bin/sh\nif [ "$1" = "--cleanup" ]; then\n'
+                                      '  printf "cleanup\\n" >> "$EXECUTED"; exit "$CLEANUP_EXIT"\n'
+                                      'fi\nprintf "partial suite diagnostics\\n"\nexit 17\n')
                 else:
                     script.write_text('#!/bin/sh\nprintf "partial suite diagnostics\\n"\nexit 17\n')
                 script.chmod(0o755)
                 fake_bin = root / "bin"
                 fake_bin.mkdir()
-                for command, marker in (("ssh", "cleanup"), ("gh", "dispatch")):
-                    fake = fake_bin / command
-                    fake.write_text(f'#!/bin/sh\nprintf "{marker}\\n" >> "$EXECUTED"\n')
-                    fake.chmod(0o755)
+                fake = fake_bin / "ssh"
+                fake.write_text('#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n'
+                                'case "$*" in *fixture-user@clean-node*) exit 0 ;; esac\n'
+                                'exit "$CLEANUP_EXIT"\n')
+                fake.chmod(0o755)
+                nodes = "failure-node clean-node" if suite == "table" else "fixture-node"
                 env = {
                     **os.environ, "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}",
-                    "EXECUTED": str(root / "executed"), "RUSTFS_NODES": "fixture-node",
+                    "EXECUTED": str(root / "executed"), "RUSTFS_NODES": nodes,
                     "RUSTFS_SSH_USER": "fixture-user", "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb",
-                    "GH_TOKEN": "local-fixture", "GITHUB_EVENT_NAME": "repository_dispatch", "GITHUB_RUN_ID": "314159",
+                    "GH_TOKEN": "local-fixture", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_RUN_ID": "314159",
+                    "CLEANUP_EXIT": str(cleanup_exit),
                 }
                 source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
                 steps = named_steps(yaml_block(source.splitlines(), self.JOBS[suite], 2))
-                context = {"github.event_name": "repository_dispatch", "steps.test.outcome": "failure"}
+                context = {"github.event_name": "workflow_dispatch", "steps.test.outcome": "failure"}
                 if suite == "table":
                     context["steps.chain_package.outputs.package_url || inputs.package_url"] = env["RUSTFS_NIGHTLY_PACKAGE_URL"]
                 for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", source):
@@ -496,14 +467,45 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertEqual(failed.returncode, 17, failed.stderr)
                 self.assertIn("partial suite diagnostics", failed.stdout)
                 cleanup = execute("Cleanup environment (after)")
-                self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
-                handoff_name = next(
-                    name for name in steps if name.startswith("Continue functional chain")
-                )
-                handoff = execute(handoff_name)
-                self.assertEqual(handoff.returncode, 0, handoff.stderr)
-                markers = (root / "executed").read_text().splitlines()
-                self.assertEqual(markers, ["cleanup", "dispatch"])
+                self.assertEqual(cleanup.returncode, cleanup_exit, cleanup.stderr)
+                self.assertEqual((root / "executed").read_text().splitlines(), ["cleanup"] * len(nodes.split()))
+
+    def test_final_verdict_rejects_failed_outcomes_even_when_report_is_valid(self) -> None:
+        cases = [(suite_outcome, report_outcome, 0)
+                 for suite_outcome in ("success", "failure", "skipped", "cancelled")
+                 for report_outcome in ("success", "failure", "skipped", "cancelled")]
+        cases.append(("success", "success", 9))
+        for suite, job_id in {**self.JOBS, "security": "security-test"}.items():
+            source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
+            steps = named_steps(yaml_block(source, job_id, 2))
+            verdict = steps["Enforce suite and report result"]
+            report_name = "Upload report to dashboard (reports/YYYY-MM-DD.md)" if suite == "performance" else "Generate report"
+            report_id = next(line.split(": ", 1)[1] for line in steps[report_name] if line.startswith("        id: "))
+            self.assertIn("          REPORT_OUTCOME: ${{ steps." + report_id + ".outcome }}", verdict)
+            suite_id = "benchmark" if suite == "performance" else "pool_test" if suite == "pool-expand" else "test"
+            self.assertIn("          SUITE_OUTCOME: ${{ steps." + suite_id + ".outcome }}", verdict)
+            with tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                validator = root / "python3"
+                validator.write_text('#!/bin/sh\nprintf "%s\\n" "$*" > "$VALIDATOR_CALL"\nexit "$VALIDATOR_EXIT"\n')
+                validator.chmod(0o755)
+                marker = root / "called"
+                for suite_outcome, report_outcome, report_exit in cases:
+                    with self.subTest(suite=suite, suite_outcome=suite_outcome,
+                                      report_outcome=report_outcome, report_exit=report_exit):
+                        marker.unlink(missing_ok=True)
+                        result = subprocess.run(
+                            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", shell_body(verdict)],
+                            env={**os.environ, "PATH": f"{root}{os.pathsep}{os.environ['PATH']}",
+                                 "SUITE_OUTCOME": suite_outcome, "REPORT_OUTCOME": report_outcome,
+                                 "VALIDATOR_CALL": str(marker), "VALIDATOR_EXIT": str(report_exit)},
+                            capture_output=True, text=True,
+                        )
+                        outcomes_passed = suite_outcome == report_outcome == "success"
+                        self.assertEqual(result.returncode == 0, outcomes_passed and report_exit == 0, result.stderr)
+                        self.assertEqual(marker.exists(), outcomes_passed)
+                        if marker.exists():
+                            self.assertIn("functional_chain_evidence.py verify-report --suite ", marker.read_text())
 
 
 class FaultToleranceWorkflowContractTests(unittest.TestCase):
@@ -721,7 +723,7 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
         self.context.update({
             "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
             "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
-            "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
+            "github.event_name": "workflow_dispatch", "steps.test.outcome": "success",
             "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
         })
         self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
@@ -761,6 +763,13 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
                 if suite in FunctionalWorkflowTests.DIRECT_TESTS:
                     self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts"))
                 for name, lines in self.steps.items():
+                    if name == "Retain environment ownership and cleanup receipt":
+                        self.assertIn("        if: ${{ always() }}", lines)
+                        suite_id = "s3" if suite == "s3-compat" else suite
+                        self.assertIn("          path: ${{ runner.temp }}/environment-" + suite_id + ".json", lines)
+                        self.assertIn("          if-no-files-found: error", lines)
+                        self.assertFalse(any("*" in line for line in lines))
+                        continue
                     if name == "Upload chain evidence":
                         self.assertIn("        if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", lines)
                         self.assertIn("        if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}", self.steps["Record chain evidence"])

scripts/test_validation_registry.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Selection, ownership and environment failure boundaries for periodic suites."""
+import copy
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+import functional_environment as environment
+import functional_chain_evidence as evidence
+import validation_registry as registry
+
+
+class RegistryTests(unittest.TestCase):
+    def setUp(self):
+        self.registry = registry.load_registry()
+
+    def test_all_workflow_responsibilities_and_suite_exits_are_wired(self):
+        self.assertTrue(registry.check_wiring(self.registry))
+        self.assertEqual(len(self.registry["suites"]), 12)
+        self.assertEqual(sum(item["origin"] == "baseline" for item in self.registry["workflows"]), 60)
+        self.assertEqual(len(self.registry["workflows"]), len(list((registry.ROOT / ".github/workflows").glob("*.yml"))))
+
+    def test_manual_and_periodic_plan_choose_identical_responsibilities(self):
+        for cadence in ("nightly", "weekly", "monthly"):
+            automatic = registry.select_plan(self.registry, cadence, event="schedule")
+            manual = registry.select_plan(self.registry, cadence, event="workflow_dispatch")
+            self.assertEqual(automatic, manual)
+            self.assertEqual(automatic["timezone"], "UTC")
+            self.assertTrue(all(registry.workflow_entry(path)["policy"] == "active"
+                                for path in automatic["periodic_workflows"]))
+
+    def test_single_suite_has_no_implicit_successor_or_optional_escape(self):
+        for suite in registry.suite_ids():
+            plan = registry.select_plan(self.registry, requested=suite)
+            self.assertEqual(plan["suites"], [suite])
+            self.assertEqual(plan["optional_suites"], [])
+        for invalid in ("s3,", "s3,s3", "unknown", " s3", "s3, kms"):
+            with self.assertRaises(ValueError):
+                registry.select_plan(self.registry, requested=invalid)
+        with self.assertRaises(ValueError):
+            registry.select_plan(self.registry, requested="s3", event="schedule")
+
+    def test_disabled_and_cyclic_responsibilities_cannot_be_scheduled(self):
+        value = copy.deepcopy(self.registry)
+        path = value["suites"][0]["workflow"]
+        registry.workflow_entry(path, value)["policy"] = "disabled"
+        with self.assertRaisesRegex(ValueError, "disabled"):
+            registry.select_plan(value)
+        value = copy.deepcopy(self.registry)
+        value["suites"][0]["dependencies"] = ["s3"]
+        value["suites"][1]["dependencies"] = ["upgrade"]
+        with self.assertRaisesRegex(ValueError, "cyclic"):
+            registry.select_plan(value)
+
+    def test_watchdog_uses_registry_scope_source_and_alert_ownership(self):
+        item = registry.workflow_entry(".github/workflows/audit.yml")
+        event = {"repository": {"full_name": "rustfs/rustfs", "default_branch": "main"},
+                 "workflow_run": {"path": item["workflow"], "name": item["name"], "event": "schedule",
+                                  "head_branch": "main", "head_repository": {"full_name": "rustfs/rustfs"},
+                                  "conclusion": "cancelled"}}
+        self.assertTrue(registry.watchdog_event(event))
+        for field, value in (("event", "workflow_dispatch"), ("head_branch", "other"),
+                             ("path", ".github/workflows/cla.yml"), ("conclusion", "success"),
+                             ("conclusion", "failure"), ("head_repository", {"full_name": "other/repo"})):
+            changed = copy.deepcopy(event)
+            changed["workflow_run"][field] = value
+            self.assertFalse(registry.watchdog_event(changed))
+        disabled = copy.deepcopy(self.registry)
+        registry.workflow_entry(item["workflow"], disabled)["policy"] = "disabled"
+        self.assertFalse(registry.watchdog_event(event, disabled))
+
+
+class SelectionEvidenceTests(unittest.TestCase):
+    def test_nonfirst_single_suite_is_verified_and_extraneous_results_are_rejected(self):
+        chain = {"run_id": 1, "attempt": 2}
+        plan = registry.select_plan(registry.load_registry(), requested="kms")
+        needs = {name: {"result": "success" if name == "kms" else "skipped"} for name in evidence.SUITES}
+        record = {"schema": 1, "suite": "kms", "chain": chain, "valid": True, "report_sha256": "a" * 64,
+                  "counts": {"PASS": 1, "FAIL": 0, "RUNNING": 0, "UNSUPPORTED": 0, "SKIP": 0}}
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            (root / "kms.json").write_text(json.dumps(record))
+            result = evidence.aggregate(chain, root, needs, plan=plan)
+            self.assertEqual([item["suite"] for item in result["suites"]], ["kms"])
+            for status in ("failure", "cancelled", "skipped", "missing"):
+                with self.assertRaises(ValueError):
+                    evidence.aggregate(chain, root, {**needs, "kms": {"result": status}}, plan=plan)
+            with self.assertRaises(ValueError):
+                evidence.aggregate(chain, root, {**needs, "s3": {"result": "success"}}, plan=plan)
+            with self.assertRaises(ValueError):
+                evidence.aggregate(chain, root, needs, allow_skipped=["kms"], plan=plan)
+            record["chain"]["attempt"] = 1
+            (root / "kms.json").write_text(json.dumps(record))
+            with self.assertRaises(ValueError):
+                evidence.aggregate({"run_id": 1, "attempt": 2}, root, needs, plan=plan)
+
+
+class EnvironmentTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.env = {"GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "1", "RUSTFS_NODES": "node1 node2",
+                    "FUNCTIONAL_ENVIRONMENT_RECEIPT": str(self.root / "receipt.json")}
+
+    def execute(self, node, user, mode, identity):
+        import base64
+        home = self.root / node
+        home.mkdir(exist_ok=True)
+        encoded = base64.b64encode(json.dumps(identity).encode()).decode()
+        subprocess.run([sys.executable, "-c", environment.REMOTE, mode, encoded],
+                       env={**os.environ, "HOME": str(home)}, check=True, capture_output=True, text=True)
+
+    def test_failed_or_skipped_cleanup_quarantines_the_entire_next_attempt(self):
+        environment.transition("enter", "s3", self.env, execute=self.execute)
+        for outcome in ("failure", "cancelled", "skipped", ""):
+            with self.assertRaises(ValueError):
+                environment.transition("finish", "s3", self.env, outcome, self.execute)
+            self.assertEqual(json.loads((self.root / "receipt.json").read_text())["state"], "quarantined")
+        with self.assertRaises(subprocess.CalledProcessError):
+            environment.transition("enter", "kms", {**self.env, "GITHUB_RUN_ID": "124"}, execute=self.execute)
+        environment.transition("finish", "s3", self.env, "success", self.execute)
+        receipt = environment.transition("enter", "kms", {**self.env, "GITHUB_RUN_ID": "124"}, execute=self.execute)
+        self.assertEqual(receipt["state"], "owned")
+
+    def test_foreign_attempt_cannot_release_and_partial_admission_is_not_reused(self):
+        calls = []
+        def interrupted(node, user, mode, identity):
+            calls.append((node, mode))
+            if node == "node2" and mode == "enter":
+                raise OSError("node unavailable")
+            self.execute(node, user, mode, identity)
+        with self.assertRaises(OSError):
+            environment.transition("enter", "s3", self.env, execute=interrupted)
+        self.assertEqual(calls[:2], [("node1", "check"), ("node2", "check")])
+        with self.assertRaises(subprocess.CalledProcessError):
+            environment.transition("enter", "kms", self.env, execute=self.execute)
+        with self.assertRaises(subprocess.CalledProcessError):
+            environment.transition("finish", "s3", {**self.env, "GITHUB_RUN_ATTEMPT": "2"}, "success", self.execute)
+        self.assertTrue((self.root / "node1/.rustfs-functional-environment/active.json").is_file())
+        environment.transition("recover", "s3", self.env, "success", self.execute)
+        environment.transition("enter", "kms", self.env, execute=self.execute)
+
+    def test_node_identity_is_validated_before_ssh(self):
+        for name in ("node;uname", "-oProxyCommand=anything", "node1 node1"):
+            execute = mock.Mock()
+            with self.assertRaises(ValueError):
+                environment.transition("enter", "s3", {**self.env, "RUSTFS_NODES": name}, execute=execute)
+            execute.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

scripts/validation_registry.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""One responsibility registry for suite selection and periodic health monitors."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+
+ROOT = Path(__file__).resolve().parents[1]
+REGISTRY = ROOT / ".github/scheduled-validations.json"
+
+
+def require(condition, message):
+    if not condition:
+        raise ValueError(message)
+
+
+def load_registry(path=REGISTRY):
+    value = json.loads(Path(path).read_text())
+    require(isinstance(value, dict) and type(value.get("schema")) is int and value["schema"] == 1,
+            "unsupported validation registry")
+    require(set(value.get("plans", {})) == {"nightly", "weekly", "monthly"}, "missing validation cadence")
+    for plan in value["plans"].values():
+        require(isinstance(plan, dict) and plan.get("timezone") == "UTC"
+                and isinstance(plan.get("cadences"), list) and plan["cadences"]
+                and set(plan["cadences"]) <= {"nightly", "weekly", "monthly"}, "invalid cadence plan")
+    workflows = value.get("workflows")
+    suites = value.get("suites")
+    require(isinstance(workflows, list) and workflows and isinstance(suites, list) and suites,
+            "empty validation responsibilities")
+    paths = set()
+    for item in workflows:
+        require(isinstance(item, dict), "invalid workflow responsibility")
+        path = item.get("workflow")
+        require(isinstance(path, str) and re.fullmatch(r"\.github/workflows/[a-z0-9-]+\.yml", path)
+                and path not in paths, "invalid or duplicate workflow responsibility")
+        paths.add(path)
+        require(item.get("owner") and item.get("name") and item.get("roles") and item.get("events"),
+                f"incomplete responsibility: {path}")
+        require(item.get("timezone") == "UTC" and item.get("policy") in ("active", "paused", "disabled"),
+                f"invalid execution policy: {path}")
+        require(item.get("monitor") in ("workflow", "complete-chain", "functional-chain", "none"),
+                f"invalid monitor: {path}")
+        if item["monitor"] in ("workflow", "complete-chain"):
+            require(type(item.get("max_age_hours")) is int and item["max_age_hours"] > 0,
+                    f"missing freshness budget: {path}")
+        if item["monitor"] == "none":
+            require(item.get("monitor_reason"), f"missing monitor exclusion: {path}")
+    ids = []
+    for suite in suites:
+        require(isinstance(suite, dict) and isinstance(suite.get("id"), str)
+                and re.fullmatch(r"[a-z][a-z0-9-]*", suite["id"]) and suite["id"] not in ids,
+                "invalid or duplicate suite")
+        ids.append(suite["id"])
+        require(suite.get("workflow") in paths and suite.get("cadence") in ("nightly", "weekly", "monthly")
+                and suite.get("timezone") == "UTC" and suite.get("resource_lock") and suite.get("environment")
+                and suite.get("owner") and suite.get("evidence") and type(suite.get("required")) is bool
+                and isinstance(suite.get("dependencies"), list), "incomplete suite responsibility")
+        require(suite["required"] or suite.get("optional_reason"), "optional suite needs a reason")
+    for suite in suites:
+        require(set(suite["dependencies"]) <= set(ids) and suite["id"] not in suite["dependencies"],
+                "unknown or self-dependent suite")
+    return value
+
+
+def suite_ids(registry=None):
+    return tuple(item["id"] for item in (registry or load_registry())["suites"])
+
+
+def workflow_entry(path, registry=None):
+    entries = [item for item in (registry or load_registry())["workflows"] if item["workflow"] == path]
+    require(len(entries) == 1, f"unregistered workflow: {path}")
+    return entries[0]
+
+
+def monitored_workflows(registry=None):
+    return [item for item in (registry or load_registry())["workflows"]
+            if item["monitor"] in ("workflow", "complete-chain")]
+
+
+def select_plan(registry, cadence="nightly", requested="", event="workflow_dispatch"):
+    require(cadence in registry["plans"], "unknown validation plan")
+    require(event in ("workflow_dispatch", "workflow_run", "schedule"), "unsupported plan event")
+    require(not requested or event == "workflow_dispatch", "automatic plans cannot override suite selection")
+    by_id = {item["id"]: item for item in registry["suites"]}
+    explicit = bool(requested and requested != "all")
+    names = requested.split(",") if explicit else [item["id"] for item in registry["suites"]
+              if item["cadence"] in registry["plans"][cadence]["cadences"]]
+    require(names and all(name == name.strip() and name in by_id for name in names)
+            and len(names) == len(set(names)), "unknown, empty or duplicate selected suite")
+    selected = set()
+
+    def add(name, ancestors=()):
+        require(name not in ancestors, "cyclic suite dependency")
+        for dependency in by_id[name]["dependencies"]:
+            add(dependency, ancestors + (name,))
+        selected.add(name)
+
+    for name in names:
+        add(name)
+    ordered = [name for name in by_id if name in selected]
+    for name in ordered:
+        require(workflow_entry(by_id[name]["workflow"], registry)["policy"] == "active",
+                f"selected suite is disabled or paused: {name}")
+    return {"schema": 1, "cadence": cadence, "timezone": registry["plans"][cadence]["timezone"],
+            "suites": ordered, "optional_suites": [] if explicit else [name for name in ordered if not by_id[name]["required"]],
+            "environments": sorted({by_id[name]["environment"] for name in ordered}),
+            "resource_locks": sorted({by_id[name]["resource_lock"] for name in ordered}),
+            "periodic_workflows": [item["workflow"] for item in registry["workflows"]
+                                   if item["cadence"] in registry["plans"][cadence]["cadences"]
+                                   and item["policy"] == "active" and item["monitor"] in ("workflow", "complete-chain")]}
+
+
+def validate_plan(plan, registry=None):
+    registry = registry or load_registry()
+    require(isinstance(plan, dict) and plan.get("schema") == 1, "invalid suite selection plan")
+    # Explicit single-suite invocations have no optional lanes; an automatic
+    # plan can omit only optional lanes declared in the registry.
+    full = select_plan(registry, plan.get("cadence", ""))
+    selected = plan.get("suites")
+    require(isinstance(selected, list) and selected
+            and selected == [name for name in suite_ids(registry) if name in selected], "invalid selected suites")
+    optional = plan.get("optional_suites")
+    require(isinstance(optional, list) and len(optional) == len(set(optional))
+            and set(optional) <= set(full["optional_suites"]) & set(selected), "invalid optional suites")
+    for suite in registry["suites"]:
+        if suite["id"] in selected:
+            require(set(suite["dependencies"]) <= set(selected), "plan omits a suite dependency")
+    return plan
+
+
+def watchdog_event(event, registry=None):
+    run = event.get("workflow_run", {})
+    matches = [item for item in monitored_workflows(registry)
+               if item["workflow"] == run.get("path") and item["name"] == run.get("name")]
+    if len(matches) != 1:
+        return False
+    item = matches[0]
+    expected_event = "workflow_run" if item["monitor"] == "complete-chain" else "schedule"
+    repository = event.get("repository", {})
+    return (item["policy"] == "active" and run.get("event") == expected_event
+            and run.get("head_branch") == repository.get("default_branch")
+            and (run.get("head_repository") or {}).get("full_name") == repository.get("full_name")
+            and run.get("conclusion") != "success"
+            and (run.get("conclusion") != "failure" or item.get("failure_owner") == "watchdog"))
+
+
+def check_wiring(registry, root=ROOT):
+    declared = {item["workflow"] for item in registry["workflows"]}
+    actual = {str(path.relative_to(root)) for path in (root / ".github/workflows").glob("*.yml")}
+    require(declared == actual, "registry workflow coverage differs: " + ", ".join(sorted(declared ^ actual)))
+    for item in registry["workflows"]:
+        source = (root / item["workflow"]).read_text()
+        name = re.search(r"^name: (.+)$", source, re.M)
+        require(name and name.group(1).strip("\"'") == item["name"], "workflow name differs from registry")
+        trigger = re.search(r"^on:\n(.*?)(?=^[^ #\n]|\Z)", source, re.M | re.S)
+        require(trigger is not None, "workflow has no event declaration")
+        events = re.findall(r"^  (\w+):", trigger.group(1), re.M)
+        schedules = re.findall(r"^\s+- cron:\s*['\"]([^'\"]+)", trigger.group(1), re.M)
+        require(events == item["events"] and schedules == item["schedule"],
+                "workflow events or cadence differ from registry: " + item["workflow"])
+    for suite in registry["suites"]:
+        source = (root / suite["workflow"]).read_text()
+        require("repository_dispatch" not in source and "Continue functional chain" not in source,
+                "retired functional relay remains wired")
+        require("group: " + suite["resource_lock"] in source and "cancel-in-progress: false" in source,
+                "suite resource lock differs from registry")
+        require("functional_chain_evidence.py record --suite " + suite["id"] in source,
+                "suite evidence exit differs from registry")
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("mode", choices=("check", "select", "watchdog"))
+    parser.add_argument("--registry", type=Path, default=REGISTRY)
+    parser.add_argument("--cadence", default=os.environ.get("VALIDATION_PLAN") or "nightly")
+    parser.add_argument("--suites", default=os.environ.get("VALIDATION_SUITES", ""))
+    parser.add_argument("--event", default=os.environ.get("GITHUB_EVENT_NAME", "workflow_dispatch"))
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+    registry = load_registry(args.registry)
+    if args.mode == "check":
+        check_wiring(registry)
+        print(f"Validated {len(registry['workflows'])} workflow responsibilities and {len(registry['suites'])} suites")
+    elif args.mode == "watchdog":
+        selected = watchdog_event(json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()), registry)
+        with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+            output.write("selected=" + str(selected).lower() + "\n")
+    else:
+        plan = select_plan(registry, args.cadence, args.suites, args.event)
+        encoded = json.dumps(plan, separators=(",", ":"))
+        if args.output:
+            args.output.write_text(encoded + "\n")
+        if os.environ.get("GITHUB_OUTPUT"):
+            with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+                output.write("plan=" + encoded + "\n")
+                for key in ("suites", "environments", "optional_suites"):
+                    output.write(key + "=" + json.dumps(plan[key], separators=(",", ":")) + "\n")
+        print(encoded)
+
+
+if __name__ == "__main__":
+    main()