fix(ci): serialize performance on shared functional VMs

.github/actions/quick-checks/action.yml

@@ -1,115 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Quick Checks
-description: Run the shared compile-free RustFS quality checks.
-
-runs:
-  using: composite
-  steps:
-    - name: Install quality tools
-      uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
-      with:
-        tool: |
-          ripgrep@15.2.0
-          shellcheck@0.11.0
-
-    - name: Install actionlint
-      shell: bash
-      run: |
-        actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
-        curl --fail --location --silent --show-error \
-          --output "$actionlint_dir/actionlint.tar.gz" \
-          https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
-        echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8  $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
-        tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
-        rm "$actionlint_dir/actionlint.tar.gz"
-        echo "$actionlint_dir" >> "$GITHUB_PATH"
-
-    - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      with:
-        components: rustfmt
-
-    - name: Check workflow syntax and shell scripts
-      shell: bash
-      run: shellcheck --version && actionlint
-
-    - name: Check code formatting
-      shell: bash
-      run: cargo fmt --all --check
-
-    - name: Check unsafe code allowances
-      shell: bash
-      run: ./scripts/check_unsafe_code_allowances.sh
-
-    - name: Check layered dependencies
-      shell: bash
-      run: ./scripts/check_layer_dependencies.sh
-
-    - name: Check architecture migration rules
-      shell: bash
-      run: ./scripts/check_architecture_migration_rules.sh
-
-    - name: Check logging guardrails
-      shell: bash
-      run: ./scripts/check_logging_guardrails.sh
-
-    - name: Check error other(format!) ratchet
-      shell: bash
-      run: ./scripts/check_error_other_format_ratchet.sh
-
-    - name: Check tokio io-uring feature guard
-      shell: bash
-      run: ./scripts/check_no_tokio_io_uring.sh
-
-    - name: Check extension schema boundaries
-      shell: bash
-      run: ./scripts/check_extension_schema_boundaries.sh
-
-    - name: Check body-cache whitelist guard
-      shell: bash
-      run: ./scripts/check_body_cache_whitelist.sh
-
-    - name: Check s3s footprint ratchet
-      shell: bash
-      run: ./scripts/check_s3s_footprint.sh
-
-    - name: Check cryptographic capability wording
-      shell: bash
-      run: ./scripts/check_fips_wording.sh
-
-    - name: Check no embedded secret material
-      shell: bash
-      run: ./scripts/check_embedded_secrets.sh
-
-    - name: Run script contract tests
-      shell: bash
-      run: make script-tests
-
-    - name: Check test wiring
-      shell: bash
-      run: python3 ./scripts/check_test_wiring.py
-
-    - name: Check no planning docs committed
-      shell: bash
-      run: ./scripts/check_no_planning_docs.sh
-
-    - name: Check CI paths stay in sync
-      shell: bash
-      run: ./scripts/check_ci_paths_sync.sh
-
-    - name: Check io_uring lane --lib precondition
-      shell: bash
-      run: ./scripts/check_uring_lane_lib_only.sh

.github/workflows/ci-docs-only.yml

@@ -12,10 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Reports the existing required checks for paths excluded by ci.yml.
-# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
-# action to keep validation coverage aligned. Keep this paths list in sync with
-# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
+# Companion to ci.yml for required status checks.
+#
+# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
+# requires a check named "Test and Lint" — without this workflow a docs-only PR
+# would wait on it forever. This workflow triggers on exactly the paths ci.yml
+# ignores and reports success under the same job name. Mixed PRs trigger both
+# workflows and the real check still gates: a required check with any failing
+# run blocks the merge.
+# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
+#
+# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
+# required too (rustfs/backlog#1599). Until that change lands this job is
+# inert; mirroring it first is what lets the ruleset change happen without
+# stranding docs-only PRs on a check nobody reports.
+#
+# Keep the paths list below in sync with the pull_request paths-ignore list
+# in ci.yml, and keep the quick-checks steps below byte-identical to the
+# quick-checks job in ci.yml.
 
 name: Continuous Integration (docs only)
 
@@ -45,6 +59,19 @@ permissions:
   contents: read
 
 jobs:
+  # Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
+  # check, ci.yml gates every expensive job behind it, so a mixed PR reports
+  # two check runs with this name: the real one (45-51s) and this companion.
+  # GitHub has no written contract for how it picks between same-named
+  # required check runs ("latest wins" vs "any failure blocks"), so instead of
+  # relying on ordering we make both runs execute the same commands against
+  # the same merge ref — their conclusions are then necessarily identical and
+  # the choice does not matter. Keep these steps byte-identical to the
+  # quick-checks job in ci.yml (a guard script that asserts this, and the paths
+  # sync below, is tracked in rustfs/backlog#1603).
+  #
+  # For a genuinely docs-only PR this adds no strictness (no code changed, so
+  # fmt and the guards always pass) and costs ~50s of ubuntu-latest.
   quick-checks:
     name: Quick Checks
     runs-on: ubuntu-latest
@@ -55,8 +82,64 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Run shared quick checks
-        uses: ./.github/actions/quick-checks
+      - name: Install ripgrep
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        with:
+          tool: ripgrep@15.2.0
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          components: rustfmt
+
+      - name: Check code formatting
+        run: cargo fmt --all --check
+
+      - name: Check unsafe code allowances
+        run: ./scripts/check_unsafe_code_allowances.sh
+
+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
+      - name: Check architecture migration rules
+        run: ./scripts/check_architecture_migration_rules.sh
+
+      - name: Check logging guardrails
+        run: ./scripts/check_logging_guardrails.sh
+
+      - name: Check tokio io-uring feature guard
+        run: ./scripts/check_no_tokio_io_uring.sh
+
+      - name: Check extension schema boundaries
+        run: ./scripts/check_extension_schema_boundaries.sh
+
+      - name: Check body-cache whitelist guard
+        run: ./scripts/check_body_cache_whitelist.sh
+
+      - name: Check s3s footprint ratchet
+        run: ./scripts/check_s3s_footprint.sh
+
+      - name: Check cryptographic capability wording
+        run: ./scripts/check_fips_wording.sh
+
+      - name: Check no embedded secret material
+        run: ./scripts/check_embedded_secrets.sh
+
+      - name: Check test wiring
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/test_security_workflow.py
+          python3 ./scripts/check_test_wiring.py
+
+      - name: Check no planning docs committed
+        run: ./scripts/check_no_planning_docs.sh
+
+      - name: Check CI paths stay in sync
+        run: ./scripts/check_ci_paths_sync.sh
+
+      - name: Check io_uring lane --lib precondition
+        run: ./scripts/check_uring_lane_lib_only.sh
 
   test-and-lint:
     name: Test and Lint

.github/workflows/ci.yml

@@ -100,7 +100,12 @@ jobs:
       - name: Typos check with custom config file
         uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
 
-  # Fail early with compile-free checks shared with docs-only CI.
+  # Fast, compile-free checks that fail early so contributors get feedback in
+  # ~1 minute instead of waiting for the full test job.
+  #
+  # These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
+  # PR, which reports two check runs named "Quick Checks", cannot get one red
+  # and one green. Edit both jobs together.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -112,8 +117,67 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Run shared quick checks
-        uses: ./.github/actions/quick-checks
+      - name: Install ripgrep
+        uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
+        with:
+          tool: ripgrep@15.2.0
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+        with:
+          components: rustfmt
+
+      - name: Check code formatting
+        run: cargo fmt --all --check
+
+      - name: Check unsafe code allowances
+        run: ./scripts/check_unsafe_code_allowances.sh
+
+      - name: Check layered dependencies
+        run: ./scripts/check_layer_dependencies.sh
+
+      - name: Check architecture migration rules
+        run: ./scripts/check_architecture_migration_rules.sh
+
+      - name: Check logging guardrails
+        run: ./scripts/check_logging_guardrails.sh
+
+      - name: Check error other(format!) ratchet
+        run: ./scripts/check_error_other_format_ratchet.sh
+
+      - name: Check tokio io-uring feature guard
+        run: ./scripts/check_no_tokio_io_uring.sh
+
+      - name: Check extension schema boundaries
+        run: ./scripts/check_extension_schema_boundaries.sh
+
+      - name: Check body-cache whitelist guard
+        run: ./scripts/check_body_cache_whitelist.sh
+
+      - name: Check s3s footprint ratchet
+        run: ./scripts/check_s3s_footprint.sh
+
+      - name: Check cryptographic capability wording
+        run: ./scripts/check_fips_wording.sh
+
+      - name: Check no embedded secret material
+        run: ./scripts/check_embedded_secrets.sh
+
+      - name: Check test wiring
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/test_security_workflow.py
+          python3 ./scripts/check_test_wiring.py
+
+      - name: Check no planning docs committed
+        run: ./scripts/check_no_planning_docs.sh
+
+      - name: Check CI paths stay in sync
+        run: ./scripts/check_ci_paths_sync.sh
+
+      - name: Check io_uring lane --lib precondition
+        run: ./scripts/check_uring_lane_lib_only.sh
 
   test-and-lint:
     name: Test and Lint

.github/workflows/rustfs-functional-chain.yml

@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-performance-test.yml

@@ -49,17 +49,16 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
+    # Chain handoff: dispatched when the replication suite finishes.
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-performance-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:

.github/workflows/rustfs-replication-test.yml

@@ -34,8 +34,7 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
+    # Chain handoff: dispatched when the security suite finishes.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -62,9 +61,6 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
-    # A failed replication run must not break the chain or the workflow: the
-    # failure is reported to rustfs/backlog instead (see the issue step).
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
@@ -349,13 +345,50 @@ jobs:
             '
           done
 
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
+      - name: "Continue functional chain (next: Performance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=replication'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            trap 'rm -f "${BODY_FILE}"' EXIT
+            {
+              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

scripts/check_test_wiring.py

@@ -5,9 +5,7 @@ from __future__ import annotations
 
 import hashlib
 import json
-import os
 import re
-import subprocess
 import sys
 import tempfile
 import tomllib
@@ -483,20 +481,18 @@ def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None:
     return lines[start:end]
 
 
-def workflow_step_block(
-    job_lines: list[str], value: str, key: str = "uses", indent: int = 6
-) -> tuple[int, list[str]] | None:
+def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None:
     uses_index = next(
         (
             index
             for index, line in enumerate(job_lines)
             if (
-                line.split("#", 1)[0].strip() == f"- {key}: {value}"
-                and len(line) - len(line.lstrip()) == indent
+                line.split("#", 1)[0].strip() == f"- uses: {action}"
+                and len(line) - len(line.lstrip()) == 6
             )
             or (
-                line.split("#", 1)[0].strip() == f"{key}: {value}"
-                and len(line) - len(line.lstrip()) == indent + 2
+                line.split("#", 1)[0].strip() == f"uses: {action}"
+                and len(line) - len(line.lstrip()) == 8
             )
         ),
         None,
@@ -524,67 +520,6 @@ def workflow_step_block(
     return start, job_lines[start:end]
 
 
-def yaml_scalar_continues(lines: list[str], index: int, indent: int) -> bool:
-    following = next(
-        (line for line in lines[index + 1:] if line.strip() and not line.lstrip().startswith("#")), None
-    )
-    return following is not None and len(following) - len(following.lstrip()) > indent
-
-
-def check_quick_checks(root: Path) -> list[str]:
-    errors: list[str] = []
-    bypass_key = r'''(?:if|continue-on-error|needs|"if"|"continue-on-error"|"needs"|'if'|'continue-on-error'|'needs')\s*:'''
-    for name in ("ci.yml", "ci-docs-only.yml"):
-        relative = f".github/workflows/{name}"
-        path = root / relative
-        job = yaml_block(path.read_text().splitlines(), "quick-checks", 2) if path.is_file() else None
-        if job is None:
-            errors.append(f"{relative}: missing Quick Checks job")
-            continue
-        conditions = [index for index, line in enumerate(job) if re.match(rf"^    {bypass_key}", line)]
-        expected = ["if: github.event_name != 'pull_request' || github.event.action != 'closed'"] if name == "ci.yml" else []
-        if [job[index].strip() for index in conditions] != expected or any(
-            yaml_scalar_continues(job, index, 4) for index in conditions
-        ):
-            errors.append(f"{relative}: Quick Checks job must not add dependencies, bypass failures, or change its event condition")
-        checkout = workflow_step_block(job, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0")
-        action = workflow_step_block(job, "./.github/actions/quick-checks")
-        if checkout is None or action is None:
-            errors.append(f"{relative}: Quick Checks requires checkout and the shared quick-checks action")
-            continue
-        if checkout[0] >= action[0]:
-            errors.append(f"{relative}: checkout must run before shared Quick Checks")
-        if "          persist-credentials: false" not in checkout[1]:
-            errors.append(f"{relative}: Quick Checks checkout must disable persisted credentials")
-        for step in (checkout, action):
-            if any(re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]):
-                errors.append(f"{relative}: Quick Checks checkout and shared action must run without bypasses")
-
-    relative = ".github/actions/quick-checks/action.yml"
-    path = root / relative
-    runs = yaml_block(path.read_text().splitlines(), "runs", 0) if path.is_file() else None
-    if runs is None or "  using: composite" not in runs:
-        errors.append(f"{relative}: missing composite action")
-        return errors
-    steps = yaml_block(runs, "steps", 2) or []
-    for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"):
-        step = workflow_step_block(steps, command, key="run", indent=4)
-        if step is None:
-            errors.append(f"{relative}: missing direct execution of {command}")
-            continue
-        if "      shell: bash" not in step[1] or any(
-            re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]
-        ):
-            errors.append(f"{relative}: {command} must use bash without a condition or continue-on-error")
-        run_index = next(
-            index for index, line in enumerate(step[1])
-            if line.split("#", 1)[0].rstrip() in (f"      run: {command}", f"    - run: {command}")
-        )
-        if yaml_scalar_continues(step[1], run_index, 6):
-            errors.append(f"{relative}: {command} must remain a single-line run scalar")
-    return errors
-
-
 def alert_step_errors(
     job_lines: list[str],
     expected_action_if: str | None,
@@ -885,143 +820,10 @@ def validate(root: Path) -> list[str]:
     errors.extend(check_workflow_readiness(root))
     errors.extend(check_profile_definitions(root))
     errors.extend(check_scheduled_alerts(root))
-    errors.extend(check_quick_checks(root))
     return errors
 
 
 class SelfTests(unittest.TestCase):
-    def test_quick_checks_rejects_caller_and_execution_bypasses(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            caller = (
-                "jobs:\n  quick-checks:\n    steps:\n"
-                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
-                "        with:\n          persist-credentials: false\n"
-                "      - uses: ./.github/actions/quick-checks\n"
-            )
-            action = (
-                "runs:\n  using: composite\n  steps:\n"
-                "    - uses: taiki-e/install-action@pinned\n"
-                "      with:\n        tool: actionlint@1.7.12\n"
-                "    - name: Lint workflows\n      shell: bash\n      run: shellcheck --version && actionlint\n"
-                "    - name: Error format ratchet\n      shell: bash\n"
-                "      run: ./scripts/check_error_other_format_ratchet.sh\n"
-                "    - name: Script tests\n      shell: bash\n      run: make script-tests\n"
-            )
-            sources = {
-                ".github/workflows/ci.yml": caller.replace(
-                    "    steps:", "    if: github.event_name != 'pull_request' || github.event.action != 'closed'\n    steps:"
-                ),
-                ".github/workflows/ci-docs-only.yml": caller,
-                ".github/actions/quick-checks/action.yml": action,
-            }
-            for relative, source in sources.items():
-                path = root / relative
-                path.parent.mkdir(parents=True, exist_ok=True)
-                path.write_text(source)
-            self.assertEqual(check_quick_checks(root), [])
-            for relative in (".github/workflows/ci.yml", ".github/workflows/ci-docs-only.yml"):
-                source = sources[relative]
-                mutations = {
-                    "different action": source.replace("./.github/actions/quick-checks", "./.github/actions/other"),
-                    "conditional call": source + "        if: false\n",
-                    "ignored call failure": source + "        continue-on-error: true\n",
-                    "conditional checkout": source.replace("        with:", "        if: false\n        with:"),
-                    "ignored job failure": source.replace("    steps:", "    continue-on-error: true\n    steps:"),
-                    "changed job condition": (
-                        source.replace("github.event_name != 'pull_request' || github.event.action != 'closed'", "false")
-                        if relative.endswith("/ci.yml") else source.replace("    steps:", "    if: false\n    steps:")
-                    ),
-                    "persisted credentials": source.replace("persist-credentials: false", "persist-credentials: true"),
-                    "late checkout": source.replace("      - uses: ./.github/actions/quick-checks\n", "").replace(
-                        "    steps:\n", "    steps:\n      - uses: ./.github/actions/quick-checks\n"
-                    ),
-                    "missing job": source.replace("  quick-checks:", "  other-checks:"),
-                }
-                for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'):
-                    mutations[f"quoted call {key}"] = source + f"        {key}\n"
-                    mutations[f"quoted checkout {key}"] = source.replace("        with:", f"        {key}\n        with:")
-                    job_source = source.replace(
-                        "    if: github.event_name != 'pull_request' || github.event.action != 'closed'\n", ""
-                    ) if "if" in key else source
-                    mutations[f"quoted job {key}"] = job_source.replace("    steps:", f"    {key}\n    steps:")
-                for dependency in ("needs: prerequisite", "needs: [prerequisite]", "needs:\n      - prerequisite", "'needs' : [prerequisite]", '"needs": [prerequisite]'):
-                    for condition in ("false", "true"):
-                        prerequisite = f"\n  prerequisite:\n    if: {condition}\n    runs-on: ubuntu-latest\n    steps:\n      - run: exit 1\n"
-                        mutations[f"job dependency {dependency} if {condition}"] = source.replace("    steps:", f"    {dependency}\n    steps:") + prerequisite
-                if relative.endswith("/ci.yml"):
-                    for separator in ("", "\n", "      # continued condition\n"):
-                        mutations[f"continued job condition {separator!r}"] = source.replace(
-                            "    steps:", f"{separator}      && false\n    steps:"
-                        )
-                for case, mutated in mutations.items():
-                    with self.subTest(path=relative, case=case):
-                        (root / relative).write_text(mutated)
-                        self.assertTrue(check_quick_checks(root))
-                (root / relative).write_text(source)
-            relative = ".github/actions/quick-checks/action.yml"
-            mutations = {
-                "not composite": action.replace("using: composite", "using: node24"),
-                "only installed actionlint": action.replace("run: shellcheck --version && actionlint", "run: echo actionlint"),
-                "missing shellcheck preflight": action.replace("shellcheck --version && ", ""),
-                "missing ratchet": action.replace("run: ./scripts/check_error_other_format_ratchet.sh", "run: echo skipped"),
-                "missing script tests": action.replace("run: make script-tests", "run: echo skipped"),
-                "swallowed script failure": action.replace("run: make script-tests", "run: make script-tests || true"),
-                "swallowed lint failure": action.replace("&& actionlint", "&& actionlint || true"),
-                "swallowed ratchet failure": action.replace("ratchet.sh", "ratchet.sh || true"),
-                "conditional lint": action.replace("run: shellcheck", "if: false\n      run: shellcheck"),
-                "ignored ratchet failure": action.replace("run: ./scripts/", "continue-on-error: true\n      run: ./scripts/"),
-                "non-failing shell": action.replace("shell: bash", "shell: bash {0}"),
-                "run text in step name": action.replace(
-                    "name: Lint workflows", "name: |\n        run: shellcheck --version && actionlint"
-                ).replace("\n      run: shellcheck --version && actionlint\n", "\n      run: shellcheck --version && actionlint\n        || true\n"),
-            }
-            for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"):
-                for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'):
-                    mutations[f"quoted {command} {key}"] = action.replace(f"run: {command}", f"{key}\n      run: {command}")
-                for separator in ("", "\n", "        # continued command\n"):
-                    mutations[f"continued {command} {separator!r}"] = action.replace(
-                        f"run: {command}\n", f"run: {command}\n{separator}        || true\n"
-                    )
-            for case, mutated in mutations.items():
-                with self.subTest(case=case):
-                    (root / relative).write_text(mutated)
-                    self.assertTrue(check_quick_checks(root))
-            (root / relative).unlink()
-            self.assertTrue(check_quick_checks(root))
-
-    def test_quick_checks_commands_propagate_failure(self) -> None:
-        runs = yaml_block((ROOT / ".github/actions/quick-checks/action.yml").read_text().splitlines(), "runs", 0)
-        steps = yaml_block(runs or [], "steps", 2) or []
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            (root / "scripts").mkdir()
-            commands = ("shellcheck", "actionlint", "./scripts/check_error_other_format_ratchet.sh")
-            (root / "Makefile").write_text(".PHONY: script-tests\nscript-tests:\n\texit 17\n")
-            for failing in (*commands, "make script-tests"):
-                with self.subTest(command=failing):
-                    run = "shellcheck --version && actionlint" if failing in ("shellcheck", "actionlint") else failing
-                    step = workflow_step_block(steps, run, key="run", indent=4)
-                    self.assertIsNotNone(step)
-                    run_index = next(index for index, line in enumerate(step[1]) if line.startswith("      run:"))
-                    self.assertFalse(yaml_scalar_continues(step[1], run_index, 6))
-                    body = step[1][run_index].removeprefix("      run: ")
-                    for command in commands:
-                        shim = root / command
-                        shim.write_text(f"#!/bin/sh\nexit {17 if command == failing else 0}\n")
-                        shim.chmod(0o755)
-                    result = subprocess.run(
-                        ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", body],
-                        cwd=root, env=dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}"),
-                        capture_output=True, text=True,
-                    )
-                    self.assertEqual(result.returncode, 2 if failing == "make script-tests" else 17, result.stderr)
-
-    def test_validate_includes_quick_checks(self) -> None:
-        error = "Quick Checks wiring regression"
-        with mock.patch(__name__ + ".check_quick_checks", return_value=[error]):
-            self.assertIn(error, validate(ROOT))
-
     def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -1256,7 +1058,6 @@ class SelfTests(unittest.TestCase):
                 mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
                 mock.patch(__name__ + ".check_ilm_build_budget", return_value=[]),
                 mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
-                mock.patch(__name__ + ".check_quick_checks", return_value=[]),
             ):
                 self.assertEqual(len(validate(root)), 1)
 
@@ -1697,7 +1498,7 @@ def main() -> int:
         for error in errors:
             print(f"ERROR: {error}", file=sys.stderr)
         return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and Quick Checks are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired")
     return 0
 
 

scripts/test_python_bin.sh

@@ -62,14 +62,20 @@ exit 1
 STUB
 chmod +x "$TMP_ROOT/bin/python3"
 
-ln -s "$(command -v bash)" "$TMP_ROOT/bin/bash"
-if PATH="$TMP_ROOT/bin" RUSTFS_PYTHON="" "$RESOLVER" -c 'pass' \
-    >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
-    fail "resolver succeeded with no usable interpreter on PATH"
+SANDBOX_PATH="$TMP_ROOT/bin:/usr/bin:/bin"
+if PATH="$SANDBOX_PATH" command -v uv >/dev/null 2>&1; then
+    # uv is reachable even from the sandbox PATH, so the resolver would
+    # legitimately fall back to it instead of failing. Skip this case.
+    echo "ℹ️  uv is on the sandbox PATH; skipping the no-interpreter case"
+else
+    if PATH="$SANDBOX_PATH" "$RESOLVER" -c 'pass' \
+        >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
+        fail "resolver succeeded with no usable interpreter on PATH"
+    fi
+    grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
+        || fail "missing-interpreter failure did not name the requirement"
+    grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
+        || fail "missing-interpreter failure did not point at the override"
 fi
-grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
-    || fail "missing-interpreter failure did not name the requirement"
-grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
-    || fail "missing-interpreter failure did not point at the override"
 
 echo "✅ scripts/python_bin.sh resolver checks passed"

scripts/test_security_workflow.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Run the security workflow's evidence and result steps without remote VMs."""
+"""Exercise functional chain dispatch and security evidence without remote VMs."""
 
 from __future__ import annotations
 
@@ -18,16 +18,20 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
 CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
 
 
+def named_steps(job: list[str]) -> dict[str, list[str]]:
+    starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
+    return {
+        job[start].split(": ", 1)[1].strip('"'): job[start:end]
+        for start, end in zip(starts, starts[1:] + [len(job)])
+    }
+
+
 class SecurityWorkflowTests(unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
         self.assertIsNotNone(self.job)
-        starts = [i for i, line in enumerate(self.job) if line.startswith("      - name: ")]
-        self.steps = {
-            self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end]
-            for start, end in zip(starts, starts[1:] + [len(self.job)])
-        }
+        self.steps = named_steps(self.job)
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
         self.directory = Path(self.temp.name)
@@ -192,6 +196,110 @@ class SecurityWorkflowTests(unittest.TestCase):
         self.assertNotIn("OLD RUN REPORT", body.read_text())
         self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
+    def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
+        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
+            with self.subTest(suite=suite):
+                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
+                # Workflow-level concurrency covers every job, including cleanup,
+                # regardless of trigger or the runner hosting the job.
+                self.assertEqual([
+                    line.strip() for line in yaml_block(source, "concurrency", 0)
+                    if line.strip() and not line.lstrip().startswith("#")
+                ], [
+                    "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
+                ])
+                self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
+                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
+                cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
+                cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
+                self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
+
+    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
+        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
+            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
+                self.setUp()
+                fake_bin = self.directory / "bin"
+                fake_bin.mkdir()
+                commands = {
+                    "gh": '''#!/usr/bin/env bash
+set -euo pipefail
+if [ "$1" = api ]; then
+  printf '%s\\n' "$*" >> "$DISPATCHES"
+  attempt=$(wc -l < "$DISPATCHES")
+  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
+elif [ "$1 $2" = 'issue create' ]; then
+  printf 'issue\\n' >> "$EXECUTED"
+  while [ "$#" -gt 0 ]; do
+    if [ "$1" = --body-file ]; then
+      cat "$2" > "$CAPTURE_BODY"
+      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
+    fi
+    shift
+  done
+  exit "$ISSUE_EXIT"
+else
+  exit 99
+fi
+''',
+                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
+                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
+                }
+                for name, contents in commands.items():
+                    command = fake_bin / name
+                    command.write_text(contents)
+                    command.chmod(0o755)
+                dispatches = self.directory / "dispatches"
+                executed = self.directory / "executed"
+                body = self.directory / "issue-body.md"
+                body_path = self.directory / "issue-body-path"
+                self.env.update(
+                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
+                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
+                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
+                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
+                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
+                )
+                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
+                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
+                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
+                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
+                started = self.run_step("Dispatch first suite (upgrade)")
+                self.assertEqual(started.returncode, 0, started.stderr)
+                self.assertEqual(dispatches.read_text().splitlines(), [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
+                ])
+                dispatches.unlink()
+
+                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
+                job = yaml_block(replication.splitlines(), "replication-test", 2)
+                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
+                self.steps = named_steps(job)
+                handoff = "Continue functional chain (next: Performance)"
+                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
+                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
+                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
+                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
+                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
+                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
+                failed = self.run_step("Run replication suite")
+                self.assertEqual(failed.returncode, 17, failed.stderr)
+                cleaned = self.run_step("Cleanup environment (after)")
+                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
+                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
+                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
+                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
+                forwarded = self.run_step(handoff)
+                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
+                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
+                self.assertEqual(calls, [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
+                ] * (min(failed_attempts + 1, 3) if token else 0))
+                if failed_attempts == 3:
+                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
+                    self.assertIn("rustfs-chain-performance", body.read_text())
+                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
+                    self.assertFalse(Path(body_path.read_text().strip()).exists())
+
 
 if __name__ == "__main__":
     unittest.main()