fix(ci): serialize performance on shared functional VMs

.github/pull_request_template.md

@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
 
 ## Summary of Changes
 <!--
-Briefly explain what changed and why reviewers should accept it.
-Focus on behavior, compatibility, and review-relevant context.
+Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
 -->
 
 ## Verification
 <!--
-List the commands or checks you ran, for example:
-- `make pre-commit`
+Give 1–3 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
 
-Use N/A only when verification is not applicable.
+Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
+
+List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
 -->
 
 ## Impact

.github/workflows/rustfs-functional-chain.yml

@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication, with performance on its own runner in parallel) and guarantees
-# the chain keeps moving even when individual suites fail.
+# replication -> performance). Each suite attempts the next handoff even
+# when its tests fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,16 +59,3 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
-
-      - name: Dispatch performance suite (parallel, own runner)
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
-            exit 1
-          fi
-          gh api --method POST repos/rustfs/rustfs/dispatches \
-            -f event_type='rustfs-chain-performance' \
-            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-performance-test.yml

@@ -49,17 +49,16 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
-    # pf-testing runner, in parallel with the shared-VM chain).
+    # Chain handoff: dispatched when the replication suite finishes.
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# Dedicated pf-testing runner/environment: own concurrency group so perf runs
-# never block (or are blocked by) the pool-expansion / heal tests.
+# The default performance nodes overlap the other suites' remote VMs, even
+# though the runner differs. Hold the shared lock through cleanup as well.
 concurrency:
-  group: rustfs-performance-test
+  group: rustfs-shared-functional-tests
   cancel-in-progress: false
 
 defaults:

.github/workflows/rustfs-replication-test.yml

@@ -34,8 +34,7 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes. This is the
-    # last link of the functional chain.
+    # Chain handoff: dispatched when the security suite finishes.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -62,9 +61,6 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
-    # A failed replication run must not break the chain or the workflow: the
-    # failure is reported to rustfs/backlog instead (see the issue step).
-    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
@@ -349,13 +345,50 @@ jobs:
             '
           done
 
-      - name: Chain complete
-        # Replication is the last link of the functional chain: nothing to
-        # dispatch after it. This step just records that the chain finished.
+      - name: "Continue functional chain (next: Performance)"
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          echo "Functional chain complete: replication (final suite) finished."
-          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
+          set -uo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+            exit 1
+          fi
+          DISPATCHED=0
+          for attempt in 1 2 3; do
+            if gh api --method POST repos/rustfs/rustfs/dispatches \
+              -f event_type='rustfs-chain-performance' \
+              -F 'client_payload[from_suite]=replication'; then
+              echo "dispatched next suite Performance (attempt ${attempt})"
+              DISPATCHED=1
+              break
+            fi
+            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
+            sleep "${attempt}0"
+          done
+          if [ "${DISPATCHED:-0}" -ne 1 ]; then
+            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
+            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
+            BODY_FILE="$(mktemp)"
+            trap 'rm -f "${BODY_FILE}"' EXIT
+            {
+              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
+              echo ""
+              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+              echo "- Expected next event: 'rustfs-chain-performance'"
+              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
+              echo "- Recovery: re-dispatch manually with"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
+              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
+            } > "${BODY_FILE}"
+            gh issue create -R rustfs/backlog --title "${TITLE}" \
+              --body-file "${BODY_FILE}" --label functional-test \
+              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
+              || echo "could not file the stall alert issue either; check the token" >&2
+            exit 1
+          fi
 
       - name: Notify on failure
         if: failure()

scripts/test_security_workflow.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Run the security workflow's evidence and result steps without remote VMs."""
+"""Exercise functional chain dispatch and security evidence without remote VMs."""
 
 from __future__ import annotations
 
@@ -18,16 +18,20 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
 CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
 
 
+def named_steps(job: list[str]) -> dict[str, list[str]]:
+    starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
+    return {
+        job[start].split(": ", 1)[1].strip('"'): job[start:end]
+        for start, end in zip(starts, starts[1:] + [len(job)])
+    }
+
+
 class SecurityWorkflowTests(unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
         self.assertIsNotNone(self.job)
-        starts = [i for i, line in enumerate(self.job) if line.startswith("      - name: ")]
-        self.steps = {
-            self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end]
-            for start, end in zip(starts, starts[1:] + [len(self.job)])
-        }
+        self.steps = named_steps(self.job)
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
         self.directory = Path(self.temp.name)
@@ -192,6 +196,110 @@ class SecurityWorkflowTests(unittest.TestCase):
         self.assertNotIn("OLD RUN REPORT", body.read_text())
         self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
+    def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
+        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
+            with self.subTest(suite=suite):
+                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
+                # Workflow-level concurrency covers every job, including cleanup,
+                # regardless of trigger or the runner hosting the job.
+                self.assertEqual([
+                    line.strip() for line in yaml_block(source, "concurrency", 0)
+                    if line.strip() and not line.lstrip().startswith("#")
+                ], [
+                    "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
+                ])
+                self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
+                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
+                cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
+                cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
+                self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
+
+    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
+        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
+            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
+                self.setUp()
+                fake_bin = self.directory / "bin"
+                fake_bin.mkdir()
+                commands = {
+                    "gh": '''#!/usr/bin/env bash
+set -euo pipefail
+if [ "$1" = api ]; then
+  printf '%s\\n' "$*" >> "$DISPATCHES"
+  attempt=$(wc -l < "$DISPATCHES")
+  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
+elif [ "$1 $2" = 'issue create' ]; then
+  printf 'issue\\n' >> "$EXECUTED"
+  while [ "$#" -gt 0 ]; do
+    if [ "$1" = --body-file ]; then
+      cat "$2" > "$CAPTURE_BODY"
+      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
+    fi
+    shift
+  done
+  exit "$ISSUE_EXIT"
+else
+  exit 99
+fi
+''',
+                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
+                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
+                }
+                for name, contents in commands.items():
+                    command = fake_bin / name
+                    command.write_text(contents)
+                    command.chmod(0o755)
+                dispatches = self.directory / "dispatches"
+                executed = self.directory / "executed"
+                body = self.directory / "issue-body.md"
+                body_path = self.directory / "issue-body-path"
+                self.env.update(
+                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
+                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
+                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
+                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
+                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
+                )
+                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
+                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
+                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
+                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
+                started = self.run_step("Dispatch first suite (upgrade)")
+                self.assertEqual(started.returncode, 0, started.stderr)
+                self.assertEqual(dispatches.read_text().splitlines(), [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
+                ])
+                dispatches.unlink()
+
+                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
+                job = yaml_block(replication.splitlines(), "replication-test", 2)
+                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
+                self.steps = named_steps(job)
+                handoff = "Continue functional chain (next: Performance)"
+                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
+                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
+                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
+                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
+                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
+                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
+                failed = self.run_step("Run replication suite")
+                self.assertEqual(failed.returncode, 17, failed.stderr)
+                cleaned = self.run_step("Cleanup environment (after)")
+                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
+                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
+                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
+                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
+                forwarded = self.run_step(handoff)
+                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
+                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
+                self.assertEqual(calls, [
+                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
+                ] * (min(failed_attempts + 1, 3) if token else 0))
+                if failed_attempts == 3:
+                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
+                    self.assertIn("rustfs-chain-performance", body.read_text())
+                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
+                    self.assertFalse(Path(body_path.read_text().strip()).exists())
+
 
 if __name__ == "__main__":
     unittest.main()