From a6b5da64f27c2c0c82fe8ce5592a0e9e80138655 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 6 Sep 2026 00:15:03 +0800 Subject: [PATCH] fix(ci): serialize performance on shared functional VMs (#7204) --- .github/workflows/rustfs-functional-chain.yml | 17 +-- .github/workflows/rustfs-performance-test.yml | 9 +- .github/workflows/rustfs-replication-test.yml | 50 ++++++-- scripts/test_security_workflow.py | 112 +++++++++++++++++- 4 files changed, 157 insertions(+), 31 deletions(-) diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index 6ce828c1a..10b9c67ad 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -14,8 +14,8 @@ # Functional chain driver: runs the ten functional suites in a fixed order # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security -> -# replication, with performance on its own runner in parallel) and guarantees -# the chain keeps moving even when individual suites fail. +# replication -> performance). Each suite attempts the next handoff even +# when its tests fail. # # Each suite workflow can still be dispatched standalone (workflow_dispatch); # only chain-triggered runs forward to the next suite via repository_dispatch, @@ -59,16 +59,3 @@ jobs: gh api --method POST repos/rustfs/rustfs/dispatches \ -f event_type='rustfs-chain-upgrade' \ -F 'client_payload[from_suite]=nightly-build' - - - name: Dispatch performance suite (parallel, own runner) - env: - GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} - run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then - echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2 - exit 1 - fi - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-performance' \ - -F 'client_payload[from_suite]=nightly-build' diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index 148e5ccf3..d698b27e0 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -49,17 +49,16 @@ on: type: boolean default: true repository_dispatch: - # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own - # pf-testing runner, in parallel with the shared-VM chain). + # Chain handoff: dispatched when the replication suite finishes. types: [rustfs-chain-performance] permissions: contents: read -# Dedicated pf-testing runner/environment: own concurrency group so perf runs -# never block (or are blocked by) the pool-expansion / heal tests. +# The default performance nodes overlap the other suites' remote VMs, even +# though the runner differs. Hold the shared lock through cleanup as well. concurrency: - group: rustfs-performance-test + group: rustfs-shared-functional-tests cancel-in-progress: false defaults: diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index ae8f9dc50..0faaf7809 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -34,8 +34,7 @@ on: - site default: all repository_dispatch: - # Chain handoff: dispatched when the security suite finishes. This is the - # last link of the functional chain. + # Chain handoff: dispatched when the security suite finishes. types: [rustfs-chain-replication] permissions: @@ -330,13 +329,50 @@ jobs: ' done - - name: Chain complete - # Replication is the last link of the functional chain: nothing to - # dispatch after it. This step just records that the chain finished. + - name: "Continue functional chain (next: Performance)" if: ${{ always() && github.event_name == 'repository_dispatch' }} + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - echo "Functional chain complete: replication (final suite) finished." - echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}" + set -uo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 + exit 1 + fi + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-performance' \ + -F 'client_payload[from_suite]=replication'; then + echo "dispatched next suite Performance (attempt ${attempt})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2 + sleep "${attempt}0" + done + if [ "${DISPATCHED:-0}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2 + TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})" + BODY_FILE="$(mktemp)" + trap 'rm -f "${BODY_FILE}"' EXIT + { + echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts." + echo "" + echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Expected next event: 'rustfs-chain-performance'" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'" + FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" + } > "${BODY_FILE}" + gh issue create -R rustfs/backlog --title "${TITLE}" \ + --body-file "${BODY_FILE}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index 4b4b063dd..b96b3e3f5 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Exercise functional workflow failures and security evidence without remote VMs.""" +"""Exercise functional failures, chain dispatch, and security evidence without remote VMs.""" from __future__ import annotations @@ -272,6 +272,110 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase): self.assertNotIn("OLD RUN REPORT", body.read_text()) self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None: + for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"): + with self.subTest(suite=suite): + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines() + # Workflow-level concurrency covers every job, including cleanup, + # regardless of trigger or the runner hosting the job. + self.assertEqual([ + line.strip() for line in yaml_block(source, "concurrency", 0) + if line.strip() and not line.lstrip().startswith("#") + ], [ + "group: rustfs-shared-functional-tests", "cancel-in-progress: false", + ]) + self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2)) + self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2)) + cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)" + cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name] + self.assertTrue(any(line.startswith(" if:") and "always()" in line for line in cleanup)) + + def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None: + for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")): + with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)): + self.setUp() + fake_bin = self.directory / "bin" + fake_bin.mkdir() + commands = { + "gh": '''#!/usr/bin/env bash +set -euo pipefail +if [ "$1" = api ]; then + printf '%s\\n' "$*" >> "$DISPATCHES" + attempt=$(wc -l < "$DISPATCHES") + [ "$attempt" -gt "$FAILED_ATTEMPTS" ] +elif [ "$1 $2" = 'issue create' ]; then + printf 'issue\\n' >> "$EXECUTED" + while [ "$#" -gt 0 ]; do + if [ "$1" = --body-file ]; then + cat "$2" > "$CAPTURE_BODY" + printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH" + fi + shift + done + exit "$ISSUE_EXIT" +else + exit 99 +fi +''', + "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n', + "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n', + } + for name, contents in commands.items(): + command = fake_bin / name + command.write_text(contents) + command.chmod(0o755) + dispatches = self.directory / "dispatches" + executed = self.directory / "executed" + body = self.directory / "issue-body.md" + body_path = self.directory / "issue-body-path" + self.env.update( + PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches), + EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path), + FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit), + RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user", + RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb", + ) + self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"}) + driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text() + self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2)) + self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"]) + started = self.run_step("Dispatch first suite (upgrade)") + self.assertEqual(started.returncode, 0, started.stderr) + self.assertEqual(dispatches.read_text().splitlines(), [ + "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build", + ]) + dispatches.unlink() + + replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text() + job = yaml_block(replication.splitlines(), "replication-test", 2) + self.assertFalse(any(line.startswith(" continue-on-error:") for line in job)) + self.steps = named_steps(job) + handoff = "Continue functional chain (next: Performance)" + self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff]) + self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff])) + self.assertIn(" if: always()", self.steps["Cleanup environment (after)"]) + self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff)) + suite = self.directory / "auto-testing/rustfs-replication-test.sh" + suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n') + failed = self.run_step("Run replication suite") + self.assertEqual(failed.returncode, 17, failed.stderr) + cleaned = self.run_step("Cleanup environment (after)") + self.assertEqual(cleaned.returncode, 0, cleaned.stderr) + self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"]) + self.env["FAILED_ATTEMPTS"] = str(failed_attempts) + self.context["secrets.PF_TESTING_GH_TOKEN"] = token + forwarded = self.run_step(handoff) + self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr) + calls = dispatches.read_text().splitlines() if dispatches.exists() else [] + self.assertEqual(calls, [ + "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication", + ] * (min(failed_attempts + 1, 3) if token else 0)) + if failed_attempts == 3: + self.assertIn("could not hand off from **replication** to **Performance**", body.read_text()) + self.assertIn("rustfs-chain-performance", body.read_text()) + self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1) + self.assertFalse(Path(body_path.read_text().strip()).exists()) + class FunctionalWorkflowTests(unittest.TestCase): JOBS = { @@ -305,7 +409,7 @@ class FunctionalWorkflowTests(unittest.TestCase): "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}", )) if suite != "performance": - handoff = steps["Chain complete"] if suite == "replication" else next( + handoff = next( value for name, value in steps.items() if name.startswith("Continue functional chain") ) self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff) @@ -348,13 +452,13 @@ class FunctionalWorkflowTests(unittest.TestCase): self.assertIn("partial suite diagnostics", failed.stdout) cleanup = execute("Cleanup environment (after)") self.assertEqual(cleanup.returncode, 0, cleanup.stderr) - handoff_name = "Chain complete" if suite == "replication" else next( + handoff_name = next( name for name in steps if name.startswith("Continue functional chain") ) handoff = execute(handoff_name) self.assertEqual(handoff.returncode, 0, handoff.stderr) markers = (root / "executed").read_text().splitlines() - self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"]) + self.assertEqual(markers, ["cleanup", "dispatch"]) class FunctionalCaseReportTests(unittest.TestCase):