From 782e80000d43b0ef803367332579ffa8b6c0a8de Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 5 Sep 2026 19:34:52 +0800 Subject: [PATCH] fix(ci): serialize performance on shared functional VMs --- .github/workflows/rustfs-functional-chain.yml | 17 +-- .github/workflows/rustfs-performance-test.yml | 9 +- .github/workflows/rustfs-replication-test.yml | 53 ++++++-- scripts/test_security_workflow.py | 120 +++++++++++++++++- 4 files changed, 163 insertions(+), 36 deletions(-) diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index 6ce828c1a..10b9c67ad 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -14,8 +14,8 @@ # Functional chain driver: runs the ten functional suites in a fixed order # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security -> -# replication, with performance on its own runner in parallel) and guarantees -# the chain keeps moving even when individual suites fail. +# replication -> performance). Each suite attempts the next handoff even +# when its tests fail. # # Each suite workflow can still be dispatched standalone (workflow_dispatch); # only chain-triggered runs forward to the next suite via repository_dispatch, @@ -59,16 +59,3 @@ jobs: gh api --method POST repos/rustfs/rustfs/dispatches \ -f event_type='rustfs-chain-upgrade' \ -F 'client_payload[from_suite]=nightly-build' - - - name: Dispatch performance suite (parallel, own runner) - env: - GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} - run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then - echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2 - exit 1 - fi - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-performance' \ - -F 'client_payload[from_suite]=nightly-build' diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index 6d960052c..7a8faae46 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -49,17 +49,16 @@ on: type: boolean default: true repository_dispatch: - # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own - # pf-testing runner, in parallel with the shared-VM chain). + # Chain handoff: dispatched when the replication suite finishes. types: [rustfs-chain-performance] permissions: contents: read -# Dedicated pf-testing runner/environment: own concurrency group so perf runs -# never block (or are blocked by) the pool-expansion / heal tests. +# The default performance nodes overlap the other suites' remote VMs, even +# though the runner differs. Hold the shared lock through cleanup as well. concurrency: - group: rustfs-performance-test + group: rustfs-shared-functional-tests cancel-in-progress: false defaults: diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index b17864f72..5cba8d29f 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -34,8 +34,7 @@ on: - site default: all repository_dispatch: - # Chain handoff: dispatched when the security suite finishes. This is the - # last link of the functional chain. + # Chain handoff: dispatched when the security suite finishes. types: [rustfs-chain-replication] permissions: @@ -62,9 +61,6 @@ env: jobs: replication-test: runs-on: smoke-testing - # A failed replication run must not break the chain or the workflow: the - # failure is reported to rustfs/backlog instead (see the issue step). - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -349,13 +345,50 @@ jobs: ' done - - name: Chain complete - # Replication is the last link of the functional chain: nothing to - # dispatch after it. This step just records that the chain finished. + - name: "Continue functional chain (next: Performance)" if: ${{ always() && github.event_name == 'repository_dispatch' }} + env: + GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - echo "Functional chain complete: replication (final suite) finished." - echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}" + set -uo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 + exit 1 + fi + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-performance' \ + -F 'client_payload[from_suite]=replication'; then + echo "dispatched next suite Performance (attempt ${attempt})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2 + sleep "${attempt}0" + done + if [ "${DISPATCHED:-0}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2 + TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})" + BODY_FILE="$(mktemp)" + trap 'rm -f "${BODY_FILE}"' EXIT + { + echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts." + echo "" + echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Expected next event: 'rustfs-chain-performance'" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'" + FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}" + } > "${BODY_FILE}" + gh issue create -R rustfs/backlog --title "${TITLE}" \ + --body-file "${BODY_FILE}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index ae82d3fb1..827e015be 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run the security workflow's evidence and result steps without remote VMs.""" +"""Exercise functional chain dispatch and security evidence without remote VMs.""" from __future__ import annotations @@ -18,16 +18,20 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml" CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |" +def named_steps(job: list[str]) -> dict[str, list[str]]: + starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")] + return { + job[start].split(": ", 1)[1].strip('"'): job[start:end] + for start, end in zip(starts, starts[1:] + [len(job)]) + } + + class SecurityWorkflowTests(unittest.TestCase): def setUp(self) -> None: self.source = WORKFLOW.read_text() self.job = yaml_block(self.source.splitlines(), "security-test", 2) self.assertIsNotNone(self.job) - starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")] - self.steps = { - self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end] - for start, end in zip(starts, starts[1:] + [len(self.job)]) - } + self.steps = named_steps(self.job) self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.directory = Path(self.temp.name) @@ -192,6 +196,110 @@ class SecurityWorkflowTests(unittest.TestCase): self.assertNotIn("OLD RUN REPORT", body.read_text()) self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None: + for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"): + with self.subTest(suite=suite): + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines() + # Workflow-level concurrency covers every job, including cleanup, + # regardless of trigger or the runner hosting the job. + self.assertEqual([ + line.strip() for line in yaml_block(source, "concurrency", 0) + if line.strip() and not line.lstrip().startswith("#") + ], [ + "group: rustfs-shared-functional-tests", "cancel-in-progress: false", + ]) + self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2)) + self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2)) + cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)" + cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name] + self.assertTrue(any(line.startswith(" if:") and "always()" in line for line in cleanup)) + + def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None: + for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")): + with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)): + self.setUp() + fake_bin = self.directory / "bin" + fake_bin.mkdir() + commands = { + "gh": '''#!/usr/bin/env bash +set -euo pipefail +if [ "$1" = api ]; then + printf '%s\\n' "$*" >> "$DISPATCHES" + attempt=$(wc -l < "$DISPATCHES") + [ "$attempt" -gt "$FAILED_ATTEMPTS" ] +elif [ "$1 $2" = 'issue create' ]; then + printf 'issue\\n' >> "$EXECUTED" + while [ "$#" -gt 0 ]; do + if [ "$1" = --body-file ]; then + cat "$2" > "$CAPTURE_BODY" + printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH" + fi + shift + done + exit "$ISSUE_EXIT" +else + exit 99 +fi +''', + "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n', + "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n', + } + for name, contents in commands.items(): + command = fake_bin / name + command.write_text(contents) + command.chmod(0o755) + dispatches = self.directory / "dispatches" + executed = self.directory / "executed" + body = self.directory / "issue-body.md" + body_path = self.directory / "issue-body-path" + self.env.update( + PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches), + EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path), + FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit), + RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user", + RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb", + ) + self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"}) + driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text() + self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2)) + self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"]) + started = self.run_step("Dispatch first suite (upgrade)") + self.assertEqual(started.returncode, 0, started.stderr) + self.assertEqual(dispatches.read_text().splitlines(), [ + "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build", + ]) + dispatches.unlink() + + replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text() + job = yaml_block(replication.splitlines(), "replication-test", 2) + self.assertFalse(any(line.startswith(" continue-on-error:") for line in job)) + self.steps = named_steps(job) + handoff = "Continue functional chain (next: Performance)" + self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff]) + self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff])) + self.assertIn(" if: always()", self.steps["Cleanup environment (after)"]) + self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff)) + suite = self.directory / "auto-testing/rustfs-replication-test.sh" + suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n') + failed = self.run_step("Run replication suite") + self.assertEqual(failed.returncode, 17, failed.stderr) + cleaned = self.run_step("Cleanup environment (after)") + self.assertEqual(cleaned.returncode, 0, cleaned.stderr) + self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"]) + self.env["FAILED_ATTEMPTS"] = str(failed_attempts) + self.context["secrets.PF_TESTING_GH_TOKEN"] = token + forwarded = self.run_step(handoff) + self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr) + calls = dispatches.read_text().splitlines() if dispatches.exists() else [] + self.assertEqual(calls, [ + "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication", + ] * (min(failed_attempts + 1, 3) if token else 0)) + if failed_attempts == 3: + self.assertIn("could not hand off from **replication** to **Performance**", body.read_text()) + self.assertIn("rustfs-chain-performance", body.read_text()) + self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1) + self.assertFalse(Path(body_path.read_text().strip()).exists()) + if __name__ == "__main__": unittest.main()