diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 995cc9b23..666c83b4c 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -54,9 +54,6 @@ env: jobs: heal-test: runs-on: smoke-testing - # Requirement: a failing suite must not fail the workflow; failures - # are filed to rustfs/backlog and the chain continues. - continue-on-error: true timeout-minutes: 480 # Standalone manual run, or one link of the nightly functional chain # (storage -> heal -> pool). Pool expansion no longer re-runs heal. diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 642c4b5e7..8647c9268 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -49,7 +49,6 @@ env: jobs: kms-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -109,7 +108,6 @@ jobs: - name: Run KMS suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-kms.log run: | diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml index 6d960052c..ff3e2978d 100644 --- a/.github/workflows/rustfs-performance-test.yml +++ b/.github/workflows/rustfs-performance-test.yml @@ -84,9 +84,6 @@ env: jobs: performance-test: runs-on: pf-testing - # Requirement: a failing benchmark must not fail the workflow; - # failures are filed to rustfs/backlog. - continue-on-error: true timeout-minutes: 900 # Run on manual dispatch, or when the nightly build completed successfully. # Skipped when nightly failed. diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index d7002c446..3c9836b2d 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -76,9 +76,6 @@ jobs: pool-expansion-test: name: Pool expansion / decommission test runs-on: smoke-testing - # Requirement: a failing suite must not fail the workflow; failures - # are filed to rustfs/backlog and the chain continues. - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} env: diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index b17864f72..74c57b6d5 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -62,9 +62,6 @@ env: jobs: replication-test: runs-on: smoke-testing - # A failed replication run must not break the chain or the workflow: the - # failure is reported to rustfs/backlog instead (see the issue step). - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -116,7 +113,6 @@ jobs: - name: Run replication suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-replication.log run: | diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index d3fff002b..80856194c 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -37,7 +37,6 @@ env: jobs: s3-compat-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -88,7 +87,6 @@ jobs: - name: Run S3 compatibility suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-s3-compat.log run: | diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 1dceda80d..767f734dc 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -46,7 +46,6 @@ env: jobs: storage-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -97,7 +96,6 @@ jobs: - name: Run storage engine suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-storage.log run: | diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 4ac80e609..5d9a2c1d7 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -61,9 +61,6 @@ env: jobs: tier-test: runs-on: smoke-testing - # Requirement: a failing suite must not fail the workflow; failures - # are filed to rustfs/backlog and the chain continues. - continue-on-error: true timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index 0c8c72cd0..0b4c19af1 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -79,7 +79,6 @@ env: jobs: upgrade-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 420 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: @@ -142,7 +141,6 @@ jobs: - name: Run upgrade compatibility suite id: test - continue-on-error: true env: LOG_FILE: /tmp/rustfs-upgrade.log GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 93770b74d..99d589d64 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -91,6 +91,12 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline. +## Packaged functional acceptance + +`rustfs-functional-chain.yml` dispatches the packaged-build suites in `rustfs-*-test.yml` on the shared lab runners. A failing suite step or job must fail its workflow. Report collection, cleanup, and dispatch of the next suite can still run with `always()`; continuing diagnostics does not make the failed suite successful. + +Workflow status preserves errors that the test scripts report. It does not establish complete execution or a common package identity across the chain: inspect the current run's case results, package identity, and test-script revision as well. A script that returns zero after a failed tool invocation needs its own result check. + ## Release validation Post-merge and tag-driven; not a substitute for a PR gate. diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index ae82d3fb1..ea2d75487 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run the security workflow's evidence and result steps without remote VMs.""" +"""Exercise functional workflow failures and security evidence without remote VMs.""" from __future__ import annotations @@ -18,16 +18,32 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml" CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |" +def named_steps(job: list[str]) -> dict[str, list[str]]: + starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")] + return { + job[start].split(": ", 1)[1].strip('"'): job[start:end] + for start, end in zip(starts, starts[1:] + [len(job)]) + } + + +def shell_body(lines: list[str]) -> str: + start = lines.index(" run: |") + 1 + shell_lines = [] + for line in lines[start:]: + if line.strip() and not line.startswith(" "): + break + shell_lines.append(line[10:]) + if not shell_lines: + raise ValueError("missing literal shell body") + return "\n".join(shell_lines) + + class SecurityWorkflowTests(unittest.TestCase): def setUp(self) -> None: self.source = WORKFLOW.read_text() self.job = yaml_block(self.source.splitlines(), "security-test", 2) self.assertIsNotNone(self.job) - starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")] - self.steps = { - self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end] - for start, end in zip(starts, starts[1:] + [len(self.job)]) - } + self.steps = named_steps(self.job) self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.directory = Path(self.temp.name) @@ -83,15 +99,8 @@ class SecurityWorkflowTests(unittest.TestCase): def run_step(self, name: str) -> subprocess.CompletedProcess[str]: lines = self.steps[name] - start = lines.index(" run: |") + 1 - shell_lines = [] - for line in lines[start:]: - if line.strip() and not line.startswith(" "): - break - shell_lines.append(line[10:]) - self.assertTrue(shell_lines, f"missing literal shell body: {name}") result = subprocess.run( - ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))], + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))], cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, ) for line in lines: @@ -193,5 +202,89 @@ class SecurityWorkflowTests(unittest.TestCase): self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) +class FunctionalWorkflowTests(unittest.TestCase): + JOBS = { + "kms": "kms-test", "storage": "storage-test", "s3-compat": "s3-compat-test", + "upgrade": "upgrade-test", "replication": "replication-test", "heal": "heal-test", + "tier": "tier-test", "pool-expand": "pool-expansion-test", "performance": "performance-test", + } + DIRECT_TESTS = { + "kms": "Run KMS suite", "storage": "Run storage engine suite", + "s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite", + "replication": "Run replication suite", + } + + def test_failure_and_always_step_wiring(self) -> None: + for suite, job_id in self.JOBS.items(): + with self.subTest(suite=suite): + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() + job = yaml_block(source.splitlines(), job_id, 2) + self.assertIsNotNone(job) + self.assertNotRegex("\n".join(job), r'''(?m)^ ["']?continue-on-error["']?\s*:''') + steps = named_steps(job) + if suite in self.DIRECT_TESTS: + test = steps[self.DIRECT_TESTS[suite]] + self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''') + self.assertIn(" if: always()", steps["Generate report"]) + cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"] + condition = next(line.strip() for line in cleanup if line.startswith(" if:")) + self.assertIn(condition, ( + "if: always()", + "if: ${{ always() && inputs.cleanup_after != 'false' }}", + "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}", + )) + if suite != "performance": + handoff = steps["Chain complete"] if suite == "replication" else next( + value for name, value in steps.items() if name.startswith("Continue functional chain") + ) + self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff) + + def test_failed_suite_preserves_exit_and_cleanup_and_dispatch_execute(self) -> None: + for suite, test_name in self.DIRECT_TESTS.items(): + with self.subTest(suite=suite), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "auto-testing").mkdir() + script = root / f"auto-testing/rustfs-{suite}-test.sh" + script.write_text('#!/bin/sh\nprintf "partial suite diagnostics\\n"\nexit 17\n') + script.chmod(0o755) + fake_bin = root / "bin" + fake_bin.mkdir() + for command, marker in (("ssh", "cleanup"), ("gh", "dispatch")): + fake = fake_bin / command + fake.write_text(f'#!/bin/sh\nprintf "{marker}\\n" >> "$EXECUTED"\n') + fake.chmod(0o755) + env = { + **os.environ, "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}", + "EXECUTED": str(root / "executed"), "RUSTFS_NODES": "fixture-node", + "RUSTFS_SSH_USER": "fixture-user", "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", + "GH_TOKEN": "local-fixture", "GITHUB_EVENT_NAME": "repository_dispatch", "GITHUB_RUN_ID": "314159", + } + source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() + steps = named_steps(yaml_block(source.splitlines(), self.JOBS[suite], 2)) + context = {"github.event_name": "repository_dispatch", "steps.test.outcome": "failure"} + for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", source): + if expression.startswith("inputs.") and re.fullmatch(r"inputs\.\w+", expression): + context[expression] = "" + def execute(name): + lines = steps[name] + rendered = re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: context[match[1]], shell_body(lines)) + return subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered], + cwd=root, env={**env, "LOG_FILE": str(root / "suite.log")}, capture_output=True, text=True, + ) + failed = execute(test_name) + self.assertEqual(failed.returncode, 17, failed.stderr) + self.assertIn("partial suite diagnostics", failed.stdout) + cleanup = execute("Cleanup environment (after)") + self.assertEqual(cleanup.returncode, 0, cleanup.stderr) + handoff_name = "Chain complete" if suite == "replication" else next( + name for name in steps if name.startswith("Continue functional chain") + ) + handoff = execute(handoff_name) + self.assertEqual(handoff.returncode, 0, handoff.stderr) + markers = (root / "executed").read_text().splitlines() + self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"]) + + if __name__ == "__main__": unittest.main()