mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 16:50:45 +00:00
hector
589b499e3b
* ci(ft): update the lane gate to the 54-case manifest with scenario F auto-testing #118/#119 added FT scenario F and case B5, so the lane now runs 54 cases over scenarios A,B,C,C2,D,E,F and passes 54/54 — but the workflow gate still pinned the 44-case A-E manifest and failed the report step. Align the gate and the contract-test fixture with the 54-case manifest. * ci(heal): give heal a 90-minute run budget inside a 120-minute job The heal task was still progressing (scanned 494 -> 571+, failed=0) when the 45-minute step cap killed it, so the verification steps never ran and the lane produced invalid evidence (chain 40796474356). The step budget is now 90 minutes, and the job-level budget rises from 60 to 120 minutes so the enclosing job cannot cancel the raised step budget. The timeout contract now records heal as the exception to the shared 60/45 baseline. * ci(chain): force the performance probe offline while pf-testing is broken The pf-testing environment is temporarily broken, so every nightly chain would end red in the performance lane. performance is required=false in scheduled-validations.json, so forcing the probe's performance_ready output to offline takes the chain through its designed skip path: the lane job is skipped, aggregate receives --allow-skipped performance, and the report records the optional-lane skip with its reason. Marked TEMPORARY; restore the probe once the environment is healthy again. * test(table): drop a stale workflow-comment pin left by #9059 #9059 rewrote the table suite step comments but kept the assertion for the old '# Red only for harness/environment breakdowns' line, so test_security_workflow.py fails against current main. The semantics stay pinned by the continue-on-error and HARNESS_OK assertions.
1200 lines
71 KiB
Python
1200 lines
71 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise functional failures, cleanup isolation, and evidence without remote VMs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from check_test_wiring import yaml_block
|
|
from functional_case_report import generate_report
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
|
|
CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
|
|
|
|
|
|
def named_steps(job: list[str]) -> dict[str, list[str]]:
|
|
starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")]
|
|
return {
|
|
job[start].split(": ", 1)[1].strip('"'): job[start:end]
|
|
for start, end in zip(starts, starts[1:] + [len(job)])
|
|
}
|
|
|
|
|
|
def shell_body(lines: list[str]) -> str:
|
|
start = lines.index(" run: |") + 1
|
|
shell_lines = []
|
|
for line in lines[start:]:
|
|
if line.strip() and not line.startswith(" "):
|
|
break
|
|
shell_lines.append(line[10:])
|
|
if not shell_lines:
|
|
raise ValueError("missing literal shell body")
|
|
return "\n".join(shell_lines)
|
|
|
|
|
|
class WorkflowSteps:
|
|
def uploaded_files(self) -> set[Path]:
|
|
upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines))
|
|
start = upload.index(" path: |") + 1
|
|
paths = []
|
|
for line in upload[start:]:
|
|
if not line.startswith(" "):
|
|
break
|
|
paths.extend(Path(path) for path in glob.glob(self.render(line.strip())))
|
|
return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()}
|
|
|
|
def render(self, value: str) -> str:
|
|
return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
|
|
|
|
def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
|
|
result = {}
|
|
for line in yaml_block(lines, "env", indent) or []:
|
|
if line.strip() and not line.lstrip().startswith("#"):
|
|
key, value = line.strip().split(": ", 1)
|
|
result[key] = self.render(value.strip("'\""))
|
|
return result
|
|
|
|
def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
|
|
lines = self.steps[name]
|
|
result = subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
|
|
cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
|
|
)
|
|
for line in lines:
|
|
if line.startswith(" id: "):
|
|
self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
|
|
if Path(self.env["GITHUB_ENV"]).exists():
|
|
for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
|
|
key, value = line.split("=", 1)
|
|
self.env[key] = value
|
|
self.context[f"env.{key}"] = value
|
|
return result
|
|
|
|
|
|
class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.source = WORKFLOW.read_text()
|
|
self.job = yaml_block(self.source.splitlines(), "security-test", 2)
|
|
self.assertIsNotNone(self.job)
|
|
self.steps = named_steps(self.job)
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.directory = Path(self.temp.name)
|
|
self.context = {
|
|
"runner.temp": self.temp.name,
|
|
"github.server_url": "https://github.com",
|
|
"github.repository": "rustfs/rustfs",
|
|
"github.run_id": "314159",
|
|
"github.run_attempt": "2",
|
|
"github.sha": "0123456789abcdef0123456789abcdef01234567",
|
|
"github.event_name": "workflow_dispatch",
|
|
"github.workspace": self.temp.name,
|
|
"inputs.package_url": "",
|
|
"inputs.rustfs_version": "test-version",
|
|
"inputs.topology": "all",
|
|
"inputs.oidc_live": "false",
|
|
"steps.evidence.outcome": "skipped",
|
|
"steps.test.outcome": "skipped",
|
|
"steps.report.outcome": "skipped",
|
|
}
|
|
self.env = {
|
|
**os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"),
|
|
"GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name,
|
|
}
|
|
for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
|
|
self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
|
|
self.context["env.SECURITY_ARTIFACTS_DIR"] = ""
|
|
self.artifacts = self.directory / "rustfs-security-314159-2"
|
|
suite = self.directory / "auto-testing/rustfs-security-test.sh"
|
|
suite.parent.mkdir()
|
|
ansi = {"ANSI_GREEN": "\033[1;32m", "ANSI_RED": "\033[1;31m", "ANSI_YELLOW": "\033[1;33m", "ANSI_RESET": "\033[0m"}
|
|
suite.write_text(
|
|
'#!/usr/bin/env bash\nset -euo pipefail\n'
|
|
'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
|
|
# Verdict lines carry ANSI color escapes and are printed to stdout
|
|
# (captured via tee into the artifacts suite.log), exactly like the
|
|
# real suite output the report step has to grep through: a color
|
|
# tag before the verdict and a reset escape between the tag and
|
|
# the case id.
|
|
'printf "%s\\n" "${ANSI_GREEN}[PASS]${ANSI_RESET} IAM-101 ok" "${ANSI_RED}[FAIL]${ANSI_RESET} STS-105 broken" "${ANSI_YELLOW}[SKIP]${ANSI_RESET} OIDC-103 skipped" | tee "$log_dir/suite.log"\n'
|
|
'echo "CURRENT SUITE LOG" >> "$log_dir/suite.log"\n'
|
|
'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n'
|
|
'case "$FAKE_REPORT" in\n'
|
|
f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
|
|
' empty) : > "$REPORT_FILE" ;;\n'
|
|
'esac\n'
|
|
'echo "UNWRAPPED SUITE SUMMARY" >> "$GITHUB_STEP_SUMMARY"\n'
|
|
'exit "$FAKE_EXIT"\n'
|
|
)
|
|
self.env.update(ansi)
|
|
|
|
def test_workflow_wiring(self) -> None:
|
|
names = list(self.steps)
|
|
self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts"))
|
|
self.assertNotIn(" continue-on-error: true", self.job)
|
|
self.assertIn(" continue-on-error: true", self.steps["Run security suite"])
|
|
for name in ("Initialize security evidence", "Generate report"):
|
|
self.assertNotIn(" continue-on-error: true", self.steps[name])
|
|
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"])
|
|
self.assertNotIn("/tmp/rustfs-security", self.source)
|
|
for name in ("Upload functional report to dashboard", "Manage backlog issues (dedup / label / auto-close)"):
|
|
expected = "${{ env.SECURITY_ARTIFACTS_DIR }}/report.md" if name.startswith("Upload") else '--report-file "${SECURITY_ARTIFACTS_DIR}/report.md"'
|
|
self.assertIn(expected, "\n".join(self.steps[name]))
|
|
for name in ("Upload functional report to dashboard", "Upload report and logs"):
|
|
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
|
|
artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
|
|
self.assertIn(" path: |", artifact_settings)
|
|
self.assertIn(" if-no-files-found: error", artifact_settings)
|
|
|
|
def test_suite_report_and_result_matrix(self) -> None:
|
|
for outcome, mode, exit_code in (
|
|
("success", "present", 0), ("failure", "present", 7), ("failure", "missing", 7),
|
|
("success", "missing", 0), ("success", "empty", 0),
|
|
("skipped", "missing", 0), ("skipped", "present", 0),
|
|
("cancelled", "missing", 0), ("cancelled", "present", 0),
|
|
):
|
|
with self.subTest(outcome=outcome, report=mode):
|
|
self.setUp()
|
|
initialized = self.run_step("Initialize security evidence")
|
|
self.assertEqual(initialized.returncode, 0, initialized.stderr)
|
|
self.assertEqual(self.env["SECURITY_ARTIFACTS_DIR"], str(self.artifacts))
|
|
self.env.update(FAKE_REPORT=mode, FAKE_EXIT=str(exit_code))
|
|
if outcome != "skipped" or mode == "present":
|
|
suite = self.run_step("Run security suite")
|
|
self.assertEqual(suite.returncode, exit_code, suite.stderr)
|
|
logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log"))
|
|
self.assertEqual(len(logs), 1)
|
|
self.assertEqual(logs[0].read_text().splitlines()[-1], "CURRENT SUITE LOG")
|
|
self.context["steps.test.outcome"] = outcome
|
|
# A suite that never ran (skipped with no report, or a
|
|
# cancelled run) leaves no suite.log behind, so there are no
|
|
# verdict lines and the report step stays red.
|
|
ran = outcome != "skipped" or mode == "present"
|
|
report = self.run_step("Generate report")
|
|
# The report step is red only for harness/environment breakdowns:
|
|
# a failed suite that still produced verdict lines stays green,
|
|
# while skipped/cancelled never reach a verdict at all.
|
|
success = outcome == "success" and mode == "present"
|
|
green = ran and (outcome == "success" or outcome == "failure")
|
|
self.assertEqual(report.returncode == 0, green, report.stderr)
|
|
contents = (self.artifacts / "report.md").read_text()
|
|
for expected in (
|
|
"https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2",
|
|
f"Workflow Commit: {self.context['github.sha']}", "Trigger: workflow_dispatch",
|
|
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}",
|
|
):
|
|
self.assertIn(expected, contents)
|
|
# The verdict counters must see through the ANSI escapes in the
|
|
# suite log: 1 passed, 1 failed, 1 skipped (only when the suite
|
|
# actually ran and left a suite.log behind).
|
|
if ran:
|
|
self.assertIn("Product result: 1 passed, 1 failed, 1 skipped", contents)
|
|
else:
|
|
self.assertIn("Product result: 0 passed, 0 failed", contents)
|
|
self.assertEqual(CASE_ROW in contents, mode == "present")
|
|
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, mode == "present")
|
|
if mode == "present":
|
|
raw = (self.artifacts / "suite-report.md").read_text()
|
|
self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n")
|
|
summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
|
|
self.assertEqual(summary, contents)
|
|
self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
|
|
expected = {self.artifacts / "report.md"}
|
|
if outcome != "skipped" or mode == "present":
|
|
expected.add(self.artifacts / "suite.log")
|
|
self.assertIn("CURRENT SUITE STDOUT\nCURRENT SUITE STDERR", (self.artifacts / "suite.log").read_text())
|
|
if mode in ("present", "empty"):
|
|
expected.add(self.artifacts / "suite-report.md")
|
|
(self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY")
|
|
scratch = Path(str(self.artifacts) + "-scratch")
|
|
(scratch / "case.out").write_text("FAKE-SECRET-CANARY")
|
|
self.assertEqual(self.uploaded_files(), expected)
|
|
|
|
|
|
def test_oidc_negative_responses_never_print_issued_credentials(self):
|
|
source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
|
|
for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")):
|
|
start = source.index('[[ "${' + variable + '}" == 403 ]]')
|
|
end = source.index("\n", source.index("grep -q '<Code>AccessDenied</Code>'", start))
|
|
guard = source[start:end]
|
|
credential_xml = "<Credentials><AccessKeyId>FAKE-ACCESS-CANARY</AccessKeyId><SecretAccessKey>FAKE-SECRET-CANARY</SecretAccessKey><SessionToken>FAKE-SESSION-CANARY</SessionToken></Credentials>"
|
|
for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1),
|
|
("403", "<Code>AccessDenied</Code>", 0)):
|
|
with self.subTest(variable=variable, status=status, expected=expected):
|
|
(self.directory / filename).write_text(body)
|
|
result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard],
|
|
env={**self.env, variable: status, "WORK_DIR": str(self.directory)},
|
|
capture_output=True, text=True)
|
|
self.assertEqual(result.returncode, expected, result.stderr)
|
|
for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
|
|
self.assertNotIn(canary, result.stdout + result.stderr)
|
|
if status == "200":
|
|
self.assertIn("HTTP 403, got 200", result.stderr)
|
|
|
|
def test_oidc_incomplete_credentials_report_only_the_missing_field(self):
|
|
source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
|
|
start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY")
|
|
end = source.index("\n)\n", start) + 3
|
|
extract = source[start:end]
|
|
values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY",
|
|
"SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z",
|
|
"SubjectFromWebIdentityToken": "alice"}
|
|
for missing in (None, "Expiration", "SubjectFromWebIdentityToken"):
|
|
with self.subTest(missing=missing):
|
|
xml = "<Credentials>" + "".join(f"<{key}>{value}</{key}>" for key, value in values.items() if key != missing) + "</Credentials>"
|
|
(self.directory / "sts-good.xml").write_text(xml)
|
|
result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract],
|
|
env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True)
|
|
self.assertEqual(result.returncode, 1 if missing else 0, result.stderr)
|
|
for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
|
|
self.assertNotIn(canary, result.stdout + result.stderr)
|
|
if missing:
|
|
self.assertIn(f"missing required STS field: {missing}", result.stderr)
|
|
|
|
def test_existing_evidence_directory_is_rejected(self) -> None:
|
|
for suffix in ("", "-scratch"):
|
|
self.setUp()
|
|
existing = Path(str(self.artifacts) + suffix)
|
|
existing.mkdir()
|
|
stale = existing / "suite-report.md"
|
|
stale.write_text("OLD RUN REPORT")
|
|
self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
|
|
self.assertEqual(stale.read_text(), "OLD RUN REPORT")
|
|
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
|
|
(self.artifacts / "report.md").write_text("OLD RUN REPORT")
|
|
self.context.update({
|
|
"env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
|
|
})
|
|
fake_bin = self.directory / "bin"
|
|
fake_bin.mkdir()
|
|
gh = fake_bin / "gh"
|
|
gh.write_text(
|
|
'#!/usr/bin/env bash\nset -euo pipefail\n'
|
|
'if [ "$1 $2" = "issue create" ]; then\n'
|
|
' while [ "$#" -gt 0 ]; do\n'
|
|
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
|
|
' shift\n'
|
|
' done\n'
|
|
'fi\n'
|
|
)
|
|
gh.chmod(0o755)
|
|
body = self.directory / "issue-body.md"
|
|
self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
|
|
result = self.run_step("Manage backlog issues (dedup / label / auto-close)")
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
# The manager lives in the private auto-testing checkout; without it
|
|
# the step must skip without publishing anything.
|
|
self.assertIn("issue_manager.py not found", result.stdout + result.stderr)
|
|
self.assertFalse(body.exists())
|
|
|
|
def test_all_suites_hold_the_expected_lock_for_manual_and_chain_runs(self) -> None:
|
|
for suite in (*FunctionalWorkflowTests.JOBS, "security"):
|
|
with self.subTest(suite=suite):
|
|
source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
|
|
# Workflow-level concurrency covers every job, including cleanup,
|
|
# regardless of trigger or the runner hosting the job. Performance
|
|
# uses a separate fleet and therefore has a dedicated group.
|
|
expected_group = "rustfs-performance-suite" if suite == "performance" else "rustfs-shared-functional-tests-v2"
|
|
self.assertEqual([
|
|
line.strip() for line in yaml_block(source, "concurrency", 0)
|
|
if line.strip() and not line.lstrip().startswith("#")
|
|
], [
|
|
f"group: {expected_group}", "cancel-in-progress: false",
|
|
])
|
|
self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
|
|
self.assertIsNotNone(yaml_block(source, "workflow_call", 2))
|
|
self.assertIsNone(yaml_block(source, "repository_dispatch", 2))
|
|
self.assertNotIn("Continue functional chain", "\n".join(source))
|
|
cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
|
|
cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
|
|
self.assertTrue(any(line.startswith(" if:") and "always()" in line for line in cleanup))
|
|
|
|
def test_reusable_chain_continues_selected_suites_after_predecessor_failure(self) -> None:
|
|
source = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text().splitlines()
|
|
ordered = ("upgrade", "s3", "kms", "tier", "storage", "heal", "pool", "security",
|
|
"replication", "fault-tolerance", "table")
|
|
for index, suite in enumerate((*ordered, "performance")):
|
|
with self.subTest(suite=suite):
|
|
job = yaml_block(source, suite, 2)
|
|
self.assertIsNotNone(job)
|
|
dependencies = "prepare" if index == 0 or suite == "performance" else f"prepare, {ordered[index - 1]}"
|
|
self.assertIn(f" needs: [{dependencies}]", job)
|
|
condition = (
|
|
" if: ${{ always() && needs.prepare.result == 'success' && "
|
|
f"contains(fromJSON(needs.prepare.outputs.suites || '[]'), '{suite}')"
|
|
)
|
|
if suite == "performance":
|
|
condition += " && needs.prepare.outputs.performance_ready == 'online'"
|
|
self.assertIn(condition + " }}", job)
|
|
self.assertFalse(any("continue-on-error:" in line for line in job))
|
|
self.assertIn(" chain_manifest: ${{ needs.prepare.outputs.manifest }}", job)
|
|
aggregate = yaml_block(source, "complete-chain", 2)
|
|
self.assertIn(" if: ${{ always() && needs.prepare.result != 'skipped' }}", aggregate)
|
|
steps = named_steps(aggregate)
|
|
for name in ("Summarize every suite including failures and missing evidence",
|
|
"Retain chain report regardless of test verdict", "Verify every required suite"):
|
|
self.assertIn(" if: always()", steps[name])
|
|
names = list(steps)
|
|
self.assertLess(names.index("Retain chain report regardless of test verdict"),
|
|
names.index("Verify every required suite"))
|
|
|
|
|
|
class FunctionalWorkflowTests(unittest.TestCase):
|
|
JOBS = {
|
|
"kms": "kms-test", "storage": "storage-test", "s3-compat": "s3-compat-test",
|
|
"upgrade": "upgrade-test", "replication": "replication-test", "heal": "heal-test",
|
|
"tier": "tier-test", "pool-expand": "pool-expansion-test", "performance": "performance-test",
|
|
"table": "table-test", "fault-tolerance": "fault-tolerance-test",
|
|
}
|
|
DIRECT_TESTS = {
|
|
"kms": "Run KMS suite", "storage": "Run storage engine suite",
|
|
"s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite",
|
|
"replication": "Run replication suite",
|
|
"table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2, D, E)",
|
|
}
|
|
|
|
def test_failure_and_always_step_wiring(self) -> None:
|
|
for suite, job_id in {**self.JOBS, "security": "security-test"}.items():
|
|
if suite in getattr(self, "EXCLUDED", ()):
|
|
continue
|
|
with self.subTest(suite=suite):
|
|
source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
|
|
job = yaml_block(source.splitlines(), job_id, 2)
|
|
self.assertIsNotNone(job)
|
|
self.assertNotRegex("\n".join(job), r'''(?m)^ ["']?continue-on-error["']?\s*:''')
|
|
steps = named_steps(job)
|
|
if suite in self.DIRECT_TESTS:
|
|
test = steps[self.DIRECT_TESTS[suite]]
|
|
# Collect reports after a suite failure; the final verdict
|
|
# still rejects failed, cancelled, and skipped suites.
|
|
self.assertRegex("\n".join(test), r'''(?m)^ continue-on-error: true$''')
|
|
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
|
|
cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
|
|
condition = next(line.strip() for line in cleanup if line.startswith(" if:"))
|
|
if suite == "pool-expand":
|
|
self.assertEqual(condition, "if: ${{ always() && steps.topology.outcome == 'success' && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}")
|
|
self.assertIn(" id: topology", steps["Validate pool topology before destructive cleanup"])
|
|
self.assertLess(list(steps).index("Validate pool topology before destructive cleanup"),
|
|
list(steps).index("Cleanup environment (before)"))
|
|
else:
|
|
self.assertIn(condition, (
|
|
"if: ${{ always() && steps.environment.outcome == 'success' }}",
|
|
"if: ${{ always() && inputs.cleanup_after != 'false' && steps.environment.outcome == 'success' }}",
|
|
"if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') && steps.environment.outcome == 'success' }}",
|
|
))
|
|
self.assertIn(" id: environment_cleanup", cleanup)
|
|
self.assertNotIn(" continue-on-error: true", cleanup)
|
|
release = steps["Release environment only after successful cleanup"]
|
|
self.assertIn(" if: ${{ always() && steps.environment.outcome == 'success' }}", release)
|
|
self.assertIn(" CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}", release)
|
|
self.assertTrue(any('finish --suite ' in line and '--cleanup-outcome "$CLEANUP_OUTCOME"' in line
|
|
for line in release))
|
|
names = list(steps)
|
|
admission = next(name for name, lines in steps.items() if " id: environment" in lines)
|
|
self.assertLess(names.index(admission), names.index("Reset test environment (before)" if suite == "performance" else "Cleanup environment (before)"))
|
|
self.assertLess(names.index("Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"),
|
|
names.index("Release environment only after successful cleanup"))
|
|
verdict = steps["Enforce suite and report result"]
|
|
self.assertIn(" if: ${{ always() }}", verdict)
|
|
self.assertNotIn(" continue-on-error: true", verdict)
|
|
self.assertEqual(names[-1], "Enforce suite and report result")
|
|
self.assertLess(names.index("Upload chain evidence"), names.index("Enforce suite and report result"))
|
|
|
|
def test_failed_suite_preserves_exit_and_cleanup_failures(self) -> None:
|
|
for suite, test_name, cleanup_exit in ((suite, name, status)
|
|
for suite, name in self.DIRECT_TESTS.items() for status in (0, 23)):
|
|
with self.subTest(suite=suite, cleanup_exit=cleanup_exit), tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / "auto-testing").mkdir()
|
|
script = root / f"auto-testing/rustfs-{suite}-test.sh"
|
|
if suite == "fault-tolerance":
|
|
# FT cleanup must propagate its own status independently of
|
|
# the preceding suite failure.
|
|
script.write_text('#!/bin/sh\nif [ "$1" = "--cleanup" ]; then\n'
|
|
' printf "cleanup\\n" >> "$EXECUTED"; exit "$CLEANUP_EXIT"\n'
|
|
'fi\nprintf "partial suite diagnostics\\n"\nexit 17\n')
|
|
else:
|
|
script.write_text('#!/bin/sh\nprintf "partial suite diagnostics\\n"\nexit 17\n')
|
|
script.chmod(0o755)
|
|
fake_bin = root / "bin"
|
|
fake_bin.mkdir()
|
|
fake = fake_bin / "ssh"
|
|
fake.write_text('#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n'
|
|
'case "$*" in *fixture-user@clean-node*) exit 0 ;; esac\n'
|
|
'exit "$CLEANUP_EXIT"\n')
|
|
fake.chmod(0o755)
|
|
nodes = "failure-node clean-node" if suite == "table" else "fixture-node"
|
|
env = {
|
|
**os.environ, "PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}",
|
|
"EXECUTED": str(root / "executed"), "RUSTFS_NODES": nodes,
|
|
"RUSTFS_SSH_USER": "fixture-user", "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb",
|
|
"GH_TOKEN": "local-fixture", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_RUN_ID": "314159",
|
|
"CLEANUP_EXIT": str(cleanup_exit),
|
|
}
|
|
source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
|
|
steps = named_steps(yaml_block(source.splitlines(), self.JOBS[suite], 2))
|
|
context = {"github.event_name": "workflow_dispatch", "steps.test.outcome": "failure"}
|
|
if suite == "table":
|
|
context["steps.chain_package.outputs.package_url || inputs.package_url"] = env["RUSTFS_NIGHTLY_PACKAGE_URL"]
|
|
for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", source):
|
|
if expression.startswith("inputs.") and re.fullmatch(r"inputs\.\w+", expression):
|
|
context[expression] = ""
|
|
elif "steps.chain_package.outputs.package_url" in expression:
|
|
# composite fallback expression used by the FT suite
|
|
context[expression] = ""
|
|
elif "inputs." in expression:
|
|
# composite conditions (e.g. always() && inputs.cleanup_after != 'false'):
|
|
# blank every inputs.* token so the literal guard stays meaningful
|
|
context[expression] = re.sub(r"inputs\.\w+", "", expression)
|
|
def execute(name):
|
|
lines = steps[name]
|
|
rendered = re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: context[match[1]], shell_body(lines))
|
|
return subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered],
|
|
cwd=root, env={**env, "LOG_FILE": str(root / "suite.log")}, capture_output=True, text=True,
|
|
)
|
|
failed = execute(test_name)
|
|
self.assertEqual(failed.returncode, 17, failed.stderr)
|
|
self.assertIn("partial suite diagnostics", failed.stdout)
|
|
cleanup = execute("Cleanup environment (after)")
|
|
self.assertEqual(cleanup.returncode, cleanup_exit, cleanup.stderr)
|
|
self.assertEqual((root / "executed").read_text().splitlines(), ["cleanup"] * len(nodes.split()))
|
|
|
|
def test_final_verdict_rejects_failed_outcomes_even_when_report_is_valid(self) -> None:
|
|
cases = [(suite_outcome, report_outcome, 0)
|
|
for suite_outcome in ("success", "failure", "skipped", "cancelled")
|
|
for report_outcome in ("success", "failure", "skipped", "cancelled")]
|
|
cases.append(("success", "success", 9))
|
|
for suite, job_id in {**self.JOBS, "security": "security-test"}.items():
|
|
source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
|
|
steps = named_steps(yaml_block(source, job_id, 2))
|
|
verdict = steps["Enforce suite and report result"]
|
|
report_name = "Upload report to dashboard (reports/YYYY-MM-DD.md)" if suite == "performance" else "Generate report"
|
|
report_id = next(line.split(": ", 1)[1] for line in steps[report_name] if line.startswith(" id: "))
|
|
self.assertIn(" REPORT_OUTCOME: ${{ steps." + report_id + ".outcome }}", verdict)
|
|
suite_id = "benchmark" if suite == "performance" else "pool_test" if suite == "pool-expand" else "test"
|
|
self.assertIn(" SUITE_OUTCOME: ${{ steps." + suite_id + ".outcome }}", verdict)
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
validator = root / "python3"
|
|
validator.write_text('#!/bin/sh\nprintf "%s\\n" "$*" > "$VALIDATOR_CALL"\nexit "$VALIDATOR_EXIT"\n')
|
|
validator.chmod(0o755)
|
|
marker = root / "called"
|
|
for suite_outcome, report_outcome, report_exit in cases:
|
|
with self.subTest(suite=suite, suite_outcome=suite_outcome,
|
|
report_outcome=report_outcome, report_exit=report_exit):
|
|
marker.unlink(missing_ok=True)
|
|
result = subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", shell_body(verdict)],
|
|
env={**os.environ, "PATH": f"{root}{os.pathsep}{os.environ['PATH']}",
|
|
"SUITE_OUTCOME": suite_outcome, "REPORT_OUTCOME": report_outcome,
|
|
"VALIDATOR_CALL": str(marker), "VALIDATOR_EXIT": str(report_exit)},
|
|
capture_output=True, text=True,
|
|
)
|
|
outcomes_passed = suite_outcome == report_outcome == "success"
|
|
self.assertEqual(result.returncode == 0, outcomes_passed and report_exit == 0, result.stderr)
|
|
self.assertEqual(marker.exists(), outcomes_passed)
|
|
if marker.exists():
|
|
self.assertIn("functional_chain_evidence.py verify-report --suite ", marker.read_text())
|
|
|
|
|
|
class FaultToleranceWorkflowContractTests(unittest.TestCase):
|
|
TESTING_SHA = "a" * 40
|
|
|
|
def setUp(self) -> None:
|
|
source = (ROOT / ".github/workflows/rustfs-fault-tolerance-test.yml").read_text()
|
|
job = yaml_block(source.splitlines(), "fault-tolerance-test", 2)
|
|
self.assertIsNotNone(job)
|
|
self.steps = named_steps(job)
|
|
self.report_body = shell_body(self.steps["Generate report"])
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.directory = Path(self.temp.name)
|
|
self.artifacts = self.directory / "artifacts"
|
|
(self.artifacts / "evidence").mkdir(parents=True)
|
|
self.log = self.artifacts / "suite.log"
|
|
self.log.write_text("FT-REPORT: fixture\n")
|
|
self.results = self.artifacts / "results.json"
|
|
self.output = self.directory / "github-output"
|
|
self.env = {
|
|
**os.environ,
|
|
"AUTO_TESTING_SHA": self.TESTING_SHA,
|
|
"FUNCTIONAL_ARTIFACTS_DIR": str(self.artifacts),
|
|
"LOG_FILE": str(self.log),
|
|
"REPORT_FILE": str(self.artifacts / "report.md"),
|
|
"RESULTS_FILE": str(self.results),
|
|
"GITHUB_OUTPUT": str(self.output),
|
|
"GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"),
|
|
"GITHUB_SERVER_URL": "https://github.com",
|
|
"GITHUB_REPOSITORY": "rustfs/rustfs",
|
|
"GITHUB_RUN_ID": "314159",
|
|
}
|
|
|
|
def write_result(self, *, seen: int = 54, complete: bool = True,
|
|
duplicate: bool = False, testing_sha: str | None = None,
|
|
unrelated_seen: bool = False, missing_verdict: bool = False) -> None:
|
|
expected = [f"CASE-{index:02d}" for index in range(54)]
|
|
seen_cases = expected[:seen]
|
|
if unrelated_seen:
|
|
seen_cases = [f"OTHER-{index:02d}" for index in range(seen)]
|
|
case_ids = list(seen_cases)
|
|
if unrelated_seen:
|
|
case_ids = expected[:seen]
|
|
if duplicate:
|
|
case_ids[-1] = case_ids[0]
|
|
payload = {
|
|
"schema_version": 1,
|
|
"auto_testing_sha": testing_sha or self.TESTING_SHA,
|
|
"selected_scenarios": ["A", "B", "C", "C2", "D", "E", "F"],
|
|
"complete": complete,
|
|
"expected_cases": expected,
|
|
"seen_cases": seen_cases,
|
|
"missing_cases": expected[seen:],
|
|
"harness_errors": [],
|
|
"counts": {
|
|
"expected": 54,
|
|
"seen": seen,
|
|
"unexpected": 1,
|
|
"known_divergence": 0,
|
|
"harness_errors": 0,
|
|
},
|
|
"cases": [
|
|
{"case_id": case_id, "verdict": "UNEXPECTED" if index == 0 else "pass", "detail": "fixture"}
|
|
for index, case_id in enumerate(case_ids)
|
|
],
|
|
}
|
|
if missing_verdict:
|
|
payload["cases"][0].pop("verdict")
|
|
self.results.write_text(json.dumps(payload))
|
|
|
|
def run_report(self) -> subprocess.CompletedProcess[str]:
|
|
replacements = {
|
|
"inputs.package_url || 'nightly (R2 latest)'": "fixture-package",
|
|
"inputs.strict || 'false'": "false",
|
|
}
|
|
rendered = re.sub(
|
|
r"\$\{\{\s*(.*?)\s*\}\}",
|
|
lambda match: replacements[match[1]],
|
|
self.report_body,
|
|
)
|
|
return subprocess.run(
|
|
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered],
|
|
cwd=self.directory,
|
|
env=self.env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
def test_complete_product_failure_is_valid_harness_evidence(self) -> None:
|
|
self.write_result()
|
|
result = self.run_report()
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(self.output.read_text().splitlines()[-1], "complete=true")
|
|
|
|
def test_partial_or_missing_results_fail_the_harness_gate(self) -> None:
|
|
self.write_result(seen=17, complete=False)
|
|
partial = self.run_report()
|
|
self.assertNotEqual(partial.returncode, 0)
|
|
self.assertEqual(self.output.read_text().splitlines()[-1], "complete=false")
|
|
|
|
self.results.unlink()
|
|
self.output.unlink()
|
|
missing = self.run_report()
|
|
self.assertNotEqual(missing.returncode, 0)
|
|
self.assertEqual(self.output.read_text().splitlines()[-1], "complete=false")
|
|
|
|
def test_duplicate_cases_or_revision_mismatch_fail_the_harness_gate(self) -> None:
|
|
self.write_result(duplicate=True)
|
|
self.assertNotEqual(self.run_report().returncode, 0)
|
|
|
|
self.output.unlink()
|
|
self.write_result(testing_sha="b" * 40)
|
|
self.assertNotEqual(self.run_report().returncode, 0)
|
|
|
|
def test_missing_verdict_or_unrelated_seen_cases_fail_the_harness_gate(self) -> None:
|
|
self.write_result(missing_verdict=True)
|
|
self.assertNotEqual(self.run_report().returncode, 0)
|
|
|
|
self.output.unlink()
|
|
self.write_result(unrelated_seen=True)
|
|
self.assertNotEqual(self.run_report().returncode, 0)
|
|
|
|
def test_backlog_manager_only_runs_after_complete_report(self) -> None:
|
|
condition = next(
|
|
line.strip() for line in self.steps["Manage backlog issues (dedup / label / auto-close)"]
|
|
if line.startswith(" if:")
|
|
)
|
|
self.assertEqual(
|
|
condition,
|
|
"if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}",
|
|
)
|
|
|
|
def test_auto_testing_ref_is_resolved_once_and_recorded(self) -> None:
|
|
checkout = "\n".join(self.steps["Checkout auto-testing scripts"])
|
|
record = "\n".join(self.steps["Record auto-testing revision"])
|
|
self.assertIn("steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main'", checkout)
|
|
self.assertIn("git -C auto-testing rev-parse HEAD", record)
|
|
self.assertIn("AUTO_TESTING_SHA", record)
|
|
|
|
|
|
class FunctionalCaseReportTests(unittest.TestCase):
|
|
def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
log = root / "suite.log"
|
|
if text is not None:
|
|
log.write_text(text)
|
|
valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
|
|
return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
|
|
|
|
def test_repeated_case_executions_preserve_failure_and_context(self):
|
|
# log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
|
|
log = subprocess.check_output(["bash", "-c", r'''
|
|
log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
|
|
log '== topology: single-single kms-backend: local =='
|
|
printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
|
|
log '== topology: single-multi kms-backend: vault-kv2 =='
|
|
printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
|
|
printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
|
|
'''], text=True)
|
|
valid, cases, _ = self.report(log)
|
|
self.assertFalse(valid)
|
|
self.assertEqual(cases.count("| KMS-101 |"), 3)
|
|
self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
|
|
self.assertIn("roundtrip (topology: single-single kms-backend: local) | FAIL |", cases)
|
|
self.assertIn("roundtrip (topology: single-multi kms-backend: vault-kv2) | PASS |", cases)
|
|
self.assertNotIn("\\n", cases)
|
|
|
|
def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
|
|
for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
|
|
"--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
|
|
"--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
|
|
with self.subTest(log=text):
|
|
valid, cases, _ = self.report(text)
|
|
self.assertFalse(valid)
|
|
self.assertIn("## Case Summary", cases)
|
|
valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
|
|
self.assertTrue(valid)
|
|
self.assertIn("suite: bucket replication", cases)
|
|
self.assertIn("- UNSUPPORTED: 1\n", cases)
|
|
|
|
def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
|
|
case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
|
|
for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
|
|
("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
|
|
with self.subTest(matrix=suffix):
|
|
valid, _, matrix = self.report(case + suffix, matrix=True)
|
|
self.assertEqual(valid, expected)
|
|
self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
|
|
self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
|
|
|
|
def test_s3_case_identifiers_include_digits(self):
|
|
valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
|
|
self.assertTrue(valid)
|
|
self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
|
|
|
|
|
|
class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
|
|
# "table" and "fault-tolerance" follow the release-branch semantics and
|
|
# their own report shapes (table: functional_case_report; FT: FT-CASE
|
|
# lines + chain evidence). They are excluded from the legacy matrices via
|
|
# EXCLUDED (report matrix tail + wiring legacy-issue section) while the
|
|
# upload-allowlist matrix still covers them through the full list.
|
|
SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
|
|
EXCLUDED = ("performance", "table", "fault-tolerance")
|
|
|
|
def prepare(self, suite: str) -> None:
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.directory = Path(self.temp.name)
|
|
self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
|
|
self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
|
|
self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
|
|
self.context.update({
|
|
"github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
|
|
"github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
|
|
"github.event_name": "workflow_dispatch", "steps.test.outcome": "success",
|
|
"secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
|
|
})
|
|
self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
|
|
self.env = {
|
|
**os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
|
|
"GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
|
|
"RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
|
|
}
|
|
for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
|
|
self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
|
|
(self.directory / "scripts").mkdir()
|
|
(self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
|
|
fake_bin = self.directory / "bin"
|
|
fake_bin.mkdir()
|
|
(fake_bin / "python3").symlink_to(sys.executable)
|
|
for command, body in (
|
|
("ssh", 'printf "fixture-version\\n"\n'),
|
|
("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
|
|
' while [ "$#" -gt 0 ]; do\n'
|
|
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
|
|
' shift\n'
|
|
' done\n'
|
|
'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
|
|
):
|
|
script = fake_bin / command
|
|
script.write_text("#!/bin/sh\n" + body)
|
|
script.chmod(0o755)
|
|
self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
|
|
|
|
def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
|
|
for suite, suffix in ((suite, suffix) for suite in self.SUITES if suite not in ("table", "fault-tolerance") for suffix in ("", "-scratch")):
|
|
with self.subTest(suite=suite, collision=suffix or "artifact"):
|
|
self.prepare(suite)
|
|
self.assertNotIn("/tmp/rustfs-", self.source)
|
|
names = list(self.steps)
|
|
self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts"))
|
|
if suite in FunctionalWorkflowTests.DIRECT_TESTS:
|
|
self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts"))
|
|
for name, lines in self.steps.items():
|
|
if name == "Retain environment ownership and cleanup receipt":
|
|
self.assertIn(" if: ${{ always() }}", lines)
|
|
suite_id = "s3" if suite == "s3-compat" else suite
|
|
self.assertIn(" path: ${{ runner.temp }}/environment-" + suite_id + ".json", lines)
|
|
self.assertIn(" if-no-files-found: error", lines)
|
|
self.assertFalse(any("*" in line for line in lines))
|
|
continue
|
|
if name == "Upload chain evidence":
|
|
self.assertIn(" if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", lines)
|
|
self.assertIn(" if: ${{ always() && inputs.chain_manifest != '' && steps.evidence.outcome == 'success' }}", self.steps["Record chain evidence"])
|
|
self.assertIn(" if-no-files-found: error", lines)
|
|
continue
|
|
if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
|
|
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
|
|
if any("uses: actions/upload-artifact@" in line for line in lines):
|
|
self.assertIn(" path: |", lines)
|
|
self.assertIn(" if-no-files-found: error", lines)
|
|
existing = Path(str(self.artifacts) + suffix)
|
|
existing.mkdir()
|
|
for filename in ("report.md", "suite.log"):
|
|
(existing / filename).write_text("OLD RUN EVIDENCE")
|
|
self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log"))
|
|
initialized = self.run_step("Initialize functional evidence")
|
|
self.assertNotEqual(initialized.returncode, 0)
|
|
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
|
|
manager = self.run_step("Manage backlog issues (dedup / label / auto-close)")
|
|
self.assertEqual(manager.returncode, 0, manager.stderr)
|
|
self.assertIn("issue_manager.py not found", manager.stdout + manager.stderr)
|
|
self.assertFalse(Path(self.env["CAPTURE_BODY"]).exists())
|
|
self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
|
|
|
|
def test_reports_use_only_current_complete_suite_evidence(self):
|
|
for suite in (s for s in self.SUITES if s not in self.EXCLUDED):
|
|
good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
|
|
partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
|
|
if suite == "s3-compat":
|
|
good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
|
|
if suite == "upgrade":
|
|
good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
|
|
if suite == "heal":
|
|
good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
|
|
partial = "[HEAL-STEP] 1 fixture PASS\n"
|
|
for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
|
|
("success", None), ("skipped", None), ("cancelled", good)):
|
|
with self.subTest(suite=suite, outcome=outcome, log=log):
|
|
self.prepare(suite)
|
|
stale = self.directory / "old-suite.log"
|
|
stale.write_text("OLD RUN EVIDENCE\n" + good)
|
|
self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
|
|
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
|
|
self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
|
|
self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch")
|
|
if log is not None:
|
|
Path(self.env["LOG_FILE"]).write_text(log)
|
|
self.context["steps.test.outcome"] = outcome
|
|
report = self.run_step("Generate report")
|
|
success = outcome == "success" and log == good
|
|
# Report steps are red only for harness/environment breakdowns;
|
|
# a failure outcome with recorded case rows stays green
|
|
# (performance is unchanged and still gates on the suite result).
|
|
green = (outcome == "success" or (outcome == "failure" and log in (good, partial))) if suite != "performance" else success
|
|
self.assertEqual(report.returncode == 0, green, report.stderr)
|
|
contents = Path(self.env["REPORT_FILE"]).read_text()
|
|
self.assertNotIn("OLD RUN EVIDENCE", contents)
|
|
self.assertEqual("| PASS |" in contents, success if suite == "performance" else log in (good, partial))
|
|
for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
|
|
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
|
|
self.assertIn(value, contents)
|
|
self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
|
|
evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
|
|
if log in (good, partial):
|
|
self.assertIn("| PASS |", evidence)
|
|
self.assertNotIn("OLD RUN EVIDENCE", evidence)
|
|
|
|
def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
|
|
for suite in self.SUITES:
|
|
with self.subTest(suite=suite):
|
|
self.prepare(suite)
|
|
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
|
|
if suite == "heal":
|
|
self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log"))
|
|
scripts = self.directory / "auto-testing"
|
|
scripts.mkdir()
|
|
filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
|
|
script = scripts / filename
|
|
script.write_text(
|
|
'#!/bin/bash\nset -euo pipefail\nlog=""\n'
|
|
'while [ "$#" -gt 0 ]; do\n'
|
|
' if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
|
|
' shift\n'
|
|
'done\n'
|
|
'[ "$log" = "$LOG_FILE" ] || exit 31\n'
|
|
'printf "CURRENT SUITE LOG\\n" > "$log"\n'
|
|
'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
|
|
'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
|
|
'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
|
|
' mkdir -p "$RUSTFS_RESULT_DIR"\n'
|
|
' printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
|
|
'fi\n'
|
|
)
|
|
script.chmod(0o755)
|
|
name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
|
|
"Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
|
|
)
|
|
result = self.run_step(name)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual((self.artifacts / "suite.log").read_text().splitlines()[-1], "CURRENT SUITE LOG")
|
|
self.assertEqual(list(self.artifacts.glob("fixture.*")), [])
|
|
if suite == "performance":
|
|
self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
|
|
|
|
def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
|
|
extra = {
|
|
"kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
|
|
"upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"], "table": ["cases.md"], "fault-tolerance": ["results.json"],
|
|
"performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
|
|
"results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
|
|
}
|
|
for suite in self.SUITES:
|
|
with self.subTest(suite=suite):
|
|
self.prepare(suite)
|
|
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
|
|
expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]}
|
|
for path in expected:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("PARTIAL FAILURE DIAGNOSTIC")
|
|
for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"):
|
|
directory.mkdir(exist_ok=True)
|
|
(directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}')
|
|
self.assertEqual(self.uploaded_files(), expected)
|
|
self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files()))
|
|
|
|
def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self):
|
|
self.prepare("kms")
|
|
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
|
|
scripts = self.directory / "auto-testing"
|
|
scripts.mkdir()
|
|
# Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487.
|
|
# The Docker boundary returns synthetic credentials; setup fails before vault_stop.
|
|
(scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash
|
|
set -Eeuo pipefail
|
|
TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")"
|
|
trap 'rm -rf "${TEST_TMP}"' EXIT
|
|
VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}"
|
|
VAULT_CONTAINER="rustfs-vault"
|
|
heal_run() { "$@"; }
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi
|
|
shift
|
|
done
|
|
mkdir -p "${VAULT_DATA_DIR}"
|
|
tmp_init="${TEST_TMP}/vault-init.json"
|
|
tmp_err="${TEST_TMP}/vault-init.stderr"
|
|
heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \
|
|
vault operator init -key-shares=1 -key-threshold=1 -format=json \
|
|
> "${tmp_init}" 2>"${tmp_err}"
|
|
cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null
|
|
printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}"
|
|
exit 42
|
|
''')
|
|
docker = self.directory / "bin/docker"
|
|
docker.write_text("""#!/bin/sh
|
|
[ "$1" = exec ] || exit 99
|
|
printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}'
|
|
""")
|
|
docker.chmod(0o755)
|
|
result = self.run_step("Run KMS suite")
|
|
self.assertEqual(result.returncode, 42, result.stderr)
|
|
vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json"
|
|
self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]})
|
|
self.assertNotIn(self.artifacts, vault_init.parents)
|
|
self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), [])
|
|
self.assertNotEqual(self.run_step("Generate report").returncode, 0)
|
|
self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")})
|
|
self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text())
|
|
for path in self.uploaded_files():
|
|
self.assertNotIn("FAKE-ROOT-CANARY", path.read_text())
|
|
self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text())
|
|
|
|
def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
|
|
self.prepare("heal")
|
|
self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
|
|
script = self.directory / "auto-testing/rustfs_heal_test.sh"
|
|
script.parent.mkdir()
|
|
# Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
|
|
script.write_text(r'''#!/bin/bash
|
|
set -euo pipefail
|
|
SELECTED_STEPS=()
|
|
PREFLIGHT=0
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
|
|
--log-file) LOG_FILE="$2"; shift ;;
|
|
--preflight) PREFLIGHT=1 ;;
|
|
esac
|
|
shift
|
|
done
|
|
if [ "$PREFLIGHT" -eq 1 ]; then
|
|
printf '\n' >> "$INVOKED_STEPS"
|
|
exit 0
|
|
fi
|
|
printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
|
|
emit_step_result() {
|
|
local n="$1" desc="$2" status="$3"
|
|
printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
|
|
}
|
|
{
|
|
for step in "${SELECTED_STEPS[@]}"; do
|
|
emit_step_result "$step" "fixture step $step" PASS
|
|
done
|
|
want_all=1
|
|
for s in 1 2 3 4 5 6 7; do
|
|
[[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
|
|
done
|
|
if [ "${want_all}" -eq 1 ]; then
|
|
printf '[HEAL-RESULT] PASS all steps passed\n'
|
|
fi
|
|
} >> "$LOG_FILE"
|
|
''')
|
|
script.chmod(0o755)
|
|
self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
|
|
for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
|
|
result = self.run_step(name)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
|
|
log = Path(self.env["LOG_FILE"]).read_text()
|
|
self.assertNotIn("[HEAL-RESULT]", log)
|
|
self.assertEqual(log.count("[HEAL-STEP]"), 7)
|
|
report = self.run_step("Generate report")
|
|
self.assertEqual(report.returncode, 0, report.stderr)
|
|
failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
|
|
for step in range(1, 8)]
|
|
failed_logs += [
|
|
log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
|
|
log + "[HEAL-STEP] 3 later step failure FAIL\n",
|
|
log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
|
|
log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
|
|
]
|
|
for failed_log in failed_logs:
|
|
with self.subTest(log=failed_log):
|
|
Path(self.env["LOG_FILE"]).write_text(failed_log)
|
|
# A failing heal run exits non-zero; the report step stays green
|
|
# because the steps ran, and the per-step table is always published.
|
|
self.context["steps.test.outcome"] = "failure"
|
|
report = self.run_step("Generate report")
|
|
self.assertEqual(report.returncode, 0, report.stderr)
|
|
contents = Path(self.env["REPORT_FILE"]).read_text()
|
|
self.assertIn("Test Step Outcome: failure", contents)
|
|
self.assertIn("| PASS |", contents)
|
|
if "original failure" in failed_log:
|
|
self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
|
|
if "later step failure" in failed_log:
|
|
self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
|
|
|
|
def test_tier_gate_rejects_failed_suites_and_invalid_structured_results(self):
|
|
self.prepare("tier")
|
|
self.artifacts.mkdir()
|
|
self.context["env.TIER_ARTIFACTS_DIR"] = str(self.artifacts)
|
|
gate_file = self.artifacts / "rustfs-tier-gate.rc"
|
|
for evidence_outcome, test_outcome, gate_result, success in (
|
|
("success", "success", "0\n", True),
|
|
("success", "failure", "0\n", False),
|
|
("success", "cancelled", "0\n", False),
|
|
("success", "skipped", "0\n", False),
|
|
("failure", "success", "0\n", False),
|
|
("success", "success", None, False),
|
|
("success", "success", "", False),
|
|
("success", "success", "1\n", False),
|
|
("success", "success", "invalid\n", False),
|
|
):
|
|
with self.subTest(evidence=evidence_outcome, suite=test_outcome, structured=gate_result):
|
|
self.context["steps.evidence.outcome"] = evidence_outcome
|
|
self.context["steps.test.outcome"] = test_outcome
|
|
gate_file.unlink(missing_ok=True)
|
|
if gate_result is not None:
|
|
gate_file.write_text(gate_result)
|
|
result = self.run_step("Enforce tier suite result")
|
|
self.assertEqual(result.returncode == 0, success, result.stderr)
|
|
|
|
def test_tier_backlog_manager_never_closes_issues_after_evidence_or_gate_failure(self):
|
|
self.prepare("tier")
|
|
self.artifacts.mkdir()
|
|
self.env["TIER_ARTIFACTS_DIR"] = str(self.artifacts)
|
|
capture = self.directory / "manager-args.json"
|
|
self.env["CAPTURE_MANAGER"] = str(capture)
|
|
manager = self.directory / "auto-testing/scripts/issue_manager.py"
|
|
manager.parent.mkdir(parents=True)
|
|
manager.write_text("import json, os, sys\nfrom pathlib import Path\n"
|
|
"Path(os.environ['CAPTURE_MANAGER']).write_text(json.dumps(sys.argv[1:]))\n")
|
|
for test, verify, gate, expected in (
|
|
("success", "success", "success", "success"),
|
|
("failure", "success", "success", "failure"),
|
|
("success", "failure", "success", "failure"),
|
|
("success", "success", "failure", "failure"),
|
|
("cancelled", "failure", "failure", "cancelled"),
|
|
):
|
|
with self.subTest(suite=test, evidence=verify, gate=gate):
|
|
self.context.update({"steps.test.outcome": test, "steps.evidence_verify.outcome": verify,
|
|
"steps.gate.outcome": gate})
|
|
result = self.run_step("Manage backlog issues (dedup / label / auto-close)")
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
args = json.loads(capture.read_text())
|
|
self.assertEqual(args[args.index("--outcome") + 1], expected)
|
|
capture.unlink()
|
|
self.artifacts.rmdir()
|
|
self.artifacts.symlink_to(self.directory, target_is_directory=True)
|
|
result = self.run_step("Manage backlog issues (dedup / label / auto-close)")
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertFalse(capture.exists())
|
|
|
|
def test_tier_failure_issue_never_reads_rejected_or_symlinked_evidence(self):
|
|
for rejected in (True, False):
|
|
with self.subTest(rejected=rejected):
|
|
self.prepare("tier")
|
|
evidence = self.directory / "untrusted-evidence"
|
|
evidence.mkdir()
|
|
(evidence / "rustfs-tier-report.md").write_text("UNTRUSTED EVIDENCE MUST NOT BE READ")
|
|
if rejected:
|
|
self.artifacts = evidence
|
|
else:
|
|
self.artifacts.symlink_to(evidence, target_is_directory=True)
|
|
self.context.update({
|
|
"env.TIER_ARTIFACTS_DIR": str(self.artifacts),
|
|
"steps.evidence.outcome": "failure" if rejected else "success",
|
|
"steps.evidence_verify.outcome": "failure", "steps.gate.outcome": "failure",
|
|
})
|
|
result = self.run_step("File failure issue in rustfs/backlog")
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
body = Path(self.env["CAPTURE_BODY"]).read_text()
|
|
self.assertNotIn("UNTRUSTED EVIDENCE MUST NOT BE READ", body)
|
|
self.assertIn("its contents were not read", body)
|
|
|
|
def test_performance_results_version_and_report_are_bound_to_the_run(self):
|
|
self.prepare("performance")
|
|
initialized = self.run_step("Initialize functional evidence")
|
|
self.assertEqual(initialized.returncode, 0, initialized.stderr)
|
|
self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
|
|
self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
|
|
version = self.run_step("Collect RustFS version info")
|
|
self.assertEqual(version.returncode, 0, version.stderr)
|
|
self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
|
|
old_summary = self.directory / "old-results/summary.md"
|
|
old_summary.parent.mkdir()
|
|
old_summary.write_text("OLD RUN EVIDENCE")
|
|
upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
|
|
self.assertNotEqual(self.run_step(upload).returncode, 0)
|
|
self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
|
|
results = Path(self.env["RUSTFS_RESULT_DIR"])
|
|
results.mkdir()
|
|
(results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
|
|
report = self.run_step(upload)
|
|
self.assertEqual(report.returncode, 0, report.stderr)
|
|
contents = Path(self.env["REPORT_FILE"]).read_text()
|
|
for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
|
|
"CURRENT PERFORMANCE RESULTS", "fixture-version"):
|
|
self.assertIn(value, contents)
|
|
self.assertNotIn("OLD RUN EVIDENCE", contents)
|
|
|
|
def test_performance_commands_bind_runner_selection_and_preserve_failures(self) -> None:
|
|
self.prepare("performance")
|
|
source = self.source.splitlines()
|
|
job = yaml_block(source, "performance-test", 2)
|
|
runner = WorkflowSteps()
|
|
runner.directory = self.directory / "workspace with spaces"
|
|
scripts = runner.directory / "auto-testing"
|
|
scripts.mkdir(parents=True)
|
|
wrapper = scripts / "rustfs_performance_test.sh"
|
|
wrapper.write_text(f"#!{sys.executable}\nimport json, os, sys\n" +
|
|
"print(json.dumps({'args': sys.argv[1:], 'env': {key: os.environ.get(key) for key in " +
|
|
"('RUSTFS_BENCH_SCRIPT', 'RUSTFS_WARP_METHODS', 'RUSTFS_WARP_SIZES', " +
|
|
"'RUSTFS_WARP_DURATION', 'RUSTFS_WARP_CONCURRENCY', 'WARP_METHODS', " +
|
|
"'WARP_SIZES', 'WARP_DURATION', 'WARP_CONCURRENCY')}}))\n" +
|
|
"sys.exit(int(os.environ['FAKE_BENCH_EXIT']))\n")
|
|
wrapper.chmod(0o755)
|
|
runner.steps = named_steps(job)
|
|
for methods, sizes, duration, concurrency in (
|
|
("get", "1KiB", "1s", "7"), ("all", "all", "5m", "64"), ("", "", "5m", "64")
|
|
):
|
|
runner.context = {"github.workspace": str(runner.directory), "inputs.test_method": methods,
|
|
"inputs.object_size": sizes, "inputs.warp_duration || '5m'": duration,
|
|
"inputs.warp_concurrency || '64'": concurrency}
|
|
runner.env = {**self.env, "RUSTFS_BENCH_SCRIPT": "/unverified/home-script.sh",
|
|
"RUSTFS_WARP_METHODS": "put", "RUSTFS_WARP_SIZES": "64MiB",
|
|
"RUSTFS_WARP_DURATION": "99h", "RUSTFS_WARP_CONCURRENCY": "2",
|
|
"WARP_DURATION": "88h", "WARP_CONCURRENCY": "3", "WARP_METHODS": "mixed", "WARP_SIZES": "32MiB",
|
|
"LOG_FILE": str(self.directory / "suite.log")}
|
|
runner.env.update(runner.step_env(job, indent=4))
|
|
for step, number in (("Run benchmark (GET/PUT/MIXED)", "5"), ("Analyze results", "6")):
|
|
for code in (0, 42):
|
|
with self.subTest(methods=methods, sizes=sizes, step=step, exit=code):
|
|
runner.env["FAKE_BENCH_EXIT"] = str(code)
|
|
result = runner.run_step(step)
|
|
self.assertEqual(result.returncode, code, result.stderr)
|
|
invocation = json.loads(result.stdout)
|
|
expected = ["--step", number, "-y", "--log-file", runner.env["LOG_FILE"]]
|
|
self.assertEqual(invocation["args"], expected)
|
|
self.assertEqual(invocation["env"]["RUSTFS_BENCH_SCRIPT"], str(scripts / "rustfs_performance_testing.sh"))
|
|
self.assertEqual(invocation["env"]["RUSTFS_WARP_METHODS"], methods)
|
|
self.assertEqual(invocation["env"]["RUSTFS_WARP_SIZES"], sizes)
|
|
self.assertEqual(invocation["env"]["RUSTFS_WARP_DURATION"], duration)
|
|
self.assertEqual(invocation["env"]["RUSTFS_WARP_CONCURRENCY"], concurrency)
|
|
if number == "6":
|
|
self.assertEqual(invocation["env"]["WARP_METHODS"], methods)
|
|
self.assertEqual(invocation["env"]["WARP_SIZES"], sizes)
|
|
self.assertEqual(invocation["env"]["WARP_DURATION"], duration)
|
|
self.assertEqual(invocation["env"]["WARP_CONCURRENCY"], concurrency)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
|
|
class TableSuiteTests(unittest.TestCase):
|
|
"""Contract tests for the S3 Tables (Iceberg REST Catalog) suite workflow."""
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
def test_workflow_wiring(self) -> None:
|
|
source = (self.ROOT / ".github/workflows/rustfs-table-test.yml").read_text()
|
|
self.assertIn("group: rustfs-shared-functional-tests-v2", source)
|
|
self.assertIn("runs-on: smoke-testing", source)
|
|
# New gate semantics: case failures stay green, harness breakdowns red.
|
|
self.assertRegex(source, r"(?m)^ continue-on-error: true")
|
|
self.assertIn("HARNESS_OK", source)
|
|
self.assertIn("Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed", source)
|
|
self.assertIn('python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}"', source)
|
|
# Case rows come from the shared generator (ANSI-tolerant parser).
|
|
# Issue manager, dashboard upload and artifacts use the shared contract.
|
|
self.assertIn("--suite table --category table", source)
|
|
self.assertIn("functional-reports/${SUITE}/${DATE}.md", source)
|
|
self.assertIn("rustfs-table-test-${{ github.run_id }}-${{ github.run_attempt }}", source)
|
|
# The suite is invoked with the product smoke script for TBL-101.
|
|
self.assertIn("--smoke-script", source)
|
|
self.assertIn("scripts/table-catalog/pyiceberg_smoke.py", source)
|