#!/usr/bin/env python3 """Exercise candidate substitution and complete-chain acceptance boundaries.""" import copy import hashlib import io import json import os from pathlib import Path import subprocess import tempfile import unittest from unittest import mock import zipfile import functional_chain_evidence as evidence import resolve_functional_candidate as candidate class RaceNativeToolsWorkflowTests(unittest.TestCase): def shell_step(self, name): from check_test_wiring import yaml_block source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text() job = yaml_block(source.splitlines(), 'native-heal-regressions', 2) start = job.index(' - name: ' + name) end = next((index for index in range(start + 1, len(job)) if job[index].startswith(' - name: ')), len(job)) step = job[start:end] return '\n'.join(line[10:] for line in step[step.index(' run: |') + 1:] if not line.strip() or line.startswith(' ')) def fixture(self, root, *, new_helper): directory = root / 'auto-testing/scripts/validation_heal' directory.mkdir(parents=True) stub = '''import json, os, pathlib, sys kind = pathlib.Path(__file__).stem with pathlib.Path('calls.jsonl').open('a') as out: out.write(json.dumps({'kind': kind, 'args': sys.argv[1:]}) + '\\n') if sys.argv[1] == 'supports': sys.exit(int(os.environ['SUPPORT_STATUS'])) sys.exit(int(os.environ.get(kind.upper() + '_FAIL', '0'))) ''' (directory / 'native_regressions.py').write_text(stub) if new_helper: (directory / 'race_native.py').write_text(stub) bin_dir = root / 'bin' bin_dir.mkdir() git = bin_dir / 'git' git.write_text('#!/bin/sh\nprintf "%s\\n" "$FAKE_HEAD"\n') git.chmod(0o755) env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, CANDIDATE_SHA='a' * 40, FAKE_HEAD='b' * 40, SUPPORT_STATUS='2', PATH=str(bin_dir) + os.pathsep + os.environ['PATH']) env.pop('BASH_ENV', None) return env def test_actual_native_build_routes_old_new_unknown_and_failed_collectors(self): shell = self.shell_step('Build and enumerate the unmodified candidate test executable') for mode, helper, support, failure, head, expected_code, expected_calls in [ ('old-harness', False, 2, None, 'b', 0, [('native_regressions', 'build')]), ('legacy', True, 2, None, 'b', 0, [('race_native', 'supports'), ('native_regressions', 'build')]), ('race', True, 0, None, 'b', 0, [('race_native', 'supports'), ('race_native', 'build')]), ('support-error', True, 42, None, 'b', 42, [('race_native', 'supports')]), ('invalid-support', True, 1, None, 'b', 1, [('race_native', 'supports')]), ('race-build-error', True, 0, 'RACE_NATIVE_FAIL', 'b', 42, [('race_native', 'supports'), ('race_native', 'build')]), ('legacy-build-error', True, 2, 'NATIVE_REGRESSIONS_FAIL', 'b', 42, [('race_native', 'supports'), ('native_regressions', 'build')]), ('wrong-helper', True, 0, None, 'c', 1, []), ]: with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode): root = Path(temp) env = self.fixture(root, new_helper=helper) env.update(SUPPORT_STATUS=str(support), FAKE_HEAD=head * 40) if failure: env[failure] = '42' result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell], cwd=root, env=env, capture_output=True, text=True) self.assertEqual(result.returncode, expected_code, result.stderr) calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else [] self.assertEqual([(row['kind'], row['args'][0]) for row in calls], expected_calls) for row in calls: if row['args'][0] == 'supports': self.assertEqual(row['args'], ['supports', '--commit', 'a' * 40]) else: expected = ['build', '--source', 'candidate', '--commit', 'a' * 40] if row['kind'] == 'race_native': expected += ['--testing-commit', 'b' * 40] self.assertEqual(row['args'], expected + ['--out', 'native-tools']) def test_actual_native_run_rejects_ambiguous_formats_and_propagates_failures(self): shell = self.shell_step('Run exact native regressions separately from fleet validation') for mode, helper, manifests, failure, expected_code, expected_kind in [ ('legacy', False, ['native-regressions.json'], None, 0, 'native_regressions'), ('race', True, ['race-native.json'], None, 0, 'race_native'), ('ambiguous', True, ['native-regressions.json', 'race-native.json'], None, 1, None), ('legacy-error', True, ['native-regressions.json'], 'NATIVE_REGRESSIONS_FAIL', 42, 'native_regressions'), ('race-error', True, ['race-native.json'], 'RACE_NATIVE_FAIL', 42, 'race_native'), ('missing-race-helper', False, ['race-native.json'], None, 2, None), ]: with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode): root = Path(temp) env = self.fixture(root, new_helper=helper) if failure: env[failure] = '42' (root / 'native-tools').mkdir() for manifest in manifests: (root / 'native-tools' / manifest).write_text('{}') result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell], cwd=root, env=env, capture_output=True, text=True) self.assertEqual(result.returncode, expected_code, result.stderr) calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] if (root / 'calls.jsonl').exists() else [] self.assertEqual(len(calls), int(expected_kind is not None)) if expected_kind: self.assertEqual(calls, [{'kind': expected_kind, 'args': ['run', '--directory', 'native-tools', '--out', 'native-evidence']}]) def test_native_upload_retains_both_fixed_executable_kinds(self): source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text() self.assertIn(' native-tools/ecstore-native-tests\n', source) self.assertIn(' native-tools/ecstore-race-native-tests\n', source) class PersistenceToolsWorkflowTests(unittest.TestCase): def test_supplement_preserves_old_pins_and_propagates_actual_collector_failure(self): from check_test_wiring import yaml_block source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text() job = yaml_block(source.splitlines(), 'native-heal-regressions', 2) start = job.index(' - name: Run existing F07 directory fsync failure regressions') end = job.index(' - name: Upload native executable identities and original diagnostics') step = job[start:end] shell = '\n'.join(line[10:] for line in step[step.index(' run: |') + 1:] if not line.strip() or line.startswith(' ')) for mode in ('old-harness', 'success', 'run-failure', 'verify-failure'): with tempfile.TemporaryDirectory() as temp, self.subTest(mode=mode): root = Path(temp) if mode != 'old-harness': collector = root / 'auto-testing/scripts/validation_heal/persistence_native.py' collector.parent.mkdir(parents=True) collector.write_text('''import json, os, pathlib, sys with pathlib.Path('calls.jsonl').open('a') as out: out.write(json.dumps(sys.argv[1:]) + '\\n') sys.exit(42 if os.environ['COLLECTOR_MODE'] == sys.argv[1] + '-failure' else 0) ''') env = dict(os.environ, AUTO_TESTING_SHA='b' * 40, COLLECTOR_MODE=mode) env.pop('BASH_ENV', None) result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shell], cwd=root, env=env, capture_output=True, text=True) self.assertEqual(result.returncode == 0, mode in ('old-harness', 'success'), result.stderr) if mode == 'old-harness': report = json.loads((root / 'native-persistence-evidence/not-available.json').read_text()) self.assertIs(report['complete'], False) self.assertIs(report['requiredNativeCoverageComplete'], False) self.assertIs(report['canonicalCasePassClaimed'], False) self.assertFalse((root / 'calls.jsonl').exists()) else: calls = [json.loads(line) for line in (root / 'calls.jsonl').read_text().splitlines()] self.assertEqual([call[0] for call in calls], ['run'] if mode == 'run-failure' else ['run', 'verify']) for call in calls: self.assertEqual(call[-2:], ['--testing-commit', 'b' * 40]) class InterruptedCleanupGuardTests(unittest.TestCase): def test_actual_workflow_guard_rejects_live_success_and_unrelated_runs(self): source = (candidate.ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text() step = source[source.index(' - name: Audit and finalize retained validation environment'):] guard = step.split(" python3 - <<'PY'", 1)[1].split(" PY", 1)[0] guard = "\n".join(line[10:] for line in guard.splitlines()) for state, conclusion, branch, workflow, accepted in ( ("completed", "failure", "topic", "RustFS Functional Chain", True), ("completed", "cancelled", "topic", "RustFS Functional Chain", True), ("in_progress", "cancelled", "topic", "RustFS Functional Chain", False), ("completed", "success", "topic", "RustFS Functional Chain", False), ("completed", "cancelled", "other", "RustFS Functional Chain", False), ("completed", "cancelled", "topic", "Other Workflow", False), ): with self.subTest(state=state, conclusion=conclusion, branch=branch, workflow=workflow), tempfile.TemporaryDirectory() as temp: Path(temp, "failed-run.json").write_text(json.dumps({"status": state, "conclusion": conclusion, "headBranch": branch, "workflowName": workflow})) with mock.patch.dict(os.environ, VALIDATION_ROOT=temp, GITHUB_REF_NAME="topic"): if accepted: exec(compile(guard, "workflow-cleanup-guard", "exec"), {}) else: with self.assertRaises(ValueError): exec(compile(guard, "workflow-cleanup-guard", "exec"), {}) class CandidateTests(unittest.TestCase): def setUp(self): self.run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40} self.manifest = {"schema": 2, "workflow_sha": "a" * 40, "source_sha": "b" * 40, "source_ref": "release", "build_run_id": 123, "build_run_attempt": 2, "package_sha256": "c" * 64, "package_url": "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/123/2/" + "c" * 64 + "/rustfs.deb"} def archive(self, names=None): output = io.BytesIO() with zipfile.ZipFile(output, "w") as archive: for name in names or ["nightly-candidate-123-2.json"]: archive.writestr(name, json.dumps(self.manifest)) payload = output.getvalue() artifact = {"id": 789, "name": "nightly-candidate-123-2", "expired": False, "size_in_bytes": len(payload), "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), "workflow_run": {"id": 123, "head_sha": "a" * 40}} return payload, artifact def test_distinct_build_source_preserves_both_identities(self): payload, artifact = self.archive() result = candidate.read_manifest(payload, artifact, self.run) self.assertEqual(result["source_sha"], "b" * 40) self.assertEqual(result["workflow_sha"], "a" * 40) self.assertEqual(result["source_ref"], "release") def test_legacy_requires_the_build_and_workflow_sha_to_agree(self): self.manifest["schema"] = 1 del self.manifest["workflow_sha"], self.manifest["source_ref"] with self.assertRaisesRegex(ValueError, "legacy"): candidate.validate_manifest(self.manifest, self.run) self.manifest["source_sha"] = self.run["head_sha"] candidate.validate_manifest(self.manifest, self.run) def test_manifest_substitutions_fail(self): for key, value in (("workflow_sha", "d" * 40), ("build_run_id", 124), ("build_run_attempt", 1), ("package_sha256", "d" * 64), ("package_url", "https://example.com/package.deb"), ("schema", True), ("source_ref", "release\nFORGED=value")): with self.subTest(key=key), self.assertRaises(ValueError): candidate.validate_manifest({**self.manifest, key: value}, self.run) def test_artifact_substitutions_and_archive_members_fail(self): payload, artifact = self.archive() for key, value in (("expired", True), ("name", "nightly-candidate-123-1"), ("size_in_bytes", 1), ("digest", "sha256:" + "d" * 64), ("workflow_run", {"id": 124, "head_sha": "a" * 40})): with self.subTest(key=key), self.assertRaises(ValueError): candidate.read_manifest(payload, {**artifact, key: value}, self.run) for names in (["../nightly-candidate-123-2.json"], ["nightly-candidate-123-2.json", "extra.json"]): payload, artifact = self.archive(names) with self.assertRaises(ValueError): candidate.read_manifest(payload, artifact, self.run) def test_resolver_uses_attempt_metadata_and_does_not_resolve_moving_branch(self): payload, artifact = self.archive() run = {**self.run, "path": ".github/workflows/nightly-gnu.yml", "head_branch": "main", "head_repository": {"full_name": "rustfs/rustfs"}, "event": "schedule", "status": "completed", "conclusion": "success", "run_started_at": "2026-09-12T00:00:00Z"} with mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]) as api: result = candidate.resolve(123, 2) self.assertTrue(api.call_args_list[0].args[0].endswith("/attempts/2")) self.assertFalse(any("branches/" in call.args[0] for call in api.call_args_list)) self.assertEqual(result["artifact_id"], 789) self.assertEqual(result["manifest"]["source_sha"], "b" * 40) def test_validation_manifest_binds_requested_source_and_private_channel(self): manifest = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"], "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")} run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"} payload = io.BytesIO() with zipfile.ZipFile(payload, "w") as archive: archive.writestr("validation-candidate-123-2.json", json.dumps(manifest)) raw = payload.getvalue() artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False, "size_in_bytes": len(raw), "digest": "sha256:" + hashlib.sha256(raw).hexdigest(), "workflow_run": {"id": 123, "head_sha": "a" * 40}} result = candidate.read_manifest(raw, artifact, run) self.assertEqual(result["channel"], "validation") self.assertIn("/packages/validation/runs/123/2/", result["package_url"]) def test_validation_resolver_requires_manual_workflow_dispatch(self): payload, artifact = self.archive() run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40, "path": ".github/workflows/rustfs-validation-package.yml", "head_branch": "main", "head_repository": {"full_name": "rustfs/rustfs"}, "event": "schedule", "status": "completed", "conclusion": "success", "run_started_at": "2026-10-12T00:00:00Z"} with mock.patch.object(candidate, "api", side_effect=[run]): with self.assertRaisesRegex(ValueError, "manually dispatched"): candidate.resolve(123, 2, allow_validation=True) def test_validation_workflow_rejects_legacy_manifest_schema(self): manifest = {**self.manifest} payload_buffer = io.BytesIO() with zipfile.ZipFile(payload_buffer, "w") as archive: archive.writestr("validation-candidate-123-2.json", json.dumps(manifest)) payload = payload_buffer.getvalue() artifact = {"id": 789, "name": "validation-candidate-123-2", "expired": False, "size_in_bytes": len(payload), "digest": "sha256:" + hashlib.sha256(payload).hexdigest(), "workflow_run": {"id": 123, "head_sha": "a" * 40}} run = {**self.run, "path": ".github/workflows/rustfs-validation-package.yml"} with self.assertRaisesRegex(ValueError, "schema-3"): candidate.read_manifest(payload, artifact, run) def test_private_candidate_resolution_is_explicit_and_attempt_bound(self): self.manifest.update(schema=3, channel="validation", source_ref=self.manifest["source_sha"], package_url=self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")) payload, artifact = self.archive(["validation-candidate-123-2.json"]) artifact["name"] = "validation-candidate-123-2" base = {**self.run, "head_branch": "cxymds/validation-runner", "head_repository": {"full_name": "rustfs/rustfs"}, "event": "workflow_dispatch", "status": "completed", "conclusion": "success", "run_started_at": "2026-10-12T00:00:00Z"} for path in candidate.VALIDATION_BUILD_WORKFLOWS: run = {**base, "path": path} with self.subTest(path=path), mock.patch.object(candidate, "api", side_effect=[run, {"artifacts": [artifact]}, payload]): resolved = candidate.resolve(123, 2, allow_validation=True) self.assertEqual(resolved["manifest"]["source_sha"], "b" * 40) self.assertEqual(resolved["workflow_sha"], "a" * 40) with mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError): candidate.resolve(123, 2) for field, value in (("id", 124), ("run_attempt", 1), ("event", "push"), ("status", "in_progress"), ("conclusion", "failure"), ("path", ".github/workflows/other.yml"), ("head_repository", {"full_name": "untrusted/rustfs"})): run = {**base, "path": ".github/workflows/rustfs-functional-chain.yml", field: value} with self.subTest(field=field), mock.patch.object(candidate, "api", return_value=run), self.assertRaises(ValueError): candidate.resolve(123, 2, allow_validation=True) def test_private_manifest_rejects_alias_channel_source_and_schema_substitution(self): valid = {**self.manifest, "schema": 3, "channel": "validation", "source_ref": self.manifest["source_sha"], "package_url": self.manifest["package_url"].replace("/packages/nightly/", "/packages/validation/")} for key, value in (("channel", "nightly"), ("source_ref", "main"), ("source_sha", "e" * 40), ("package_url", self.manifest["package_url"]), ("workflow_sha", "e" * 40), ("build_run_attempt", 3), ("schema", True)): with self.subTest(key=key), self.assertRaises(ValueError): candidate.validate_manifest({**valid, key: value}, self.run) def test_package_only_mode_rejects_conflicting_fleet_and_tool_requests(self): from check_test_wiring import yaml_block workflow = Path(__file__).resolve().parents[1] / ".github/workflows/rustfs-functional-chain.yml" job = yaml_block(workflow.read_text().splitlines(), "validation-package-request", 2) begin = job.index(" run: |") + 1 script = "\n".join(line[10:] for line in job[begin:] if not line.strip() or line.startswith(" ")) import os inputs = {"validation_package_sha": "a" * 40} env = {**os.environ, "PACKAGE_INPUTS": json.dumps(inputs)} result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) for field in ("build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha", "validation_tools_only", "validation_native_tools", "validation_shared_tools", "validation_manager_tools", "validation_cleanup_run_id", "validation_tools_run_id"): with self.subTest(field=field): env["PACKAGE_INPUTS"] = json.dumps({**inputs, field: "not-empty"}) result = subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True) self.assertNotEqual(result.returncode, 0) for value in ("main", "a" * 39, "a" * 40 + "\nFORGED=value"): env["PACKAGE_INPUTS"] = json.dumps({"validation_package_sha": value}) self.assertNotEqual(subprocess.run(["bash", "-e", "-c", script], env=env, capture_output=True).returncode, 0) class EvidenceTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.directory = Path(self.temp.name) self.chain = {"run_id": 456, "attempt": 1, "candidate": {"source_sha": "a" * 40, "workflow_sha": "b" * 40}} self.needs = {suite: {"result": "success"} for suite in evidence.SUITES} for suite in evidence.SUITES: value = {"schema": 1, "suite": suite, "chain": self.chain, "valid": True, "report_sha256": "c" * 64, "counts": {"PASS": 1, "FAIL": 0, "SKIP": 0, "UNSUPPORTED": 0, "RUNNING": 0}} (self.directory / (suite + ".json")).write_text(json.dumps(value)) def test_complete_chain_retains_source_identity(self): result = evidence.aggregate(self.chain, self.directory, self.needs) self.assertTrue(result["complete"]) self.assertEqual(result["chain"], self.chain) self.assertEqual([suite["suite"] for suite in result["suites"]], [ "upgrade", "s3", "kms", "tier", "storage", "heal", "pool", "security", "replication", "fault-tolerance", "table", "performance"]) def test_release_suites_cannot_be_missing_failed_or_have_missing_artifacts(self): for suite in ("fault-tolerance", "table"): with self.subTest(suite=suite): missing = {key: value for key, value in self.needs.items() if key != suite} with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, missing) with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, {**self.needs, suite: {"result": "failure"}}) path = self.directory / (suite + ".json") original = path.read_text() path.unlink() with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, self.needs) path.write_text(original) def test_failure_report_preserves_counts_without_weakening_success_gate(self): path = self.directory / "s3.json" record = json.loads(path.read_text()) record.update(valid=False, counts={**record["counts"], "FAIL": 2}, error="failed cases") path.write_text(json.dumps(record)) needs = {**self.needs, "s3": {"result": "failure"}, "performance": {"result": "cancelled"}} (self.directory / "performance.json").unlink() result = evidence.summarize(self.chain, self.directory, needs) self.assertFalse(result["complete"]) self.assertEqual(result["lanes"][1]["evidence"]["counts"]["FAIL"], 2) self.assertEqual(result["lanes"][-1]["result"], "cancelled") self.assertIsNone(result["lanes"][-1]["evidence"]) with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, needs) def test_preparation_failure_still_has_all_required_lanes(self): result = evidence.summarize(None, self.directory / "absent", {"prepare": {"result": "failure"}}) self.assertFalse(result["complete"]) self.assertEqual([lane["suite"] for lane in result["lanes"]], list(evidence.SUITES)) self.assertIn("| performance | missing | missing |", evidence.render_summary(result)) self.assertIn("Preparation: failure", evidence.render_summary(result)) def test_malformed_or_cross_attempt_record_is_not_reused(self): for payload in ("broken json", "[]", json.dumps({"suite": "s3", "chain": {"attempt": 9}})): (self.directory / "s3.json").write_text(payload) result = evidence.summarize(self.chain, self.directory, self.needs) self.assertFalse(result["complete"]) self.assertIsNone(result["lanes"][1]["evidence"]) def test_missing_failed_cancelled_or_skipped_lane_never_passes(self): for state in ("failure", "cancelled", "skipped", "pending"): with self.subTest(state=state), self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, {**self.needs, "s3": {"result": state}}) del self.needs["s3"] with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, self.needs) def test_partial_rerun_missing_artifact_and_zero_test_fail(self): path = self.directory / "s3.json" original = json.loads(path.read_text()) values = [{**original, "chain": {**self.chain, "attempt": 2}}, {**original, "counts": {**original["counts"], "PASS": 0}}, {**original, "counts": {**original["counts"], "FAIL": 1}}, {**original, "valid": False}, {**original, "report_sha256": ""}] for value in values: path.write_text(json.dumps(value)) with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, self.needs) path.unlink() with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, self.needs) def test_reports_count_case_status_not_cleanup_status(self): text = "| Topology | Case | Name | Status | Cleanup |\n| --- | --- | --- | --- | --- |\n| sns | TIER-1 | test | UNSUPPORTED | PASS |\n| sns | TIER-2 | test | FAIL | PASS |\n" counts = evidence.report_counts(text) self.assertEqual(counts["PASS"], 0) self.assertEqual(counts["FAIL"], 1) self.assertEqual(counts["UNSUPPORTED"], 1) self.assertEqual(evidence.report_counts("")["PASS"], 0) def test_performance_needs_real_complete_metrics(self): header = "method\tsize\tthroughput\tobj_per_s\treq_avg\treq_p50\treq_p90\treq_p99\n" row = "put\t1MiB\t100MiB/s\t100\t1ms\t1ms\t2ms\t3ms\n" self.assertEqual(evidence.report_counts(header + row, True)["PASS"], 1) self.assertEqual(evidence.report_counts(header, True)["PASS"], 0) with self.assertRaises(ValueError): evidence.report_counts(header + row + row, True) with self.assertRaises(ValueError): evidence.report_counts(header + "put\t1MiB\t\t\t\t\t\t\n", True) def test_fault_tolerance_requires_complete_known_verdicts(self): passing = "FT-CASE: A-read verdict=pass observed expected result\n" divergent = "FT-CASE: C2-read verdict=known-divergence lock majority unavailable\n" summary = "FT-SUMMARY: unexpected=0 known-divergence=1 strict=0\n" counts = evidence.fault_tolerance_counts(passing + divergent + summary) self.assertEqual(counts["PASS"], 1) self.assertEqual(counts["UNSUPPORTED"], 1) for invalid in (passing, passing + passing + summary, passing + divergent + summary.replace("strict=0", "strict=1"), passing + divergent + summary.replace("known-divergence=1", "known-divergence=0"), passing.replace("verdict=pass", "verdict=UNKNOWN") + summary): with self.subTest(invalid=invalid), self.assertRaises(ValueError): evidence.fault_tolerance_counts(invalid) class EnvelopeTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) (self.root / ".config").mkdir() (self.root / ".config/functional-script-revision.txt").write_text("d" * 40) manifest = {"schema": 2, "workflow_sha": "a" * 40, "source_sha": "b" * 40, "source_ref": "release", "build_run_id": 123, "build_run_attempt": 2, "package_sha256": "c" * 64, "package_url": "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/123/2/" + "c" * 64 + "/rustfs.deb"} self.chain = {"schema": 1, "run_id": 456, "attempt": 3, "workflow_sha": "e" * 40, "testing_sha": "d" * 40, "candidate": {"manifest": manifest, "artifact_id": 789, "artifact_digest": "sha256:" + "f" * 64, "workflow_sha": "a" * 40, "workflow_ref": "main", "build_started_at": "2026-09-12T00:00:00Z"}} self.env = {"CHAIN_MANIFEST": json.dumps(self.chain), "GITHUB_RUN_ID": "456", "GITHUB_RUN_ATTEMPT": "3", "GITHUB_SHA": "e" * 40, "GITHUB_ENV": str(self.root / "env"), "GITHUB_OUTPUT": str(self.root / "output"), "CHAIN_JOB_STATUS": "success", "CHAIN_TEST_OUTCOME": "success", "CHAIN_REPORT_OUTCOME": "success"} def test_consumer_exports_the_same_package_and_checksum_to_installers(self): with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40): evidence.consume(evidence.current_chain()) variables = dict(line.split("=", 1) for line in (self.root / "env").read_text().splitlines()) self.assertEqual(variables["PACKAGE_SHA256"], "c" * 64) self.assertEqual(variables["TO_SHA256"], "c" * 64) self.assertEqual(variables["RUSTFS_NIGHTLY_PACKAGE_URL"], self.chain["candidate"]["manifest"]["package_url"]) def test_partial_rerun_and_wrong_lane_checkout_fail_before_install(self): for changes, head in (({"GITHUB_RUN_ATTEMPT": "4"}, "e" * 40), ({}, "f" * 40), ({"GITHUB_RUN_ID": "457"}, "e" * 40)): with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, {**self.env, **changes}), mock.patch.object(evidence.subprocess, "check_output", return_value=head), self.assertRaises(ValueError): evidence.current_chain() self.assertFalse((self.root / "env").exists()) def test_testing_sha_fallback_is_accepted_while_garbage_is_rejected(self): # prepare's >24h staleness fallback legitimately sets testing_sha to # auto-testing main HEAD, which differs from the committed pin; only # the sha format is an invariant now. for testing_sha, ok in (("d" * 40, True), ("1" * 40, True), ("xyz", False), ("", False)): chain = dict(self.chain, testing_sha=testing_sha) env = dict(self.env, CHAIN_MANIFEST=json.dumps(chain)) if ok: with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40): evidence.consume(evidence.current_chain()) else: with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40), self.assertRaises(ValueError): evidence.current_chain() def test_report_or_swallowed_test_failure_cannot_produce_valid_evidence(self): report = self.root / "cases.md" report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n") for index, (key, status) in enumerate((("CHAIN_TEST_OUTCOME", "failure"), ("CHAIN_REPORT_OUTCOME", "failure"), ("CHAIN_JOB_STATUS", "cancelled"))): output = self.root / str(index) / "kms.json" Path(self.env["GITHUB_OUTPUT"]).unlink(missing_ok=True) with mock.patch.dict(evidence.os.environ, {**self.env, key: status}), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40), self.assertRaises(ValueError): evidence.record(self.chain, "kms", report, output) self.assertFalse(json.loads(output.read_text())["valid"]) self.assertIn("error", json.loads(output.read_text())) self.assertEqual(Path(self.env["GITHUB_OUTPUT"]).read_text(), "written=true\n") output = self.root / "success" / "kms.json" with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40): evidence.record(self.chain, "kms", report, output) self.assertTrue(json.loads(output.read_text())["valid"]) def test_collision_or_failed_write_never_authorizes_evidence_upload(self): report = self.root / "cases.md" report.write_text("| Case | Status |\n| --- | --- |\n| KMS-1 | PASS |\n") output = self.root / "stale" / "kms.json" output.parent.mkdir() output.write_text("OLD RUN EVIDENCE") with mock.patch.dict(evidence.os.environ, self.env), mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40): with self.assertRaises(FileExistsError): evidence.record(self.chain, "kms", report, output) self.assertEqual(output.read_text(), "OLD RUN EVIDENCE") self.assertFalse(Path(self.env["GITHUB_OUTPUT"]).exists()) with mock.patch.object(Path, "write_text", side_effect=OSError("disk full")), self.assertRaises(OSError): evidence.record(self.chain, "kms", report, self.root / "new" / "kms.json") self.assertFalse(Path(self.env["GITHUB_OUTPUT"]).exists()) def test_record_uses_workspace_testing_checkout_for_nested_lane(self): lane_checkout = self.root / "rustfs-repo" lane_checkout.mkdir() private_checkout = self.root / "auto-testing" private_checkout.mkdir() subprocess.run(["git", "init", "--quiet", str(private_checkout)], check=True) subprocess.run(["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgSign=false", "commit", "--allow-empty", "-qm", "fixture"], cwd=private_checkout, check=True) private_head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=private_checkout, text=True).strip() chain = {**self.chain, "testing_sha": private_head} report = self.root / "security.md" report.write_text("| Case | Status |\n| --- | --- |\n| SEC-1 | PASS |\n") env = {**self.env, "GITHUB_WORKSPACE": str(self.root)} output = self.root / "nested-success" / "security.json" with mock.patch.object(evidence, "ROOT", lane_checkout), mock.patch.dict(evidence.os.environ, env): evidence.record(chain, "security", report, output) self.assertTrue(json.loads(output.read_text())["valid"]) wrong_pin_output = self.root / "nested-wrong-pin" / "security.json" with self.assertRaisesRegex(ValueError, "suite used a different private script revision"): evidence.record({**chain, "testing_sha": "f" * 40}, "security", report, wrong_pin_output) self.assertFalse(json.loads(wrong_pin_output.read_text())["valid"]) fallback_output = self.root / "local-success" / "security.json" with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, self.env): evidence.os.environ.pop("GITHUB_WORKSPACE", None) evidence.record(chain, "security", report, fallback_output) self.assertTrue(json.loads(fallback_output.read_text())["valid"]) def test_unknown_status_cannot_hide_among_passing_cases(self): text = "| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n| KMS-2 | fixture | NOT RUN |\n" with self.assertRaises(ValueError): evidence.report_counts(text) def test_release_suite_records_require_successful_execution_and_complete_reports(self): reports = { "table": "| Case | Name | Status |\n| --- | --- | --- |\n| TBL-101 | Iceberg smoke | PASS |\n", "fault-tolerance": "FT-CASE: A-read verdict=pass read succeeded\nFT-SUMMARY: unexpected=0 known-divergence=0 strict=0\n", } for suite, text in reports.items(): report = self.root / (suite + ".txt") report.write_text(text) for status in ("success", "failure"): with self.subTest(suite=suite, status=status): output = self.root / (suite + "-" + status) / (suite + ".json") with mock.patch.dict(evidence.os.environ, {**self.env, "CHAIN_TEST_OUTCOME": status}), \ mock.patch.object(evidence.subprocess, "check_output", return_value="d" * 40): if status == "success": evidence.record(self.chain, suite, report, output) else: with self.assertRaises(ValueError): evidence.record(self.chain, suite, report, output) self.assertEqual(json.loads(output.read_text())["valid"], status == "success") def test_driver_passes_one_manifest_and_runs_every_lane_after_failure(self): from check_test_wiring import yaml_block lines = (candidate.ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text().splitlines() previous = None for suite in evidence.SUITES: job = "\n".join(yaml_block(lines, suite, 2)) self.assertIn("needs: [prepare" + (", " + previous if previous else "") + "]", job) if suite == "performance": # gated on the preflight probe: runs only when its fleet is online self.assertIn("if: ${{ always() && needs.prepare.result == 'success' && needs.prepare.outputs.performance_ready == 'online' }}", job) else: self.assertIn("if: ${{ always() && needs.prepare.result == 'success' }}", job) self.assertIn("chain_manifest: ${{ needs.prepare.outputs.manifest }}", job) previous = suite complete = "\n".join(yaml_block(lines, "complete-chain", 2)) self.assertIn("needs: [prepare, " + ", ".join(evidence.SUITES) + "]", complete) self.assertIn("functional_chain_evidence.py aggregate", complete) self.assertIn("functional_chain_evidence.py summarize", complete) self.assertIn("functional-chain-report-", complete) self.assertIn("needs.prepare.result != 'skipped'", complete) prepare = "\n".join(yaml_block(lines, "prepare", 2)) self.assertIn("Check shared functional fleet runner before scheduling suites", prepare) self.assertIn("check_functional_runners.py smoke-testing", prepare) self.assertIn("Probe performance fleet runner", prepare) self.assertIn("check_functional_runners.py pf-testing", prepare) self.assertIn("performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}", prepare) def test_every_lane_retains_failed_evidence_and_identifies_its_own_attempt(self): paths = list((candidate.ROOT / ".github/workflows").glob("rustfs-*-test.yml")) lanes = [path for path in paths if "name: Upload chain evidence" in path.read_text()] self.assertEqual(len(lanes), len(evidence.SUITES)) for path in lanes: with self.subTest(path=path.name): text = path.read_text() self.assertIn("if: ${{ always() && steps.chain_record.outputs.written == 'true' }}", text) self.assertIn("python3 auto-testing/scripts/issue_manager.py handle", text) self.assertIn('--run-id "${GITHUB_RUN_ID}"', text) self.assertIn('--attempt "${GITHUB_RUN_ATTEMPT}"', text) run_url = next(line for line in text.splitlines() if "--run-url" in line) self.assertIn('/attempts/${GITHUB_RUN_ATTEMPT}#summary"', run_url) def test_table_issue_manager_receives_outcome_and_case_evidence(self): text = (candidate.ROOT / ".github/workflows/rustfs-table-test.yml").read_text() self.assertIn("--outcome '${{ steps.test.outcome }}'", text) self.assertIn('--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"', text) def test_pool_topology_is_checked_before_cleanup_with_four_node_defaults(self): text = (candidate.ROOT / ".github/workflows/rustfs-pool-expand-test.yml").read_text() self.assertIn("http://rustfs-node4:9000", text) self.assertIn("vars.RUSTFS_NODES || 'vm000 vm001 vm002'", text) self.assertLess(text.index("name: Validate pool topology"), text.index("name: Cleanup environment (before)")) self.assertIn("steps.topology.outcome == 'success' && inputs.cleanup_after", text) class RunnerTests(unittest.TestCase): def test_offline_or_missing_runner_fails_before_dispatch(self): import check_functional_runners as runners for inventory in ([], [{"status": "offline", "labels": [{"name": "pf-testing"}]}]): with mock.patch.object(runners, "api", return_value={"runners": inventory}), self.assertRaisesRegex(ValueError, "pf-testing"): runners.check(["pf-testing"]) def test_busy_online_runner_is_available_and_inventory_is_paginated(self): import check_functional_runners as runners pages = [{"runners": [{"status": "online", "labels": []}] * 100}, {"runners": [{"status": "online", "busy": True, "labels": [{"name": "pf-testing"}]}]}] with mock.patch.object(runners, "api", side_effect=pages) as api: runners.check(["pf-testing"]) self.assertIn("page=2", api.call_args.args[0]) def test_unavailable_inventory_does_not_assume_online(self): import check_functional_runners as runners with mock.patch.object(runners, "api", side_effect=OSError("forbidden")), self.assertRaises(OSError): runners.check(["pf-testing"]) def test_release_lanes_use_pinned_scripts_and_verified_package_before_recording(self): for suite, report in (("fault-tolerance", "suite.log"), ("table", "cases.md")): with self.subTest(suite=suite): workflow = (candidate.ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() self.assertIn(" workflow_call:", workflow) self.assertIn(" workflow_dispatch:", workflow) self.assertIn("group: rustfs-shared-functional-tests-v2", workflow) expected_ref = ( "ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}" if suite == "fault-tolerance" else "ref: ${{ steps.chain.outputs.testing_sha || 'main' }}" ) self.assertIn(expected_ref, workflow) self.assertIn("steps.chain_package.outputs.package_url || inputs.package_url", workflow) self.assertLess(workflow.index("prepare_functional_package.py prepare"), workflow.index("id: test")) self.assertIn("prepare_functional_package.py cleanup", workflow) self.assertIn(f"functional_chain_evidence.py record --suite {suite}", workflow) self.assertIn('--report "${FUNCTIONAL_ARTIFACTS_DIR}/' + report + '"', workflow) self.assertIn("CHAIN_TEST_OUTCOME: ${{ steps.test.outcome }}", workflow) self.assertIn("CHAIN_REPORT_OUTCOME: ${{ steps.chain_report.outcome }}", workflow) class WorkflowTimeoutTests(unittest.TestCase): def test_non_performance_suites_have_hard_and_step_deadlines(self): from check_test_wiring import yaml_block from test_security_workflow import FunctionalWorkflowTests, named_steps jobs = {**FunctionalWorkflowTests.JOBS, "security": "security-test"} jobs.pop("performance") self.assertEqual(len(jobs), 11) for suite, job_id in jobs.items(): with self.subTest(suite=suite): source = (candidate.ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() job = yaml_block(source.splitlines(), job_id, 2) job_timeout = 360 if suite == "pool-expand" else 60 self.assertIn(f" timeout-minutes: {job_timeout}", job) steps = named_steps(job) primary = [step for step in steps.values() if any( line in (" id: test", " id: pool_test") for line in step)] self.assertEqual(len(primary), 1) if suite == "pool-expand": self.assertIn(" timeout-minutes: ${{ fromJSON(inputs.pool_timeout_minutes || '240') }}", primary[0]) for event in ("workflow_call", "workflow_dispatch"): event_block = yaml_block(source.splitlines(), event, 2) timeout_input = yaml_block(event_block, "pool_timeout_minutes", 6) self.assertIsNotNone(timeout_input, event) self.assertIn(" default: '240'", timeout_input) self.assertIn(" required: false", timeout_input) else: self.assertIn(" timeout-minutes: 45", primary[0]) cleanup = "Cleanup environment" for phase in ("before", "after"): self.assertIn(" timeout-minutes: 5", steps[f"{cleanup} ({phase})"]) self.assertTrue(any("always()" in line for line in steps[f"{cleanup} (after)"])) for name, step in steps.items(): if name.startswith(("Generate report", "Upload functional report", "File failure issue", "Upload report", "Upload test logs", "Manage backlog issues")): self.assertIn(" timeout-minutes: 2", step, name) if suite in ("heal", "pool-expand"): install = next(step for name, step in steps.items() if name.startswith("Install RustFS package")) self.assertIn(" timeout-minutes: 5", install) self.assertIn(" timeout-minutes: 5", steps["Preflight checks"]) def test_performance_is_exempt_from_functional_timeouts(self): from check_test_wiring import yaml_block source = (candidate.ROOT / ".github/workflows/rustfs-performance-test.yml").read_text() job = "\n".join(yaml_block(source.splitlines(), "performance-test", 2)) self.assertIn(" timeout-minutes: 900", job) self.assertNotIn(" timeout-minutes:", job) self.assertIn("RUSTFS_WARP_DURATION: ${{ inputs.warp_duration || '5m' }}", job) self.assertNotIn("RUSTFS_WARP_SLEEP:", job) self.assertIn("RUSTFS_WARP_METHODS: ${{ inputs.test_method }}", job) self.assertIn("RUSTFS_WARP_SIZES: ${{ inputs.object_size }}", job) self.assertIn("default: '5m'", source) if __name__ == "__main__": unittest.main()