mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 08:40:23 +00:00
Chris An
191c95a469
* ci: enforce affected validation and rehearse bounded shards * test(ci): keep regression output stable under forced color
439 lines
26 KiB
Python
439 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Archive consumers require exact provenance and complete selected executions."""
|
|
|
|
import copy
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import shutil
|
|
from string import Template
|
|
import tempfile
|
|
import subprocess
|
|
import sys
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import xml.etree.ElementTree as ET
|
|
|
|
import ci_test_artifact as artifact
|
|
|
|
|
|
def listing(cases):
|
|
suites = {}
|
|
for binary, name, ignored in cases:
|
|
suite = suites.setdefault(binary, {"status": "listed", "testcases": {}})
|
|
suite["testcases"][name] = {"ignored": ignored,
|
|
"filter-match": {"status": "mismatch" if ignored else "matches"}}
|
|
return {"rust-suites": suites}
|
|
|
|
|
|
def junit(cases):
|
|
root = ET.Element("testsuites", tests=str(len(cases)), failures="0", errors="0")
|
|
suites = {}
|
|
for binary, name, outcome in cases:
|
|
if binary not in suites:
|
|
suites[binary] = ET.SubElement(root, "testsuite", name=binary, failures="0", errors="0")
|
|
case = ET.SubElement(suites[binary], "testcase", name=name, classname=binary)
|
|
if outcome:
|
|
ET.SubElement(case, outcome)
|
|
return root
|
|
|
|
|
|
class ArtifactIdentityTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.archive = self.root / "tests.tar.zst"
|
|
self.archive.write_bytes(b"controlled archive bytes")
|
|
self.identity = {"source": {"head": "a" * 40, "sha256": "b" * 64}, "tree": "c" * 40,
|
|
"workspace_root": "/workspace/rustfs", "lock_sha256": "d" * 64,
|
|
"rustc": "rustc 1.92.0", "nextest": "cargo-nextest 0.9.138",
|
|
"configuration": "workspace", "profile": "test", "features": [],
|
|
"build_environment": {key: "" for key in artifact.BUILD_ENVIRONMENT},
|
|
"system": "Linux", "machine": "x86_64", "repository": "owner/repo",
|
|
"run_id": "123"}
|
|
self.digest = hashlib.sha256(self.archive.read_bytes()).hexdigest()
|
|
self.size = self.archive.stat().st_size
|
|
self.record = {"schema": 1, "identity": copy.deepcopy(self.identity),
|
|
"package_args": ["--workspace", "--exclude", "e2e_test"],
|
|
"sha256": self.digest, "size_bytes": self.size, "libc": ["glibc", "2.39"]}
|
|
|
|
def verify(self, record=None, expected=None, digest=None, size=None, libc=("glibc", "2.39")):
|
|
artifact.verify_record(record or self.record, expected or self.identity,
|
|
digest or self.digest, self.size if size is None else size, libc)
|
|
|
|
def test_exact_record_accepts_same_or_newer_glibc(self):
|
|
self.verify()
|
|
self.verify(libc=("glibc", "2.40"))
|
|
|
|
def test_every_provenance_dimension_rejects_mismatch(self):
|
|
mutations = {"source": {"head": "e" * 40, "sha256": "f" * 64}, "tree": "e" * 40,
|
|
"workspace_root": "/different/checkout", "lock_sha256": "e" * 64,
|
|
"rustc": "rustc 1.93.0", "nextest": "cargo-nextest 0.9.140",
|
|
"configuration": "e2e", "profile": "release", "features": ["rio-v2"],
|
|
"build_environment": {**self.identity["build_environment"], "RUSTFLAGS": "-C opt-level=3"},
|
|
"system": "Darwin", "machine": "aarch64", "repository": "other/repo",
|
|
"run_id": "124"}
|
|
for key, value in mutations.items():
|
|
with self.subTest(key=key), self.assertRaisesRegex(ValueError, "identity mismatch"):
|
|
self.verify(expected={**self.identity, key: value})
|
|
|
|
def test_failed_job_rerun_reuses_the_archive_but_another_run_does_not(self):
|
|
with patch.object(artifact, "source_identity", return_value=self.identity["source"]), \
|
|
patch.object(artifact, "output", return_value="fixture"), \
|
|
patch.object(artifact, "file_hash", return_value="d" * 64), \
|
|
patch.dict(os.environ, {"GITHUB_RUN_ID": "123", "GITHUB_RUN_ATTEMPT": "1"}, clear=True):
|
|
producer = artifact.identity("workspace")
|
|
with patch.dict(os.environ, {"GITHUB_RUN_ATTEMPT": "2"}):
|
|
self.assertEqual(artifact.identity("workspace"), producer)
|
|
with patch.dict(os.environ, {"GITHUB_RUN_ID": "124"}):
|
|
self.assertNotEqual(artifact.identity("workspace"), producer)
|
|
|
|
def test_modified_truncated_and_empty_archives_are_rejected(self):
|
|
for key, value in (("schema", 2), ("sha256", "f" * 64), ("size_bytes", self.size + 1)):
|
|
with self.subTest(key=key), self.assertRaises(ValueError):
|
|
self.verify(record={**self.record, key: value})
|
|
with self.assertRaises(ValueError):
|
|
self.verify(size=0)
|
|
with self.assertRaises(ValueError):
|
|
self.verify(digest="0" * 64)
|
|
|
|
def test_abi_refuses_older_or_different_or_unparseable_libc(self):
|
|
for libc in (("glibc", "2.38"), ("musl", "1.2.5"), ("glibc", "unknown")):
|
|
with self.subTest(libc=libc), self.assertRaises(ValueError):
|
|
self.verify(libc=libc)
|
|
for producer in ([], ["glibc"], ["glibc", "not-a-version"]):
|
|
with self.subTest(producer=producer), self.assertRaises(ValueError):
|
|
self.verify(record={**self.record, "libc": producer})
|
|
musl = {**self.record, "libc": ["musl", "1.2.5"]}
|
|
self.verify(record=musl, libc=("musl", "1.2.5"))
|
|
with self.assertRaises(ValueError):
|
|
self.verify(record=musl, libc=("musl", "1.2.6"))
|
|
|
|
def test_verify_binds_actual_bytes_and_package_plan(self):
|
|
artifact.manifest_path(self.archive).write_text(json.dumps(self.record))
|
|
with patch.object(artifact, "identity", return_value=self.identity), \
|
|
patch.object(artifact.platform, "libc_ver", return_value=("glibc", "2.39")), \
|
|
patch.dict(os.environ, {}, clear=True):
|
|
self.assertEqual(artifact.verify(self.archive, "workspace"), self.record)
|
|
with patch.dict(os.environ, {"CI_PACKAGES": '["rustfs"]'}), self.assertRaisesRegex(ValueError, "package selection"):
|
|
artifact.verify(self.archive, "workspace")
|
|
self.archive.write_bytes(b"corrupt")
|
|
with self.assertRaisesRegex(ValueError, "corrupt"):
|
|
artifact.verify(self.archive, "workspace")
|
|
|
|
def test_package_selection_rejects_invalid_and_keeps_e2e_separate(self):
|
|
with patch.dict(os.environ, {"CI_PACKAGES": '["rustfs-utils", "rustfs", "rustfs"]'}):
|
|
self.assertEqual(artifact.package_args("workspace"), ["-p", "rustfs", "-p", "rustfs-utils"])
|
|
self.assertEqual(artifact.package_args("e2e"), ["-p", "e2e_test"])
|
|
locked = [{"name": "zstd", "version": "0.13.3"},
|
|
{"name": "zstd", "version": "0.14.3", "source": "registry+https://github.com/rust-lang/crates.io-index"},
|
|
{"name": "rustfs", "version": "0.1.0"}]
|
|
with patch.object(artifact, "locked_packages", return_value=locked), \
|
|
patch.dict(os.environ, {"CI_PACKAGES": '["zstd", "rustfs"]'}):
|
|
self.assertEqual(artifact.package_args("workspace"), ["-p", "rustfs", "-p", "zstd@0.13.3"])
|
|
for clash in ([*locked[:2], {"name": "zstd", "version": "0.14.3"}], [locked[1], {**locked[1], "version": "0.15.0"}]):
|
|
with self.subTest(clash=clash), patch.object(artifact, "locked_packages", return_value=clash), \
|
|
self.assertRaisesRegex(ValueError, "unambiguously"):
|
|
artifact.package_args("workspace")
|
|
for value in ('[]', '{}', '[null]', '["--all-features"]', '["e2e_test"]', '[""]'):
|
|
with self.subTest(value=value), patch.dict(os.environ, {"CI_PACKAGES": value}), self.assertRaises(ValueError):
|
|
artifact.package_args("workspace")
|
|
|
|
def test_source_drift_during_archive_build_cannot_write_receipt(self):
|
|
after = {**self.identity, "source": {"head": "e" * 40, "sha256": "f" * 64}}
|
|
with patch.object(artifact, "identity", side_effect=[self.identity, after]), \
|
|
patch.object(artifact.subprocess, "run"), patch.dict(os.environ, {}, clear=True), \
|
|
self.assertRaisesRegex(ValueError, "changed while compiling"):
|
|
artifact.build(self.archive, "workspace")
|
|
self.assertFalse(artifact.manifest_path(self.archive).exists())
|
|
|
|
def test_codegen_overrides_are_bound_but_build_concurrency_is_not(self):
|
|
overrides = {"CARGO_PROFILE_TEST_OVERFLOW_CHECKS": "false", "CARGO_PROFILE_TEST_DEBUG_ASSERTIONS": "false",
|
|
"CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS": "-C opt-level=3",
|
|
"CARGO_BUILD_RUSTFLAGS": "-C debuginfo=0", "RUSTFS_BUILD_VERSION": "fixture",
|
|
"CFLAGS": "-O3", "CARGO_BUILD_JOBS": "2"}
|
|
with patch.dict(os.environ, overrides, clear=True):
|
|
bound = artifact.build_environment()
|
|
for key, value in overrides.items():
|
|
if key != "CARGO_BUILD_JOBS":
|
|
self.assertEqual(bound[key], value)
|
|
with patch.dict(os.environ, {"CARGO_BUILD_JOBS": "4"}):
|
|
self.assertEqual(artifact.build_environment(), bound)
|
|
with patch.dict(os.environ, {"CARGO_PROFILE_TEST_OVERFLOW_CHECKS": "true"}):
|
|
self.assertNotEqual(artifact.build_environment(), bound)
|
|
|
|
def test_native_target_and_original_target_directory_are_required(self):
|
|
for overrides in ({"CARGO_BUILD_TARGET": "aarch64-unknown-linux-gnu"},
|
|
{"CARGO_TARGET_DIR": str(self.root / "different-target")}):
|
|
with self.subTest(overrides=overrides), patch.dict(os.environ, overrides, clear=True), self.assertRaises(ValueError):
|
|
artifact.identity("workspace")
|
|
|
|
def test_lint_command_keeps_all_targets_deny_warnings_and_failure(self):
|
|
arguments = ["ci_test_artifact.py", "cargo", "lint", "--configuration", "workspace"]
|
|
expected = ["cargo", "clippy", "--locked", "-p", "rustfs", "-p", "e2e_test", "--all-targets", "--", "-D", "warnings"]
|
|
with patch.object(sys, "argv", arguments), patch.dict(os.environ, {"CI_PACKAGES": '["rustfs"]'}):
|
|
with patch.object(artifact.subprocess, "run") as run:
|
|
self.assertEqual(artifact.main(), 0)
|
|
self.assertEqual(run.call_args.args[0], expected)
|
|
self.assertTrue(run.call_args.kwargs["check"])
|
|
with patch.object(artifact.subprocess, "run", side_effect=subprocess.CalledProcessError(1, expected)), \
|
|
patch.object(sys, "stderr", io.StringIO()):
|
|
self.assertEqual(artifact.main(), 1)
|
|
for environment in ({}, {"CI_PACKAGES": ""}):
|
|
with self.subTest(environment=environment), patch.object(sys, "argv", arguments), \
|
|
patch.dict(os.environ, environment, clear=True), patch.object(artifact.subprocess, "run") as run:
|
|
self.assertEqual(artifact.main(), 0)
|
|
self.assertEqual(run.call_args.args[0], ["cargo", "clippy", "--locked", "--workspace", "--all-targets", "--", "-D", "warnings"])
|
|
|
|
def test_doctests_are_an_independent_command(self):
|
|
with patch.object(sys, "argv", ["ci_test_artifact.py", "cargo", "doc", "--configuration", "workspace"]), \
|
|
patch.dict(os.environ, {}, clear=True), patch.object(artifact.subprocess, "run") as run:
|
|
self.assertEqual(artifact.main(), 0)
|
|
self.assertEqual(run.call_args.args[0], ["cargo", "test", "--locked", "--workspace", "--exclude", "e2e_test", "--doc"])
|
|
|
|
def test_real_lint_rejects_a_warning_even_when_archive_tests_pass(self):
|
|
(self.root / 'rust-toolchain.toml').write_text((artifact.ROOT / 'rust-toolchain.toml').read_text())
|
|
(self.root / 'Cargo.toml').write_text('[workspace]\nmembers = ["probe", "e2e_test"]\n')
|
|
for name in ('probe', 'e2e_test'):
|
|
package = self.root / name
|
|
(package / 'src').mkdir(parents=True)
|
|
(package / 'Cargo.toml').write_text(f'[package]\nname="{name}"\nversion="0.1.0"\nedition="2021"\n')
|
|
(package / 'src/lib.rs').write_text('')
|
|
source = self.root / 'probe/src/lib.rs'
|
|
source.write_text('pub fn answer() -> u8 { let unused = 1; 2 }\n#[test] fn value() { assert_eq!(answer(), 2); }\n')
|
|
archive = self.root / 'lint-probe.tar.zst'
|
|
for arguments in (['archive', '--offline', '--archive-file', str(archive)],
|
|
['run', '--color', 'never', '--archive-file', str(archive)]):
|
|
result = subprocess.run(['cargo', 'nextest', *arguments], cwd=self.root, text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn('1 passed', result.stderr)
|
|
with patch.object(artifact, 'ROOT', self.root), patch.dict(os.environ, {'CI_PACKAGES': '["probe"]'}), \
|
|
patch.object(sys, 'argv', ['ci_test_artifact.py', 'cargo', 'lint', '--configuration', 'workspace']):
|
|
self.assertEqual(artifact.main(), 1)
|
|
source.write_text(source.read_text().replace('let unused = 1; ', ''))
|
|
self.assertEqual(artifact.main(), 0)
|
|
|
|
|
|
class ExecutionEvidenceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.listing_file = self.root / "listing.json"
|
|
self.junit_file = self.root / "junit.xml"
|
|
self.listing = listing([("crate-a", "one", False), ("crate-b", "two", False), ("crate-b", "ignored", True)])
|
|
self.report = junit([("crate-a", "one", None), ("crate-b", "two", None)])
|
|
|
|
def verify(self, listing_value=None, report=None, *, core=False):
|
|
self.listing_file.write_text(json.dumps(self.listing if listing_value is None else listing_value))
|
|
ET.ElementTree(self.report if report is None else report).write(self.junit_file)
|
|
return artifact.verify_execution(self.listing_file, self.junit_file, core=core)
|
|
|
|
def test_exact_selected_set_and_known_ignored_cases(self):
|
|
self.assertEqual(self.verify(), 2)
|
|
self.assertEqual(self.verify(report=junit([("crate-a", "one", None), ("crate-b", "two", None),
|
|
("crate-b", "ignored", "skipped")])), 2)
|
|
|
|
def test_missing_extra_or_misnamed_execution_is_rejected(self):
|
|
cases = [("crate-a", "one", None), ("crate-b", "two", None)]
|
|
for altered in (cases[:1], cases + [("crate-a", "extra", None)],
|
|
[("wrong-binary", "one", None), cases[1]]):
|
|
with self.subTest(cases=altered), self.assertRaises(ValueError):
|
|
self.verify(report=junit(altered))
|
|
|
|
def test_selected_skip_failure_error_or_duplicate_is_rejected(self):
|
|
for outcome in ("skipped", "failure", "error"):
|
|
with self.subTest(outcome=outcome), self.assertRaises(ValueError):
|
|
self.verify(report=junit([("crate-a", "one", outcome), ("crate-b", "two", None)]))
|
|
with self.assertRaisesRegex(ValueError, "duplicate"):
|
|
self.verify(report=junit([("crate-a", "one", None), ("crate-a", "one", None), ("crate-b", "two", None)]))
|
|
self.report.find("testsuite/testcase").set("classname", "different")
|
|
with self.assertRaisesRegex(ValueError, "misplaced"):
|
|
self.verify()
|
|
|
|
def test_extra_skipped_case_must_exist_in_listing(self):
|
|
with self.assertRaises(ValueError):
|
|
self.verify(report=junit([("crate-a", "one", None), ("crate-b", "two", None),
|
|
("unknown-binary", "unknown-case", "skipped")]))
|
|
|
|
def test_setup_scripts_must_pass_without_counting_as_selected_tests(self):
|
|
cases = [("crate-a", "one", None), ("crate-b", "two", None)]
|
|
script = ("@setup-script:prepare", "prepare", None)
|
|
self.assertEqual(self.verify(report=junit([*cases, script])), 2)
|
|
for outcome in ("skipped", "failure", "error"):
|
|
with self.subTest(outcome=outcome), self.assertRaises(ValueError):
|
|
self.verify(report=junit([*cases, (*script[:2], outcome)]))
|
|
with self.assertRaisesRegex(ValueError, "missing"):
|
|
self.verify(report=junit([cases[0], script]))
|
|
|
|
def test_malformed_or_duplicate_setup_script_evidence_is_rejected(self):
|
|
cases = [("crate-a", "one", None), ("crate-b", "two", None)]
|
|
script = ("@setup-script:prepare", "prepare", None)
|
|
for extra in ([script, script], [("@setup-script:", "", None)],
|
|
[(script[0], "wrong-name", None)]):
|
|
with self.subTest(extra=extra), self.assertRaises(ValueError):
|
|
self.verify(report=junit([*cases, *extra]))
|
|
report = junit([*cases, script])
|
|
report.append(copy.deepcopy(report[-1]))
|
|
with self.assertRaises(ValueError):
|
|
self.verify(report=report)
|
|
report = junit(cases)
|
|
ET.SubElement(report, "testsuite", name=script[0], failures="0", errors="0")
|
|
with self.assertRaises(ValueError):
|
|
self.verify(report=report)
|
|
report = junit([*cases, script])
|
|
report[-1].find("testcase").set("classname", "wrong-suite")
|
|
with self.assertRaisesRegex(ValueError, "misplaced"):
|
|
self.verify(report=report)
|
|
|
|
def test_empty_unlisted_or_failed_top_level_report_is_rejected(self):
|
|
with self.assertRaises(ValueError):
|
|
self.verify(listing_value={"rust-suites": {}})
|
|
with self.assertRaises(ValueError):
|
|
self.verify(listing_value=listing([("crate-a", "ignored", True)]))
|
|
self.listing["rust-suites"]["crate-a"]["status"] = "not-listed"
|
|
with self.assertRaises(ValueError):
|
|
self.verify()
|
|
self.listing["rust-suites"]["crate-a"]["status"] = "listed"
|
|
for attribute in ("failures", "errors"):
|
|
report = copy.deepcopy(self.report)
|
|
report.set(attribute, "1")
|
|
with self.subTest(attribute=attribute), self.assertRaises(ValueError):
|
|
self.verify(report=report)
|
|
|
|
def test_core_requires_exactly_three_selected_proofs(self):
|
|
names = sorted(artifact.CORE_TESTS)
|
|
core_listing = listing([("e2e", name, False) for name in names])
|
|
core_report = junit([("e2e", name, None) for name in names])
|
|
self.assertEqual(self.verify(core_listing, core_report, core=True), 3)
|
|
for cases in ([name for name in names if name != names[0]], names + ["unrelated"]):
|
|
with self.subTest(cases=cases), self.assertRaises(ValueError):
|
|
self.verify(listing([("e2e", name, False) for name in cases]),
|
|
junit([("e2e", name, None) for name in cases]), core=True)
|
|
# Reusing one proof's name in another binary must not expand the suite.
|
|
duplicate_name = listing([("e2e", name, False) for name in names] + [("other-binary", names[0], False)])
|
|
duplicate_report = junit([("e2e", name, None) for name in names] + [("other-binary", names[0], None)])
|
|
with self.assertRaises(ValueError):
|
|
self.verify(duplicate_name, duplicate_report, core=True)
|
|
|
|
|
|
class NextestSetupScriptTests(unittest.TestCase):
|
|
def test_workflow_consumers_reuse_extraction_and_validate_setup_script_reports(self):
|
|
from ci_gate import workflow_document
|
|
|
|
workflow = workflow_document((artifact.ROOT / ".github/workflows/ci.yml").read_text())
|
|
|
|
def commands(job):
|
|
result = []
|
|
for step in workflow["jobs"][job]["steps"]:
|
|
for line in step.get("run", "").replace("\\\n", " ").splitlines():
|
|
if "cargo nextest " not in line:
|
|
continue
|
|
words = shlex.split(line)
|
|
for start in range(len(words) - 1):
|
|
if words[start:start + 2] == ["cargo", "nextest"]:
|
|
end = next((index for index in range(start, len(words))
|
|
if words[index] in ("|", ">", "2>&1")), len(words))
|
|
result.append(words[start:end])
|
|
break
|
|
return result
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / "src").mkdir()
|
|
(root / ".config").mkdir()
|
|
(root / "Cargo.toml").write_text(
|
|
'[package]\nname = "rustfs"\nversion = "0.1.0"\nedition = "2021"\n'
|
|
'[lib]\nname = "lifecycle_integration_test"\n[workspace]\n'
|
|
)
|
|
(root / "rust-toolchain.toml").write_text((artifact.ROOT / "rust-toolchain.toml").read_text())
|
|
(root / "src/lib.rs").write_text(
|
|
'#[test]\nfn setup_is_applied() {\n'
|
|
' assert_eq!(std::env::var("EVIDENCE_SETUP").unwrap(), "ready");\n}\n'
|
|
'mod lifecycle_transition_api_test {\n'
|
|
' #[test]\n #[ignore]\n fn ignored_case() {}\n}\n'
|
|
)
|
|
config = root / ".config/nextest.toml"
|
|
config.write_text(
|
|
'experimental = ["setup-scripts"]\n'
|
|
'[scripts.setup.prepare]\ncommand = ["sh", "prepare.sh"]\n'
|
|
'[[profile.default.scripts]]\nfilter = "all()"\nsetup = "prepare"\n'
|
|
+ ''.join(f'[profile.{profile}.junit]\npath = "junit.xml"\n'
|
|
for profile in ("default", "ci", "e2e-smoke", "e2e-full", "core-contract"))
|
|
)
|
|
script = root / "prepare.sh"
|
|
script.write_text('echo EVIDENCE_SETUP=ready >> "$NEXTEST_ENV"\n')
|
|
environment = {**os.environ, "CARGO_TARGET_DIR": str(root / "target"), "CARGO_NET_OFFLINE": "true",
|
|
"GITHUB_WORKSPACE": str(root), "RUNNER_TEMP": str(root),
|
|
"NEXTEST_ARCHIVE": str(root / "e2e-tests.tar.zst")}
|
|
|
|
def nextest(*arguments):
|
|
return subprocess.run(["cargo", "nextest", *arguments], cwd=root, env=environment,
|
|
capture_output=True, text=True, timeout=120)
|
|
|
|
archive = root / "built-tests.tar.zst"
|
|
archived = nextest("archive", "--archive-file", str(archive))
|
|
self.assertEqual(archived.returncode, 0, archived.stderr)
|
|
# Any accidental compilation after the archive build must fail.
|
|
(root / "src/lib.rs").write_text('compile_error!("consumer attempted to rebuild");\n')
|
|
listing_file = root / "listing.json"
|
|
for job in ("workspace-tests", "core-contract", "e2e-tests", "e2e-full"):
|
|
with self.subTest(job=job):
|
|
shutil.rmtree(root / "target")
|
|
consumer_archive = root / ("workspace-tests.tar.zst" if job == "workspace-tests" else "e2e-tests.tar.zst")
|
|
shutil.copyfile(archive, consumer_archive)
|
|
selected = [[Template(word).substitute(environment) for word in command] for command in commands(job)]
|
|
self.assertEqual(len(selected), 3 if job == "workspace-tests" else 2)
|
|
self.assertEqual(selected[0][:3], ["cargo", "nextest", "list"])
|
|
listed = nextest(*selected[0][2:])
|
|
self.assertEqual(listed.returncode, 0, listed.stderr)
|
|
listing_file.write_text(listed.stdout)
|
|
self.assertTrue((root / "target/nextest/binaries-metadata.json").is_file())
|
|
# The remaining commands cannot extract again, even for ignored tests.
|
|
consumer_archive.unlink()
|
|
for command in selected[1:]:
|
|
executed = nextest(*command[2:])
|
|
self.assertEqual(executed.returncode, 0, executed.stderr)
|
|
profile = command[command.index("--profile") + 1]
|
|
report = root / f"target/nextest/{profile}/junit.xml"
|
|
suites = ET.parse(report).getroot().findall("testsuite")
|
|
self.assertIn("@setup-script:prepare", [suite.get("name") for suite in suites])
|
|
if "--run-ignored" in command:
|
|
cases = [case.get("name") for suite in suites if not suite.get("name", "").startswith("@")
|
|
for case in suite.findall("testcase") if case.find("skipped") is None]
|
|
self.assertEqual(cases, ["lifecycle_transition_api_test::ignored_case"])
|
|
else:
|
|
self.assertEqual(artifact.verify_execution(listing_file, report), 1)
|
|
|
|
verified = subprocess.run(
|
|
[sys.executable, str(artifact.ROOT / "scripts/ci_test_artifact.py"), "evidence",
|
|
str(listing_file), str(report)], capture_output=True, text=True, timeout=30,
|
|
)
|
|
self.assertEqual(verified.returncode, 0, verified.stderr)
|
|
self.assertIn("Verified 1 test executions", verified.stdout)
|
|
|
|
for setup in ('echo EVIDENCE_SETUP=wrong >> "$NEXTEST_ENV"\n', 'exit 1\n'):
|
|
report.unlink()
|
|
script.write_text(setup)
|
|
failed = nextest(*command[2:])
|
|
self.assertNotEqual(failed.returncode, 0)
|
|
with self.assertRaisesRegex(ValueError, "successful nextest run"):
|
|
artifact.verify_execution(listing_file, report)
|
|
|
|
for name in ("cargo-metadata.json", "binaries-metadata.json"):
|
|
metadata = root / "target/nextest" / name
|
|
content = metadata.read_bytes()
|
|
metadata.unlink()
|
|
missing = nextest(*command[2:])
|
|
self.assertNotEqual(missing.returncode, 0)
|
|
self.assertIn(name, missing.stderr)
|
|
metadata.write_bytes(content)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|