From 9aebcefa9c725912bf541048ff5be23fa37bdef7 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 20:32:52 +0800 Subject: [PATCH] test(scanner): harden measured ABBA evidence claims (#7491) Reject measured Scanner/Heal ABBA manifests whose mixed-version evidence uses the same baseline and candidate source revision or binary hash. Require crash fault modes and profile artifact names to match the exact supported sets, rejecting missing, duplicate, and unknown values. Update harness fixtures and regression coverage for same-build mixed-version claims and exact-set release evidence fields. Co-authored-by: zhi22915 --- scripts/scanner_abba.py | 37 +++++++++++++++++++++--------------- scripts/test_scanner_abba.py | 30 ++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py index cdf53c564..6bb594f9b 100644 --- a/scripts/scanner_abba.py +++ b/scripts/scanner_abba.py @@ -173,6 +173,18 @@ def release_evidence_true(value, name): require(value is True, f"missing release_evidence.{name}") +def release_evidence_exact_strings(value, expected, name): + require(isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value), + f"invalid release_evidence.{name}") + observed = set(value) + require(len(observed) == len(value), f"duplicate release_evidence.{name}") + missing = sorted(set(expected) - observed) + require(not missing, f"missing release_evidence.{name}: {', '.join(missing)}") + unknown = sorted(observed - set(expected)) + require(not unknown, f"unknown release_evidence.{name}: {', '.join(unknown)}") + return value + + def validate_release_evidence_manifest(manifest): if manifest["evidence"] != "measured": return @@ -210,17 +222,17 @@ def validate_release_evidence_manifest(manifest): crash = evidence.get("crash_restart") require(isinstance(crash, dict), "missing release_evidence.crash_restart") - fault_modes = crash.get("fault_modes") - require( - isinstance(fault_modes, list) - and all(mode in fault_modes for mode in RELEASE_FAULT_MODES) - and all(isinstance(mode, str) and mode.strip() for mode in fault_modes), - "missing release_evidence.crash_restart.fault_modes", - ) + release_evidence_exact_strings(crash.get("fault_modes"), RELEASE_FAULT_MODES, "crash_restart.fault_modes") release_evidence_true(crash.get("unclean_shutdown_marker"), "crash_restart.unclean_shutdown_marker") mixed = evidence.get("mixed_version") require(isinstance(mixed, dict), "missing release_evidence.mixed_version") + baseline_revision = manifest["baseline"]["revision"] + candidate_revision = manifest["candidate"]["revision"] + require(baseline_revision != candidate_revision, + "release_evidence.mixed_version requires distinct baseline and candidate revisions") + require(manifest["baseline"]["sha256"] != manifest["candidate"]["sha256"], + "release_evidence.mixed_version requires distinct baseline and candidate binaries") revisions = mixed.get("participating_revisions") require( isinstance(revisions, list) @@ -229,20 +241,15 @@ def validate_release_evidence_manifest(manifest): for revision in revisions), "invalid release_evidence.mixed_version.participating_revisions", ) - for revision in (manifest["baseline"]["revision"], manifest["candidate"]["revision"]): + for revision in (baseline_revision, candidate_revision): require(revision in revisions, "release_evidence.mixed_version omits tested build revision") for key in ("reader", "writer", "rollback_payload"): require(mixed.get(key) is True, f"missing release_evidence.mixed_version.{key}") profile = evidence.get("profile") require(isinstance(profile, dict), "missing release_evidence.profile") - artifacts = profile.get("required_artifacts") - require( - isinstance(artifacts, list) - and all(item in artifacts for item in RELEASE_PROFILE_ARTIFACTS) - and all(isinstance(item, str) and item.strip() for item in artifacts), - "missing release_evidence.profile.required_artifacts", - ) + release_evidence_exact_strings(profile.get("required_artifacts"), RELEASE_PROFILE_ARTIFACTS, + "profile.required_artifacts") for key in ("collector_config_sha256", "profiler_config_sha256"): require(sha(profile.get(key)), f"invalid release_evidence.profile.{key}") diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py index afb93baec..116528048 100755 --- a/scripts/test_scanner_abba.py +++ b/scripts/test_scanner_abba.py @@ -168,7 +168,14 @@ class ScannerAbbaTest(unittest.TestCase): def measured_manifest(self): manifest = copy.deepcopy(self.manifest) manifest.update(evidence="measured", duration_seconds=900) - manifest["candidate"]["revision"] = "b" * 40 + candidate_binary = self.root / "candidate-python" + candidate_binary.write_bytes(self.binary.read_bytes() + b"\n") + candidate_binary.chmod(0o755) + manifest["candidate"] = { + "binary": str(candidate_binary), + "sha256": harness.digest(candidate_binary), + "revision": "b" * 40, + } manifest["release_evidence"] = { "topology": { "nodes": 3, @@ -548,6 +555,12 @@ class ScannerAbbaTest(unittest.TestCase): "missing crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update( fault_modes=["process-restart"], ), + "unknown crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update( + fault_modes=["process-restart", "process-crash-restart", "kernel-panic"], + ), + "duplicate crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update( + fault_modes=["process-restart", "process-restart", "process-crash-restart"], + ), "clean crash marker": lambda manifest: manifest["release_evidence"]["crash_restart"].update( unclean_shutdown_marker=False, ), @@ -555,9 +568,24 @@ class ScannerAbbaTest(unittest.TestCase): "missing candidate": lambda manifest: manifest["release_evidence"]["mixed_version"].update( participating_revisions=["a" * 40, "c" * 40], ), + "same mixed revision": lambda manifest: manifest["candidate"].update( + revision=manifest["baseline"]["revision"], + ), + "same mixed binary": lambda manifest: manifest["candidate"].update( + binary=manifest["baseline"]["binary"], + sha256=manifest["baseline"]["sha256"], + ), "missing profile": lambda manifest: manifest["release_evidence"]["profile"].update( required_artifacts=["allocation-profile", "flamegraph", "rss-samples"], ), + "unknown profile": lambda manifest: manifest["release_evidence"]["profile"].update( + required_artifacts=["allocation-profile", "flamegraph", "rss-samples", "save-frequency", "heapdump"], + ), + "duplicate profile": lambda manifest: manifest["release_evidence"]["profile"].update( + required_artifacts=[ + "allocation-profile", "flamegraph", "rss-samples", "save-frequency", "flamegraph", + ], + ), "bad profile hash": lambda manifest: manifest["release_evidence"]["profile"].update( profiler_config_sha256="not-a-sha", ),