From d85b8a893113f9a94cbad7bdf14b3269a746f05b Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 01:20:38 +0800 Subject: [PATCH] test(scanner): report heal release gate status Co-Authored-By: heihutu Co-Authored-By: zhi22915 --- docs/testing/ci-gates.md | 13 +++ scripts/check_test_wiring.py | 101 +++++++++++++++++++++- scripts/run_scanner_heal_evidence_case.sh | 21 +++-- 3 files changed, 126 insertions(+), 9 deletions(-) diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index b40f1ee5e..cff8dc072 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -228,6 +228,19 @@ these. The external `rustfs/auto-testing` functional workflows propagate suite failures. Their workflow status does not establish this registry's required case coverage, build provenance, or object-level oracles. +For automation, `--check-scanner-heal-release "$RUN_DIR"` emits one compact +JSON decision and exits nonzero while blocked. `verified_cases` contains only +cases that pass the complete receipt, build provenance, nextest/JUnit and real +oracle checks; `rejected_cases` names registered cases that do not, and +`pending_gates` names the unimplemented release requirements. Approval requires +every registered case to verify, `pending_gates` to be empty, and a future +registry schema capable of representing the complete release matrix. Schema 1 +is deliberately marked `release_schema_capable: false`: it models only the +single-version, unversioned-object restart/crash cases and cannot represent +mixed-version, rollback, EC8+4 or performance evidence. A focused run, +synthetic harness, compile-only result, skipped/retried test, ordinary CI +success, or removal of pending text therefore cannot become a release approval. + Run parser/receipt regressions with `scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures validate the checker only and produce no runtime or performance evidence. diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 0d43c274d..8fa09c6fa 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -1094,6 +1094,38 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li return [f"scanner/heal evidence rejected: {error}"] +def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object]: + """Return a compact release decision without weakening case validation.""" + registry = read_json(root / ".config/scanner-heal-required-tests.json") + evidence_integer(registry.get("schema"), "registry schema", 1, 1) + cases = registry.get("cases") + require(isinstance(cases, dict) and cases, "invalid scanner/heal registry") + pending = registry.get("release_pending") + require(isinstance(pending, dict), "invalid scanner/heal release requirements") + for gate, reason in pending.items(): + require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None, + "invalid scanner/heal release gate") + require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}") + + verified_cases = [] + rejected_cases = [] + for case_id in sorted(cases): + if check_scanner_heal_evidence(root, directory, case_id): + rejected_cases.append(case_id) + else: + verified_cases.append(case_id) + + return { + "schema": 1, + "decision": "blocked", + "release_approved": False, + "release_schema_capable": False, + "verified_cases": verified_cases, + "rejected_cases": rejected_cases, + "pending_gates": sorted(pending), + } + + def validate(root: Path) -> list[str]: errors: list[str] = [] errors.extend(check_core_fixtures(root)) @@ -1311,6 +1343,61 @@ class SelfTests(unittest.TestCase): self.assertTrue(any(error.startswith("pending R-D:") for error in errors)) self.assertTrue(any(error.startswith("pending R-L:") for error in errors)) + status = scanner_heal_release_status(root, run_dir) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertEqual(status["rejected_cases"], []) + self.assertEqual(len(status["pending_gates"]), 21) + + def test_scanner_heal_case_only_schema_cannot_approve_release(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["release_pending"] = {} + write_json(root / ".config/scanner-heal-required-tests.json", registry) + + status = scanner_heal_release_status(root, run_dir) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertFalse(status["release_schema_capable"]) + self.assertEqual(status["rejected_cases"], []) + self.assertEqual(status["pending_gates"], []) + + def test_scanner_heal_release_status_rejects_synthetic_case(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["release_pending"] = {} + write_json(root / ".config/scanner-heal-required-tests.json", registry) + path = run_dir / "background-target-crash.json" + oracle = read_json(path) + oracle["evidence"] = "synthetic" + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0, root) + + status = scanner_heal_release_status(root, run_dir) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertEqual(status["rejected_cases"], ["background-target-crash"]) + self.assertEqual(status["pending_gates"], []) + + def test_scanner_heal_release_status_rejects_focused_case_run(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["release_pending"] = {} + write_json(root / ".config/scanner-heal-required-tests.json", registry) + (run_dir / "background-target-crash.json").unlink() + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0, root) + + status = scanner_heal_release_status(root, run_dir) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertEqual(status["verified_cases"], ["background-target-restart"]) + self.assertEqual(status["rejected_cases"], ["background-target-crash"]) + def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None: with tempfile.TemporaryDirectory() as tmp: root, run_dir = self.scanner_heal_fixture(Path(tmp)) @@ -2158,7 +2245,8 @@ def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 - if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"]): + if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"], + ["--check-scanner-heal-release"]): try: if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal": begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4])) @@ -2173,7 +2261,16 @@ def main() -> int: if not errors: print(f"Case evidence verified: {sys.argv[3]}; this does not approve release") return 1 if errors else 0 - raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, or --check-scanner-heal DIR CASE|release") + if len(sys.argv) == 3 and sys.argv[1] == "--check-scanner-heal-release": + try: + status = scanner_heal_release_status(ROOT, Path(sys.argv[2])) + except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: + print(json.dumps({"schema": 1, "decision": "invalid", "release_approved": False, + "error": str(error)}, sort_keys=True, separators=(",", ":"))) + return 2 + print(json.dumps(status, sort_keys=True, separators=(",", ":"))) + return 0 if status["release_approved"] else 1 + raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, or --check-scanner-heal-release DIR") except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error: print(f"ERROR: {error}", file=sys.stderr) return 1 diff --git a/scripts/run_scanner_heal_evidence_case.sh b/scripts/run_scanner_heal_evidence_case.sh index 2c8b0503c..ff7c30086 100755 --- a/scripts/run_scanner_heal_evidence_case.sh +++ b/scripts/run_scanner_heal_evidence_case.sh @@ -90,15 +90,22 @@ PY release_gate_must_remain_blocked() { local run_dir="$1" - local output="$run_dir/release-check.txt" - if "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$run_dir" release >"$output" 2>&1; then + local output="$run_dir/release-status.json" + if "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release "$run_dir" >"$output"; then echo "release gate unexpectedly approved a single Scanner/Heal evidence run" >&2 return 1 fi - if ! grep -Eq 'required test not selected:|pending [A-Z0-9-]+:' "$output"; then - echo "release gate did not explain why the Scanner/Heal release remains blocked" >&2 - return 1 - fi + "$PYTHON_BIN" - "$output" <<'PY' +import json +import pathlib +import sys + +status = json.loads(pathlib.Path(sys.argv[1]).read_text()) +if status.get("decision") != "blocked" or status.get("release_approved") is not False: + raise SystemExit("release status did not record a blocked decision") +if status.get("release_schema_capable") is not False: + raise SystemExit("case-only evidence schema unexpectedly became release-capable") +PY } run_self_test() { @@ -242,5 +249,5 @@ fi "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$RUN_DIR" "$CASE_ID" release_gate_must_remain_blocked "$RUN_DIR" echo "Scanner/Heal evidence case verified: $CASE_ID" -echo "Release gate remains blocked; details: $RUN_DIR/release-check.txt" +echo "Release gate remains blocked; status: $RUN_DIR/release-status.json" echo "Evidence directory: $RUN_DIR"