From d7c42d409985da231828939e6619d2e6dfb02e59 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:25:36 +0800 Subject: [PATCH] fix(ci): accept the testing-sha staleness fallback in chain evidence (#8056) The 09-22 nightly chain failed all 12 lanes in seconds at the 'Bind functional candidate' step: ValueError: private script pin differs from chain resolve_functional_candidate.py's >24h staleness fallback (added by #8026, made functional by #8041's token fix) legitimately sets manifest.testing_sha to auto-testing main HEAD, but current_chain() still required it to equal .config/functional-script-revision.txt - a check written for the pre-fallback world where the two could never diverge. Once the fallback finally fired, prepare produced testing_sha 21edcf4 while the pin file still holds 27e9584 and every lane aborted before checking out the test scripts. Drop the pin-file comparison and keep what the lane actually needs to guarantee: testing_sha is a valid commit sha (current_chain), the lane checked out exactly that sha (record: private_head == testing_sha, kept as-is), and the health checker validates the same format instead of re-reading the pin file. Tests updated: a fallback testing_sha that differs from the pin is accepted; a non-sha testing_sha is rejected. Verified: python3 -m unittest test_functional_chain test_functional_chain_health -> 39 tests OK. --- scripts/functional_chain_evidence.py | 7 ++++++- scripts/functional_chain_health.py | 5 +++-- scripts/test_functional_chain.py | 14 ++++++++++++++ scripts/test_functional_chain_health.py | 8 +++++++- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/functional_chain_evidence.py b/scripts/functional_chain_evidence.py index d070c7d6a..3f3cbbdb3 100644 --- a/scripts/functional_chain_evidence.py +++ b/scripts/functional_chain_evidence.py @@ -28,7 +28,12 @@ def current_chain(): require(sha(chain["workflow_sha"]) and chain["workflow_sha"] == os.environ["GITHUB_SHA"], "chain workflow source mismatch") head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() require(head == chain["workflow_sha"], "lane checkout differs from chain workflow source") - require(chain["testing_sha"] == (ROOT / ".config/functional-script-revision.txt").read_text().strip() and sha(chain["testing_sha"]), "private script pin differs from chain") + # testing_sha is either the committed pin or auto-testing main HEAD via + # resolve_functional_candidate.py's >24h staleness fallback, so pin + # equality is no longer an invariant (the 09-21 chain died on exactly + # that check once the fallback finally fired). Lanes check out exactly + # this sha, which is what the format check guards. + require(sha(chain["testing_sha"]), "private script revision is not a valid commit sha") candidate = chain["candidate"] require(isinstance(candidate, dict) and set(candidate) == {"manifest", "artifact_id", "artifact_digest", "workflow_sha", "workflow_ref", "build_started_at"}, "invalid candidate envelope") manifest = candidate["manifest"] diff --git a/scripts/functional_chain_health.py b/scripts/functional_chain_health.py index d32deaf1b..efcf54a05 100644 --- a/scripts/functional_chain_health.py +++ b/scripts/functional_chain_health.py @@ -32,8 +32,9 @@ def validate_summary(summary, run): candidate = chain["candidate"] manifest = candidate["manifest"] require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed") - config = api(f"repos/{REPOSITORY}/contents/.config/functional-script-revision.txt?ref={run['head_sha']}") - require(base64.b64decode(config["content"]).decode().strip() == chain["testing_sha"], "private pin differs from workflow source") + # testing_sha may legitimately be auto-testing main HEAD via the prepare + # step's >24h staleness fallback (checked for sha format above), so pin + # equality is not an invariant; drop the pin-file comparison. completed = timestamp(summary["completed_at"]) require(timestamp(run["run_started_at"]) <= completed <= datetime.now(timezone.utc) + timedelta(minutes=5), "invalid completion timestamp") source_ref = manifest.get("source_ref", candidate["workflow_ref"]) diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index 4d737781a..b7fde514a 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -226,6 +226,20 @@ class EnvelopeTests(unittest.TestCase): evidence.current_chain() self.assertFalse((self.root / "env").exists()) + def test_testing_sha_fallback_is_accepted_while_garbage_is_rejected(self): + # prepare's >24h staleness fallback legitimately sets testing_sha to + # auto-testing main HEAD, which differs from the committed pin; only + # the sha format is an invariant now. + for testing_sha, ok in (("d" * 40, True), ("1" * 40, True), ("xyz", False), ("", False)): + chain = dict(self.chain, testing_sha=testing_sha) + env = dict(self.env, CHAIN_MANIFEST=json.dumps(chain)) + if ok: + with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40): + evidence.consume(evidence.current_chain()) + else: + with mock.patch.object(evidence, "ROOT", self.root), mock.patch.dict(evidence.os.environ, env), mock.patch.object(evidence.subprocess, "check_output", return_value="e" * 40), self.assertRaises(ValueError): + evidence.current_chain() + def test_report_or_swallowed_test_failure_cannot_produce_valid_evidence(self): report = self.root / "cases.md" report.write_text("| Case | Name | Status |\n| --- | --- | --- |\n| KMS-1 | fixture | PASS |\n") diff --git a/scripts/test_functional_chain_health.py b/scripts/test_functional_chain_health.py index 225bb99b4..b03aedf7a 100644 --- a/scripts/test_functional_chain_health.py +++ b/scripts/test_functional_chain_health.py @@ -39,8 +39,14 @@ class HealthTests(unittest.TestCase): def test_substituted_producer_pin_attempt_or_empty_suite_fails(self): with self.assertRaises(ValueError): self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40}) + # The prepare step's >24h staleness fallback legitimately sets + # testing_sha to auto-testing main HEAD, so the pin FILE is no longer + # consulted at all; the sha FORMAT of the chain's testing_sha is the + # remaining invariant. + wrong = copy.deepcopy(self.summary) + wrong["chain"]["testing_sha"] = "short" with self.assertRaises(ValueError): - self.validate(config={"content": base64.b64encode(b"wrong pin").decode()}) + self.validate(wrong) wrong = copy.deepcopy(self.summary) wrong["chain"]["attempt"] = 1 with self.assertRaises(ValueError):