diff --git a/scripts/release_control/reconcile_release_convergence.py b/scripts/release_control/reconcile_release_convergence.py index 343aa4da4..2e06705ce 100644 --- a/scripts/release_control/reconcile_release_convergence.py +++ b/scripts/release_control/reconcile_release_convergence.py @@ -24,6 +24,7 @@ PRIVATE_PROMOTION_RUN = re.compile( PRIVATE_REPOSITORY = "rcourtman/pulse-pro" PRIVATE_PROMOTION_PATH = ".github/workflows/promote-paid-runtime-release.yml" PAID_RUNTIME_JOB = "Converge paid-runtime broker / promote" +CONVERGENCE_VERDICT_JOB = "Customer Promotion Convergence Verdict" CREDENTIAL_CONTAINMENT_JOB = "Require credential containment" CREDENTIAL_BLOCK_MARKER = "credential containment gate: BLOCKED" CREDENTIAL_CONTAINMENT_PATHS = ( @@ -355,6 +356,26 @@ class GitHub: ] if len(paid_jobs) != 1: return False + # Containment may explain the paid-runtime failure and the aggregate + # verdict it necessarily makes red. It cannot explain another failed + # customer surface, an interrupted finalizer, or incomplete job + # evidence; those remain ordinary convergence debt and must retain the + # unattended retry path. + expected_failures = sorted((PAID_RUNTIME_JOB, CONVERGENCE_VERDICT_JOB)) + if any( + not isinstance(value, dict) + or not isinstance(value.get("name"), str) + or not value.get("name") + or value.get("status") != "completed" + or value.get("conclusion") not in {"success", "failure", "skipped"} + for value in jobs + ): + return False + failed_jobs = sorted( + value["name"] for value in jobs if value["conclusion"] == "failure" + ) + if failed_jobs != expected_failures: + return False paid_job_id = positive_int(paid_jobs[0].get("id"), "paid-runtime job ID") annotations = flatten_pages( self.pages( diff --git a/scripts/release_control/reconcile_release_convergence_test.py b/scripts/release_control/reconcile_release_convergence_test.py index f44eb431e..7bf947df5 100644 --- a/scripts/release_control/reconcile_release_convergence_test.py +++ b/scripts/release_control/reconcile_release_convergence_test.py @@ -259,6 +259,8 @@ class CredentialContainmentTests(unittest.TestCase): containment="failure", containment_log=subject.CREDENTIAL_BLOCK_MARKER, containment_state_changed=False, + extra_public_job=None, + verdict="failure", ): private_run_id = 700 paid_job_id = 800 @@ -279,8 +281,16 @@ class CredentialContainmentTests(unittest.TestCase): { "id": paid_job_id, "name": subject.PAID_RUNTIME_JOB, + "status": "completed", "conclusion": "failure", - } + }, + { + "id": paid_job_id + 1, + "name": subject.CONVERGENCE_VERDICT_JOB, + "status": "completed", + "conclusion": verdict, + }, + *([extra_public_job] if extra_public_job else []), ] } ] @@ -362,6 +372,53 @@ class CredentialContainmentTests(unittest.TestCase): github = self.github(containment="success") self.assertFalse(github.unchanged_credential_containment_block(100)) + def test_other_public_surface_failure_remains_retriable(self): + github = self.github( + extra_public_job={ + "id": 802, + "name": "Converge Helm Pages / release", + "status": "completed", + "conclusion": "failure", + } + ) + self.assertFalse(github.unchanged_credential_containment_block(100)) + + def test_interrupted_public_job_remains_retriable(self): + github = self.github( + extra_public_job={ + "id": 802, + "name": "Release global customer-promotion lease", + "status": "completed", + "conclusion": "cancelled", + } + ) + self.assertFalse(github.unchanged_credential_containment_block(100)) + + def test_missing_failed_aggregate_verdict_remains_retriable(self): + github = self.github(verdict="success") + self.assertFalse(github.unchanged_credential_containment_block(100)) + + def test_duplicate_failed_aggregate_verdict_remains_retriable(self): + github = self.github( + extra_public_job={ + "id": 802, + "name": subject.CONVERGENCE_VERDICT_JOB, + "status": "completed", + "conclusion": "failure", + } + ) + self.assertFalse(github.unchanged_credential_containment_block(100)) + + def test_malformed_public_job_remains_retriable(self): + github = self.github( + extra_public_job={ + "id": 802, + "status": "completed", + "conclusion": "failure", + } + ) + self.assertFalse(github.unchanged_credential_containment_block(100)) + def test_containment_job_error_without_block_marker_remains_retriable(self): github = self.github(containment_log="checkout failed") self.assertFalse(github.unchanged_credential_containment_block(100))