diff --git a/.github/workflows/release-convergence.yml b/.github/workflows/release-convergence.yml index 736580dfee..fd4fa02505 100644 --- a/.github/workflows/release-convergence.yml +++ b/.github/workflows/release-convergence.yml @@ -130,18 +130,35 @@ jobs: fi activation_owner_run_id="$(jq -r '.convergence_run_id' "${marker}")" if [ "${activation_owner_run_id}" != "${EXPECTED_CONVERGENCE_RUN_ID}" ]; then - original_status="$( + # The immutable marker outlives Actions history: runs are + # deleted by retention or by hand, and GitHub only deletes a + # run once it has completed or is older than two weeks, far + # beyond every job timeout here. A definite 404 is therefore + # a terminal owner. Any other read failure is unknown and + # keeps waiting. + owner_read_error="$(mktemp)" + if ! original_status="$( gh api \ "repos/${{ github.repository }}/actions/runs/${activation_owner_run_id}" \ - --jq '.status' 2>/dev/null || true - )" - if [ "${original_status}" != "completed" ]; then + --jq '.status' 2>"${owner_read_error}" + )"; then + original_status="" + if grep -q 'HTTP 404' "${owner_read_error}"; then + original_status="deleted" + fi + fi + rm -f "${owner_read_error}" + if [ "${original_status}" != "completed" ] && [ "${original_status}" != "deleted" ]; then rm -f "${marker}" echo "Original convergence owner ${activation_owner_run_id} is ${original_status:-unavailable}; successor adoption is not yet allowed." sleep 2 continue fi - echo "[OK] Adopting committed ${TAG} from completed convergence owner ${activation_owner_run_id}." + if [ "${original_status}" = "deleted" ]; then + echo "[OK] Adopting committed ${TAG}; convergence owner ${activation_owner_run_id} was deleted from Actions history and is terminal." + else + echo "[OK] Adopting committed ${TAG} from completed convergence owner ${activation_owner_run_id}." + fi fi activation_marker_sha256="$(sha256sum "${marker}" | awk '{print $1}')" server_image_digest="$(jq -r '.server_image_digest' "${marker}")" diff --git a/.github/workflows/retry-release-convergence.yml b/.github/workflows/retry-release-convergence.yml index ca1d8afc14..23785326e3 100644 --- a/.github/workflows/retry-release-convergence.yml +++ b/.github/workflows/retry-release-convergence.yml @@ -6,17 +6,17 @@ run-name: Reconcile release convergence debt # when the failed run predates a repair, so the complete convergence workflow # reacquires the global lease and replays every idempotent surface. Successful # surfaces are safe to repeat; retrying only failed jobs would bypass the lease. +# +# The hourly schedule is the retry mechanism. There is deliberately no +# workflow_run trigger: create-release.yml and this reconciler dispatch +# convergence with GITHUB_TOKEN, and GitHub does not deliver workflow_run for +# runs started that way. A workflow_run job only ever fired for convergence +# runs an operator started by hand; between 29 Aug and 24 Oct 2026 it never +# fired at all, while dozens of failed bot-dispatched runs were recovered by +# this schedule. on: - workflow_run: - workflows: - - Release Convergence - types: - - completed - branches: - - main schedule: - # workflow_run delivery is not durable. Reconcile the current stable and - # preview channel heads hourly so a dropped event cannot strand aliases. + # Reconcile the current stable and preview channel heads hourly. - cron: '37 * * * *' workflow_dispatch: @@ -29,28 +29,7 @@ concurrency: cancel-in-progress: false jobs: - retry_committed_convergence: - if: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'success' }} - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - name: Checkout current reconciliation controls - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Reconcile the failed convergence - env: - GH_TOKEN: ${{ github.token }} - PRO_REPOSITORY_TOKEN: ${{ secrets.WORKFLOW_PAT }} - RUN_ID: ${{ github.event.workflow_run.id }} - run: >- - python3 scripts/release_control/reconcile_release_convergence.py - --repository "${GITHUB_REPOSITORY}" - --run-id "${RUN_ID}" - - reconcile_missed_event: - if: ${{ github.event_name != 'workflow_run' }} + reconcile_channel_heads: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 462cda664b..3d26247b6c 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -3476,13 +3476,19 @@ artifact-selection behaviour. Reading a floating ref or a clobbered constant record is forbidden because cached prior bytes could authorize stale ownership. Convergence retry is a current-control reconciliation, not unconditional - replay of the failed run's checkout. A terminal failure delivered through - `workflow_run` must be reconciled, and an hourly or manual pass must inspect - the current immutable stable and preview channel heads so a missed event - cannot strand mutable aliases. Discovery must not fall back behind a mutable + replay of the failed run's checkout. An hourly or manual pass must inspect + the current immutable stable and preview channel heads; that schedule is the + retry mechanism, because GitHub does not deliver `workflow_run` for + convergence runs dispatched with `GITHUB_TOKEN`, and no event-driven retry + path may be presented as one. Discovery must not fall back behind a mutable channel head. Before mutation, reconciliation must bind the exact mainline workflow-dispatch identity, release, source run, activation marker and - original convergence owner; reject malformed marker digests; suppress a + original convergence owner. The immutable marker outlives Actions history: + an original owner that GitHub definitively reports as not found is terminal, + because GitHub deletes only completed runs or runs older than two weeks, so + both the reconciler and the successor's adoption wait proceed from the + marker alone, while any other owner read failure keeps waiting or fails. + Reconciliation must also reject malformed marker digests; suppress a newer matching run; and enforce one aggregate attempt budget across reruns and successor runs. If the failed run used the current default-branch commit, the complete run is rerun. If its checkout predates current controls, a fresh diff --git a/scripts/release_control/reconcile_release_convergence.py b/scripts/release_control/reconcile_release_convergence.py index e85168963d..44dd7f2caf 100644 --- a/scripts/release_control/reconcile_release_convergence.py +++ b/scripts/release_control/reconcile_release_convergence.py @@ -637,24 +637,26 @@ def reconcile(github: GitHub, run_id: int, max_attempts: int) -> None: ) try: owner_run = github.api(f"repos/{repository}/actions/runs/{owner_run_id}") - except GitHubNotFound as exc: - # The immutable marker outlives Actions history. Once its owner run is - # deleted, release-convergence.yml can never adopt the tag (it waits - # for that owner to read "completed"), so a retry would only time out. - raise ReconciliationError( - f"activation marker owner run {owner_run_id} for committed {tag} no longer " - "exists in Actions history; convergence cannot adopt this release until an " - "operator resolves it. No retry was dispatched." - ) from exc - require_run_identity( - owner_run, - repository=repository, - workflow_id=convergence_workflow_id, - path=CONVERGENCE_PATH, - display_title=title, - ) - if owner_run.get("status") != "completed": - raise ReconciliationError("activation marker owner is not terminal") + except GitHubNotFound: + # The immutable marker outlives Actions history. GitHub only deletes a + # run once it has completed or is older than two weeks, beyond every + # convergence job timeout, so a deleted owner is terminal and + # release-convergence.yml adopts the tag from it. The marker itself + # was validated above; there is no run left to bind its identity to. + print( + f"Activation marker owner run {owner_run_id} for committed {tag} was " + "deleted from Actions history; treating it as terminal." + ) + else: + require_run_identity( + owner_run, + repository=repository, + workflow_id=convergence_workflow_id, + path=CONVERGENCE_PATH, + display_title=title, + ) + if owner_run.get("status") != "completed": + raise ReconciliationError("activation marker owner is not terminal") default_commit = github.api(f"repos/{repository}/commits/{default_branch}").get("sha") if not isinstance(default_commit, str) or EXACT_SHA.fullmatch(default_commit) is None: diff --git a/scripts/release_control/reconcile_release_convergence_test.py b/scripts/release_control/reconcile_release_convergence_test.py index f75bb5e061..c68e6e15d0 100644 --- a/scripts/release_control/reconcile_release_convergence_test.py +++ b/scripts/release_control/reconcile_release_convergence_test.py @@ -798,21 +798,45 @@ class ReconciliationTests(unittest.TestCase): subject.reconcile(github, github.run_id, 5) self.assertEqual([], github.posts) - def test_deleted_activation_owner_is_named_without_retry(self): - github = FakeGitHub() + @staticmethod + def delete_owner_run(github, error): fake_api = github.api def api(endpoint): if endpoint.endswith(f"/actions/runs/{github.owner_run_id}"): - raise subject.GitHubNotFound("GitHub object was not found") + raise error return fake_api(endpoint) github.api = api - with self.assertRaises(subject.ReconciliationError) as raised: + + def test_deleted_activation_owner_is_terminal_for_fresh_controls(self): + # v6.5.0's owner run was deleted from Actions on 13 Oct 2026; the + # immutable marker still names it. Deletion implies a terminal run. + github = FakeGitHub() + self.delete_owner_run(github, subject.GitHubNotFound("GitHub object was not found")) + with contextlib.redirect_stdout(io.StringIO()) as output: + subject.reconcile(github, github.run_id, 5) + self.assertIn("owner run 90 for committed v6.5.0-rc.1 was deleted", output.getvalue()) + self.assertEqual(1, len(github.posts)) + endpoint, payload = github.posts[0] + self.assertTrue(endpoint.endswith("/release-convergence.yml/dispatches")) + self.assertEqual("200", payload["inputs"]["source_release_run_id"]) + + def test_deleted_activation_owner_reruns_current_controls(self): + github = FakeGitHub(current_controls=True) + self.delete_owner_run(github, subject.GitHubNotFound("GitHub object was not found")) + with contextlib.redirect_stdout(io.StringIO()): + subject.reconcile(github, github.run_id, 5) + self.assertEqual( + [(f"repos/{github.repository}/actions/runs/{github.run_id}/rerun", None)], + github.posts, + ) + + def test_unreadable_activation_owner_is_not_treated_as_deleted(self): + github = FakeGitHub() + self.delete_owner_run(github, subject.ReconciliationError("GitHub command failed")) + with self.assertRaises(subject.ReconciliationError): subject.reconcile(github, github.run_id, 5) - self.assertNotIsInstance(raised.exception, subject.GitHubNotFound) - self.assertIn("owner run 90 for committed v6.5.0-rc.1", str(raised.exception)) - self.assertIn("no longer exists", str(raised.exception)) self.assertEqual([], github.posts) def test_one_channel_failure_does_not_starve_the_other(self): diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index e4298e4344..5b8546419a 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -904,13 +904,14 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn(surface_name, convergence_verdict) self.assertIn("the committed release remains public", convergence_verdict) - self.assertIn("github.event.workflow_run.conclusion != 'success'", retry) + # GitHub does not deliver workflow_run for GITHUB_TOKEN-dispatched + # convergence runs, so the hourly schedule is the retry mechanism. + self.assertNotIn("workflow_run", yaml.safe_load(retry)[True]) self.assertIn("cron: '37 * * * *'", retry) self.assertIn("workflow_dispatch:", retry) self.assertIn("reconcile_release_convergence.py", retry) - self.assertIn('--run-id "${RUN_ID}"', retry) self.assertIn("--latest", retry) - self.assertEqual(2, retry.count("PRO_REPOSITORY_TOKEN: ${{ secrets.WORKFLOW_PAT }}")) + self.assertEqual(1, retry.count("PRO_REPOSITORY_TOKEN: ${{ secrets.WORKFLOW_PAT }}")) self.assertNotIn("rerun-failed-jobs", retry) self.assertIn('actions/runs/{run_id}/rerun', reconciler) self.assertIn('release-convergence.yml/dispatches', reconciler) @@ -1141,10 +1142,14 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("does not match the expected immutable release identity", await_commit) self.assertIn("EXPECTED_CONVERGENCE_RUN_ID: ${{ github.run_id }}", await_commit) self.assertIn('original_status}" != "completed"', await_commit) + # A deleted owner is terminal: GitHub only deletes completed runs or + # runs older than two weeks. Other read failures keep waiting. + self.assertIn("grep -q 'HTTP 404'", await_commit) + self.assertIn('original_status="deleted"', await_commit) self.assertIn("successor adoption is not yet allowed", await_commit) self.assertIn("activation_marker_sha256", await_commit) - self.assertIn("--run-id", retry) + self.assertIn("--latest", retry) self.assertIn("DISPLAY_TITLE = re.compile", reconciler) self.assertIn("source_run_id = int(source_text)", reconciler) self.assertIn('actions/runs/{source_run_id}', reconciler)