From d897fc52ce6e30f33b57e9bc28437ca8acbcb594 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:27:09 +0100 Subject: [PATCH] Retry the release listing so a GitHub 504 cannot fail a release The first beta's release workflow failed in create_release at "Locate existing release". That step reads up to 100 releases with every asset, about 25 MB, in one call, and GitHub answered it with an HTTP 504. The step has no pipefail, so the failed read reached jq as empty input and the job ended with exit code 5. Timing the call from a workstation, one of three tries returned an error body, so this is not rare. Retry the read up to five times with the same backoff the asset upload uses, and accept only a complete listing of valid pages. A failed or truncated read must never look like "no existing release": that answer would stage a duplicate draft for a tag that already has one. Exhausting the attempts fails the job with an explicit error. The listing call and the downstream handling of an existing draft are unchanged. Run the real step script in the release policy test against doubles for gh and sleep: two 504s then a complete listing finds the draft after backoff of 15s and 30s; a read that exits 0 but is truncated every time fails after five attempts, with the capped backoff and no outputs; and a complete listing without the tag still means no existing release. Against the previous step the test fails with the production symptom, exit 5 and the 504 message. The create-release pin list and the contract record the retry. Contract-Neutral: Release workflow reliability only; no published artifact, tag, signing, install or customer contract changes. --- .github/workflows/create-release.yml | 31 +++++++- .../subsystems/deployment-installability.md | 7 ++ .../installtests/build_release_assets_test.go | 3 + .../release_promotion_policy_test.py | 71 +++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index c332b02334..04739e1458 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -1027,7 +1027,36 @@ jobs: WORKFLOW_OUTPUT_1: ${{ needs.prepare.outputs.tag }} run: | TAG="${WORKFLOW_OUTPUT_1}" - EXISTING_RELEASE=$(gh api "repos/${{ github.repository }}/releases?per_page=100" --paginate | jq -sc --arg tag "$TAG" 'add | map(select(.tag_name == $tag)) | first // empty') + # The listing carries every release's assets and is about 25 MB, and GitHub answers + # it with an HTTP 504 often enough to fail a release on its own. Retry a bounded + # number of times and accept only a complete listing: a failed or truncated read + # must never look like "no existing release", which would stage a duplicate draft. + RELEASES_FILE="$(mktemp)" + attempt=1 + max_attempts=5 + wait_seconds=15 + while true; do + if gh api "repos/${{ github.repository }}/releases?per_page=100" --paginate > "$RELEASES_FILE" && + jq -s -e 'length > 0 and all(.[]; type == "array")' "$RELEASES_FILE" > /dev/null; then + break + fi + + if [ "$attempt" -ge "$max_attempts" ]; then + echo "::error::releases listing failed after ${max_attempts} attempts" + exit 1 + fi + + echo "releases listing failed on attempt ${attempt}/${max_attempts}; retrying in ${wait_seconds}s" + sleep "$wait_seconds" + attempt=$((attempt + 1)) + if [ "$wait_seconds" -lt 120 ]; then + wait_seconds=$((wait_seconds * 2)) + if [ "$wait_seconds" -gt 120 ]; then + wait_seconds=120 + fi + fi + done + EXISTING_RELEASE=$(jq -sc --arg tag "$TAG" 'add | map(select(.tag_name == $tag)) | first // empty' "$RELEASES_FILE") RELEASE_ID=$(echo "$EXISTING_RELEASE" | jq -r '.id // empty') RELEASE_URL=$(echo "$EXISTING_RELEASE" | jq -r '.html_url // empty') RELEASE_IS_DRAFT=$(echo "$EXISTING_RELEASE" | jq -r '.draft // false') diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 2ac6f51014..aa9c536c47 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -5003,6 +5003,13 @@ on the reviewed `golang:1.26.9-alpine` digest it does not pull the image, prove its registry contents, or qualify a deployed control plane. The release publisher's prebuilt control-plane target is a separate path and does not rebuild its binary from this stage. +The release workflow locates an existing release for its tag with one listing +of up to 100 releases and all their assets, about 25 MB, which GitHub answers +with an HTTP 504 often enough to fail a release by itself (observed on the first +beta). That read is retried up to five times with backoff and accepts only a +complete listing of valid pages, so a failed or truncated read can never be +mistaken for "no existing release" and stage a duplicate draft. Exhausting the +attempts fails the job with an explicit error. As of 2026-10-11, the governed release floor is Go `1.26.9`. It supersedes `1.26.8`, whose standard library is reachable through twelve vulnerable Pulse call paths reported by `govulncheck` after the 2026-10-08 advisories, including diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 7520e38860..7b0bddc419 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -1007,6 +1007,9 @@ func TestCreateReleaseUploadsPowerShellInstaller(t *testing.T) { `gh release upload failed after ${max_attempts} attempts`, `release/release-build-provenance.sigstore.json`, `gh api "repos/${{ github.repository }}/releases?per_page=100" --paginate`, + `jq -s -e 'length > 0 and all(.[]; type == "array")' "$RELEASES_FILE"`, + `releases listing failed on attempt ${attempt}/${max_attempts}; retrying in ${wait_seconds}s`, + `releases listing failed after ${max_attempts} attempts`, `git push origin "refs/tags/${TAG}"`, `--rawfile body "$NOTES_FILE"`, `--input "$RELEASE_PAYLOAD"`, diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index 98ee5dcc49..8dfe96a5fc 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -2787,6 +2787,77 @@ class ReleasePromotionPolicyTest(unittest.TestCase): normalize_ws(contract), ) + + def test_create_release_locates_its_release_through_a_bounded_complete_listing_retry(self) -> None: + """Run the real step script against doubles: GitHub 504s must be retried, never read as absence.""" + workflow = yaml.safe_load(read(".github/workflows/create-release.yml")) + step = next(s for s in workflow["jobs"]["create_release"]["steps"] + if s.get("name") == "Locate existing release") + script = step["run"].replace("${{ github.repository }}", "owner/repo") + pages = ( + '[{"id": 1, "tag_name": "v6.5.0", "draft": false, "assets": []}]\n' + '[{"id": 42, "tag_name": "v6.6.0-beta.1", "draft": true, "html_url": "https://example.test/42",' + ' "target_commitish": "abc", "published_at": null, "assets": [{"name": "checksums.txt"}]}]\n' + ) + fake_gh = ( + '#!/bin/bash\n' + 'n=$(cat "$FIX/count" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$FIX/count"\n' + 'case "$MODE" in\n' + ' flaky) if [ "$n" -le 2 ]; then\n' + ' echo "gh: We could not respond to your request in time (HTTP 504)" >&2; exit 1; fi\n' + ' cat "$FIX/pages.json" ;;\n' + ' truncated) printf \'[{"id": 42, "tag_name": \' ; exit 0 ;;\n' + ' ok) cat "$FIX/pages.json" ;;\n' + 'esac\n' + ) + fake_sleep = '#!/bin/bash\necho "$1" >> "$FIX/sleeps"\n' + + def outputs(text: str) -> dict: + # write_github_output.py writes name< dict: + with tempfile.TemporaryDirectory() as directory: + fix = Path(directory) + (fix / "bin").mkdir() + for name, body in (("gh", fake_gh), ("sleep", fake_sleep)): + (fix / "bin" / name).write_text(body) + (fix / "bin" / name).chmod(0o755) + (fix / "pages.json").write_text(pages) + output = fix / "github_output" + output.write_text("") + env = {**os.environ, "FIX": str(fix), "MODE": mode, "WORKFLOW_OUTPUT_1": tag, + "GH_TOKEN": "unused", "GITHUB_OUTPUT": str(output), + "PATH": f"{fix / 'bin'}{os.pathsep}{os.environ['PATH']}"} + result = subprocess.run(["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script], + cwd=REPO_ROOT, env=env, text=True, capture_output=True, timeout=60) + read_text = lambda name: (fix / name).read_text() if (fix / name).exists() else "" + return {"result": result, "output": output.read_text(), + "calls": int(read_text("count").strip() or 0), + "sleeps": read_text("sleeps").split()} + + # Two GitHub 504s, then a complete listing: retried with backoff, then the draft is found. + flaky = run("flaky", "v6.6.0-beta.1") + self.assertEqual(flaky["result"].returncode, 0, flaky["result"].stderr) + self.assertEqual((flaky["calls"], flaky["sleeps"]), (3, ["15", "30"])) + self.assertEqual(outputs(flaky["output"]).get("release_id"), "42") + self.assertEqual(outputs(flaky["output"]).get("release_is_draft"), "true") + + # A read that exits 0 but is truncated every time is never an empty answer: five attempts, + # capped backoff, an explicit error, and no outputs written. + truncated = run("truncated", "v6.6.0-beta.1") + self.assertNotEqual(truncated["result"].returncode, 0) + self.assertIn("releases listing failed after 5 attempts", truncated["result"].stdout) + self.assertEqual((truncated["calls"], truncated["sleeps"]), (5, ["15", "30", "60", "120"])) + self.assertEqual(truncated["output"], "") + + # A complete listing that does not contain the tag still means "no existing release". + absent = run("ok", "v6.7.0") + self.assertEqual(absent["result"].returncode, 0, absent["result"].stderr) + self.assertEqual((absent["calls"], absent["sleeps"]), (1, [])) + self.assertEqual(outputs(absent["output"]).get("release_id"), "") + def test_v620_owner_soak_waiver_is_version_bound(self) -> None: policy = normalize_ws( read("docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md")