From 9c28f097b3fc1a7232665232fa9543327cc836e8 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:28:14 +0100 Subject: [PATCH] Warm the pulse-agent build before the pinned release API shards The first beta's release workflow failed twice in backend_tests: TestHostAgentCredentialFileHandoff, added on 2026-10-11 and so new to every release job, compiles pulse-agent itself and was killed at exactly 180s with "build agent from this source: signal: killed". The release runner starts each API shard with a small pinned GOMAXPROCS, and the test's child compiler inherits it. A cold agent build is about 250 CPU-seconds, so under the pin it serializes past the test's fixed three-minute limit. The release job also has no Go build cache, and the race-enabled test binary the runner compiles does not warm this plain build. Pull request jobs keep a warm cache, which is why the test passed there. Reproduced on pulse-dev with the shard's conditions, a prebuilt test binary, GOMAXPROCS=1 and an empty cache: the test fails at 180.02s with the identical message. With the agent built once beforehand, unpinned, in 99s, the same pinned run passes in 16s. Build the agent once in the runner, after compiling the test binary and before any shard starts, with the full CPU count. The runner test pins that order so the warm-up cannot drift after the shards, and the contract records why. Against the previous runner the pin fails with the expected message. The release preflight and policy suites pass. Contract-Neutral: Release test runner ordering only; no published artifact, tag, signing, install or customer contract changes. --- .../internal/subsystems/deployment-installability.md | 9 +++++++++ scripts/installtests/build_release_assets_test.go | 11 +++++++++++ scripts/run-release-backend-tests.sh | 9 +++++++++ 3 files changed, 29 insertions(+) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 120f3d48a3..325905cb47 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -5058,6 +5058,15 @@ beta's release), so each is retried up to four times with doubling backoff from half-printed failed attempt cannot corrupt the verified digests. A moved tag or an invalid attestation fails identically on every attempt and is delayed by the backoff, never masked. +The release backend test runner warms the `pulse-agent` build cache once, after +compiling the race-enabled API test binary and before any shard starts, with +the full CPU count. `TestHostAgentCredentialFileHandoff` compiles the agent +itself under a fixed three-minute limit, and each API shard runs with a small +pinned `GOMAXPROCS` that the test's child compiler inherits. A cold agent build +is about 250 CPU-seconds, so under the pin it fails at 180 seconds with +`signal: killed` (reproduced; the first beta's release failed on it twice), +while a warmed cache makes the same pinned run a 16-second test. The race +build of the test binary does not warm that plain build. As of 2026-10-11, the governed release floor is Go `1.26.9`. It supersedes `1.26.8`, whose standard library is reachable through twelve vulnerable Pulse call paths reported by `govulncheck` after the 2026-10-08 advisories, including diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 7b0bddc419..78f73b83c4 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -4621,8 +4621,19 @@ func TestReleaseBackendRaceGateUsesCompleteWorkerPartition(t *testing.T) { t.Fatalf("read run-release-backend-tests.sh: %v", err) } backendScript := string(backendScriptBytes) + // The credential handoff test builds pulse-agent under a fixed three-minute limit. The + // pinned shards would serialize that cold build past it, so the runner must warm the + // agent build after compiling the test binary and before any shard starts. + compileIndex := strings.Index(backendScript, "go test -c -race") + warmIndex := strings.Index(backendScript, "go build -o /dev/null ./cmd/pulse-agent") + firstShardIndex := strings.Index(backendScript, `run_api_shard "$shard_index"`) + if compileIndex < 0 || warmIndex < compileIndex || firstShardIndex < warmIndex { + t.Fatalf("release backend runner must warm the pulse-agent build between compiling the API test binary (%d) and starting the shards (%d); warm-up at %d", + compileIndex, firstShardIndex, warmIndex) + } for _, needle := range []string{ "go test -c -race", + "Warming the pulse-agent build cache", "python3 scripts/shard_go_tests.py", `--max-regex-bytes "$MAX_REGEX_BYTES"`, `MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}"`, diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.sh index 6c792bcc26..9b998e1566 100755 --- a/scripts/run-release-backend-tests.sh +++ b/scripts/run-release-backend-tests.sh @@ -206,6 +206,15 @@ echo " Regex bytes: $MAX_REGEX_BYTES max" echo "Compiling the race-enabled internal/api test binary once..." go test -c -race -o "$API_BINARY" ./internal/api + +# TestHostAgentCredentialFileHandoff compiles pulse-agent itself, with a fixed three-minute +# limit. Every API shard runs with a small pinned GOMAXPROCS that the test's child compiler +# inherits, so a cold build of roughly 250 CPU-seconds is serialized past that limit: run +# alone with GOMAXPROCS=1 and a cold cache it fails at 180s with "signal: killed", while a +# cache warmed beforehand makes it a 16s test. The race-enabled test binary does not warm +# this plain build, so compile the agent once here, before any shard starts and unpinned. +echo "Warming the pulse-agent build cache for tests that compile the agent..." +go build -o /dev/null ./cmd/pulse-agent ( cd internal/api "$API_BINARY" -test.list '^Test'