From d0f138cf267184260ad01fe517f7610c7769255c Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 19 Sep 2026 21:24:38 +0100 Subject: [PATCH] fix(release): give the rehearsal backend an explicit package timeout The accelerated rehearsal profile runs the backend as one serial 'go test -json -p 1 ./...', whose Go default per-package timeout is 10m. internal/api exceeded that under ordinary shared-worker load (606.9s at loadavg 5.55 on 2026-09-19) and the run failed with 'panic: test timed out after 10m0s' despite zero test assertions failing. The release profile already budgets 30m per package via run-release-backend-tests.sh. Add PULSE_RELEASE_PREFLIGHT_REHEARSAL_TIMEOUT (default 30m) and pass it as -timeout, and update the worker contract test that pins the exact rehearsal command. Change-source: pulse-maintainer Change-source: pulse-maintainer --- scripts/release-preflight-worker.sh | 9 ++++++++- .../release_control/internal/release_preflight_test.py | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/release-preflight-worker.sh b/scripts/release-preflight-worker.sh index b2626311e6..901f77e1a7 100755 --- a/scripts/release-preflight-worker.sh +++ b/scripts/release-preflight-worker.sh @@ -42,6 +42,13 @@ RUN_DIR="${WORKER_ROOT}/tmp/${RUN_ID}" # argument". Set PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR only for a deliberately # isolated, equally short directory. GO_TMP_DIR="${PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR:-}" +# The rehearsal runs the backend as one serial `go test` process so its streamed +# events stay readable. Go's default per-package timeout is 10m, which the +# shared worker exceeded on internal/api under ordinary concurrent load (606.9s +# on 2026-09-19), killing an otherwise green package with "panic: test timed +# out". Match the release profile's per-package budget so host contention cannot +# turn a healthy candidate into a release-gate failure. +REHEARSAL_BACKEND_TIMEOUT="${PULSE_RELEASE_PREFLIGHT_REHEARSAL_TIMEOUT:-30m}" TIMINGS_FILE="${RUN_DIR}/timings.tsv" TEST_DATA_DIR="${WORKER_ROOT}/test-data/${PROFILE}" # The smoke stacks publish the Pulse server and agent ports on the host. A @@ -227,7 +234,7 @@ run_backend() ( # Keep readable Output and the original verdict via pipefail; instrument # only the known stress-test window, without changing tests or thresholds. backend_serial() { - env PULSE_DATA_DIR="$TEST_DATA_DIR" go test -json -p 1 ./... | + env PULSE_DATA_DIR="$TEST_DATA_DIR" go test -json -p 1 -timeout "$REHEARSAL_BACKEND_TIMEOUT" ./... | python3 ./scripts/release-go-test-events.py } phase backend-serial backend_serial diff --git a/scripts/release_control/internal/release_preflight_test.py b/scripts/release_control/internal/release_preflight_test.py index da70a5d452..e9f757d97f 100755 --- a/scripts/release_control/internal/release_preflight_test.py +++ b/scripts/release_control/internal/release_preflight_test.py @@ -320,6 +320,7 @@ echo CONTINUED PROFILE=rehearsal TEST_DATA_DIR=unused ACTUAL_GO=fixture +REHEARSAL_BACKEND_TIMEOUT=30m rm() {{ :; }} mkdir() {{ :; }} python3() {{ @@ -330,7 +331,7 @@ python3() {{ fi }} env() {{ - [ "$*" = 'PULSE_DATA_DIR=unused go test -json -p 1 ./...' ] || return 42 + [ "$*" = 'PULSE_DATA_DIR=unused go test -json -p 1 -timeout 30m ./...' ] || return 42 printf '%s\\n' '{{"Action":"output","Output":"synthetic verdict\\n"}}' return {code} }}