#!/usr/bin/env bash set -euo pipefail usage() { cat <<'EOF' Usage: release-preflight-worker.sh <40-character-source-sha> Runs the portable, high-cost release checks for an exact pushed commit in a dedicated Linux amd64 checkout. The checkout and dependency caches persist between runs; release credentials and signing keys are neither required nor accepted. EOF } SOURCE_SHA="${1:-}" PROFILE="${2:-}" if [ "$SOURCE_SHA" = "--help" ] || [ "$SOURCE_SHA" = "-h" ]; then usage exit 0 fi if [[ ! "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "Error: source SHA must be a lowercase 40-character Git commit id." >&2 exit 2 fi if [ "$PROFILE" != "rehearsal" ] && [ "$PROFILE" != "release" ]; then echo "Error: profile must be rehearsal or release." >&2 exit 2 fi WORKER_ROOT="${PULSE_RELEASE_PREFLIGHT_ROOT:-/opt/pulse-release-worker}" REPOSITORY_URL="${PULSE_RELEASE_PREFLIGHT_REPOSITORY_URL:-https://github.com/rcourtman/Pulse.git}" REPOSITORY_DIR="${WORKER_ROOT}/repo" CACHE_DIR="${WORKER_ROOT}/cache" RECEIPT_DIR="${WORKER_ROOT}/receipts" RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${SOURCE_SHA:0:12}-${PROFILE}" RUN_DIR="${WORKER_ROOT}/tmp/${RUN_ID}" # Go places test temp directories under GOTMPDIR, and several packages bind # unix sockets under t.TempDir at the 108-byte sun_path limit. GitHub runners # leave GOTMPDIR unset so those fixtures resolve under /tmp; the worker must # match that layout exactly or the socket tests fail with "bind: invalid # argument". Set PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR only for a deliberately # isolated, equally short directory. GO_TMP_DIR="${PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR:-}" TIMINGS_FILE="${RUN_DIR}/timings.tsv" TEST_DATA_DIR="${WORKER_ROOT}/test-data/${PROFILE}" # The smoke stacks publish the Pulse server and agent ports on the host. A # worker may also host long-running Pulse instances (pulse-dev keeps the # dogfood instance on 7655 and a second instance on 17655), and a collision # fails the smoke only after every other stage has passed. Honour an explicit # override, otherwise take the first candidate pair with both ports free. smoke_port_pair_is_free() { local listeners listeners="$(ss -Hltn 2>/dev/null | awk '{print $4}' | sed 's/.*://' | sort -u)" ! printf '%s\n' "$listeners" | grep -qx "$1" && ! printf '%s\n' "$listeners" | grep -qx "$2" } if [ -n "${PULSE_RELEASE_PREFLIGHT_E2E_PORT:-}" ]; then PULSE_E2E_PORT="$PULSE_RELEASE_PREFLIGHT_E2E_PORT" PULSE_E2E_AGENT_PORT="${PULSE_RELEASE_PREFLIGHT_E2E_AGENT_PORT:-$((PULSE_E2E_PORT + 1))}" else PULSE_E2E_PORT="" for candidate in 27655 28655 29655 31655; do if smoke_port_pair_is_free "$candidate" "$((candidate + 1))"; then PULSE_E2E_PORT="$candidate" PULSE_E2E_AGENT_PORT="$((candidate + 1))" break fi done if [ -z "$PULSE_E2E_PORT" ]; then echo "Error: no free host port pair for the smoke stack; set PULSE_RELEASE_PREFLIGHT_E2E_PORT." >&2 exit 3 fi fi export PULSE_E2E_PORT PULSE_E2E_AGENT_PORT export PULSE_E2E_BASE_URL="http://localhost:${PULSE_E2E_PORT}" # The worker is invoked over a non-login ssh shell, so /etc/profile.d is not # sourced and an infra-managed mise toolchain (Node 24, Go) would be shadowed by # any stale system binary on PATH. Activate mise shims when they are installed. if [ -x "$HOME/.local/bin/mise" ]; then eval "$("$HOME/.local/bin/mise" activate bash --shims)" fi for command_name in git go node npm docker curl flock timeout python3; do if ! command -v "$command_name" >/dev/null 2>&1; then echo "Error: required worker command is missing: ${command_name}" >&2 exit 3 fi done if ! docker compose version >/dev/null 2>&1; then echo "Error: Docker Compose v2 is required on the worker." >&2 exit 3 fi if [ "$(node -p "process.versions.node.split('.')[0]")" != "24" ]; then echo "Error: the worker must use Node.js 24 to match the release workflows." >&2 exit 3 fi playwright_container_user() { local security_options security_options="$(docker info --format '{{json .SecurityOptions}}')" || return python3 - "$security_options" "$(id -u):$(id -g)" <<'PY' import json import sys options = json.loads(sys.argv[1]) if not isinstance(options, list) or not all(isinstance(value, str) for value in options): raise SystemExit("Docker security options must be a JSON list of strings.") # Container root maps to the daemon owner on rootless Docker. Reusing that # owner's host UID instead selects an unrelated subordinate host identity. print("0:0" if "name=rootless" in options else sys.argv[2]) PY } PLAYWRIGHT_CONTAINER_USER="$(playwright_container_user)" mkdir -p \ "$CACHE_DIR/go-build" \ "$CACHE_DIR/go-mod" \ "$CACHE_DIR/npm" \ "$RECEIPT_DIR" \ "$RUN_DIR" \ "$(dirname "$TEST_DATA_DIR")" exec 9>"${WORKER_ROOT}/worker.lock" if ! flock -n 9; then echo "Error: another release preflight is already using this worker." >&2 exit 5 fi WALL_STARTED="$(date +%s)" # A preflight compiles and tests only. Keep publication and signing authority # out of the worker even if its login shell happens to define these names. unset GH_TOKEN GITHUB_TOKEN PULSE_LICENSE_PRIVATE_KEY PULSE_UPDATE_SIGNING_KEY export GOCACHE="$CACHE_DIR/go-build" export GOMODCACHE="$CACHE_DIR/go-mod" if [ -n "$GO_TMP_DIR" ]; then mkdir -p "$GO_TMP_DIR" export GOTMPDIR="$GO_TMP_DIR" fi export npm_config_cache="$CACHE_DIR/npm" # Match the canonical workflow's isolated single-repository checkout. Tests # that explicitly require private sibling repositories use this signal to # apply their documented hosted-CI skip instead of inventing local evidence. export GITHUB_ACTIONS=true export CI=true phase() { local name="$1" shift local started finished started="$(date +%s)" echo echo "==> ${name}" "$@" finished="$(date +%s)" printf '%s\t%s\n' "$name" "$((finished - started))" >> "$TIMINGS_FILE" echo "<== ${name}: $((finished - started))s" } cleanup() { if [ -n "$GO_TMP_DIR" ]; then rm -rf "$GO_TMP_DIR" fi if [ -d "$REPOSITORY_DIR/tests/integration" ]; then ( cd "$REPOSITORY_DIR/tests/integration" docker compose -f docker-compose.test.yml down -v >/dev/null 2>&1 || true ) fi } trap cleanup EXIT if [ ! -d "$REPOSITORY_DIR/.git" ]; then phase clone git clone "$REPOSITORY_URL" "$REPOSITORY_DIR" fi phase fetch git -C "$REPOSITORY_DIR" fetch --force --no-tags origin "$SOURCE_SHA" FETCHED_SHA="$(git -C "$REPOSITORY_DIR" rev-parse 'FETCH_HEAD^{commit}')" if [ "$FETCHED_SHA" != "$SOURCE_SHA" ]; then echo "Error: origin returned ${FETCHED_SHA}, expected ${SOURCE_SHA}." >&2 exit 4 fi git -C "$REPOSITORY_DIR" checkout --detach --force "$SOURCE_SHA" git -C "$REPOSITORY_DIR" clean -ffdx cd "$REPOSITORY_DIR" EXPECTED_GO="$(awk '/^toolchain go/ { sub(/^toolchain /, ""); print; exit }' go.mod)" ACTUAL_GO="$(go env GOVERSION)" if [ -n "$EXPECTED_GO" ] && [ "$ACTUAL_GO" != "$EXPECTED_GO" ]; then echo "Error: worker Go toolchain is ${ACTUAL_GO}; exact-SHA source requires ${EXPECTED_GO}." >&2 exit 3 fi phase frontend-dependencies npm --prefix frontend-modern ci phase frontend-build npm --prefix frontend-modern run build rm -rf internal/api/frontend-modern mkdir -p internal/api/frontend-modern cp -R frontend-modern/dist internal/api/frontend-modern/ run_frontend_static_quality() { phase frontend-lint npm --prefix frontend-modern run lint phase frontend-headers npm --prefix frontend-modern run lint:headers phase frontend-duplication npm --prefix frontend-modern run lint:cpd phase frontend-types npm --prefix frontend-modern run type-check } run_frontend_tests() { phase frontend-tests npm --prefix frontend-modern test } # Keep evidence in stdout: the exact preflight launcher retains its log even # when its disposable worker directory is removed after failure. This subshell # retains errexit semantics; do not wrap run_backend in an `if` or `||` list. run_backend() ( backend_resource_snapshot() { python3 ./scripts/release-resource-snapshot.py "$1" || echo "RELEASE_RESOURCE_SNAPSHOT unavailable boundary=$1" >&2 } trap 'status=$?; backend_resource_snapshot after; echo "RELEASE_BACKEND_EXIT ${status}"; exit "$status"' EXIT backend_resource_snapshot before echo "RELEASE_BACKEND_TOOLCHAIN ${ACTUAL_GO}" rm -rf "$TEST_DATA_DIR" mkdir -p "$TEST_DATA_DIR" if [ "$PROFILE" = "rehearsal" ]; then # -json streams test events instead of waiting for the package buffer. # Keep readable Output and the original verdict via pipefail; instrument # only the known stress-test window, without changing tests or thresholds. backend_serial() { env PULSE_DATA_DIR="$TEST_DATA_DIR" go test -json -p 1 ./... | python3 ./scripts/release-go-test-events.py } phase backend-serial backend_serial else phase backend-race-sharded ./scripts/run-release-backend-tests.sh \ --data-root "$TEST_DATA_DIR" \ --api-shards auto fi ) run_playwright() { docker run --rm \ --network host \ --ipc host \ --user "$PLAYWRIGHT_CONTAINER_USER" \ --env CI=true \ --env HOME=/tmp \ --env "PULSE_E2E_DIAGNOSTIC=${PULSE_E2E_DIAGNOSTIC:-}" \ --env "PLAYWRIGHT_BASE_URL=${PULSE_E2E_BASE_URL}" \ --volume "$REPOSITORY_DIR/tests/integration:/work" \ --workdir /work \ "$PLAYWRIGHT_IMAGE" \ node /work/node_modules/@playwright/test/cli.js "$@" } run_integration_prep() { phase integration-dependencies npm --prefix tests/integration ci PLAYWRIGHT_VERSION="$(node -p "require('./tests/integration/node_modules/@playwright/test/package.json').version")" PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" phase playwright-image docker pull "$PLAYWRIGHT_IMAGE" # Check the real mount and locked CLI before the expensive test suites and # image build. Never let npx download a different browser test version. phase playwright-runtime run_playwright --version phase mock-github-image docker build --tag pulse-mock-github:test tests/integration/mock-github-server } # Static frontend checks and integration preparation are bounded enough to # overlap safely. Keep the full frontend and race-enabled backend test suites # serial: both saturate this worker, and concurrent execution can turn healthy # monitoring tests into load-induced release-gate failures. parallel_pids=() run_frontend_static_quality & parallel_pids+=("$!") run_integration_prep & parallel_pids+=("$!") remaining="${#parallel_pids[@]}" while [ "$remaining" -gt 0 ]; do if wait -n; then remaining=$((remaining - 1)) else status=$? kill "${parallel_pids[@]}" >/dev/null 2>&1 || true wait "${parallel_pids[@]}" >/dev/null 2>&1 || true exit "$status" fi done run_frontend_tests run_backend PLAYWRIGHT_VERSION="$(node -p "require('./tests/integration/node_modules/@playwright/test/package.json').version")" PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" VERSION="$(tr -d '\r\n' < VERSION)" if [ "$PROFILE" = "rehearsal" ]; then phase pulse-image docker build \ --build-arg "VERSION=${VERSION}" \ --platform linux/amd64 \ --target runtime \ --tag pulse:test \ . else phase pulse-image docker build \ --build-arg GO_BUILD_TAGS= \ --build-arg "VERSION=${VERSION}" \ --platform linux/amd64 \ --target e2e_runtime \ --tag pulse:test \ . fi run_rehearsal_smoke() { cd tests/integration export MOCK_CHECKSUM_ERROR=false export MOCK_NETWORK_ERROR=false export MOCK_RATE_LIMIT=false export MOCK_STALE_RELEASE=false export PULSE_E2E_DIAGNOSTIC=1 docker compose -f docker-compose.test.yml up -d --wait timeout 60 sh -c 'until curl -fsS ${PULSE_E2E_BASE_URL}/api/health >/dev/null; do sleep 2; done' run_playwright test tests/00-diagnostic.spec.ts --project=chromium --reporter=list local status status="$(curl -s -o "$RUN_DIR/update-status.json" -w '%{http_code}' ${PULSE_E2E_BASE_URL}/api/updates/status || true)" case "$status" in 200|401|403) ;; *) echo "Unexpected /api/updates/status response: ${status}" >&2 cat "$RUN_DIR/update-status.json" >&2 || true return 1 ;; esac docker compose -f docker-compose.test.yml down -v } run_release_smoke() { cd tests/integration export MOCK_CHECKSUM_ERROR=false export MOCK_NETWORK_ERROR=false export MOCK_RATE_LIMIT=false export MOCK_STALE_RELEASE=false export PULSE_E2E_BOOTSTRAP_TOKEN=0123456789abcdef0123456789abcdef0123456789abcdef docker compose -f docker-compose.test.yml up -d timeout 60 sh -c 'until docker inspect --format="{{json .State.Health.Status}}" pulse-mock-github | grep -q healthy; do sleep 2; done' timeout 60 sh -c 'until docker inspect --format="{{json .State.Health.Status}}" pulse-test-server | grep -q healthy; do sleep 2; done' timeout 60 sh -c 'until curl -fsS ${PULSE_E2E_BASE_URL}/api/health >/dev/null; do sleep 2; done' run_playwright test tests/95-release-smoke.spec.ts --project=chromium --reporter=list docker compose -f docker-compose.test.yml down -v } if [ "$PROFILE" = "rehearsal" ]; then phase rehearsal-smoke run_rehearsal_smoke else phase release-smoke run_release_smoke fi FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" WALL_SECONDS="$(($(date +%s) - WALL_STARTED))" TOTAL_SECONDS="$(awk -F '\t' '{ total += $2 } END { print total + 0 }' "$TIMINGS_FILE")" RECEIPT_PATH="${RECEIPT_DIR}/${RUN_ID}.json" { printf '{\n' printf ' "schema_version": 2,\n' printf ' "source_sha": "%s",\n' "$SOURCE_SHA" printf ' "profile": "%s",\n' "$PROFILE" printf ' "architecture": "%s",\n' "$(uname -m)" printf ' "finished_at": "%s",\n' "$FINISHED_AT" printf ' "wall_seconds": %s,\n' "$WALL_SECONDS" printf ' "total_phase_seconds": %s,\n' "$TOTAL_SECONDS" printf ' "result": "success"\n' printf '}\n' } > "$RECEIPT_PATH" echo echo "Exact-SHA release preflight passed." echo "Source SHA: ${SOURCE_SHA}" echo "Profile: ${PROFILE}" echo "Wall time: ${WALL_SECONDS}s" echo "Phase time: ${TOTAL_SECONDS}s" echo "Receipt: ${RECEIPT_PATH}"