mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 10:35:51 +00:00
rcourtman
8b73085e82
The exact rehearsal passed its test suites and image build but its browser could not enter the private bind mount. A host UID is remapped to an unrelated subordinate identity inside rootless Docker. Use the daemon owner's container identity for rootless Docker and the host UID/GID for rootful Docker. Execute the locked Playwright CLI directly and probe it during integration preparation, before expensive suites, so missing dependencies or inaccessible mounts fail early without fetching an unqualified CLI version.
363 lines
13 KiB
Bash
Executable File
363 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: release-preflight-worker.sh <40-character-source-sha> <rehearsal|release>
|
|
|
|
Runs the portable, high-cost release checks for an exact pushed commit in a
|
|
dedicated Linux amd64 checkout. The checkout and dependency caches persist
|
|
between runs; release credentials and signing keys are neither required nor
|
|
accepted.
|
|
EOF
|
|
}
|
|
|
|
SOURCE_SHA="${1:-}"
|
|
PROFILE="${2:-}"
|
|
|
|
if [ "$SOURCE_SHA" = "--help" ] || [ "$SOURCE_SHA" = "-h" ]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
if [[ ! "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo "Error: source SHA must be a lowercase 40-character Git commit id." >&2
|
|
exit 2
|
|
fi
|
|
if [ "$PROFILE" != "rehearsal" ] && [ "$PROFILE" != "release" ]; then
|
|
echo "Error: profile must be rehearsal or release." >&2
|
|
exit 2
|
|
fi
|
|
|
|
WORKER_ROOT="${PULSE_RELEASE_PREFLIGHT_ROOT:-/opt/pulse-release-worker}"
|
|
REPOSITORY_URL="${PULSE_RELEASE_PREFLIGHT_REPOSITORY_URL:-https://github.com/rcourtman/Pulse.git}"
|
|
REPOSITORY_DIR="${WORKER_ROOT}/repo"
|
|
CACHE_DIR="${WORKER_ROOT}/cache"
|
|
RECEIPT_DIR="${WORKER_ROOT}/receipts"
|
|
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${SOURCE_SHA:0:12}-${PROFILE}"
|
|
RUN_DIR="${WORKER_ROOT}/tmp/${RUN_ID}"
|
|
# Go places test temp directories under GOTMPDIR, and several packages bind
|
|
# unix sockets under t.TempDir at the 108-byte sun_path limit. GitHub runners
|
|
# leave GOTMPDIR unset so those fixtures resolve under /tmp; the worker must
|
|
# match that layout exactly or the socket tests fail with "bind: invalid
|
|
# argument". Set PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR only for a deliberately
|
|
# isolated, equally short directory.
|
|
GO_TMP_DIR="${PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR:-}"
|
|
TIMINGS_FILE="${RUN_DIR}/timings.tsv"
|
|
TEST_DATA_DIR="${WORKER_ROOT}/test-data/${PROFILE}"
|
|
# The smoke stacks publish the Pulse server and agent ports on the host. A
|
|
# worker may also host long-running Pulse instances (pulse-dev keeps the
|
|
# dogfood instance on 7655 and a second instance on 17655), and a collision
|
|
# fails the smoke only after every other stage has passed. Honour an explicit
|
|
# override, otherwise take the first candidate pair with both ports free.
|
|
smoke_port_pair_is_free() {
|
|
local listeners
|
|
listeners="$(ss -Hltn 2>/dev/null | awk '{print $4}' | sed 's/.*://' | sort -u)"
|
|
! printf '%s\n' "$listeners" | grep -qx "$1" && ! printf '%s\n' "$listeners" | grep -qx "$2"
|
|
}
|
|
if [ -n "${PULSE_RELEASE_PREFLIGHT_E2E_PORT:-}" ]; then
|
|
PULSE_E2E_PORT="$PULSE_RELEASE_PREFLIGHT_E2E_PORT"
|
|
PULSE_E2E_AGENT_PORT="${PULSE_RELEASE_PREFLIGHT_E2E_AGENT_PORT:-$((PULSE_E2E_PORT + 1))}"
|
|
else
|
|
PULSE_E2E_PORT=""
|
|
for candidate in 27655 28655 29655 31655; do
|
|
if smoke_port_pair_is_free "$candidate" "$((candidate + 1))"; then
|
|
PULSE_E2E_PORT="$candidate"
|
|
PULSE_E2E_AGENT_PORT="$((candidate + 1))"
|
|
break
|
|
fi
|
|
done
|
|
if [ -z "$PULSE_E2E_PORT" ]; then
|
|
echo "Error: no free host port pair for the smoke stack; set PULSE_RELEASE_PREFLIGHT_E2E_PORT." >&2
|
|
exit 3
|
|
fi
|
|
fi
|
|
export PULSE_E2E_PORT PULSE_E2E_AGENT_PORT
|
|
export PULSE_E2E_BASE_URL="http://localhost:${PULSE_E2E_PORT}"
|
|
|
|
# The worker is invoked over a non-login ssh shell, so /etc/profile.d is not
|
|
# sourced and an infra-managed mise toolchain (Node 24, Go) would be shadowed by
|
|
# any stale system binary on PATH. Activate mise shims when they are installed.
|
|
if [ -x "$HOME/.local/bin/mise" ]; then
|
|
eval "$("$HOME/.local/bin/mise" activate bash --shims)"
|
|
fi
|
|
|
|
for command_name in git go node npm docker curl flock timeout python3; do
|
|
if ! command -v "$command_name" >/dev/null 2>&1; then
|
|
echo "Error: required worker command is missing: ${command_name}" >&2
|
|
exit 3
|
|
fi
|
|
done
|
|
if ! docker compose version >/dev/null 2>&1; then
|
|
echo "Error: Docker Compose v2 is required on the worker." >&2
|
|
exit 3
|
|
fi
|
|
if [ "$(node -p "process.versions.node.split('.')[0]")" != "24" ]; then
|
|
echo "Error: the worker must use Node.js 24 to match the release workflows." >&2
|
|
exit 3
|
|
fi
|
|
|
|
playwright_container_user() {
|
|
local security_options
|
|
security_options="$(docker info --format '{{json .SecurityOptions}}')" || return
|
|
python3 - "$security_options" "$(id -u):$(id -g)" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
options = json.loads(sys.argv[1])
|
|
if not isinstance(options, list) or not all(isinstance(value, str) for value in options):
|
|
raise SystemExit("Docker security options must be a JSON list of strings.")
|
|
# Container root maps to the daemon owner on rootless Docker. Reusing that
|
|
# owner's host UID instead selects an unrelated subordinate host identity.
|
|
print("0:0" if "name=rootless" in options else sys.argv[2])
|
|
PY
|
|
}
|
|
PLAYWRIGHT_CONTAINER_USER="$(playwright_container_user)"
|
|
|
|
mkdir -p \
|
|
"$CACHE_DIR/go-build" \
|
|
"$CACHE_DIR/go-mod" \
|
|
"$CACHE_DIR/npm" \
|
|
"$RECEIPT_DIR" \
|
|
"$RUN_DIR" \
|
|
"$(dirname "$TEST_DATA_DIR")"
|
|
|
|
exec 9>"${WORKER_ROOT}/worker.lock"
|
|
if ! flock -n 9; then
|
|
echo "Error: another release preflight is already using this worker." >&2
|
|
exit 5
|
|
fi
|
|
WALL_STARTED="$(date +%s)"
|
|
|
|
# A preflight compiles and tests only. Keep publication and signing authority
|
|
# out of the worker even if its login shell happens to define these names.
|
|
unset GH_TOKEN GITHUB_TOKEN PULSE_LICENSE_PRIVATE_KEY PULSE_UPDATE_SIGNING_KEY
|
|
|
|
export GOCACHE="$CACHE_DIR/go-build"
|
|
export GOMODCACHE="$CACHE_DIR/go-mod"
|
|
if [ -n "$GO_TMP_DIR" ]; then
|
|
mkdir -p "$GO_TMP_DIR"
|
|
export GOTMPDIR="$GO_TMP_DIR"
|
|
fi
|
|
export npm_config_cache="$CACHE_DIR/npm"
|
|
# Match the canonical workflow's isolated single-repository checkout. Tests
|
|
# that explicitly require private sibling repositories use this signal to
|
|
# apply their documented hosted-CI skip instead of inventing local evidence.
|
|
export GITHUB_ACTIONS=true
|
|
export CI=true
|
|
|
|
phase() {
|
|
local name="$1"
|
|
shift
|
|
local started finished
|
|
started="$(date +%s)"
|
|
echo
|
|
echo "==> ${name}"
|
|
"$@"
|
|
finished="$(date +%s)"
|
|
printf '%s\t%s\n' "$name" "$((finished - started))" >> "$TIMINGS_FILE"
|
|
echo "<== ${name}: $((finished - started))s"
|
|
}
|
|
|
|
cleanup() {
|
|
if [ -n "$GO_TMP_DIR" ]; then
|
|
rm -rf "$GO_TMP_DIR"
|
|
fi
|
|
if [ -d "$REPOSITORY_DIR/tests/integration" ]; then
|
|
(
|
|
cd "$REPOSITORY_DIR/tests/integration"
|
|
docker compose -f docker-compose.test.yml down -v >/dev/null 2>&1 || true
|
|
)
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
if [ ! -d "$REPOSITORY_DIR/.git" ]; then
|
|
phase clone git clone "$REPOSITORY_URL" "$REPOSITORY_DIR"
|
|
fi
|
|
|
|
phase fetch git -C "$REPOSITORY_DIR" fetch --force --no-tags origin "$SOURCE_SHA"
|
|
FETCHED_SHA="$(git -C "$REPOSITORY_DIR" rev-parse 'FETCH_HEAD^{commit}')"
|
|
if [ "$FETCHED_SHA" != "$SOURCE_SHA" ]; then
|
|
echo "Error: origin returned ${FETCHED_SHA}, expected ${SOURCE_SHA}." >&2
|
|
exit 4
|
|
fi
|
|
git -C "$REPOSITORY_DIR" checkout --detach --force "$SOURCE_SHA"
|
|
git -C "$REPOSITORY_DIR" clean -ffdx
|
|
|
|
cd "$REPOSITORY_DIR"
|
|
EXPECTED_GO="$(awk '/^toolchain go/ { sub(/^toolchain /, ""); print; exit }' go.mod)"
|
|
ACTUAL_GO="$(go env GOVERSION)"
|
|
if [ -n "$EXPECTED_GO" ] && [ "$ACTUAL_GO" != "$EXPECTED_GO" ]; then
|
|
echo "Error: worker Go toolchain is ${ACTUAL_GO}; exact-SHA source requires ${EXPECTED_GO}." >&2
|
|
exit 3
|
|
fi
|
|
|
|
phase frontend-dependencies npm --prefix frontend-modern ci
|
|
phase frontend-build npm --prefix frontend-modern run build
|
|
|
|
rm -rf internal/api/frontend-modern
|
|
mkdir -p internal/api/frontend-modern
|
|
cp -R frontend-modern/dist internal/api/frontend-modern/
|
|
run_frontend_static_quality() {
|
|
phase frontend-lint npm --prefix frontend-modern run lint
|
|
phase frontend-headers npm --prefix frontend-modern run lint:headers
|
|
phase frontend-duplication npm --prefix frontend-modern run lint:cpd
|
|
phase frontend-types npm --prefix frontend-modern run type-check
|
|
}
|
|
|
|
run_frontend_tests() {
|
|
phase frontend-tests npm --prefix frontend-modern test
|
|
}
|
|
|
|
run_backend() {
|
|
rm -rf "$TEST_DATA_DIR"
|
|
mkdir -p "$TEST_DATA_DIR"
|
|
if [ "$PROFILE" = "rehearsal" ]; then
|
|
phase backend-serial env PULSE_DATA_DIR="$TEST_DATA_DIR" go test -p 1 ./...
|
|
else
|
|
phase backend-race-sharded ./scripts/run-release-backend-tests.sh \
|
|
--data-root "$TEST_DATA_DIR" \
|
|
--api-shards auto
|
|
fi
|
|
}
|
|
|
|
run_playwright() {
|
|
docker run --rm \
|
|
--network host \
|
|
--ipc host \
|
|
--user "$PLAYWRIGHT_CONTAINER_USER" \
|
|
--env CI=true \
|
|
--env HOME=/tmp \
|
|
--env "PULSE_E2E_DIAGNOSTIC=${PULSE_E2E_DIAGNOSTIC:-}" \
|
|
--env "PLAYWRIGHT_BASE_URL=${PULSE_E2E_BASE_URL}" \
|
|
--volume "$REPOSITORY_DIR/tests/integration:/work" \
|
|
--workdir /work \
|
|
"$PLAYWRIGHT_IMAGE" \
|
|
node /work/node_modules/@playwright/test/cli.js "$@"
|
|
}
|
|
|
|
run_integration_prep() {
|
|
phase integration-dependencies npm --prefix tests/integration ci
|
|
PLAYWRIGHT_VERSION="$(node -p "require('./tests/integration/node_modules/@playwright/test/package.json').version")"
|
|
PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble"
|
|
phase playwright-image docker pull "$PLAYWRIGHT_IMAGE"
|
|
# Check the real mount and locked CLI before the expensive test suites and
|
|
# image build. Never let npx download a different browser test version.
|
|
phase playwright-runtime run_playwright --version
|
|
phase mock-github-image docker build --tag pulse-mock-github:test tests/integration/mock-github-server
|
|
}
|
|
|
|
# Static frontend checks and integration preparation are bounded enough to
|
|
# overlap safely. Keep the full frontend and race-enabled backend test suites
|
|
# serial: both saturate this worker, and concurrent execution can turn healthy
|
|
# monitoring tests into load-induced release-gate failures.
|
|
parallel_pids=()
|
|
run_frontend_static_quality &
|
|
parallel_pids+=("$!")
|
|
run_integration_prep &
|
|
parallel_pids+=("$!")
|
|
|
|
remaining="${#parallel_pids[@]}"
|
|
while [ "$remaining" -gt 0 ]; do
|
|
if wait -n; then
|
|
remaining=$((remaining - 1))
|
|
else
|
|
status=$?
|
|
kill "${parallel_pids[@]}" >/dev/null 2>&1 || true
|
|
wait "${parallel_pids[@]}" >/dev/null 2>&1 || true
|
|
exit "$status"
|
|
fi
|
|
done
|
|
|
|
run_frontend_tests
|
|
run_backend
|
|
|
|
PLAYWRIGHT_VERSION="$(node -p "require('./tests/integration/node_modules/@playwright/test/package.json').version")"
|
|
PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble"
|
|
|
|
VERSION="$(tr -d '\r\n' < VERSION)"
|
|
if [ "$PROFILE" = "rehearsal" ]; then
|
|
phase pulse-image docker build \
|
|
--build-arg "VERSION=${VERSION}" \
|
|
--platform linux/amd64 \
|
|
--target runtime \
|
|
--tag pulse:test \
|
|
.
|
|
else
|
|
phase pulse-image docker build \
|
|
--build-arg GO_BUILD_TAGS= \
|
|
--build-arg "VERSION=${VERSION}" \
|
|
--platform linux/amd64 \
|
|
--target e2e_runtime \
|
|
--tag pulse:test \
|
|
.
|
|
fi
|
|
run_rehearsal_smoke() {
|
|
cd tests/integration
|
|
export MOCK_CHECKSUM_ERROR=false
|
|
export MOCK_NETWORK_ERROR=false
|
|
export MOCK_RATE_LIMIT=false
|
|
export MOCK_STALE_RELEASE=false
|
|
export PULSE_E2E_DIAGNOSTIC=1
|
|
docker compose -f docker-compose.test.yml up -d --wait
|
|
timeout 60 sh -c 'until curl -fsS ${PULSE_E2E_BASE_URL}/api/health >/dev/null; do sleep 2; done'
|
|
run_playwright test tests/00-diagnostic.spec.ts --project=chromium --reporter=list
|
|
local status
|
|
status="$(curl -s -o "$RUN_DIR/update-status.json" -w '%{http_code}' ${PULSE_E2E_BASE_URL}/api/updates/status || true)"
|
|
case "$status" in
|
|
200|401|403) ;;
|
|
*)
|
|
echo "Unexpected /api/updates/status response: ${status}" >&2
|
|
cat "$RUN_DIR/update-status.json" >&2 || true
|
|
return 1
|
|
;;
|
|
esac
|
|
docker compose -f docker-compose.test.yml down -v
|
|
}
|
|
|
|
run_release_smoke() {
|
|
cd tests/integration
|
|
export MOCK_CHECKSUM_ERROR=false
|
|
export MOCK_NETWORK_ERROR=false
|
|
export MOCK_RATE_LIMIT=false
|
|
export MOCK_STALE_RELEASE=false
|
|
export PULSE_E2E_BOOTSTRAP_TOKEN=0123456789abcdef0123456789abcdef0123456789abcdef
|
|
docker compose -f docker-compose.test.yml up -d
|
|
timeout 60 sh -c 'until docker inspect --format="{{json .State.Health.Status}}" pulse-mock-github | grep -q healthy; do sleep 2; done'
|
|
timeout 60 sh -c 'until docker inspect --format="{{json .State.Health.Status}}" pulse-test-server | grep -q healthy; do sleep 2; done'
|
|
timeout 60 sh -c 'until curl -fsS ${PULSE_E2E_BASE_URL}/api/health >/dev/null; do sleep 2; done'
|
|
run_playwright test tests/95-release-smoke.spec.ts --project=chromium --reporter=list
|
|
docker compose -f docker-compose.test.yml down -v
|
|
}
|
|
|
|
if [ "$PROFILE" = "rehearsal" ]; then
|
|
phase rehearsal-smoke run_rehearsal_smoke
|
|
else
|
|
phase release-smoke run_release_smoke
|
|
fi
|
|
|
|
FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
WALL_SECONDS="$(($(date +%s) - WALL_STARTED))"
|
|
TOTAL_SECONDS="$(awk -F '\t' '{ total += $2 } END { print total + 0 }' "$TIMINGS_FILE")"
|
|
RECEIPT_PATH="${RECEIPT_DIR}/${RUN_ID}.json"
|
|
{
|
|
printf '{\n'
|
|
printf ' "schema_version": 2,\n'
|
|
printf ' "source_sha": "%s",\n' "$SOURCE_SHA"
|
|
printf ' "profile": "%s",\n' "$PROFILE"
|
|
printf ' "architecture": "%s",\n' "$(uname -m)"
|
|
printf ' "finished_at": "%s",\n' "$FINISHED_AT"
|
|
printf ' "wall_seconds": %s,\n' "$WALL_SECONDS"
|
|
printf ' "total_phase_seconds": %s,\n' "$TOTAL_SECONDS"
|
|
printf ' "result": "success"\n'
|
|
printf '}\n'
|
|
} > "$RECEIPT_PATH"
|
|
|
|
echo
|
|
echo "Exact-SHA release preflight passed."
|
|
echo "Source SHA: ${SOURCE_SHA}"
|
|
echo "Profile: ${PROFILE}"
|
|
echo "Wall time: ${WALL_SECONDS}s"
|
|
echo "Phase time: ${TOTAL_SECONDS}s"
|
|
echo "Receipt: ${RECEIPT_PATH}"
|