test: isolate shell integration runs and scope cleanup

Shared image tags, container names and host ports let concurrent checkout qualification collide or inspect another runtime. Give each shell integration invocation its own identities and allocated listeners, and scope failure cleanup to those resources so results cannot silently qualify unrelated bytes.

Contract-Neutral: Integration harness isolation and documentation only; no user-visible frontend, product contract or release surface changes.
Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
2026-09-05 11:35:27 +01:00
parent b3e71347a3
commit f1bdc44ea5
4 changed files with 84 additions and 21 deletions
+13
View File
@@ -220,3 +220,16 @@ test('my new test', async ({ page }) => {
- Review existing test files for examples
- Check Docker logs for service issues
- Review Playwright documentation: <https://playwright.dev>
### Worktree-safe shell runner
`scripts/run-tests.sh` builds unique per-invocation image tags and uses a unique
Compose project and container names. Host ports are allocated by Docker on
loopback and discovered after startup; this runner does not honour fixed port
or base-URL overrides. Its EXIT/INT/TERM/HUP cleanup removes only that invocation's
stack, volumes and image tags. SIGKILL or host failure can leave resources behind;
the printed `pulse-e2e-…` project identifies those resources for manual cleanup.
This isolation applies to the shell runner, not direct `npm test`,
`setup.sh` or hand-written Compose commands, which retain legacy defaults.
Do not use those defaults concurrently for checkout-specific qualification.
+8 -8
View File
@@ -3,7 +3,7 @@ services:
# This is only used by the test stack and is safe to keep deterministic.
seed-bootstrap-token:
image: alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b
container_name: pulse-test-seed-bootstrap-token
container_name: ${PULSE_E2E_SEED_CONTAINER:-pulse-test-seed-bootstrap-token}
environment:
- PULSE_E2E_BOOTSTRAP_TOKEN=${PULSE_E2E_BOOTSTRAP_TOKEN:-0123456789abcdef0123456789abcdef0123456789abcdef}
- PUID=${PUID:-1000}
@@ -37,10 +37,10 @@ services:
# Mock GitHub API server for controlled testing
mock-github:
image: pulse-mock-github:test
container_name: pulse-mock-github
image: ${PULSE_E2E_MOCK_IMAGE:-pulse-mock-github:test}
container_name: ${PULSE_E2E_MOCK_CONTAINER:-pulse-mock-github}
ports:
- "${PULSE_E2E_MOCK_GITHUB_PORT:-8080}:8080"
- "127.0.0.1:${PULSE_E2E_MOCK_GITHUB_PORT:-8080}:8080"
environment:
- PORT=8080
- MOCK_BASE_URL=http://mock-github:8080
@@ -60,11 +60,11 @@ services:
# Pulse server under test
pulse-test:
image: pulse:test
container_name: pulse-test-server
image: ${PULSE_E2E_SERVER_IMAGE:-pulse:test}
container_name: ${PULSE_E2E_SERVER_CONTAINER:-pulse-test-server}
ports:
- "${PULSE_E2E_PORT:-7655}:7655"
- "${PULSE_E2E_AGENT_PORT:-7656}:7656"
- "127.0.0.1:${PULSE_E2E_PORT:-7655}:7655"
- "127.0.0.1:${PULSE_E2E_AGENT_PORT:-7656}:7656"
environment:
- TZ=UTC
# Dedicated agent-ingest listener so E2E can prove the split-port
@@ -18,7 +18,7 @@ printf '%s\\n' "$*" >> "$CALL_LOG"
if [ "$1 $2" = 'compose version' ]; then exit 0; fi
if [ "$1 $2" = 'image inspect' ]; then exit 0; fi
if [ "$1" = build ]; then
[ "$3" != "$FAIL_IMAGE" ]
[[ "$3" != "$FAIL_IMAGE":* ]]
exit $?
fi
exit 91
@@ -27,7 +27,8 @@ exit 91
env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, CALL_LOG: log, FAIL_IMAGE: failImage },
encoding: 'utf8',
});
return { ...result, calls: readFileSync(log, 'utf8').trim().split('\n') };
const project = result.stdout.match(/Isolated integration project: (pulse-e2e-[a-f0-9]+)/)?.[1];
return { ...result, project, calls: readFileSync(log, 'utf8').trim().split('\n') };
} finally {
rmSync(dir, { recursive: true, force: true });
}
@@ -38,17 +39,31 @@ test('rebuilds both test images from this checkout even when local tags exist',
assert.equal(result.status, 1); // Deliberately unknown suite: never starts services.
assert.ok(result.stdout.includes('Unknown suite: invalid-test-suite'));
assert.deepEqual(result.calls.filter(call => call.startsWith('build ')), [
`build -t pulse-mock-github:test ${repo}/tests/integration/mock-github-server`,
`build -t pulse:test --build-arg GO_BUILD_TAGS= -f ${repo}/Dockerfile ${repo}`,
`build -t pulse-mock-github:${result.project} ${repo}/tests/integration/mock-github-server`,
`build -t pulse:${result.project} --build-arg GO_BUILD_TAGS= -f ${repo}/Dockerfile ${repo}`,
]);
});
for (const image of ['pulse-mock-github:test', 'pulse:test']) {
for (const image of ['pulse-mock-github', 'pulse']) {
test(`failed ${image} build cannot fall through to a cached image`, () => {
const result = run(image);
assert.equal(result.status, 1);
assert.ok(result.calls.some(call => call.startsWith(`build -t ${image} `)));
assert.ok(result.calls.some(call => call.startsWith(`build -t ${image}:`)));
assert.ok(!result.calls.some(call => call.includes(' up ')));
assert.ok(!result.stdout.includes('Starting test environment'));
});
}
test('independent invocations never reuse image/project identities', () => {
const a = run();
const b = run();
assert.match(a.project, /^pulse-e2e-[a-f0-9]+$/);
assert.notEqual(a.project, b.project);
});
test('failed startup cleanup is scoped to its own project', () => {
const result = run('not-an-image');
const composeCalls = result.calls.filter(call => call.startsWith('compose ') && call !== 'compose version');
assert.ok(composeCalls.some(call => call.endsWith('down -v')));
for (const call of composeCalls) assert.ok(call.includes(`-p ${result.project} `));
});
+42 -7
View File
@@ -26,6 +26,17 @@ echo ""
cd "$TEST_ROOT"
REPO_ROOT="$(cd "$TEST_ROOT/../.." && pwd)"
# Unique per invocation, including simultaneous runs of the same checkout.
# Never inherit another runner's project, image tags or fixed host ports.
export PULSE_E2E_RUN_ID="pulse-e2e-$(node -e "process.stdout.write(require('crypto').randomBytes(16).toString('hex'))")"
export PULSE_E2E_SERVER_IMAGE="pulse:$PULSE_E2E_RUN_ID"
export PULSE_E2E_MOCK_IMAGE="pulse-mock-github:$PULSE_E2E_RUN_ID"
export PULSE_E2E_SERVER_CONTAINER="$PULSE_E2E_RUN_ID-server"
export PULSE_E2E_MOCK_CONTAINER="$PULSE_E2E_RUN_ID-mock"
export PULSE_E2E_SEED_CONTAINER="$PULSE_E2E_RUN_ID-seed"
export PULSE_E2E_PORT=0 PULSE_E2E_AGENT_PORT=0 PULSE_E2E_MOCK_GITHUB_PORT=0
echo "Isolated integration project: $PULSE_E2E_RUN_ID"
if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then
COMPOSE_CMD=(docker compose)
else
@@ -33,7 +44,7 @@ else
fi
compose() {
"${COMPOSE_CMD[@]}" -f docker-compose.test.yml "$@"
"${COMPOSE_CMD[@]}" -p "$PULSE_E2E_RUN_ID" -f docker-compose.test.yml "$@"
}
ensure_test_images() {
@@ -41,14 +52,24 @@ ensure_test_images() {
# sources; Docker can reuse unchanged layers, but tag existence is not
# evidence that the image includes the code we are qualifying.
echo "Building test images from: $REPO_ROOT"
docker build -t pulse-mock-github:test "$TEST_ROOT/mock-github-server"
docker build -t "$PULSE_E2E_MOCK_IMAGE" "$TEST_ROOT/mock-github-server"
# Test image drops the release build tag so the suite can enable mock
# fixtures without a demo entitlement. A build failure must stop the run,
# never fall back to a previously tagged image.
docker build -t pulse:test --build-arg GO_BUILD_TAGS="" -f "$REPO_ROOT/Dockerfile" "$REPO_ROOT"
docker build -t "$PULSE_E2E_SERVER_IMAGE" --build-arg GO_BUILD_TAGS="" -f "$REPO_ROOT/Dockerfile" "$REPO_ROOT"
}
# EXIT also handles build/start/test failures; signals preserve failure status.
cleanup() {
compose down -v || true
docker image rm "$PULSE_E2E_SERVER_IMAGE" "$PULSE_E2E_MOCK_IMAGE" >/dev/null 2>&1 || true
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
trap 'exit 129' HUP
# Function to run suite with specific mock config
run_suite() {
local name="$1"
@@ -74,9 +95,7 @@ run_suite() {
else
unset PULSE_E2E_ENTITLEMENT_PROFILE
fi
local pulse_base_url="${PULSE_BASE_URL:-http://localhost:${PULSE_E2E_PORT:-7655}}"
pulse_base_url="${pulse_base_url%/}"
local health_url="${pulse_base_url}/api/health"
export PULSE_E2E_PORT=0 PULSE_E2E_AGENT_PORT=0 PULSE_E2E_MOCK_GITHUB_PORT=0
# Start services
echo "Starting test environment..."
@@ -87,6 +106,22 @@ run_suite() {
return 1
fi
# Resolve Docker-allocated ports only after successful startup. A failed
# bind must never fall back to probing a different worktree's listener.
local binding
binding="$(compose port pulse-test 7655)" || return 1
[[ "$binding" =~ :([0-9]+)$ ]] || return 1
export PULSE_BASE_URL="http://127.0.0.1:${BASH_REMATCH[1]}"
local pulse_base_url="$PULSE_BASE_URL"
local health_url="$pulse_base_url/api/health"
binding="$(compose port pulse-test 7656)" || return 1
[[ "$binding" =~ :([0-9]+)$ ]] || return 1
export PULSE_E2E_AGENT_PORT="${BASH_REMATCH[1]}"
export PULSE_AGENT_BASE_URL="http://127.0.0.1:${BASH_REMATCH[1]}"
binding="$(compose port mock-github 8080)" || return 1
[[ "$binding" =~ :([0-9]+)$ ]] || return 1
export MOCK_GITHUB_URL="http://127.0.0.1:${BASH_REMATCH[1]}"
# Wait for services
echo "Waiting for services to be ready..."
local health_ok=0
@@ -100,7 +135,7 @@ run_suite() {
# Check if the Pulse test container is actually running and reachable.
local pulse_running
pulse_running="$(docker inspect -f '{{.State.Running}}' pulse-test-server 2>/dev/null || true)"
pulse_running="$(docker inspect -f '{{.State.Running}}' "$PULSE_E2E_SERVER_CONTAINER" 2>/dev/null || true)"
if [ "$health_ok" -ne 1 ] || [ "$pulse_running" != "true" ]; then
echo -e "${RED}❌ Services failed to start${NC}"
compose ps