mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 18:45:53 +00:00
rcourtman
2df6ebd210
Playwright HTML reports already include screenshots, videos and traces. Upload only the separate JUnit XML alongside each report to retain machine-readable results without storing failure attachments twice.
338 lines
13 KiB
YAML
338 lines
13 KiB
YAML
name: Core E2E Tests
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- pulse/v6-release
|
|
- 'release/v*'
|
|
paths:
|
|
- 'frontend-modern/**'
|
|
- 'internal/**'
|
|
- 'tests/integration/**'
|
|
- 'Dockerfile'
|
|
- '.github/workflows/test-e2e.yml'
|
|
push:
|
|
branches:
|
|
- main
|
|
- master
|
|
- pulse/v6-release
|
|
- 'release/v*'
|
|
paths:
|
|
- 'frontend-modern/**'
|
|
- 'internal/**'
|
|
- 'tests/integration/**'
|
|
- 'Dockerfile'
|
|
- '.github/workflows/test-e2e.yml'
|
|
workflow_dispatch:
|
|
|
|
# Let the in-progress run finish (pushes land here every few minutes, and
|
|
# cancelling would mean a busy main never completes a verdict); queued runs
|
|
# collapse to the newest pending one, so intermediate pushes skip.
|
|
concurrency:
|
|
group: e2e-${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
tier-selection:
|
|
name: Validate E2E tier selection
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: '24'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Validate telemetry schema version parity
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm run check:telemetry-schema-version
|
|
node --test scripts/check-telemetry-schema-version.test.mjs
|
|
|
|
- name: Install integration dependencies
|
|
working-directory: tests/integration
|
|
run: npm ci
|
|
|
|
- name: Validate stable/probation project accounting
|
|
working-directory: tests/integration
|
|
run: npm run check:e2e-tiers
|
|
|
|
- name: Validate stable failure reporting
|
|
working-directory: tests/integration
|
|
run: node --test scripts/report-stable-e2e-failures.test.mjs
|
|
|
|
e2e:
|
|
name: Playwright Core E2E (shard ${{ matrix.shard }}/8)
|
|
needs: tier-selection
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: [1, 2, 3, 4, 5, 6, 7, 8]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: '24'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Install Playwright dependencies
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm ci
|
|
npx playwright install --with-deps chromium webkit
|
|
|
|
- name: Install Pulse Account frontend dependencies
|
|
working-directory: internal/cloudcp/portal/frontend
|
|
run: npm ci
|
|
|
|
- name: Build Docker images for test environment
|
|
# GO_BUILD_TAGS="" drops the release build tag so the suite can enable
|
|
# mock fixtures; release-tag gating has its own -tags release Go tests.
|
|
run: |
|
|
docker build -t pulse:test --target e2e_runtime --build-arg GO_BUILD_TAGS="" .
|
|
docker build -t pulse-mock-github:test ./tests/integration/mock-github-server
|
|
|
|
- name: Start test containers
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: node scripts/pretest.mjs
|
|
|
|
# Two-tier run (see PROBATION_SPECS in tests/integration/e2e-tiering.mjs
|
|
# for the tier mechanism and the promotion/demotion rule). The stable tier
|
|
# is the gate; the probation tier reuses the same containers but reports
|
|
# through continue-on-error so a probation flake cannot paint main red.
|
|
- name: Run stable-tier E2E suite
|
|
id: stable
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_E2E_PERF: "1"
|
|
PULSE_E2E_REQUIRE_DEFAULT_MOCK_READY: "true"
|
|
PULSE_E2E_TIER: "stable"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npm test -- --shard=${{ matrix.shard }}/8
|
|
|
|
# Playwright's report artifacts preserve the full forensic record, but
|
|
# artifact bodies and job logs require an authenticated GitHub session.
|
|
# Project the bounded failed testcase identity into ordinary check
|
|
# annotations so a red stable gate is diagnosable from the API without
|
|
# rerunning, demoting, or skipping the failure.
|
|
- name: Report stable-tier failures
|
|
if: ${{ !cancelled() && steps.stable.outcome == 'failure' }}
|
|
working-directory: tests/integration
|
|
run: node scripts/report-stable-e2e-failures.mjs test-results/junit.xml
|
|
|
|
# Runs even when the stable tier failed (its data still counts toward
|
|
# promotion), but not when the stable tier was skipped — that means the
|
|
# environment never came up and every probation spec would fail on it.
|
|
- name: Run probation-tier E2E suite (non-gating)
|
|
id: probation
|
|
if: ${{ !cancelled() && steps.stable.conclusion != 'skipped' }}
|
|
continue-on-error: true
|
|
# A broken probation surface can otherwise consume the entire 60m job
|
|
# budget through retries and cancel the job before continue-on-error
|
|
# can make the observational result non-gating. Playwright's own
|
|
# global timeout preserves its reports; the step timeout is a backstop
|
|
# for runner/process failures outside Playwright.
|
|
timeout-minutes: 12
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_E2E_PERF: "1"
|
|
PULSE_E2E_REQUIRE_DEFAULT_MOCK_READY: "true"
|
|
PULSE_E2E_TIER: "probation"
|
|
PULSE_E2E_REPORT_DIR: playwright-report-probation
|
|
PULSE_E2E_RESULTS_DIR: test-results-probation
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: >-
|
|
npm test -- --shard=${{ matrix.shard }}/8 --pass-with-no-tests
|
|
--max-failures=5 --global-timeout=600000
|
|
|
|
- name: Report probation-tier outcome
|
|
if: ${{ !cancelled() && steps.probation.outcome != 'skipped' }}
|
|
run: |
|
|
if [ "${{ steps.probation.outcome }}" = "failure" ]; then
|
|
echo "⚠️ Probation tier FAILED on shard ${{ matrix.shard }} (non-gating; see the playwright-report-probation artifact). A probation spec's 10-consecutive-green promotion count restarts." >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "Probation tier passed on shard ${{ matrix.shard }} (non-gating)." >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Collect container logs
|
|
if: always()
|
|
working-directory: tests/integration
|
|
run: |
|
|
echo "=== Docker containers ==="
|
|
docker ps -a
|
|
echo "=== Pulse test server logs ==="
|
|
docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"
|
|
echo "=== Mock GitHub server logs ==="
|
|
docker logs pulse-mock-github 2>&1 || echo "No pulse-mock-github container"
|
|
|
|
- name: Upload Playwright report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-shard-${{ matrix.shard }}
|
|
path: tests/integration/playwright-report/
|
|
retention-days: 3
|
|
|
|
# The HTML report already contains screenshots, videos and traces.
|
|
# Keep only the separate machine-readable result here.
|
|
- name: Upload JUnit results
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: junit-results-shard-${{ matrix.shard }}
|
|
path: tests/integration/test-results/junit.xml
|
|
retention-days: 3
|
|
|
|
# continue-on-error keeps the job green on a probation failure, so
|
|
# failure() never fires for these; key off the step outcome instead.
|
|
- name: Upload probation Playwright report
|
|
if: ${{ !cancelled() && steps.probation.outcome == 'failure' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-probation-shard-${{ matrix.shard }}
|
|
path: tests/integration/playwright-report-probation/
|
|
retention-days: 3
|
|
|
|
- name: Upload probation JUnit results
|
|
if: ${{ !cancelled() && steps.probation.outcome == 'failure' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: junit-results-probation-shard-${{ matrix.shard }}
|
|
path: tests/integration/test-results-probation/junit.xml
|
|
retention-days: 3
|
|
|
|
agent-registration:
|
|
name: Agent registration lifecycle
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: '24'
|
|
package-manager-cache: false
|
|
|
|
- name: Install Playwright dependencies
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm ci
|
|
npx playwright install --with-deps chromium
|
|
|
|
- name: Install Pulse Account frontend dependencies
|
|
working-directory: internal/cloudcp/portal/frontend
|
|
run: npm ci
|
|
|
|
- name: Build test images
|
|
run: |
|
|
docker build -t pulse:test --target e2e_runtime --build-arg GO_BUILD_TAGS="" .
|
|
docker build -t pulse-mock-github:test ./tests/integration/mock-github-server
|
|
|
|
- name: Start non-mock test container
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_BASE_URL: http://localhost:7655
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MOCK_MODE: "false"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: node scripts/pretest.mjs
|
|
|
|
- name: Run agent registration lifecycle
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_BASE_URL: http://localhost:7655
|
|
PULSE_E2E_AGENT_JOURNEY: "true"
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MOCK_MODE: "false"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npx playwright test tests/journeys/04-agent-install-registration.spec.ts --project=chromium
|
|
|
|
- name: Collect container logs
|
|
if: always()
|
|
working-directory: tests/integration
|
|
run: docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"
|
|
|
|
- name: Upload Playwright report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-agent-registration
|
|
path: tests/integration/playwright-report/
|
|
retention-days: 3
|
|
|
|
- name: Upload test videos and screenshots
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: test-failures-agent-registration
|
|
path: tests/integration/test-results/
|
|
retention-days: 3
|
|
|
|
e2e-verdict:
|
|
name: E2E verdict
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 5
|
|
needs:
|
|
- tier-selection
|
|
- e2e
|
|
- agent-registration
|
|
if: always()
|
|
steps:
|
|
# The e2e shards fail only on stable-tier failures: the probation tier
|
|
# runs behind continue-on-error inside each shard, so its failures are
|
|
# reported (shard summaries + probation artifacts) without reaching
|
|
# this verdict. Mechanism and promotion rule: PROBATION_SPECS in
|
|
# tests/integration/e2e-tiering.mjs.
|
|
- name: Check E2E results
|
|
run: |
|
|
if [ "${{ needs.tier-selection.result }}" != "success" ]; then
|
|
echo "E2E tier selection is invalid (result: ${{ needs.tier-selection.result }})"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.e2e.result }}" != "success" ]; then
|
|
echo "Stable-tier E2E shards did not all pass (result: ${{ needs.e2e.result }})"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.agent-registration.result }}" != "success" ]; then
|
|
echo "Agent registration lifecycle did not pass (result: ${{ needs.agent-registration.result }})"
|
|
exit 1
|
|
fi
|
|
echo "All stable-tier E2E shards and the agent registration lifecycle passed (probation tier is non-gating)"
|