mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
rcourtman
b92893351d
* Make release dry-run diagnostics fail closed Select the installed Chromium project with retries disabled, replace the unconditional pass with fail-closed API and rendered-UI readiness assertions, retain actionable runtime evidence, and guard the release workflow contract against regression. Contract-Neutral: Release diagnostic and workflow verification hardening only; no product runtime contract changes. * Expose stable E2E failure identities Project Playwright JUnit failures into bounded GitHub annotations so repeated stable-tier failures can be diagnosed without rerunning or weakening the gate. Keep the full reports and runtime logs as the forensic record, and cover annotation parsing and escaping with deterministic tests. Contract-Neutral: This changes CI failure observability only and does not alter product runtime behavior, stable-tier membership, retries, or verdict semantics. --------- Co-authored-by: rcourtman <rcourtman@users.noreply.github.com>
318 lines
12 KiB
YAML
318 lines
12 KiB
YAML
name: Core E2E Tests
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- pulse/v6-release
|
|
paths:
|
|
- 'frontend-modern/**'
|
|
- 'internal/**'
|
|
- 'tests/integration/**'
|
|
- 'Dockerfile'
|
|
- '.github/workflows/test-e2e.yml'
|
|
push:
|
|
branches:
|
|
- main
|
|
- master
|
|
- pulse/v6-release
|
|
paths:
|
|
- 'frontend-modern/**'
|
|
- 'internal/**'
|
|
- 'tests/integration/**'
|
|
- 'Dockerfile'
|
|
- '.github/workflows/test-e2e.yml'
|
|
workflow_dispatch:
|
|
|
|
# Let the in-progress run finish (pushes land here every few minutes, and
|
|
# cancelling would mean a busy main never completes a verdict); queued runs
|
|
# collapse to the newest pending one, so intermediate pushes skip.
|
|
concurrency:
|
|
group: e2e-${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
tier-selection:
|
|
name: Validate E2E tier selection
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Install integration dependencies
|
|
working-directory: tests/integration
|
|
run: npm ci
|
|
|
|
- name: Validate stable/probation project accounting
|
|
working-directory: tests/integration
|
|
run: npm run check:e2e-tiers
|
|
|
|
- name: Validate stable failure reporting
|
|
working-directory: tests/integration
|
|
run: node --test scripts/report-stable-e2e-failures.test.mjs
|
|
|
|
e2e:
|
|
name: Playwright Core E2E (shard ${{ matrix.shard }}/8)
|
|
needs: tier-selection
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: [1, 2, 3, 4, 5, 6, 7, 8]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Install Playwright dependencies
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm ci
|
|
npx playwright install --with-deps chromium webkit
|
|
|
|
- name: Install Pulse Account frontend dependencies
|
|
working-directory: internal/cloudcp/portal/frontend
|
|
run: npm ci
|
|
|
|
- name: Build Docker images for test environment
|
|
# GO_BUILD_TAGS="" drops the release build tag so the suite can enable
|
|
# mock fixtures; release-tag gating has its own -tags release Go tests.
|
|
run: |
|
|
docker build -t pulse:test --target e2e_runtime --build-arg GO_BUILD_TAGS="" .
|
|
docker build -t pulse-mock-github:test ./tests/integration/mock-github-server
|
|
env:
|
|
PULSE_LICENSE_PUBLIC_KEY: ${{ secrets.PULSE_LICENSE_PUBLIC_KEY }}
|
|
|
|
- name: Start test containers
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: node scripts/pretest.mjs
|
|
|
|
# Two-tier run (see PROBATION_SPECS in tests/integration/e2e-tiering.mjs
|
|
# for the tier mechanism and the promotion/demotion rule). The stable tier
|
|
# is the gate; the probation tier reuses the same containers but reports
|
|
# through continue-on-error so a probation flake cannot paint main red.
|
|
- name: Run stable-tier E2E suite
|
|
id: stable
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_E2E_PERF: "1"
|
|
PULSE_E2E_REQUIRE_DEFAULT_MOCK_READY: "true"
|
|
PULSE_E2E_TIER: "stable"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npm test -- --shard=${{ matrix.shard }}/8
|
|
|
|
# Playwright's report artifacts preserve the full forensic record, but
|
|
# artifact bodies and job logs require an authenticated GitHub session.
|
|
# Project the bounded failed testcase identity into ordinary check
|
|
# annotations so a red stable gate is diagnosable from the API without
|
|
# rerunning, demoting, or skipping the failure.
|
|
- name: Report stable-tier failures
|
|
if: ${{ !cancelled() && steps.stable.outcome == 'failure' }}
|
|
working-directory: tests/integration
|
|
run: node scripts/report-stable-e2e-failures.mjs test-results/junit.xml
|
|
|
|
# Runs even when the stable tier failed (its data still counts toward
|
|
# promotion), but not when the stable tier was skipped — that means the
|
|
# environment never came up and every probation spec would fail on it.
|
|
- name: Run probation-tier E2E suite (non-gating)
|
|
id: probation
|
|
if: ${{ !cancelled() && steps.stable.conclusion != 'skipped' }}
|
|
continue-on-error: true
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_E2E_PERF: "1"
|
|
PULSE_E2E_REQUIRE_DEFAULT_MOCK_READY: "true"
|
|
PULSE_E2E_TIER: "probation"
|
|
PULSE_E2E_REPORT_DIR: playwright-report-probation
|
|
PULSE_E2E_RESULTS_DIR: test-results-probation
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npm test -- --shard=${{ matrix.shard }}/8 --pass-with-no-tests
|
|
|
|
- name: Report probation-tier outcome
|
|
if: ${{ !cancelled() && steps.probation.outcome != 'skipped' }}
|
|
run: |
|
|
if [ "${{ steps.probation.outcome }}" = "failure" ]; then
|
|
echo "⚠️ Probation tier FAILED on shard ${{ matrix.shard }} (non-gating; see the playwright-report-probation artifact). A probation spec's 10-consecutive-green promotion count restarts." >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "Probation tier passed on shard ${{ matrix.shard }} (non-gating)." >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Collect container logs
|
|
if: always()
|
|
working-directory: tests/integration
|
|
run: |
|
|
echo "=== Docker containers ==="
|
|
docker ps -a
|
|
echo "=== Pulse test server logs ==="
|
|
docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"
|
|
echo "=== Mock GitHub server logs ==="
|
|
docker logs pulse-mock-github 2>&1 || echo "No pulse-mock-github container"
|
|
|
|
- name: Upload Playwright report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-shard-${{ matrix.shard }}
|
|
path: tests/integration/playwright-report/
|
|
retention-days: 3
|
|
|
|
- name: Upload test videos and screenshots
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: test-failures-shard-${{ matrix.shard }}
|
|
path: tests/integration/test-results/
|
|
retention-days: 3
|
|
|
|
# continue-on-error keeps the job green on a probation failure, so
|
|
# failure() never fires for these; key off the step outcome instead.
|
|
- name: Upload probation Playwright report
|
|
if: ${{ !cancelled() && steps.probation.outcome == 'failure' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-probation-shard-${{ matrix.shard }}
|
|
path: tests/integration/playwright-report-probation/
|
|
retention-days: 3
|
|
|
|
- name: Upload probation test videos and screenshots
|
|
if: ${{ !cancelled() && steps.probation.outcome == 'failure' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: test-failures-probation-shard-${{ matrix.shard }}
|
|
path: tests/integration/test-results-probation/
|
|
retention-days: 3
|
|
|
|
agent-registration:
|
|
name: Agent registration lifecycle
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Install Playwright dependencies
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm ci
|
|
npx playwright install --with-deps chromium
|
|
|
|
- name: Install Pulse Account frontend dependencies
|
|
working-directory: internal/cloudcp/portal/frontend
|
|
run: npm ci
|
|
|
|
- name: Build test images
|
|
run: |
|
|
docker build -t pulse:test --target e2e_runtime --build-arg GO_BUILD_TAGS="" .
|
|
docker build -t pulse-mock-github:test ./tests/integration/mock-github-server
|
|
env:
|
|
PULSE_LICENSE_PUBLIC_KEY: ${{ secrets.PULSE_LICENSE_PUBLIC_KEY }}
|
|
|
|
- name: Start non-mock test container
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_BASE_URL: http://localhost:7655
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MOCK_MODE: "false"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: node scripts/pretest.mjs
|
|
|
|
- name: Run agent registration lifecycle
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_BASE_URL: http://localhost:7655
|
|
PULSE_E2E_AGENT_JOURNEY: "true"
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MOCK_MODE: "false"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npx playwright test tests/journeys/04-agent-install-registration.spec.ts --project=chromium
|
|
|
|
- name: Collect container logs
|
|
if: always()
|
|
working-directory: tests/integration
|
|
run: docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"
|
|
|
|
- name: Upload Playwright report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-agent-registration
|
|
path: tests/integration/playwright-report/
|
|
retention-days: 3
|
|
|
|
- name: Upload test videos and screenshots
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: test-failures-agent-registration
|
|
path: tests/integration/test-results/
|
|
retention-days: 3
|
|
|
|
e2e-verdict:
|
|
name: E2E verdict
|
|
runs-on: ubuntu-24.04
|
|
needs:
|
|
- tier-selection
|
|
- e2e
|
|
- agent-registration
|
|
if: always()
|
|
steps:
|
|
# The e2e shards fail only on stable-tier failures: the probation tier
|
|
# runs behind continue-on-error inside each shard, so its failures are
|
|
# reported (shard summaries + probation artifacts) without reaching
|
|
# this verdict. Mechanism and promotion rule: PROBATION_SPECS in
|
|
# tests/integration/e2e-tiering.mjs.
|
|
- name: Check E2E results
|
|
run: |
|
|
if [ "${{ needs.tier-selection.result }}" != "success" ]; then
|
|
echo "E2E tier selection is invalid (result: ${{ needs.tier-selection.result }})"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.e2e.result }}" != "success" ]; then
|
|
echo "Stable-tier E2E shards did not all pass (result: ${{ needs.e2e.result }})"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.agent-registration.result }}" != "success" ]; then
|
|
echo "Agent registration lifecycle did not pass (result: ${{ needs.agent-registration.result }})"
|
|
exit 1
|
|
fi
|
|
echo "All stable-tier E2E shards and the agent registration lifecycle passed (probation tier is non-gating)"
|