diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 2c9d59ad5..8c77c5472 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -5,10 +5,12 @@ on: branches: - main - pulse/v6-release + - 'release/v*' pull_request: branches: - main - pulse/v6-release + - 'release/v*' workflow_dispatch: permissions: diff --git a/.github/workflows/canonical-governance.yml b/.github/workflows/canonical-governance.yml index eb6f2383c..d6a352b4c 100644 --- a/.github/workflows/canonical-governance.yml +++ b/.github/workflows/canonical-governance.yml @@ -191,6 +191,9 @@ jobs: - name: Run Helm Pages publication retry tests run: python3 scripts/release_control/helm_pages_retry_test.py + - name: Helm publication version binding + run: python3 scripts/release_control/helm_publish_version_test.py + - name: Run status audit unit tests run: python3 scripts/release_control/status_audit_test.py diff --git a/.github/workflows/helm-pages.yml b/.github/workflows/helm-pages.yml index 970dfd3ce..3ca3e13d3 100644 --- a/.github/workflows/helm-pages.yml +++ b/.github/workflows/helm-pages.yml @@ -275,6 +275,10 @@ jobs: set -euo pipefail public_repo="https://rcourtman.github.io/Pulse" public_ready=false + public_work="$(mktemp -d)" + trap 'rm -rf "${public_work}"' EXIT + qualified_chart="dist/pulse-${VERSION}.tgz" + test -f "${qualified_chart}" for attempt in $(seq 1 30); do public_index="$(mktemp)" if curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors \ @@ -283,7 +287,17 @@ jobs: helm repo remove pulse-public >/dev/null 2>&1 || true helm repo add pulse-public "${public_repo}" --force-update helm repo update pulse-public - if helm show chart pulse-public/pulse --version "${VERSION}" >/dev/null; then + # Metadata readability is not an exact-artifact receipt. Pull via + # the consumer index and compare with the OCI-qualified package, + # not the OCI manifest digest (which hashes a different object). + rm -f "${public_work}/pulse-${VERSION}.tgz" + if helm pull pulse-public/pulse --version "${VERSION}" --destination "${public_work}" && \ + helm show chart "${public_work}/pulse-${VERSION}.tgz" >/dev/null; then + if ! cmp -s "${qualified_chart}" "${public_work}/pulse-${VERSION}.tgz"; then + echo "::error::Public Helm chart bytes differ from the qualified package; refusing a successful convergence receipt." + rm -f "${public_index}" + exit 1 + fi public_ready=true rm -f "${public_index}" break @@ -297,4 +311,4 @@ jobs: echo "::error::Public Helm repository did not expose chart ${VERSION}." exit 1 fi - echo "[OK] Public Helm repository serves pulse ${VERSION}." + echo "[OK] Public Helm repository serves the exact qualified pulse ${VERSION} package." diff --git a/.github/workflows/publish-helm-chart.yml b/.github/workflows/publish-helm-chart.yml index b73ffdc35..9b785a382 100644 --- a/.github/workflows/publish-helm-chart.yml +++ b/.github/workflows/publish-helm-chart.yml @@ -22,7 +22,7 @@ on: required: true type: string app_version: - description: "Application version to embed (defaults to chart version)." + description: "Application version (must equal chart version; defaults to it)." required: false type: string default: "" @@ -36,7 +36,7 @@ on: description: "Chart version (required when running manually, use format 4.24.0)" required: true app_version: - description: "Application version to embed (defaults to chart version)" + description: "Application version (must equal chart version; defaults to it)" required: false permissions: @@ -90,6 +90,13 @@ jobs: APP_VERSION="$CHART_VERSION" fi + # Pulse's server and agent image defaults use Chart.AppVersion. + # Source provenance alone cannot detect a mismatched image override. + if [ "$APP_VERSION" != "$CHART_VERSION" ]; then + echo "::error::Release chart app_version must equal chart_version." + exit 1 + fi + IS_PRERELEASE="false" if [[ "$APP_VERSION" =~ -rc\.[0-9]+$ ]] || [[ "$APP_VERSION" =~ -alpha\.[0-9]+$ ]] || [[ "$APP_VERSION" =~ -beta\.[0-9]+$ ]]; then IS_PRERELEASE="true" diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index d9af85e69..b400e9b64 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -5,6 +5,7 @@ on: branches: - main - pulse/v6-release + - 'release/v*' paths: - 'frontend-modern/**' - 'internal/**' @@ -16,6 +17,7 @@ on: - main - master - pulse/v6-release + - 'release/v*' paths: - 'frontend-modern/**' - 'internal/**' diff --git a/.gitignore b/.gitignore index 65b5b0b1f..002e47b3b 100644 --- a/.gitignore +++ b/.gitignore @@ -242,6 +242,7 @@ scripts/release_control/* !scripts/release_control/governance_stage_guard.py !scripts/release_control/governance_stage_guard_test.py !scripts/release_control/helm_pages_retry_test.py +!scripts/release_control/helm_publish_version_test.py !scripts/release_control/live_runtime_proof.py !scripts/release_control/live_runtime_proof_test.py !scripts/release_control/mobile_relay_auth_approvals_proof.py diff --git a/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md b/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md index d1134d31b..2fc67229e 100644 --- a/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md +++ b/docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md @@ -867,3 +867,24 @@ The final full tools package passes in 59.413s and focused file-read race proof in 1.033s, covering successful reads alongside the failed-read controls. Private worker logs are `file-read-full.log` and `file-read-race.log` under `/opt/pulse-release-worker/`. + +## Integration qualification, 2026-09-06 + +The detection and failed-read slice is committed as `61607333cc9e`. Integration +with main `3f74c0c27304` required a fresh browser receipt. Intermediate-width +inspection exposed the shared delivery-health card squeezing its explanation +under the action buttons. The shared card now wraps by available width and uses +an opaque semantic heading colour. Permanent fixtures include 900 pixels and +check control bounds and heading overflow. + +Final worker proof passes 18 delivery-ordering cases and 12 Overview refresh +cases at 1440, 900 and 390 pixels, with Overview checked in light and dark themes. +Unavailable health, retained retry/dismiss actions, pending refresh and healthy +recovery were exercised. Final card pixels were inspected at all three widths. +Frontend lint, type checking and three affected test files pass. Focused merged +Patrol/API/adapter/Docker-result proof passes. The final local scripted +`/patrol` and `/alerts` journey passes at 1440x1000, 900x1000 and 390x1000, +including nested failed-read evidence and the linked Assistant. This qualifies +the integrated presentation and context contract only. Exact staged hook and +remote landing are subsequent delivery checks. Real diagnosis and autonomous +action outcomes remain unqualified, and the provider refusal remains enforced. diff --git a/docs/qualification/delivery-health-ordering/1440-healthy-error.png b/docs/qualification/delivery-health-ordering/1440-healthy-error.png new file mode 100644 index 000000000..bcf9b507c Binary files /dev/null and b/docs/qualification/delivery-health-ordering/1440-healthy-error.png differ diff --git a/docs/qualification/delivery-health-ordering/390-healthy-degraded.png b/docs/qualification/delivery-health-ordering/390-healthy-degraded.png new file mode 100644 index 000000000..78c92fe57 Binary files /dev/null and b/docs/qualification/delivery-health-ordering/390-healthy-degraded.png differ diff --git a/docs/qualification/delivery-health-ordering/README.md b/docs/qualification/delivery-health-ordering/README.md new file mode 100644 index 000000000..0eb628c17 --- /dev/null +++ b/docs/qualification/delivery-health-ordering/README.md @@ -0,0 +1,28 @@ +# Delivery health request ordering — 5 September 2026 + +Run from repository root after root and frontend `npm ci --ignore-scripts`: + + pulse-heavy-run -- node scripts/check-delivery-health-ordering.mjs + +Twelve real Chromium cases passed at 1440×900 and 390×900. The fixture imports +the production destinations caller, health hook, warning card and CSS; only API +responses, configuration props and confirmation acceptance are scripted. +It does not serve the complete application or exercise an installed backend. +The temporary Vite fixture is never part of the product bundle. + +Four success/error orderings per width preserve latest attention and rendered +state. Two action cases per width exercise retry/dismissal overlap: an older +completion cannot clear loading while the post-action read remains pending, +and the latest healthy response clears attention. All 19 focused hook, caller +and card tests pass; reverting only the runtime hook produces nine failures in +the 16 hook/caller tests. TypeScript and changed-file ESLint pass. + +The first browser attempt failed before UI verification because the isolated +Vite configuration omitted the repository's esnext dependency target. The +retained runner corrects that and uses the frontend working directory for CSS. +`browser-result.json` is the successful rerun's output; hashes bind the runner +and runtime source. Representative screenshots show actual fixture rendering. +Warning titles appear pale in this isolated CSS context; this receipt qualifies +ordering, not whole-app contrast or accessibility. Full-shell visual review, +installed delivery, release-line applicability and exact-candidate integration +remain separate qualification work. No public release is claimed. diff --git a/docs/qualification/delivery-health-ordering/browser-result.json b/docs/qualification/delivery-health-ordering/browser-result.json new file mode 100644 index 000000000..aad95bc55 --- /dev/null +++ b/docs/qualification/delivery-health-ordering/browser-result.json @@ -0,0 +1 @@ +{"result":"passed","cases":12,"viewports":[1440,390],"scope":"real Chromium, real caller/hook/card, scripted API; not installed application or notification receipt"} diff --git a/docs/qualification/delivery-health-ordering/source-sha256.json b/docs/qualification/delivery-health-ordering/source-sha256.json new file mode 100644 index 000000000..ab6798eb7 --- /dev/null +++ b/docs/qualification/delivery-health-ordering/source-sha256.json @@ -0,0 +1,4 @@ +{ + "scripts/check-delivery-health-ordering.mjs": "69f2c34c0fb0ecf1e67f5e5c85051f63d943314d975d6eadb8f37363622dd9c9", + "frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts": "1eadf6df30b4f1130f868b8f139561ea60fdb87e1823deeeaaf2f8fbc699a00c" +} diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 2e6870d1b..39b42486c 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -10201,7 +10201,7 @@ }, { "id": "patrol-assistant-customer-outcome-qualification", - "summary": "The maintainer requested a whole-design assessment and an explicit completion goal for Patrol and Assistant. The ordered plan and source-bound evidence are in docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md. The contract separates observations, model conclusions, proposals, execution and independently verified outcomes. Assistant continues the same issue and governed action records. The explicit explanation handoff is an implemented frontend foundation, not outcome qualification. The recorded 2026-09-05 latest-report, monitoring-active, multi-ping review excluded development and deployment proof and contained 127 paid installs, 71 with Patrol enabled and 23 with Assistant calls. Fourteen reported verified resolutions came from one install. Paid includes all non-free tiers. Activity cooccurrence does not establish a successful linked task, and schema 17 outcome/provider/cost fields had no adoption in that review. Real read-only homelab testing with Claude Opus 5 exposed missing canonical evidence, history target/tier gaps, premature context loss, unsupported causal claims and multi-minute responses. Prior history and handoff changes are recorded in the plan. The current working slice removes proposal rationale promotion into Root Cause and duplicate causal inference, preserves disk SMART/risk/source freshness and explicit remaining life, supports canonical disk get/health, and corrects collector cadence and status recovery. Targeted regressions and the live disk data path pass. Affected full worker package suites pass. The single-metric downsample regression and an adjacent bounded chart regression are corrected. Compiled presence statements are reused within a bounded set, but every query reads current data in one snapshot. The chart comparison shows no significant slowdown. Final metrics/database suites, focused concurrency race checks and the existing benchmark guard pass. Single-metric downsampling remains close to the performance threshold. PR #1920 CI subsequently failed nine benchmark comparisons against the actual base 4f9de86e, including small retained reads. The prior worker comparison used local source 5288b64d and does not establish final CI performance. The exact-base correction removes unnecessary plain-read presence probes, transaction wrappers and metric sorting while retaining canonical reconciliation and current snapshots. Full metrics/database and focused race suites pass. Ten alternating worker samples cover the previously failed CI comparisons without a significant greater-than-10% regression. Final CI landing qualification remains pending. One earlier SQLite native binding fault has no established root cause, although subsequent final full and focused race proofs pass. The latest live answers still infer collector startup from timestamp coincidence, claim continuous coverage from retained spacing and infer backup recovery from an online datastore. They fail diagnostic qualification. A serialized routing-envelope leak prompted a subscription transport correction with targeted regression proof, verified without routing prose in a completed real Assistant answer. A narrow answer-table overflow defect is corrected in the shared sanitized renderer with desktop, intermediate and mobile browser proof. The selected subscription provider explicitly refused autonomous Patrol readiness. The runtime and API preserve that refusal and prevent execution. Ordinary Assistant success is not autonomous proof. Required healthy/unhealthy, dependency, missing-access, backup/capacity and approved/rejected action qualification remains incomplete. Independent volunteered Pro environments are required before claiming repeatable customer value or wider readiness. The next diagnostic slice removes successful-read completion floors, evidence checkpoint instructions and generic call-count/silent-turn wrap-up policy. Full worker chat and AI suites pass. Browser qualification exposed and corrected mirrored-alert finding selection in the shared review lookup. Ordinary/mirrored records and linked uncertainty/failed-read context pass Playwright at 1440/900/390 widths. The detection slice removes contextless evaluation and assessment sessions, the flag-count turn ladder, signal-count report quotas and post-finding prompt replacement. The original conversation retains evidence tools and accepted records alongside errors. Full chat and AI packages, focused Patrol API tests and conversation race regressions pass. Source-bound scripted browser proof against the rebuilt runtime preserves linked uncertainty and failed reads at 1440/900/390 widths. Integration with current main and final landing verification remain pending. The latest ordinary Assistant recheck ended with context cancellation before a final answer, with eleven tool results retained. Its cancellation source and exact running binary correspondence are unresolved. The subsequent unchanged-runtime ordinary Assistant check completed in about 173 seconds, correctly distinguished utilisation from pressure, but still made unsupported temporal claims. It also exposed a failed file read marked successful. The shared file-read correction now preserves error status for absent agents and nonzero exits, with a final real ordinary file-read check returning success=false and showing the failed badge, preserved explanation and honest collection limit through reload. The linked-issue browser matrix passes again on the final rebuilt runtime. The full tools package and focused file-read race proof pass. Final integrated landing proof remains pending. Real-model/action outcome qualification remains open.", + "summary": "The maintainer requested a whole-design assessment and an explicit completion goal for Patrol and Assistant. The ordered plan and source-bound evidence are in docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md. The contract separates observations, model conclusions, proposals, execution and independently verified outcomes. Assistant continues the same issue and governed action records. The explicit explanation handoff is an implemented frontend foundation, not outcome qualification. The recorded 2026-09-05 latest-report, monitoring-active, multi-ping review excluded development and deployment proof and contained 127 paid installs, 71 with Patrol enabled and 23 with Assistant calls. Fourteen reported verified resolutions came from one install. Paid includes all non-free tiers. Activity cooccurrence does not establish a successful linked task, and schema 17 outcome/provider/cost fields had no adoption in that review. Real read-only homelab testing with Claude Opus 5 exposed missing canonical evidence, history target/tier gaps, premature context loss, unsupported causal claims and multi-minute responses. Prior history and handoff changes are recorded in the plan. The current working slice removes proposal rationale promotion into Root Cause and duplicate causal inference, preserves disk SMART/risk/source freshness and explicit remaining life, supports canonical disk get/health, and corrects collector cadence and status recovery. Targeted regressions and the live disk data path pass. Affected full worker package suites pass. The single-metric downsample regression and an adjacent bounded chart regression are corrected. Compiled presence statements are reused within a bounded set, but every query reads current data in one snapshot. The chart comparison shows no significant slowdown. Final metrics/database suites, focused concurrency race checks and the existing benchmark guard pass. Single-metric downsampling remains close to the performance threshold. PR #1920 CI subsequently failed nine benchmark comparisons against the actual base 4f9de86e, including small retained reads. The prior worker comparison used local source 5288b64d and does not establish final CI performance. The exact-base correction removes unnecessary plain-read presence probes, transaction wrappers and metric sorting while retaining canonical reconciliation and current snapshots. Full metrics/database and focused race suites pass. Ten alternating worker samples cover the previously failed CI comparisons without a significant greater-than-10% regression. Final CI landing qualification remains pending. One earlier SQLite native binding fault has no established root cause, although subsequent final full and focused race proofs pass. The latest live answers still infer collector startup from timestamp coincidence, claim continuous coverage from retained spacing and infer backup recovery from an online datastore. They fail diagnostic qualification. A serialized routing-envelope leak prompted a subscription transport correction with targeted regression proof, verified without routing prose in a completed real Assistant answer. A narrow answer-table overflow defect is corrected in the shared sanitized renderer with desktop, intermediate and mobile browser proof. The selected subscription provider explicitly refused autonomous Patrol readiness. The runtime and API preserve that refusal and prevent execution. Ordinary Assistant success is not autonomous proof. Required healthy/unhealthy, dependency, missing-access, backup/capacity and approved/rejected action qualification remains incomplete. Independent volunteered Pro environments are required before claiming repeatable customer value or wider readiness. The next diagnostic slice removes successful-read completion floors, evidence checkpoint instructions and generic call-count/silent-turn wrap-up policy. Full worker chat and AI suites pass. Browser qualification exposed and corrected mirrored-alert finding selection in the shared review lookup. Ordinary/mirrored records and linked uncertainty/failed-read context pass Playwright at 1440/900/390 widths. The detection slice removes contextless evaluation and assessment sessions, the flag-count turn ladder, signal-count report quotas and post-finding prompt replacement. The original conversation retains evidence tools and accepted records alongside errors. Full chat and AI packages, focused Patrol API tests and conversation race regressions pass. Source-bound scripted browser proof against the rebuilt runtime preserves linked uncertainty and failed reads at 1440/900/390 widths. Integration with current main and final landing verification remain pending. The latest ordinary Assistant recheck ended with context cancellation before a final answer, with eleven tool results retained. Its cancellation source and exact running binary correspondence are unresolved. The subsequent unchanged-runtime ordinary Assistant check completed in about 173 seconds, correctly distinguished utilisation from pressure, but still made unsupported temporal claims. It also exposed a failed file read marked successful. The shared file-read correction now preserves error status for absent agents and nonzero exits, with a final real ordinary file-read check returning success=false and showing the failed badge, preserved explanation and honest collection limit through reload. The linked-issue browser matrix passes again on the final rebuilt runtime. The full tools package and focused file-read race proof pass. Final integrated landing proof remains pending. Real-model/action outcome qualification remains open. Integrated main 3f74c0c27304 passes focused API proof and final frontend lint, type checking and three affected test files. Permanent browser fixtures pass 18 delivery-ordering and 12 light/dark Overview cases at 1440/900/390 widths after repairing shared card wrapping and heading readability. The final linked Patrol/Assistant browser matrix also passes. Exact staged hook and remote landing remain pending. These checks do not close the real-model/action outcome gap.", "owner": "project-owner", "status": "planned", "recorded_at": "2026-09-05", diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 1c7a56452..43a12c40b 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -635,6 +635,20 @@ installer download and the agent's subsequent Pulse TLS connection. ## Shared Boundaries +### Container update receipt and independent observation + +The server's independent Docker-update verification must compare the daemon +state/running observation with the replacement state recorded by the agent, +not only its container ID. A matching running replacement needs healthy or +no-healthcheck evidence; intentionally stopped replacements remain stopped. +Missing agent readback cannot become independent confirmation. This server-side +classification does not amend the agent's mutation receipt, trigger another +update, change runner permissions, or reinterpret compensation as execution. +`TestDockerContainerUpdateIndependentObservationMustMatchState` in +`internal/api/docker_container_action_result_test.go` verifies these boundaries; +existing callback-loss reconciliation must continue without redispatch. + + The shared `PBSInstance.NodeMetricsUnavailable` field belongs exclusively to provider polling and alert evaluation. It is retained by in-process state copies but excluded from JSON; it neither grants nor revokes host-agent identity, @@ -2880,6 +2894,22 @@ traverse an agent. ## Completion Obligations +### Docker update readback is not mutation history + +The agent's post-update inspect can observe a running replacement whose health +has not settled or has deteriorated after module success. The API must retain +the completed mutation as execution history without treating replacement +identity alone as confirmed running health. Agent-attested verification requires +running state and healthy or explicit no-healthcheck evidence for running +replacements; unknown health remains inconclusive. This does not trigger an +automatic resend or rollback and preserves deliberately stopped updates. + +Verification: `TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim` +asserts both verification outcomes and unchanged successful execution. +`TestDockerContainerActionExecutorDispatchesTypedUpdate` retains typed-only +dispatch with explicit no-healthcheck readback evidence. + + Command-capable agent completion must prove more than fresh telemetry. The dedicated agent listener must admit the full bootstrap/report/WebSocket lifecycle, command sessions must be keyed by organization plus canonical bound diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index df0496f56..17ac19400 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,19 @@ ## Purpose +The shared delivery-health card wraps action groups according to available +space, retaining readable explanation width when Review, Retry, Dismiss and +Refresh appear together. Its heading uses the opaque semantic foreground, +not the translucent palette shades reserved for status backgrounds. Verify +light/dark layouts at desktop, intermediate and narrow widths, including +unavailable health, pending refresh and recovery. + +The alerts overview offers the existing delivery-status refresh control when +health is unavailable, including after a successful retained-queue action whose +follow-up health read fails. The warning remains until a verified healthy read; +a successful queue action alone is not evidence of delivery health. Normal +degraded summary presentation continues to omit refresh. + Confirmed canonical metric recovery publishes the clearing evaluation's value, observation time, and resolved metric wording in the snapshot consumed by recent-resolution reads and notification callbacks. It must not reuse the last @@ -71,6 +84,22 @@ their updated thresholds and may apply explicit resource-disable policies, but it must not treat provider-owned incidents as missing thresholds. Unrelated configuration saves preserve those incidents and their acknowledgement state until their provider evaluator supplies recovery evidence. +Storage configuration re-evaluation must use the same ordered resource ID and +alias override lookup as polling. Static and forecast capacity alerts retain +storage policy aliases in durable metadata, including through JSON and SQLite +restore; a configuration reload must not fabricate recovery by substituting +global defaults for a still-applicable datastore override. Older PBS snapshots +without alias metadata may reconstruct the canonical datastore alias only when +the recorded PBS instance, datastore name and complete legacy resource ID agree. +Hyphenated names must not be split heuristically, and a same-named datastore on +another instance must not inherit the override. Explicit policy changes retain +normal resolution semantics. +`TestPBSDatastoreOverrideLifecycleAcrossRestart` and +`TestStoragePolicyAliasesLegacyIdentity` in +`internal/alerts/canonical_stateful_test.go` pin restored incident identity, +hysteresis, confirmed recovery, refiring, event counts and instance isolation +for both persisted-alias and legacy snapshots. + When a VM or container stops, guest evaluation resolves only metric-threshold alerts whose observations are no longer meaningful. Backup-age and snapshot posture remain owned by their posture evaluator and may stay active while the @@ -2579,3 +2608,19 @@ reasoning and real remediation in `docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md`. The repeatable browser proof is `scripts/check-patrol-assistant-journey.mjs`. A passing scripted response does not establish a useful customer outcome or model qualification. + +### Delivery health requests have latest-started ownership + +Overlapping mount, configuration Retry and post-queue-action reads must not +allow an older completion to replace newer delivery health. Success, failure, +first-load completion and the refreshing flag belong only to the most recently +started health request. A stale healthy response cannot hide degraded attention; +a stale error cannot invent unavailability after recovery. Queue actions still +refresh from the server, not from their affected count. This changes no provider +acceptance, incident lifecycle or delivery guarantee. + +The hook and destinations caller regressions in +`useNotificationDeliveryHealth.test.tsx` and +`useAlertDestinationsTabState.test.tsx` pin ordering and loading ownership. +`scripts/check-delivery-health-ordering.mjs` exercises the real caller and card +in Chromium with scripted API completions; it is not installed delivery proof. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 67321df9e..0449a97aa 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -461,6 +461,21 @@ enums locally. ## Shared Boundaries +### Independent Docker update readback + +`dockerContainerUpdateExecutionResult` must not promote replacement-ID equality +alone to independent confirmation. The daemon observation must match the +agent readback's state and running flag; running replacements additionally +require state `running` and health `healthy` or `none`. A deliberately stopped +replacement remains confirmable without being started. Missing agent readback +or its state leaves independent verification inconclusive. Contradictory +readback changes verification, not the recorded execution or compensation. +No wire schema or mutation authority changes. Verification: +`TestDockerContainerUpdateIndependentObservationMustMatchState` in +`internal/api/docker_container_action_result_test.go` covers running, stopped, +restarting, unhealthy, unknown health and absent readback cases. + + GET /api/security/status includes currentUsername in authenticated and privileged responses, derived from the validated authentication snapshot rather than configured administrator identity. Public login discovery omits it. The field is identity only: it does not grant instance-settings authority or change token scopes. Scoped local sessions retain this identity when privileged authUsername is withheld. Commercial migration payloads are a shared API/cloud-paid contract. The @@ -3592,6 +3607,22 @@ counters exist to measure. ## Completion Obligations +### Docker update agent-attested running verification + +In `internal/api/docker_container_action_result.go`, a matching replacement +container ID is insufficient to confirm a running agent readback. A running +replacement must report state `running` and health `healthy` or explicit +`none`. Known unsuitable health or lifecycle states contradict verification; +missing or unrecognised running health yields `inconclusive` with +`container_health_unknown`. This does not change execution or compensation +history, introduce a wire field, or require stopped replacements to run. + +Verification: `TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim` +in `internal/api/docker_container_action_result_test.go` covers healthy, +no-healthcheck, stopped, unhealthy, starting, restarting and unknown-health +readbacks. The typed-dispatch fixture supplies explicit no-healthcheck evidence. + + The public connection ledger and action APIs must project telemetry liveness and command admission independently. An agent may be adapter-healthy while remote control is `disconnected`; in that state command policy is blocked and diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index daf9a4e82..c8c8e4772 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -15,6 +15,22 @@ ## Purpose +### Public Helm exact-package receipt + +The post-activation public Pages verification in `.github/workflows/helm-pages.yml` +must pull the requested chart version through the consumer repository and compare +its archive bytes with the package already recovered through OCI qualification. +Readable chart metadata alone is not a successful convergence receipt. The OCI +manifest digest hashes a different object and must not be compared directly with +the archive. A byte mismatch fails immediately; unavailable downloads retain the +bounded retry, with previous downloaded files removed before each attempt. +Activation bindings, publication authority and containment remain prerequisites. +The executed-shell fixtures in `scripts/release_control/helm_pages_retry_test.py` +cover matching, mismatched, missing and unavailable public packages without +network or publication writes. They are not installed Helm qualification or proof +that any historical published chart was wrong or has been repaired. + + The shell-owned multi-tenant integration suite uses a dedicated desktop-only Playwright configuration selecting the seven multi-tenant scenarios. It must reject any non-empty E2E tier identity rather than impersonating stable or @@ -4073,6 +4089,13 @@ are part of the same governed bootstrap input even when the package manifest range already permits the newer version; the lockfile must identify the resolved package version and integrity that the release build will actually consume. +Build and Test and Core E2E must admit both pushes and pull requests for +`release/v*` trains as well as main. A release proposal must not appear +qualified merely because documentation and boundary checks passed while the +branch filters excluded build, dependency-security and applicable E2E checks. +Existing path filters and per-job requirements remain in force; branch admission +is not evidence that those jobs executed or passed. + Frontend dependency-security changes use their own proof route rather than borrowing the local dev-runtime orchestration tests. The canonical `.github/workflows/build-and-test.yml` frontend job must run the complete @@ -5168,3 +5191,20 @@ passed. A read-only probe of job 101235205647 retained the expected failure text without ESC bytes. Private containment classification and successful scheduled reconciliation still require post-integration evidence; this is not customer convergence or release qualification. + +### 2026-09-05 — Bind hosted chart application version to release version + +The hosted Helm publisher rejects a supplied application version that differs +from the chart version before emitting version outputs or packaging. Both server +and agent image defaults use Chart.AppVersion, so an exact source SHA and chart +digest alone do not prevent an override from selecting another release's images. +The normal release caller already supplies equal versions; default and release +event paths remain unchanged. This does not remove users' image value overrides. + +Verification: helm_publish_version_test.py executes the actual version-resolution +shell. Four mismatches failed assertions before the fix and are rejected after it; +stable/alpha/beta/RC equal and default versions and release-event defaults pass. +All 4 tests, 7 Helm Pages retry tests and 47 promotion-policy tests pass locally. +No hosted publication or installed-image qualification is claimed. +External reference retrieved 2026-09-05: https://helm.sh/docs/topics/charts/#the-appversion-field +explains that application version is separate from chart version. diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 159c6f023..0b217c234 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -20,6 +20,19 @@ ## Purpose +The shared delivery-health card wraps action groups according to available +space, retaining readable explanation width when Review, Retry, Dismiss and +Refresh appear together. Its heading uses the opaque semantic foreground, +not the translucent palette shades reserved for status backgrounds. Verify +light/dark layouts at desktop, intermediate and narrow widths, including +unavailable health, pending refresh and recovery. + +The alerts overview offers the existing delivery-status refresh control when +health is unavailable, including after a successful retained-queue action whose +follow-up health read fails. The warning remains until a verified healthy read; +a successful queue action alone is not evidence of delivery health. Normal +degraded summary presentation continues to omit refresh. + Proxmox backup presentation treats every manifestless PBS artifact as non-recoverable. It renders the artifact as `Running` when current writer visibility is absent or a matching writer is active, and as danger-tone @@ -7117,3 +7130,15 @@ reasoning and real remediation in `docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md`. The repeatable browser proof is `scripts/check-patrol-assistant-journey.mjs`. A passing scripted response does not establish a useful customer outcome or model qualification. + +### Alert health attention preserves asynchronous ownership + +The existing delivery-health card and shared buttons consume only the latest +started health read's state. Configuration Retry can overlap a disabled card +refresh; disabling that button is not a concurrency guard. Older completions +must neither clear the latest request's busy flag nor replace its attention or +unavailable presentation. Existing danger tone, accessible alert role, labels, +confirmation and wrapping controls remain unchanged; no new primitive is added. +The focused hook/caller tests and `scripts/check-delivery-health-ordering.mjs` +cover this dependency at desktop and narrow widths using scripted health and +queue-action responses, without claiming backend notification delivery. diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md index 2cc850585..8e11b5413 100644 --- a/docs/release-control/v6/internal/subsystems/notifications.md +++ b/docs/release-control/v6/internal/subsystems/notifications.md @@ -571,11 +571,59 @@ every destination the decision webhook delivery already made for HTTP 4xx in `isRetryableWebhookError`. Dead-lettering early must not lose the notification: `RetryTerminalFailures` -remains the operator's recovery path, returning retained terminal failures to -the queue with a fresh budget once the credentials or configuration are fixed. +remains the operator's recovery path, returning eligible retained terminal +failures to the queue with a fresh budget once the credentials or configuration +are fixed. Resolution removes obsolete firing entries from that eligibility; +retry must not resurrect an incident which has already recovered. A dead-letter row records `failureClass` and a `deadLetterReason` of `failure_class_not_retryable` or `max_retries_exhausted` so the two are distinguishable in local logs. `internal/notifications/failure_class_test.go` pins the retryable split and that a deterministic failure dead-letters on its first attempt. + + +### Resolution remains final across terminal retries and restart + +Resolution cancellation covers pending, sending, failed, and dead-lettered +firing rows. A wholly obsolete row becomes cancelled; a grouped row retains +only unrelated firing alerts and their operational links. Recovery jobs are +not cancelled by this operation. Only removed pending entries contribute to +the pending-suppression return count; terminal or interrupted sends must not +be counted as proof that firing was never delivered. + +The cancellation verdict persists across queue reopen and bulk operator +retry. Per-item retry also rejects cancelled and already-sent rows atomically, +so a stale retry request cannot bypass resolution or duplicate a completed +delivery. Pending, sending, failed, and dead-lettered rows remain eligible for +the existing retry scheduler. + +Cancelling a row retains its failed-attempt audit history and announces the +changed queue-health verdict only after releasing both the database mutex and +per-alert delivery gates. Clearing obsolete retained failures does not prove +that a destination has been repaired. Nor does this operation retrospectively +identify obsolete rows whose resolution happened before this behaviour was +installed; historical backlogs still require incident reconciliation. + +`internal/notifications/queue_resolution_retry_test.go` proves failed and +dead-lettered cancellation across durable reopen and actual queue processing, +preservation of unrelated grouped firing and recovery jobs, retained failed +attempts, callback lock release and committed-state visibility, and the +per-item retry eligibility matrix. These are component proofs, not installed +receiver receipts or exactly-once delivery guarantees. + +### Disabled delivery is cancellation, not a receipt + +At processing time, globally disabled delivery or a disabled/removed destination +returns `ErrNotificationDeliverySkipped`. The queue persists that job as +cancelled with the policy reason and cancelled operational links. It does not +write a provider-attempt audit, a successful receipt, or a delivery failure, and +operator retry does not replay the cancelled job. Existing attempt history is +retained. Queue health reconciliation runs after releasing the database mutex +and alert delivery gates. + +`queue_disabled_delivery_test.go` exercises the real manager/queue boundary for +email, webhook and Apprise, firing and recovery, and global versus destination +disablement. This corrects false successful queue/audit records; it does not +establish maintenance-window expiry, stop an already-started provider request, +or repair historical false-success records. diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index bd40b36fb..562ae7917 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -271,6 +271,22 @@ command-capable profile. ## Shared Boundaries +### Shared Docker-update verification boundary + +The shared API result converter classifies independent Docker update readback +using replacement identity, state/running agreement and running health, rather +than identity alone. Both immediate execution and durable receipt reconciliation +use this converter. Contradictory observations affect verification only; +missing agent readback is inconclusive. Execution and compensation records +remain unchanged: container backup-rename compensation is not a storage backup, +recovery point, or independently verified restore. No storage selection, +retention or recovery authority is added. The focused +`TestDockerContainerUpdateIndependentObservationMustMatchState` in +`internal/api/docker_container_action_result_test.go` asserts that independent +verification changes never rewrite successful execution history, including +stopped replacements and missing readback. + + Recovery consumers of security status must distinguish currentUsername (the validated caller identity) from privileged authUsername (administrator configuration). Presence of currentUsername in a scoped response does not authorise export, import, recovery or unprotected backup transfer; existing settings capabilities and backend enforcement still gate those operations. The Patrol action broker and shared policy-writer wiring under `internal/api/` @@ -2115,6 +2131,22 @@ take a correctness dependency on its contents. ## Completion Obligations +### Recovered Docker update evidence preserves health uncertainty + +The shared API action-result boundary must not turn a matching replacement ID +into confirmed running health when the agent readback reports unhealthy, +health-check starting or restarting. Unknown running health remains +inconclusive, while the recorded completed execution and compensation facts +remain unchanged. This rule does not settle missing receipts, prove intended +stopped state, or authorise redispatch of a recovered update. + +Verification: `TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim` +in `internal/api/docker_container_action_result_test.go` pins these result +distinctions. Existing `TestIssue1649StrandedDockerUpdateReconcilesWithoutRedispatch` +and `TestRouterRecoverExecutingDockerUpdateAfterCapabilityDisappears` exercise +reconciliation without treating the verification correction as a new mutation. + + Coverage rows reconcile replacement recovery-model snapshots by their logical `key` before windowing. Refreshed names must render without discarding expanded restore evidence or keyboard focus on a surviving row. Verify with diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index 4d0c8bddf..df645715b 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,19 +1,36 @@ { "version": 1, - "base_sha": "c5d2f56dda42946ebafe09266c8e2e52319ba24b", - "verified_at": "2026-09-06T01:21:13.188Z", + "base_sha": "61607333cc9e2fe1c1712b94ce458ac45abdabd1", + "verified_at": "2026-09-06T03:17:28.710286Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/components/AI/FindingsPanel.tsx" + "frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx", + "frontend-modern/src/features/alerts/OverviewTab.tsx", + "frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts" ], "content_sha256": { - "frontend-modern/src/components/AI/FindingsPanel.tsx": "aead071f9431e5c2284014cd8e4af8fc5cde69951f3f850821808dcf885c5226" + "frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx": "dad60838804e575c0314398b304bda6d2fbd842cb2f693b3f33d168a4bce5dc1", + "frontend-modern/src/features/alerts/OverviewTab.tsx": "8f7fdc04bd0546f86152c8bb392ff3e3fb755f1f0a1c28d1ee85da3e9fe23582", + "frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts": "1eadf6df30b4f1130f868b8f139561ea60fdb87e1823deeeaaf2f8fbc699a00c" }, "routes": [ + "/qualification (actual delivery-health and Overview components in isolated Solid Router fixtures, scripted API promises)", "/patrol", "/alerts" ], "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 900, + "height": 900 + }, + { + "width": 390, + "height": 900 + }, { "width": 1440, "height": 1000 @@ -28,12 +45,13 @@ } ], "states": [ - "Ordinary and alert-mirrored findings retain the same review selection. Completed investigation remains Needs attention with unknown cause and failed access evidence.", - "Original failed read and uncertain conclusion remain visible in the nested investigation transcript and linked Assistant request. Scripted responses establish presentation and context preservation, not diagnostic competence." + "Delivery ordering: 18 scripted cases at three widths. Overview refresh: 12 scripted cases across light/dark themes and three widths. Unavailable health, retained Retry/Dismiss actions, pending disabled Refresh and healthy recovery. These fixtures do not qualify installed backend delivery or recipient outcomes.", + "Final card pixels inspected at desktop, intermediate and narrow widths in both themes. Heading is readable and actions wrap within the card. This is a focused card check, not a full dark-theme shell audit.", + "Ordinary and alert-mirrored Patrol findings preserve unknown cause and failed-read evidence in the investigation transcript and linked Assistant. Scripted responses qualify presentation and context preservation, not diagnostic competence." ], "interactions": [ - "Activity, Finding options and history, mirrored disclosure, keyboard Review, investigation thread expansion/collapse, review close at 1440/900/390 widths. Inspect final pixels for readable content and reachable controls.", - "Inbox selected issue, evidence expansion, keyboard Explain, same canonical finding with failed-read evidence and uncertainty in Assistant handoff, Escape, draft preservation, reload without resubmission.", - "Existing adjacent alert menu Escape/outside dismissal and provider failure/retry checked. No infrastructure mutations or autonomous provider calls. Private source-bound proof: tmp/patrol-assistant-journey/result.json." + "Retry and Dismiss followed by failed health read, manual Refresh while pending, healthy recovery and overlapping old/new response ordering. Geometry assertions check heading overflow and control bounds.", + "Selected issue, evidence expansion, keyboard Review and Explain, nested transcript expansion/collapse, Assistant streaming, preserved draft, reload, error/retry, menu Escape and outside dismissal at 1440/900/390 widths.", + "Private receipts: /Volumes/Development/pulse/tmp/patrol-merge-browser/final-health-ordering and final-overview-refresh, and tmp/patrol-assistant-journey/result.json. No infrastructure mutations or autonomous provider calls." ] } diff --git a/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.test.tsx b/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.test.tsx index 9f3828c05..2651c01da 100644 --- a/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.test.tsx +++ b/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.test.tsx @@ -103,6 +103,26 @@ describe('AlertDeliveryHealthCard', () => { expect(screen.getByRole('button', { name: 'Refresh delivery status' })).toBeDisabled(); }); + it('allows unavailable summary health to be rechecked without a queue mutation', () => { + const onRefresh = vi.fn(); + render(() => ( + + )); + expect(screen.getByRole('alert')).toHaveTextContent( + 'Notification delivery status is unavailable', + ); + fireEvent.click(screen.getByRole('button', { name: 'Refresh delivery status' })); + expect(onRefresh).toHaveBeenCalledOnce(); + expect(screen.getByRole('alert')).toBeTruthy(); + }); + it('keeps the overview treatment concise and points directly to delivery evidence', () => { render(() => ( diff --git a/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx b/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx index 6519ec23c..6839f5b83 100644 --- a/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx +++ b/frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx @@ -52,20 +52,20 @@ export function AlertDeliveryHealthCard(props: AlertDeliveryHealthCardProps) { return ( -
-
+
+
-
+
{props.detailsHref ? ( {getAlertDestinationsDeliveryReviewLabel()} diff --git a/frontend-modern/src/features/alerts/OverviewTab.tsx b/frontend-modern/src/features/alerts/OverviewTab.tsx index 219aeb25e..de8319f39 100644 --- a/frontend-modern/src/features/alerts/OverviewTab.tsx +++ b/frontend-modern/src/features/alerts/OverviewTab.tsx @@ -90,7 +90,7 @@ export function OverviewTab(props: { onDismissFailures={() => void deliveryHealthState.dismissTerminalFailures()} detailsHref="/alerts/notifications#notification-delivery-activity" detailLevel="summary" - showRefresh={false} + showRefresh={deliveryHealthState.deliveryHealthUnavailable()} /> diff --git a/frontend-modern/src/features/alerts/__tests__/OverviewTab.deliveryactions.test.tsx b/frontend-modern/src/features/alerts/__tests__/OverviewTab.deliveryactions.test.tsx index c177bc613..f7724875b 100644 --- a/frontend-modern/src/features/alerts/__tests__/OverviewTab.deliveryactions.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/OverviewTab.deliveryactions.test.tsx @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { cleanup, render, screen, waitFor } from '@solidjs/testing-library'; +import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library'; import { DEFAULT_LOCALE, setActiveLocale } from '@/i18n'; import type { Alert } from '@/types/api'; import type { NotificationHealth } from '@/api/notifications'; @@ -49,6 +49,8 @@ vi.mock('@/components/Alerts/InvestigateAlertButton', () => ({ InvestigateAlertButton: () => null, })); +import { notificationStore } from '@/stores/notifications'; + import { OverviewTab } from '../OverviewTab'; function degradedHealth(): NotificationHealth { @@ -107,10 +109,15 @@ describe('OverviewTab delivery health actions', () => { getDeliveryDiagnoses.mockReset(); getDeliveryDiagnoses.mockResolvedValue([]); getHealth.mockReset(); + retryTerminalFailures.mockReset(); + dismissTerminalFailures.mockReset(); + vi.mocked(notificationStore.success).mockClear(); + vi.mocked(notificationStore.error).mockClear(); }); afterEach(() => { cleanup(); + vi.restoreAllMocks(); setActiveLocale(DEFAULT_LOCALE); }); @@ -127,4 +134,110 @@ describe('OverviewTab delivery health actions', () => { expect(screen.queryByRole('button', { name: 'Refresh delivery status' })).toBeNull(); expect(screen.getByRole('alert')).toHaveTextContent('Most recent failures: connectivity (1).'); }); + for (const action of [ + { name: 'Retry retained deliveries', api: retryTerminalFailures }, + { name: 'Dismiss retained failures', api: dismissTerminalFailures }, + ]) { + it(`does not mutate or refresh when ${action.name} is cancelled`, async () => { + getHealth.mockResolvedValue(degradedHealth()); + const confirmation = vi.spyOn(window, 'confirm').mockReturnValue(false); + render(() => ); + fireEvent.click(await screen.findByRole('button', { name: action.name })); + + expect(confirmation).toHaveBeenCalledOnce(); + expect(action.api).not.toHaveBeenCalled(); + expect(getHealth).toHaveBeenCalledTimes(1); + expect(screen.getByRole('alert')).toBeTruthy(); + }); + + it(`retains attention and enables another attempt when ${action.name} fails`, async () => { + getHealth.mockResolvedValue(degradedHealth()); + vi.spyOn(window, 'confirm').mockReturnValue(true); + action.api.mockRejectedValue(new Error('queue action unavailable')); + render(() => ); + fireEvent.click(await screen.findByRole('button', { name: action.name })); + + await waitFor(() => expect(notificationStore.error).toHaveBeenCalledOnce()); + expect(notificationStore.success).not.toHaveBeenCalled(); + expect(getHealth).toHaveBeenCalledTimes(1); + expect(screen.getByRole('alert')).toHaveTextContent('connectivity (1)'); + expect(screen.getByRole('button', { name: action.name })).not.toBeDisabled(); + }); + + it(`keeps health visibly unknown after ${action.name} succeeds but refresh fails`, async () => { + getHealth + .mockResolvedValueOnce(degradedHealth()) + .mockRejectedValueOnce(new Error('health unavailable')); + action.api.mockResolvedValue({ affected: 85 }); + vi.spyOn(window, 'confirm').mockReturnValue(true); + render(() => ); + fireEvent.click(await screen.findByRole('button', { name: action.name })); + + const refresh = await screen.findByRole('button', { name: 'Refresh delivery status' }); + expect(screen.getByRole('alert')).toBeTruthy(); + expect(notificationStore.success).toHaveBeenCalledOnce(); + expect(notificationStore.error).not.toHaveBeenCalled(); + + const healthy = degradedHealth(); + healthy.overallHealthy = true; + healthy.queue = { + ...healthy.queue, + status: 'healthy', + healthy: true, + attentionRequired: 0, + deadLetter: 0, + }; + getHealth.mockResolvedValueOnce(healthy); + await waitFor(() => expect(refresh).not.toBeDisabled()); + fireEvent.click(refresh); + await waitFor(() => expect(screen.queryByRole('alert')).toBeNull()); + expect(getHealth).toHaveBeenCalledTimes(3); + expect(action.api).toHaveBeenCalledOnce(); + }); + + it(`keeps both actions disabled until ${action.name} and its health refresh finish`, async () => { + const healthy = degradedHealth(); + healthy.overallHealthy = true; + healthy.queue = { + ...healthy.queue, + status: 'healthy', + healthy: true, + attentionRequired: 0, + deadLetter: 0, + }; + let completeAction!: (value: { affected: number }) => void; + let completeHealth!: (value: NotificationHealth) => void; + action.api.mockReturnValue( + new Promise((resolve) => { + completeAction = resolve; + }), + ); + getHealth.mockResolvedValueOnce(degradedHealth()).mockReturnValueOnce( + new Promise((resolve) => { + completeHealth = resolve; + }), + ); + vi.spyOn(window, 'confirm').mockReturnValue(true); + render(() => ); + fireEvent.click(await screen.findByRole('button', { name: action.name })); + + const expectActionsDisabled = () => { + const buttons = screen.getByRole('alert').querySelectorAll('button'); + expect(buttons.length).toBe(2); + for (const button of buttons) expect(button).toBeDisabled(); + }; + expectActionsDisabled(); + expect(getHealth).toHaveBeenCalledTimes(1); + completeAction({ affected: 85 }); + await waitFor(() => expect(getHealth).toHaveBeenCalledTimes(2)); + expectActionsDisabled(); + expect(screen.getByRole('alert')).toBeTruthy(); + + completeHealth(healthy); + await waitFor(() => expect(screen.queryByRole('alert')).toBeNull()); + expect(action.api).toHaveBeenCalledOnce(); + expect(notificationStore.success).toHaveBeenCalledOnce(); + expect(notificationStore.error).not.toHaveBeenCalled(); + }); + } }); diff --git a/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx b/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx index 998bd3caa..1982eb681 100644 --- a/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx @@ -81,6 +81,40 @@ const buildAppriseConfig = (): UIAppriseConfig => ({ }); describe('useAlertDestinationsTabState', () => { + it('preserves degraded configuration Retry health when the older mount request finishes', async () => { + let finishMount!: (health: Awaited>) => void; + const healthy = { queue: { status: 'healthy', attentionRequired: 0 } } as Awaited< + ReturnType + >; + vi.mocked(NotificationsAPI.getWebhooks).mockResolvedValue([]); + vi.mocked(NotificationsAPI.getDeliveryLog).mockResolvedValue({ entries: [] } as never); + vi.mocked(NotificationsAPI.getHealth) + .mockReturnValueOnce( + new Promise((resolve) => { + finishMount = resolve; + }), + ) + .mockResolvedValueOnce({ ...healthy, queue: { ...healthy.queue, status: 'degraded' } }); + const [appriseConfig, setAppriseConfig] = createSignal(buildAppriseConfig()); + const { result } = renderHook(() => + useAlertDestinationsTabState({ + appriseConfig, + setAppriseConfig, + configLoadError: () => null, + emailConfig: () => buildEmailConfig(), + isLoadingDestinations: () => false, + isRetrying: () => false, + onRetryLoad: vi.fn(), + }), + ); + result.handleRetry(); + await waitFor(() => expect(result.deliveryNeedsAttention()).toBe(true)); + finishMount(healthy); + await Promise.resolve(); + expect(result.deliveryHealth()?.queue.status).toBe('degraded'); + expect(result.deliveryNeedsAttention()).toBe(true); + }); + beforeEach(() => { vi.mocked(AlertsAPI.getEvents).mockReset(); vi.mocked(AlertsAPI.getEvents).mockResolvedValue([]); diff --git a/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryHealth.test.tsx b/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryHealth.test.tsx index e08b50e91..6740a7691 100644 --- a/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryHealth.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryHealth.test.tsx @@ -6,7 +6,11 @@ import { NotificationsAPI } from '@/api/notifications'; import { useNotificationDeliveryHealth } from '../useNotificationDeliveryHealth'; vi.mock('@/api/notifications', () => ({ - NotificationsAPI: { getHealth: vi.fn() }, + NotificationsAPI: { + getHealth: vi.fn(), + dismissTerminalFailures: vi.fn(), + retryTerminalFailures: vi.fn(), + }, })); const healthWith = (status: string) => ({ queue: { status, failed: 3, deadLetter: 1 } }) as never; @@ -60,4 +64,107 @@ describe('useNotificationDeliveryHealth', () => { expect(state.deliveryNeedsAttention()).toBe(true); dispose(); })); + it.each([ + ['healthy', 'degraded'], + ['degraded', 'healthy'], + ['error', 'healthy'], + ['healthy', 'error'], + ])('ignores older %s completion after newer %s result', (older, newer) => + createRoot(async (dispose) => { + let resolve!: (value: Awaited>) => void; + let reject!: (error: Error) => void; + vi.mocked(NotificationsAPI.getHealth).mockReturnValueOnce( + new Promise((yes, no) => { + resolve = yes; + reject = no; + }), + ); + const state = useNotificationDeliveryHealth(); + const pending = state.loadDeliveryHealth(); + if (newer === 'error') { + vi.mocked(NotificationsAPI.getHealth).mockRejectedValueOnce(new Error('new failure')); + } else { + vi.mocked(NotificationsAPI.getHealth).mockResolvedValueOnce(healthWith(newer)); + } + await state.loadDeliveryHealth(); + if (older === 'error') reject(new Error('old failure')); + else resolve(healthWith(older)); + await pending; + expect(state.deliveryHealth()?.queue.status).toBe(newer === 'error' ? undefined : newer); + expect(state.deliveryHealthUnavailable()).toBe(newer === 'error'); + expect(state.deliveryNeedsAttention()).toBe(newer !== 'healthy'); + expect(state.refreshingDeliveryHealth()).toBe(false); + dispose(); + }), + ); + + it.each(['healthy', 'error'])( + 'keeps loading and first-load silence when older %s finishes first', + (older) => + createRoot(async (dispose) => { + let finishOld!: () => void; + let finishNew!: () => void; + vi.mocked(NotificationsAPI.getHealth) + .mockReturnValueOnce( + new Promise((resolve, reject) => { + finishOld = () => + older === 'error' + ? reject(new Error('old failure')) + : resolve(healthWith('healthy')); + }), + ) + .mockReturnValueOnce( + new Promise((resolve) => { + finishNew = () => resolve(healthWith('degraded')); + }), + ); + const state = useNotificationDeliveryHealth(); + const oldRequest = state.loadDeliveryHealth(); + const newRequest = state.loadDeliveryHealth(); + finishOld(); + await oldRequest; + expect(state.refreshingDeliveryHealth()).toBe(true); + expect(state.deliveryHealth()).toBeNull(); + expect(state.deliveryNeedsAttention()).toBe(false); + finishNew(); + await newRequest; + expect(state.refreshingDeliveryHealth()).toBe(false); + expect(state.deliveryNeedsAttention()).toBe(true); + dispose(); + }), + ); + + it.each(['dismissTerminalFailures', 'retryTerminalFailures'] as const)( + 'keeps post-%s health when a pre-action request finishes late', + (action) => + createRoot(async (dispose) => { + const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(true); + try { + const state = useNotificationDeliveryHealth(); + vi.mocked(NotificationsAPI.getHealth).mockResolvedValueOnce({ + queue: { status: 'degraded', attentionRequired: 2 }, + } as never); + await state.loadDeliveryHealth(); + let finishOld!: (health: Awaited>) => void; + vi.mocked(NotificationsAPI.getHealth) + .mockReturnValueOnce( + new Promise((resolve) => { + finishOld = resolve; + }), + ) + .mockResolvedValueOnce(healthWith('healthy')); + const pending = state.loadDeliveryHealth(); + vi.mocked(NotificationsAPI[action]).mockResolvedValueOnce({ affected: 2 } as never); + await state[action](); + expect(state.deliveryNeedsAttention()).toBe(false); + finishOld(healthWith('degraded')); + await pending; + expect(state.deliveryHealth()?.queue.status).toBe('healthy'); + expect(state.deliveryNeedsAttention()).toBe(false); + } finally { + confirmSpy.mockRestore(); + dispose(); + } + }), + ); }); diff --git a/frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts b/frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts index 44424f2d6..367e6837a 100644 --- a/frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts +++ b/frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts @@ -22,19 +22,27 @@ export function useNotificationDeliveryHealth(options?: { const [refreshingDeliveryHealth, setRefreshingDeliveryHealth] = createSignal(false); const [loadedOnce, setLoadedOnce] = createSignal(false); + // Mount, configuration retry and queue actions can overlap. Only the latest + // requested snapshot owns health and loading state, regardless of completion order. + let latestHealthRequest = 0; const loadDeliveryHealth = async () => { + const request = ++latestHealthRequest; setRefreshingDeliveryHealth(true); try { const health = await NotificationsAPI.getHealth(); + if (request !== latestHealthRequest) return; setDeliveryHealth(health); setDeliveryHealthUnavailable(health.queue.status === 'unavailable'); } catch (error) { + if (request !== latestHealthRequest) return; logger.error('Failed to load notification delivery health', error); setDeliveryHealth(null); setDeliveryHealthUnavailable(true); } finally { - setLoadedOnce(true); - setRefreshingDeliveryHealth(false); + if (request === latestHealthRequest) { + setLoadedOnce(true); + setRefreshingDeliveryHealth(false); + } } }; diff --git a/internal/alerts/canonical_stateful_test.go b/internal/alerts/canonical_stateful_test.go index 5587e38c0..7920c9563 100644 --- a/internal/alerts/canonical_stateful_test.go +++ b/internal/alerts/canonical_stateful_test.go @@ -4,7 +4,9 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -165,3 +167,115 @@ func TestStatefulAlertReFireCooldown(t *testing.T) { } }) } + +// A UI-keyed datastore override must govern actual incidents, not just the +// threshold resolver. Exercise hysteresis and recurrence across SQLite reopen, +// including a same-named datastore on another PBS instance. +func TestPBSDatastoreOverrideLifecycleAcrossRestart(t *testing.T) { + for _, legacySnapshot := range []bool{false, true} { + name := "persisted aliases" + if legacySnapshot { + name = "legacy snapshot" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + start := func() *Manager { + m := NewManagerWithDataDir(dir) + t.Cleanup(m.Stop) + m.EnableEventLog() + if !m.activeStateAuthoritative.Load() { + t.Fatal("SQLite active state is not authoritative") + } + m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive, + StorageDefault: HysteresisThreshold{Trigger: 95, Clear: 90}, + Overrides: map[string]ThresholdConfig{"pbs-primary/backups": {Usage: &HysteresisThreshold{Trigger: 80, Clear: 70}}}, + }) + disableTestTimeThresholds(m) + return m + } + storage := func(instance string, usage float64) models.Storage { + return models.Storage{ID: instance + "-backups", AliasIDs: []string{instance + "/backups"}, Name: "backups", Instance: instance, Type: "pbs", Status: "online", Total: 1000, Used: int64(usage * 10), Free: int64(1000 - usage*10), Usage: usage} + } + observe := func(m *Manager, usage float64) { + t.Helper() + for range 5 { + m.CheckStorage(storage("pbs-primary", usage)) + m.CheckStorage(storage("pbs-secondary", usage)) + } + if testHasActiveAlert(t, m, canonicalMetricStateID("pbs-secondary-backups", "usage")) { + t.Fatal("override leaked to another PBS instance") + } + } + events := func(m *Manager, fired, resolved int) { + t.Helper() + for kind, want := range map[string]int{eventlog.TypeFired: fired, eventlog.TypeResolved: resolved} { + if got := len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{kind}})); got != want { + t.Fatalf("%s events = %d, want %d", kind, got, want) + } + } + } + id := canonicalMetricStateID("pbs-primary-backups", "usage") + m := start() + observe(m, 85) + original := *testRequireActiveAlert(t, m, id) + events(m, 1, 0) + if legacySnapshot { + m.mu.Lock() + delete(m.activeAlerts[id].Metadata, storagePolicyAliasesKey) + m.mu.Unlock() + if err := m.SaveActiveAlerts(); err != nil { + t.Fatal(err) + } + } + m.Stop() + + m = start() + observe(m, 75) // Below trigger, but not below the override's clear threshold. + if got := testRequireActiveAlert(t, m, id); !got.StartTime.Equal(original.StartTime) { + t.Fatal("restart replaced the firing incident") + } + events(m, 1, 0) + observe(m, 65) + if testHasActiveAlert(t, m, id) { + t.Fatal("override recovery did not clear incident") + } + events(m, 1, 1) + m.Stop() + + m = start() + observe(m, 75) + if testHasActiveAlert(t, m, id) { + t.Fatal("resolved incident resurrected inside hysteresis band") + } + events(m, 1, 1) + observe(m, 85) + if got := testRequireActiveAlert(t, m, id); !got.StartTime.After(original.StartTime) { + t.Fatal("refire reused original incident start") + } + events(m, 2, 1) + }) + } +} + +func TestStoragePolicyAliasesLegacyIdentity(t *testing.T) { + for _, tc := range []struct { + name, instance, resource, datastore string + want bool + }{ + {"hyphenated names", "pbs-backup-east", "pbs-backup-east-daily-store", "daily-store", true}, + {"different instance", "pbs-backup-west", "pbs-backup-east-daily-store", "daily-store", false}, + {"not PBS", "backup-east", "backup-east-daily-store", "daily-store", false}, + {"missing datastore", "pbs-backup-east", "pbs-backup-east-", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + got := storagePolicyAliases(&Alert{Instance: tc.instance, ResourceID: tc.resource, ResourceName: tc.datastore}) + if tc.want { + if len(got) != 1 || got[0] != tc.instance+"/"+tc.datastore { + t.Fatalf("aliases = %v", got) + } + } else if len(got) != 0 { + t.Fatalf("invented alias: %v", got) + } + }) + } +} diff --git a/internal/alerts/capacity_forecast.go b/internal/alerts/capacity_forecast.go index b435829b3..e7c2cc7d0 100644 --- a/internal/alerts/capacity_forecast.go +++ b/internal/alerts/capacity_forecast.go @@ -215,12 +215,13 @@ var capacityForecastMetadataKeys = []string{ func (m *Manager) evaluateStorageCapacity(storage models.Storage, thresholds ThresholdConfig, trend CapacityTrendObservation) { input := &UnifiedResourceInput{ - ID: storage.ID, - Type: "storage", - Name: storage.Name, - Node: storage.Node, - Instance: storage.Instance, - Disk: &UnifiedResourceMetric{Percent: storage.Usage}, + ID: storage.ID, + StorageAliases: storage.AliasIDs, + Type: "storage", + Name: storage.Name, + Node: storage.Node, + Instance: storage.Instance, + Disk: &UnifiedResourceMetric{Percent: storage.Usage}, } m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool { if !m.config.Enabled { @@ -377,6 +378,9 @@ func (m *Manager) evaluateCapacityForecast(input *UnifiedResourceInput, threshol "forecastBucketCount": trend.BucketCount, "forecastCoverageSeconds": int64(trend.CoverageSpan / time.Second), } + if len(input.StorageAliases) > 0 { + metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...) + } _, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{ Spec: spec, Evidence: alertspecs.AlertEvidence{ diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go index efeabf819..27202d3d0 100644 --- a/internal/alerts/config_runtime.go +++ b/internal/alerts/config_runtime.go @@ -503,7 +503,7 @@ func (m *Manager) reevaluateActiveAlertsLocked() { alertsToResolve = append(alertsToResolve, alertID) continue } - thresholds := m.resolveResourceThresholds("storage", resourceID) + thresholds := m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: "storage", ResourceID: resourceID, StorageAliases: storagePolicyAliases(alert)}).Thresholds if thresholds.Disabled { alertsToResolve = append(alertsToResolve, alertID) continue diff --git a/internal/alerts/storage_crash_test.go b/internal/alerts/storage_crash_test.go new file mode 100644 index 000000000..d1628c893 --- /dev/null +++ b/internal/alerts/storage_crash_test.go @@ -0,0 +1,121 @@ +package alerts + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +// Unlike an orderly restart, process exit must not get a shutdown checkpoint +// or store Close to rescue a lifecycle transition. Keep the recovery mirror +// deliberately stale to prove SQLite, not the mirror, preserves the incident. +// This tests process interruption, not power loss or installed delivery. +func TestStorageLifecycleAcrossProcessExit(t *testing.T) { + const helperEnv = "PULSE_TEST_STORAGE_CRASH_PHASE" + storage := models.Storage{ID: "pbs-crash-backups", Name: "backups", Type: "pbs", Status: "online", Total: 1000, Used: 850, Free: 150, Usage: 85} + id := canonicalMetricStateID(storage.ID, "usage") + start := func(t *testing.T, dir string) *Manager { + t.Helper() + m := NewManagerWithDataDir(dir) + m.EnableEventLog() + if !m.activeStateAuthoritative.Load() { + t.Fatal("SQLite authority not enabled") + } + m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive, + TimeThresholds: map[string]int{"storage": 0}, StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 70}}) + return m + } + observe := func(m *Manager, s models.Storage) { + for range 5 { + m.CheckStorage(s) + } + } + if phase := os.Getenv(helperEnv); phase != "" { + if phase != "fired" && phase != "resolved" { + t.Fatal("invalid child phase") + } + m := start(t, os.Getenv("PULSE_TEST_STORAGE_CRASH_DIR")) + if phase == "resolved" { + observe(m, storage) + testRequireActiveAlert(t, m, id) + } + if err := m.SaveActiveAlerts(); err != nil { + t.Fatal(err) + } + // Freeze checkpoints, leaving either an empty or a firing JSON mirror. + // Lifecycle commits must remain independent of periodic/async saves. + m.saveMu.Lock() + if phase == "resolved" { + storage.Used, storage.Free, storage.Usage = 0, 1000, 0 + } + observe(m, storage) + if testHasActiveAlert(t, m, id) != (phase == "fired") { + t.Fatal("child did not reach expected lifecycle state") + } + os.Exit(0) // Intentionally bypass all cleanups, Stop and store Close. + } + + for _, phase := range []string{"fired", "resolved"} { + t.Run(phase, func(t *testing.T) { + dir := t.TempDir() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStorageLifecycleAcrossProcessExit$") + cmd.Env = append(os.Environ(), helperEnv+"="+phase, "PULSE_TEST_STORAGE_CRASH_DIR="+dir) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("child failed: %v\n%s", err, output) + } + mirror, err := os.ReadFile(filepath.Join(dir, "alerts", "active-alerts.json")) + if err != nil { + t.Fatal(err) + } + var stale []*Alert + if err := json.Unmarshal(mirror, &stale); err != nil { + t.Fatal(err) + } + wantStale := 0 + if phase == "resolved" { + wantStale = 1 + } + if len(stale) != wantStale { + t.Fatalf("recovery mirror was not stale: %s", mirror) + } + m := start(t, dir) + t.Cleanup(m.Stop) + if testHasActiveAlert(t, m, id) != (phase == "fired") { + t.Fatal("process exit lost firing incident or resurrected resolved incident") + } + events, err := m.AlertEvents(eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}}) + if err != nil { + t.Fatal(err) + } + want := 1 + if phase == "resolved" { + want = 2 + } + counts := make(map[string]int) + for _, event := range events { + counts[event.Type]++ + } + if counts[eventlog.TypeFired] != 1 || counts[eventlog.TypeResolved] != want-1 { + t.Fatalf("unexpected lifecycle history: %v", counts) + } + if len(events) != want { + t.Fatalf("durable lifecycle events = %d, want %d", len(events), want) + } + missing := storage + missing.Total, missing.Used, missing.Free, missing.Usage = 0, 0, 0, 0 + observe(m, missing) + if testHasActiveAlert(t, m, id) != (phase == "fired") { + t.Fatal("missing post-restart counters changed the incident state") + } + }) + } +} diff --git a/internal/alerts/storage_override_identity.go b/internal/alerts/storage_override_identity.go index a0c906114..00c19a86a 100644 --- a/internal/alerts/storage_override_identity.go +++ b/internal/alerts/storage_override_identity.go @@ -67,3 +67,32 @@ func (m *Manager) resolveStorageThresholdOverride(base ThresholdConfig, resource func (m *Manager) resolveStorageThresholdsNoLock(storage models.Storage) ThresholdConfig { return m.resolveStorageThresholdOverride(m.defaultThresholdsForResourceType("storage"), storage.ID, storage.AliasIDs) } + +// Keep the polling identity through JSON/SQLite restore so configuration saves +// cannot silently replace a datastore override with global storage defaults. +const storagePolicyAliasesKey = "storagePolicyAliases" + +func storagePolicyAliases(alert *Alert) []string { + if alert == nil { + return nil + } + switch aliases := alert.Metadata[storagePolicyAliasesKey].(type) { + case []string: + return append([]string(nil), aliases...) + case []interface{}: + result := make([]string, 0, len(aliases)) + for _, value := range aliases { + if alias, ok := value.(string); ok { + result = append(result, alias) + } + } + return result + } + // Older PBS snapshots predate alias metadata. The poller records the + // instance and datastore name separately: only reconstruct an alias when + // the complete legacy ID agrees, never split hyphenated names heuristically. + if strings.HasPrefix(alert.Instance, "pbs-") && alert.ResourceName != "" && alert.ResourceID == alert.Instance+"-"+alert.ResourceName { + return []string{alert.Instance + "/" + alert.ResourceName} + } + return nil +} diff --git a/internal/alerts/time_threshold_test.go b/internal/alerts/time_threshold_test.go index 83a182f30..a79001aff 100644 --- a/internal/alerts/time_threshold_test.go +++ b/internal/alerts/time_threshold_test.go @@ -8,7 +8,8 @@ import ( ) func TestGetTimeThresholdMappings(t *testing.T) { - manager := NewManager() + manager := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) manager.mu.Lock() manager.config.TimeThresholds = map[string]int{ @@ -42,7 +43,8 @@ func TestGetTimeThresholdMappings(t *testing.T) { } func TestGetTimeThresholdMetricOverrides(t *testing.T) { - manager := NewManager() + manager := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) manager.mu.Lock() manager.config.TimeThresholds = map[string]int{ @@ -139,7 +141,8 @@ func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) { } func TestCheckMetricUsesPendingStartTime(t *testing.T) { - manager := NewManager() + manager := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) manager.mu.Lock() manager.config.TimeThresholds["guest"] = 2 diff --git a/internal/alerts/unified_eval.go b/internal/alerts/unified_eval.go index 7e065cb9b..bcffe8100 100644 --- a/internal/alerts/unified_eval.go +++ b/internal/alerts/unified_eval.go @@ -55,6 +55,8 @@ type UnifiedResourceInput struct { NetworkIn *UnifiedResourceMetric NetworkOut *UnifiedResourceMetric Temperature *UnifiedResourceMetric + + StorageAliases []string // Durable policy lookup identities for storage alerts. } type unifiedMetricCandidate struct { @@ -259,6 +261,19 @@ func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds } opts = metricOptionsWithTags(opts, input.Tags) + if len(input.StorageAliases) > 0 { + merged := metricOptions{} + if opts != nil { + merged = *opts + } + metadata := make(map[string]interface{}, len(merged.Metadata)+1) + for k, v := range merged.Metadata { + metadata[k] = v + } + metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...) + merged.Metadata = metadata + opts = &merged + } for _, candidate := range buildUnifiedMetricCandidates(input, thresholds) { m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts) } diff --git a/internal/api/docker_container_action_executor_test.go b/internal/api/docker_container_action_executor_test.go index 6f41024b7..63de8eb8a 100644 --- a/internal/api/docker_container_action_executor_test.go +++ b/internal/api/docker_container_action_executor_test.go @@ -120,7 +120,7 @@ func (f *fakeDockerActionAgentCommander) ExecuteDockerContainerUpdate(_ context. OldImageDigest: "sha256:1111111111111111111111111111111111111111111111111111111111111111", NewImageDigest: "sha256:2222222222222222222222222222222222222222222222222222222222222222", BackupCreated: true, BackupContainer: "api_pulse_backup_20260714_000000", - After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: newID, State: "running", Running: true, StartedAt: now, ObservedAt: now}, + After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: newID, State: "running", Running: true, Health: agentexec.DockerContainerHealthNone, StartedAt: now, ObservedAt: now}, }, nil } diff --git a/internal/api/docker_container_action_result.go b/internal/api/docker_container_action_result.go index 00111231e..7d67f9e2a 100644 --- a/internal/api/docker_container_action_result.go +++ b/internal/api/docker_container_action_result.go @@ -115,7 +115,15 @@ func dockerContainerUpdateExecutionResult(resourceID, agentID string, facts agen } status := unified.ActionVerificationContradicted reason := "postcondition_contradicted" - if dockerUpdateFactsMatch(facts, facts.After.ContainerID) { + // A running replacement must also have a usable running/health + // readback; identity alone must not confirm an unhealthy update. + // Preserve updates of containers intentionally left stopped. + if facts.After.Running && !agentexec.IsDockerContainerHealth(facts.After.Health) { + status = unified.ActionVerificationInconclusive + reason = "container_health_unknown" + } else if dockerUpdateFactsMatch(facts, facts.After.ContainerID) && + (!facts.After.Running || (facts.After.State == "running" && + agentexec.DockerContainerHealthAllowsVerifiedRunningState(facts.After.Health))) { status = unified.ActionVerificationConfirmed reason = "" } @@ -145,7 +153,18 @@ func dockerContainerUpdateExecutionResult(resourceID, agentID string, facts agen } status := unified.ActionVerificationContradicted reason := "postcondition_contradicted" - if dockerUpdateFactsMatch(facts, independent.Snapshot.ContainerID) { + // Identity is necessary but not sufficient: a replacement can exit or + // become unhealthy after the agent reported success. Compare against + // its readback rather than requiring running unconditionally, because + // updates deliberately preserve stopped containers. + if !facts.ReadbackRan || facts.After.State == "" { + status = unified.ActionVerificationInconclusive + reason = "agent_readback_unavailable" + } else if dockerUpdateFactsMatch(facts, independent.Snapshot.ContainerID) && + independent.Snapshot.State == facts.After.State && + independent.Snapshot.Running == facts.After.Running && + (!independent.Snapshot.Running || (independent.Snapshot.State == "running" && + agentexec.DockerContainerHealthAllowsVerifiedRunningState(independent.Snapshot.Health))) { status = unified.ActionVerificationConfirmed reason = "" } @@ -254,8 +273,8 @@ func dockerActionEvidenceTimes(observedAt, receivedAt time.Time) (time.Time, tim } // dockerUpdateFactsMatch confirms the observed container is the replacement -// the agent claims to have created. A stopped original is recreated without -// being started, so run-state is not part of the postcondition; identity is. +// the agent claims to have created. Independent verification additionally +// compares its state with the agent readback, preserving stopped updates. func dockerUpdateFactsMatch(facts agentexec.DockerContainerUpdateResultPayload, observedContainerID string) bool { if facts.NewContainerID == "" || observedContainerID == "" { return false diff --git a/internal/api/docker_container_action_result_test.go b/internal/api/docker_container_action_result_test.go index 4df41f167..b0365279e 100644 --- a/internal/api/docker_container_action_result_test.go +++ b/internal/api/docker_container_action_result_test.go @@ -106,7 +106,7 @@ func TestDockerContainerUpdateExecutionResultClampsBoundedPositiveAgentClockSkew facts := agentexec.DockerContainerUpdateResultPayload{ Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete, MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID, - After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, ObservedAt: observedAt}, + After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, Health: agentexec.DockerContainerHealthHealthy, ObservedAt: observedAt}, } result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, receivedAt) if err != nil { @@ -198,3 +198,84 @@ func dockerResultFacts(now time.Time, started, completed, readback, matches bool } const dockerLifecycleTestID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +// A replacement identity alone is not proof that its reported running state +// survived until the independent daemon readback. +func TestDockerContainerUpdateIndependentObservationMustMatchState(t *testing.T) { + now := time.Now().UTC() + for _, tc := range []struct { + name, baseline, state, health string + running bool + want unified.ActionVerificationStatus + }{ + {"running", "running", "running", "healthy", true, unified.ActionVerificationConfirmed}, + {"no healthcheck", "running", "running", "none", true, unified.ActionVerificationConfirmed}, + {"stopped original", "created", "created", "none", false, unified.ActionVerificationConfirmed}, + {"stopped after update", "running", "exited", "", false, unified.ActionVerificationContradicted}, + {"restarting", "running", "restarting", "", true, unified.ActionVerificationContradicted}, + {"unhealthy", "running", "running", "unhealthy", true, unified.ActionVerificationContradicted}, + {"starting healthcheck", "running", "running", "starting", true, unified.ActionVerificationContradicted}, + {"unknown health", "running", "running", "", true, unified.ActionVerificationContradicted}, + {"missing agent readback", "", "running", "healthy", true, unified.ActionVerificationInconclusive}, + } { + t.Run(tc.name, func(t *testing.T) { + facts := agentexec.DockerContainerUpdateResultPayload{ + Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete, + MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID, + After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, ObservedAt: now}, + } + facts.After.State = tc.baseline + facts.After.Running = tc.baseline == "running" + facts.ReadbackRan = tc.baseline != "" + observation := &dockerContainerPostconditionObservation{ + ObserverID: "daemon-1", TrustDomain: "daemon:1", Method: "daemon_inspect", ReceivedAt: now, + Snapshot: agentexec.DockerContainerObservationSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now}, + } + result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, observation, now) + if err != nil { + t.Fatal(err) + } + if got := result.ActionResultV2.Verification; got.Status != tc.want || got.EvidenceClass != unified.ActionEvidenceIndependent { + t.Fatalf("verification = %+v, want %s / independent", got, tc.want) + } + if result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded { + t.Fatal("readback changed execution history") + } + }) + } +} + +func TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim(t *testing.T) { + now := time.Now().UTC() + for _, tc := range []struct { + name, state, health string + running bool + want unified.ActionVerificationStatus + }{ + {"healthy", "running", "healthy", true, unified.ActionVerificationConfirmed}, + {"no healthcheck", "running", "none", true, unified.ActionVerificationConfirmed}, + {"stopped preserved", "created", "none", false, unified.ActionVerificationConfirmed}, + {"unhealthy", "running", "unhealthy", true, unified.ActionVerificationContradicted}, + {"starting", "running", "starting", true, unified.ActionVerificationContradicted}, + {"restarting", "restarting", "healthy", true, unified.ActionVerificationContradicted}, + {"unknown health", "running", "", true, unified.ActionVerificationInconclusive}, + } { + t.Run(tc.name, func(t *testing.T) { + facts := agentexec.DockerContainerUpdateResultPayload{ + Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete, + MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID, + After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now}, + } + result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, now) + if err != nil { + t.Fatal(err) + } + if got := result.ActionResultV2.Verification; got.Status != tc.want || got.EvidenceClass != unified.ActionEvidenceAgentAttested { + t.Fatalf("verification = %+v, want %s / agent_attested", got, tc.want) + } + if result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded { + t.Fatal("readback changed execution history") + } + }) + } +} diff --git a/internal/notifications/notifications.go b/internal/notifications/notifications.go index 4721a784a..c043a5c58 100644 --- a/internal/notifications/notifications.go +++ b/internal/notifications/notifications.go @@ -3868,7 +3868,7 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio Str("type", baseType). Str("event", string(event)). Msg("skipping queued email notification because email delivery is disabled") - return nil + return ErrNotificationDeliverySkipped } deliveredJob = notificationDeliveryJob{ Type: "email", @@ -3891,7 +3891,7 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio Str("event", string(event)). Str("webhookID", webhookConfig.ID). Msg("skipping queued webhook notification because delivery is disabled") - return nil + return ErrNotificationDeliverySkipped } deliveredJob = notificationDeliveryJob{ Type: "webhook", @@ -3914,7 +3914,7 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio Str("type", baseType). Str("event", string(event)). Msg("skipping queued Apprise notification because delivery is disabled") - return nil + return ErrNotificationDeliverySkipped } deliveredJob = notificationDeliveryJob{ Type: "apprise", diff --git a/internal/notifications/notifications_test.go b/internal/notifications/notifications_test.go index ee27dc148..b0f0d47e7 100644 --- a/internal/notifications/notifications_test.go +++ b/internal/notifications/notifications_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "io" "net/http" "os" @@ -3367,8 +3368,8 @@ func TestProcessQueuedNotification_SkipsDisabledEmailDelivery(t *testing.T) { Alerts: []*alerts.Alert{{ID: "alert-1"}}, } - if err := nm.ProcessQueuedNotification(notif); err != nil { - t.Fatalf("expected queued email notification to be skipped without error, got %v", err) + if err := nm.ProcessQueuedNotification(notif); !errors.Is(err, ErrNotificationDeliverySkipped) { + t.Fatalf("expected queued email notification to report a policy skip, got %v", err) } } @@ -3414,8 +3415,8 @@ func TestProcessQueuedNotification_SkipsWhenNotificationsDisabled(t *testing.T) Alerts: []*alerts.Alert{{ID: "alert-1"}}, } - if err := nm.ProcessQueuedNotification(notif); err != nil { - t.Fatalf("expected queued webhook notification to be skipped without error, got %v", err) + if err := nm.ProcessQueuedNotification(notif); !errors.Is(err, ErrNotificationDeliverySkipped) { + t.Fatalf("expected queued webhook notification to report a policy skip, got %v", err) } } diff --git a/internal/notifications/ntfy_outage_test.go b/internal/notifications/ntfy_outage_test.go new file mode 100644 index 000000000..ad779027f --- /dev/null +++ b/internal/notifications/ntfy_outage_test.go @@ -0,0 +1,122 @@ +package notifications + +import ( + "encoding/json" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" +) + +// Provider rejection must preserve the firing receipt needed for recovery, +// including when an operator retries the recovery after reopening SQLite. +func TestQueuedNtfyRecoveryAfterProviderOutageAndRestart(t *testing.T) { + var unavailable atomic.Bool + var accepted atomic.Int32 + server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read body: %v", err) + } + if unavailable.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if accepted.Add(1) == 2 { + if r.Header.Get("Priority") != "default" || r.Header.Get("Title") != "RESOLVED: database" || + !strings.Contains(string(body), "is now healthy") { + t.Errorf("incorrect recovery: headers=%v body=%q", r.Header, body) + } + } + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + dir := t.TempDir() + webhook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL + "/topic", Enabled: true, Service: "ntfy"} + open := func() *NotificationManager { + m := NewNotificationManagerWithDataDir("", dir) + m.webhookClient = server.Client() + if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil { + t.Fatal(err) + } + m.AddWebhook(webhook) + t.Cleanup(m.Stop) + return m + } + m := open() + config, err := json.Marshal(webhook) + if err != nil { + t.Fatal(err) + } + alert := &alerts.Alert{ID: "cpu", ResourceName: "database", Type: "cpu", Level: alerts.AlertLevelCritical, + Message: "CPU above threshold", StartTime: time.Now().Add(-time.Minute)} + resolved := notificationDeliveryJob{Type: "webhook", Event: eventResolved, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook} + wait := func(m *NotificationManager, id string, want NotificationQueueStatus, wantAudits int) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for { + var status string + var audits int + if err := m.queue.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", id).Scan(&status); err != nil { + t.Fatal(err) + } + if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = ?", id).Scan(&audits); err != nil { + t.Fatal(err) + } + if status == string(want) && audits == wantAudits { + return + } + if time.Now().After(deadline) { + t.Fatalf("%s: status=%s audits=%d, want %s with audit", id, status, audits, want) + } + time.Sleep(time.Millisecond) + } + } + enqueue := func(id, kind string, payload *alerts.Alert) { + t.Helper() + if err := m.queue.Enqueue(&QueuedNotification{ID: id, Type: kind, Status: QueueStatusPending, + Config: config, Alerts: []*alerts.Alert{payload}, MaxAttempts: 1}); err != nil { + t.Fatal(err) + } + } + enqueue("firing", "webhook", alert.Clone()) + wait(m, "firing", QueueStatusSent, 1) + unavailable.Store(true) + recovery := alert.Clone() + annotateResolvedMetadata(recovery, time.Now()) + enqueue("recovery", "webhook_resolved", recovery) + wait(m, "recovery", QueueStatusDLQ, 1) + if got := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 1 { + t.Fatal("failed recovery consumed the firing receipt") + } + m.Stop() + m = open() + if got := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 1 { + t.Fatal("restart lost the firing receipt") + } + unavailable.Store(false) + if count, err := m.queue.RetryTerminalFailures(); err != nil || count != 1 { + t.Fatalf("retry = %d, %v; want one recovery", count, err) + } + wait(m, "recovery", QueueStatusSent, 2) + if got := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 0 { + t.Fatal("successful recovery did not consume the firing receipt") + } + if got := accepted.Load(); got != 2 { + t.Fatalf("accepted HTTP requests = %d, want firing and recovery only", got) + } + var failures, successes int + if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = 'recovery' AND success = 0").Scan(&failures); err != nil { + t.Fatal(err) + } + if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = 'recovery' AND success = 1").Scan(&successes); err != nil { + t.Fatal(err) + } + if failures != 1 || successes != 1 { + t.Fatalf("recovery audit failures=%d successes=%d, want 1 each", failures, successes) + } +} diff --git a/internal/notifications/ntfy_transition_test.go b/internal/notifications/ntfy_transition_test.go index 162c7230f..398df6f3e 100644 --- a/internal/notifications/ntfy_transition_test.go +++ b/internal/notifications/ntfy_transition_test.go @@ -1,6 +1,7 @@ package notifications import ( + "encoding/json" "io" "net/http" "reflect" @@ -14,6 +15,16 @@ import ( // Exercise the shared destination across transitions: generated firing headers // must not leak into the stored configuration and override recovery metadata. func TestNtfySeverityRecoveryTransition(t *testing.T) { + for _, queued := range []bool{false, true} { + name := "direct" + if queued { + name = "queued" + } + t.Run(name, func(t *testing.T) { testNtfySeverityRecoveryTransition(t, queued) }) + } +} + +func testNtfySeverityRecoveryTransition(t *testing.T, queued bool) { type receipt struct { header http.Header body string @@ -28,7 +39,7 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) { w.WriteHeader(http.StatusAccepted) })) defer server.Close() - manager := NewNotificationManager("") + manager := NewNotificationManagerWithDataDir("", t.TempDir()) defer manager.Stop() manager.webhookClient = server.Client() if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil { @@ -36,6 +47,7 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) { } webhook := WebhookConfig{Name: "transition", URL: server.URL + "/topic", Enabled: true, Service: "ntfy", Headers: map[string]string{"X-Static": "preserved"}} + manager.AddWebhook(webhook) originalHeaders := map[string]string{"X-Static": "preserved"} alert := &alerts.Alert{ID: "transition", Type: "cpu", ResourceID: "vm-1", ResourceName: "database", Node: "node-a", Message: "CPU above threshold", Value: 99, Threshold: 90, StartTime: time.Now().Add(-time.Minute)} @@ -54,7 +66,25 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) { alert.Level = step.level before := *alert var err error - if step.resolved { + if queued { + config, marshalErr := json.Marshal(webhook) + if marshalErr != nil { + t.Fatal(marshalErr) + } + kind := "webhook" + payload := alert.Clone() + if step.resolved { + kind += "_resolved" + annotateResolvedMetadata(payload, time.Now()) + } + if manager.queue == nil { + t.Fatal("notification queue unavailable") + } + err = manager.queue.Enqueue(&QueuedNotification{ + ID: step.name, Type: kind, Status: QueueStatusPending, + Config: config, Alerts: []*alerts.Alert{payload}, MaxAttempts: 1, + }) + } else if step.resolved { err = manager.sendResolvedWebhook(webhook, []*alerts.Alert{alert}, time.Now()) } else { err = manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert}) @@ -86,6 +116,24 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) { if !reflect.DeepEqual(*alert, before) { t.Error("source alert mutated") } + if queued { + // Receipt precedes the queue commit. Wait for completion before + // advancing the same alert identity to its next lifecycle state. + deadline := time.Now().Add(3 * time.Second) + for { + var status string + if err := manager.queue.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", step.name).Scan(&status); err != nil { + t.Fatal(err) + } + if status == string(QueueStatusSent) { + break + } + if time.Now().After(deadline) { + t.Fatalf("received HTTP request but queue status remained %s", status) + } + time.Sleep(time.Millisecond) + } + } }) } } diff --git a/internal/notifications/queue.go b/internal/notifications/queue.go index ef311465c..bbc76f7a6 100644 --- a/internal/notifications/queue.go +++ b/internal/notifications/queue.go @@ -3,6 +3,7 @@ package notifications import ( "database/sql" "encoding/json" + "errors" "fmt" "net/url" "os" @@ -19,6 +20,10 @@ import ( _ "modernc.org/sqlite" ) +// ErrNotificationDeliverySkipped distinguishes a policy cancellation from a +// successful provider delivery. Queue processors must not report skips as nil. +var ErrNotificationDeliverySkipped = errors.New("notification delivery disabled") + // defaultQueueMaxAttempts is the default number of delivery attempts // before a notification is moved to the dead-letter queue. With the // exponential backoff schedule (1s doubling, capped at 60s) eight attempts @@ -1205,7 +1210,8 @@ func (nq *NotificationQueue) scanNotification(rows *sql.Rows) (*QueuedNotificati return ¬if, nil } -// ScheduleRetry schedules a notification for retry with exponential backoff +// ScheduleRetry schedules a notification for retry with exponential backoff. +// Cancelled and delivered rows are final, even for stale operator requests. func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error { backoff := calculateBackoff(attempt) nextRetry := time.Now().Add(backoff) @@ -1231,10 +1237,10 @@ func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error { UPDATE notification_queue SET status = 'pending', next_retry_at = ?, last_attempt = ?, operational_links = ?, completed_at = NULL, last_error = NULL - WHERE id = ? + WHERE id = ? AND status IN ('pending', 'sending', 'failed', 'dlq') ` - _, err = nq.db.Exec( + result, err := nq.db.Exec( query, nextRetry.Unix(), time.Now().Unix(), @@ -1245,7 +1251,14 @@ func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error { nq.mu.Unlock() return fmt.Errorf("failed to schedule retry: %w", err) } + affected, err := result.RowsAffected() nq.mu.Unlock() + if err != nil { + return fmt.Errorf("read scheduled retry result: %w", err) + } + if affected == 0 { + return fmt.Errorf("notification %s is no longer eligible for retry", id) + } log.Debug(). Str("id", id). @@ -1729,7 +1742,13 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) { return } releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiersFromAlerts(notif.Alerts), false) - defer releaseDeliveryGates() + healthChanged := false + defer func() { + releaseDeliveryGates() + if healthChanged { + nq.notifyDeliveryHealthChanged() + } + }() // Atomically claim the pending row. A concurrent resolution may have // cancelled it while it was waiting for its per-alert delivery gate. @@ -1765,6 +1784,21 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) { err = processor(notif) + if errors.Is(err, ErrNotificationDeliverySkipped) { + // No provider attempt occurred. Preserve the cancellation and its reason, + // but do not manufacture a successful (or failed) delivery audit entry. + // Reconcile health only after releasing the per-alert gate. + nq.mu.Lock() + cancelErr := nq.updateNotificationStatusNoLock(notif.ID, QueueStatusCancelled, err.Error(), time.Now()) + nq.mu.Unlock() + if cancelErr != nil { + log.Error().Err(cancelErr).Str("id", notif.ID).Msg("Failed to cancel skipped notification") + } else { + healthChanged = true + } + return + } + success := err == nil errorMsg := "" failureClass := NotificationFailureClass("") @@ -2074,22 +2108,29 @@ func calculateBackoff(attempt int) time.Duration { // returns the number of matched firing-alert entries removed from rows that // were still waiting for delivery ('pending'). Entries in rows already // mid-send ('sending') are cancelled best-effort but not counted, because -// their delivery may still complete. +// their delivery may still complete. Failed/dead-lettered firing entries are +// also suppressed so operator retry cannot resurrect a resolved incident; +// these do not contribute to the pending-only return count. func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string) (int, error) { alertIdentifiers = normalizeAlertIdentifiers(alertIdentifiers) if len(alertIdentifiers) == 0 { return 0, nil } releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiers, true) - defer releaseDeliveryGates() - + healthChanged := false nq.mu.Lock() - defer nq.mu.Unlock() + defer func() { + nq.mu.Unlock() + releaseDeliveryGates() + if healthChanged { + nq.notifyDeliveryHealthChanged() + } + }() query := ` SELECT id, type, status, alerts, operational_links FROM notification_queue - WHERE status IN ('pending', 'sending') + WHERE status IN ('pending', 'sending', 'failed', 'dlq') ` rows, err := nq.db.Query(query) @@ -2248,7 +2289,9 @@ func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string) Str("action", "cancel_mark_notification"). Str("notifID", notifID). Msg("Failed to mark notification as cancelled") + return 0, fmt.Errorf("cancel resolved notification %s: %w", notifID, err) } + healthChanged = true } } diff --git a/internal/notifications/queue_disabled_delivery_test.go b/internal/notifications/queue_disabled_delivery_test.go new file mode 100644 index 000000000..03c602ec3 --- /dev/null +++ b/internal/notifications/queue_disabled_delivery_test.go @@ -0,0 +1,94 @@ +package notifications + +import ( + "sync/atomic" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" +) + +// Policy skips are neither provider receipts nor delivery failures. Exercise +// the real queue processor: checking only its return value hid false success. +func TestQueueDisabledDeliveryIsCancelledNotSent(t *testing.T) { + for _, kind := range []string{"email", "webhook", "apprise"} { + for _, suffix := range []string{"", "_resolved"} { + for _, enabled := range []bool{false, true} { + name := kind + suffix + "/global-disabled" + if enabled { + name = kind + suffix + "/destination-disabled" + } + t.Run(name, func(t *testing.T) { + q, err := NewNotificationQueue(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer q.Stop() + nm := &NotificationManager{enabled: enabled} + if !enabled { + nm.emailConfig.Enabled = true + nm.appriseConfig.Enabled = true + nm.webhooks = []WebhookConfig{{ID: "ops", Enabled: true}} + } + n := &QueuedNotification{ID: "disabled", Type: kind + suffix, Status: QueueStatusPending, + Links: []operationaltrust.NotificationLink{{DestinationID: "ops", OperationalRecordID: "incident", TransitionID: "firing", LifecycleState: operationaltrust.OperationalOpen, CauseKey: "cpu"}}, + Config: []byte(`{"enabled":true,"id":"ops"}`), MaxAttempts: 3, Alerts: []*alerts.Alert{{ID: "incident"}}} + if err := q.Enqueue(n); err != nil { + t.Fatal(err) + } + // Queue construction starts workers. Configure via the locked setter + // and wait for reconciliation even if a worker wins the claim. + var callbacks atomic.Int32 + reconciled := make(chan struct{}, 2) + q.SetDeliveryHealthChangedCallback(func() { + callbacks.Add(1) + release := q.acquireAlertDeliveryGates([]string{"incident"}, true) + defer release() + if _, err := q.GetQueueStats(); err != nil { + t.Error(err) + } + reconciled <- struct{}{} + }) + q.SetProcessor(nm.ProcessQueuedNotification) + q.processNotification(n) + select { + case <-reconciled: + case <-time.After(3 * time.Second): + t.Fatal("disabled delivery did not reconcile") + } + var status string + var completed *int64 + if err := q.db.QueryRow(`SELECT status, completed_at FROM notification_queue WHERE id = ?`, n.ID).Scan(&status, &completed); err != nil { + t.Fatal(err) + } + if status != string(QueueStatusCancelled) || completed == nil { + t.Errorf("status=%s completed=%v; want terminal cancellation", status, completed) + } + links, err := q.getNotificationLinks(n.ID) + if err != nil { + t.Fatal(err) + } + if len(links) != 1 || links[0].DeliveryState != operationaltrust.NotificationCancelled { + t.Errorf("links = %+v, want cancelled", links) + } + for _, table := range []string{"notification_audit", "notification_delivery_receipts"} { + var count int + if err := q.db.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Errorf("%s has %d rows for a policy skip", table, count) + } + } + if count := callbacks.Load(); count != 1 { + t.Errorf("health callbacks = %d, want 1", count) + } + if count, err := q.RetryTerminalFailures(); err != nil || count != 0 { + t.Errorf("retry = %d, %v; want no replay", count, err) + } + }) + } + } + } +} diff --git a/internal/notifications/queue_resolution_crash_test.go b/internal/notifications/queue_resolution_crash_test.go new file mode 100644 index 000000000..fb3d19330 --- /dev/null +++ b/internal/notifications/queue_resolution_crash_test.go @@ -0,0 +1,129 @@ +package notifications + +import ( + "context" + "os" + "os/exec" + "sync" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" +) + +// Exit without Stop or database Close: graceful reopen tests cannot establish +// that resolution rewrites and cancellations survive a process exit. +// This is queue durability evidence, not an installed provider receipt or a +// power-loss simulation; an interrupted provider send remains at-least-once. +func TestQueueResolutionSurvivesAbruptProcessExit(t *testing.T) { + const helperEnv = "PULSE_TEST_QUEUE_RESOLUTION_EXIT_DIR" + const exitCode = 23 + states := []NotificationQueueStatus{QueueStatusPending, QueueStatusSending, QueueStatusFailed, QueueStatusDLQ} + if dir := os.Getenv(helperEnv); dir != "" { + q, err := NewNotificationQueue(dir) + if err != nil { + t.Fatal(err) + } + enqueue := func(id, kind string, members ...string) *QueuedNotification { + t.Helper() + n := &QueuedNotification{ID: id, Type: kind, Status: QueueStatusPending, Config: []byte(`{}`), MaxAttempts: 3} + for _, member := range members { + n.Alerts = append(n.Alerts, &alerts.Alert{ID: member}) + } + if err := q.Enqueue(n); err != nil { + t.Fatal(err) + } + return n + } + for _, status := range states { + for _, grouped := range []bool{false, true} { + id := string(status) + members := []string{"recovered"} + if grouped { + id += "-group" + members = append(members, "still-firing") + } + enqueue(id, "webhook", members...) + if err := q.UpdateStatus(id, status, "destination unavailable"); err != nil { + t.Fatal(err) + } + } + } + enqueue("recovery", "webhook_resolved", "recovered") + if count, err := q.CancelByAlertIdentifiers([]string{"recovered"}); err != nil || count != 2 { + t.Fatalf("resolution = %d, %v; want two pending members", count, err) + } + os.Exit(exitCode) + } + + dir := t.TempDir() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestQueueResolutionSurvivesAbruptProcessExit$") + cmd.Env = append(os.Environ(), helperEnv+"="+dir) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("child timed out: %v\n%s", ctx.Err(), output) + } + if exited, ok := err.(*exec.ExitError); !ok || exited.ExitCode() != exitCode { + t.Fatalf("child did not reach deliberate exit: %v\n%s", err, output) + } + q, err := NewNotificationQueue(dir) + if err != nil { + t.Fatal(err) + } + defer q.Stop() + if count, err := q.RetryTerminalFailures(); err != nil || count != 2 { + t.Fatalf("terminal retry = %d, %v; want only failed/dlq surviving groups", count, err) + } + var mu sync.Mutex + delivered := make(map[string][]string) + // Exercise ordinary dispatch as well as repeated batch selection. + q.SetProcessor(func(n *QueuedNotification) error { + mu.Lock() + defer mu.Unlock() + for _, a := range n.Alerts { + delivered[n.ID] = append(delivered[n.ID], a.ID) + } + return nil + }) + q.processBatch() + q.processBatch() + deadline := time.Now().Add(3 * time.Second) + for { + stats, err := q.GetQueueStats() + if err != nil { + t.Fatal(err) + } + if stats["pending"]+stats["sending"] == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("restarted queue did not drain") + } + time.Sleep(time.Millisecond) + } + mu.Lock() + defer mu.Unlock() + if len(delivered) != 5 { + t.Fatalf("delivered = %v; want four surviving groups and recovery", delivered) + } + for _, id := range []string{"pending-group", "sending-group", "failed-group", "dlq-group", "recovery"} { + want := "still-firing" + if id == "recovery" { + want = "recovered" + } + if got := delivered[id]; len(got) != 1 || got[0] != want { + t.Errorf("%s delivered %v, want [%s] exactly once in this run", id, got, want) + } + } + for _, id := range []string{"pending", "sending", "failed", "dlq"} { + var status string + if err := q.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", id).Scan(&status); err != nil { + t.Fatal(err) + } + if status != string(QueueStatusCancelled) { + t.Errorf("%s status = %s, want cancelled", id, status) + } + } +} diff --git a/internal/notifications/queue_resolution_retry_test.go b/internal/notifications/queue_resolution_retry_test.go new file mode 100644 index 000000000..4c2cfaf7d --- /dev/null +++ b/internal/notifications/queue_resolution_retry_test.go @@ -0,0 +1,173 @@ +package notifications + +import ( + "sync" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" +) + +// A repaired destination must not replay obsolete firing alerts. Exercise the +// operator retry after reopening the database, not just the cancellation query. +func TestResolvedTerminalFiringIsNotReplayedAfterRestart(t *testing.T) { + for _, terminal := range []NotificationQueueStatus{QueueStatusFailed, QueueStatusDLQ} { + t.Run(string(terminal), func(t *testing.T) { + dir := t.TempDir() + q, err := NewNotificationQueue(dir) + if err != nil { + t.Fatal(err) + } + defer func() { _ = q.Stop() }() + for _, n := range []*QueuedNotification{ + {ID: "obsolete", Type: "webhook", Alerts: []*alerts.Alert{{ID: "healthy"}}}, + {ID: "group", Type: "webhook", Alerts: []*alerts.Alert{{ID: "healthy"}, {ID: "still-firing"}}}, + {ID: "recovery", Type: "webhook_resolved", Alerts: []*alerts.Alert{{ID: "healthy"}}}, + } { + n.Config = []byte("{}") + n.Status = QueueStatusPending + n.MaxAttempts = 3 + if err := q.Enqueue(n); err != nil { + t.Fatal(err) + } + if err := q.UpdateStatus(n.ID, terminal, "destination unavailable"); err != nil { + t.Fatal(err) + } + if err := q.RecordAudit(n, false, "destination unavailable"); err != nil { + t.Fatal(err) + } + } + healthCallbacks := 0 + q.SetDeliveryHealthChangedCallback(func() { + healthCallbacks++ + // Callback must run after both the DB mutex and alert gate + // are released, and see the committed cancellation. + release := q.acquireAlertDeliveryGates([]string{"healthy"}, true) + defer release() + stats, err := q.GetQueueStats() + if err != nil { + t.Error(err) + return + } + if stats[string(terminal)] != 2 { + t.Errorf("remaining terminal rows = %d, want 2", stats[string(terminal)]) + } + }) + count, err := q.CancelByAlertIdentifiers([]string{"healthy"}) + if err != nil { + t.Fatal(err) + } + q.SetDeliveryHealthChangedCallback(nil) + if healthCallbacks != 1 { + t.Errorf("health callbacks = %d, want 1", healthCallbacks) + } + if count != 0 { + t.Fatalf("pending suppression count = %d, want 0 for terminal rows", count) + } + if err := q.Stop(); err != nil { + t.Fatal(err) + } + q, err = NewNotificationQueue(dir) + if err != nil { + t.Fatal(err) + } + retried, err := q.RetryTerminalFailures() + if err != nil { + t.Fatal(err) + } + if retried != 2 { + t.Errorf("retried %d rows, want only surviving group and recovery", retried) + } + // Alert IDs identify a resource/condition and can recur. Resolution + // must suppress the old queue row, not permanently mute that ID. + if err := q.Enqueue(&QueuedNotification{ + ID: "new-incident", Type: "webhook", Status: QueueStatusPending, + Config: []byte("{}"), MaxAttempts: 3, + Alerts: []*alerts.Alert{{ID: "healthy"}}, + }); err != nil { + t.Fatal(err) + } + delivered := map[string][]string{} + var mu sync.Mutex + q.SetProcessor(func(n *QueuedNotification) error { + mu.Lock() + defer mu.Unlock() + for _, a := range n.Alerts { + delivered[n.ID] = append(delivered[n.ID], a.ID) + } + return nil + }) + q.processBatch() + deadline := time.Now().Add(3 * time.Second) + for { + var pending int + if err := q.db.QueryRow("SELECT count(*) FROM notification_queue WHERE status IN ('pending', 'sending')").Scan(&pending); err != nil { + t.Fatal(err) + } + if pending == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("queue did not drain") + } + time.Sleep(time.Millisecond) + } + mu.Lock() + if len(delivered) != 3 || len(delivered["new-incident"]) != 1 || + delivered["new-incident"][0] != "healthy" || len(delivered["group"]) != 1 || + delivered["group"][0] != "still-firing" || + len(delivered["recovery"]) != 1 || delivered["recovery"][0] != "healthy" { + t.Errorf("replayed payloads = %v, want still-firing, genuine recovery and the new incident", delivered) + } + mu.Unlock() + var failures int + if err := q.db.QueryRow("SELECT count(*) FROM notification_audit WHERE success = 0").Scan(&failures); err != nil { + t.Fatal(err) + } + if failures != 3 { + t.Errorf("retained failed attempts = %d, want 3", failures) + } + }) + } +} + +// A stale per-item retry request must not bypass resolution or resend a +// successful notification. The same scheduler handles transient send failures. +func TestScheduleRetryRejectsCancelledAndSent(t *testing.T) { + q, err := NewNotificationQueue(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = q.Stop() }() + for _, status := range []NotificationQueueStatus{ + QueueStatusCancelled, QueueStatusSent, QueueStatusPending, + QueueStatusSending, QueueStatusFailed, QueueStatusDLQ, + } { + t.Run(string(status), func(t *testing.T) { + id := string(status) + n := &QueuedNotification{ID: id, Type: "webhook", Status: QueueStatusPending, Config: []byte("{}")} + if err := q.Enqueue(n); err != nil { + t.Fatal(err) + } + if err := q.UpdateStatus(id, status, ""); err != nil { + t.Fatal(err) + } + err := q.ScheduleRetry(id, 0) + blocked := status == QueueStatusCancelled || status == QueueStatusSent + if (err != nil) != blocked { + t.Fatalf("retry error = %v, blocked = %v", err, blocked) + } + want := QueueStatusPending + if blocked { + want = status + } + var got NotificationQueueStatus + if err := q.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", id).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("status = %s, want %s", got, want) + } + }) + } +} diff --git a/internal/relay/client_managed_runtime_test.go b/internal/relay/client_managed_runtime_test.go index 57abe703a..d8d570514 100644 --- a/internal/relay/client_managed_runtime_test.go +++ b/internal/relay/client_managed_runtime_test.go @@ -4,8 +4,10 @@ import ( "bytes" "context" "crypto/ed25519" + "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" "errors" "io" "net" @@ -52,7 +54,7 @@ func TestManagedRuntimeRelayRegistrationReconnectDrain(t *testing.T) { pulseRoot, pulseProRelayDir := managedRelayWorkspaceRoots(t) relayBinary := buildManagedRelayBinary(t, pulseProRelayDir) const revocationFeedToken = "managed-runtime-revocation-feed-token" - revocationFeed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + revocationFeed := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/revocations" { http.NotFound(w, r) return @@ -69,6 +71,16 @@ func TestManagedRuntimeRelayRegistrationReconnectDrain(t *testing.T) { }) })) defer revocationFeed.Close() + revocationCertificate, err := x509.ParseCertificate(revocationFeed.TLS.Certificates[0].Certificate[0]) + if err != nil { + t.Fatalf("parse revocation feed certificate: %v", err) + } + revocationCAPath := filepath.Join(t.TempDir(), "revocation-feed-ca.pem") + revocationCAPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: revocationCertificate.Raw}) + if err := os.WriteFile(revocationCAPath, revocationCAPEM, 0o600); err != nil { + t.Fatalf("write revocation feed CA: %v", err) + } + t.Setenv("SSL_CERT_FILE", revocationCAPath) t.Setenv("PULSE_RELAY_LICENSE_SERVER_URL", revocationFeed.URL) t.Setenv("PULSE_RELAY_REVOCATION_FEED_TOKEN", revocationFeedToken) diff --git a/scripts/check-delivery-health-ordering.mjs b/scripts/check-delivery-health-ordering.mjs new file mode 100644 index 000000000..d2f5917f8 --- /dev/null +++ b/scripts/check-delivery-health-ordering.mjs @@ -0,0 +1,146 @@ +// Isolated real-browser component qualification; no installed backend or delivery claim. +import { createServer } from "../frontend-modern/node_modules/vite/dist/node/index.js"; +import solid from "../frontend-modern/node_modules/vite-plugin-solid/dist/esm/index.mjs"; +import { chromium } from "@playwright/test"; +import { resolve } from "node:path"; +import { mkdirSync } from "node:fs"; +import assert from "node:assert/strict"; +const root = resolve("frontend-modern"); +process.chdir(root); +const fixture = ` +import { render } from 'solid-js/web'; +import { Show } from 'solid-js'; +import { NotificationsAPI } from '/src/api/notifications'; +import { AlertsAPI } from '/src/api/alerts'; +import { useAlertDestinationsTabState } from '/src/features/alerts/useAlertDestinationsTabState'; +import { AlertDeliveryHealthCard } from '/src/features/alerts/AlertDeliveryHealthCard'; +import '/src/index.css'; +const pending = []; +NotificationsAPI.getHealth = () => new Promise((resolve, reject) => pending.push({resolve, reject})); +NotificationsAPI.getDeliveryLog = async () => []; +AlertsAPI.getEvents = async () => []; +NotificationsAPI.retryTerminalFailures = NotificationsAPI.dismissTerminalFailures = async () => ({affected: 1}); +window.confirm = () => true; +window.finish = (i, status) => status === 'error' ? pending[i].reject(new Error('scripted offline')) : pending[i].resolve({queue:{status, failed:1, deadLetter:0, attentionRequired:1}}); +window.count = () => pending.length; +function Fixture() { +const s = useAlertDestinationsTabState({emailConfig:()=>({}), appriseConfig:()=>({}), setAppriseConfig:()=>{}, configLoadError:()=> 'scripted config unavailable', isRetrying:()=>false, isLoadingDestinations:()=>false, onRetryLoad:()=>{}, webhooks:()=>[]}); +return

Delivery health ordering fixture

{String(s.refreshingDeliveryHealth())}
; +} +render(() => , document.getElementById('root')); +`; +const server = await createServer({ + root, + configFile: false, + optimizeDeps: { + noDiscovery: true, + entries: [], + esbuildOptions: { target: "esnext" }, + }, + esbuild: { target: "esnext" }, + plugins: [ + solid(), + { + name: "ordering-fixture", + configureServer(s) { + s.middlewares.use((req, res, next) => { + if (req.url === "/qualification") { + res.setHeader("Content-Type", "text/html"); + res.end( + '
', + ); + } else next(); + }); + }, + resolveId(id) { + if (id === "/ordering-fixture.tsx") return id; + }, + load(id) { + if (id === "/ordering-fixture.tsx") return fixture; + }, + }, + ], + resolve: { alias: { "@": resolve(root, "src") } }, + server: { host: "127.0.0.1", port: 5197, strictPort: true }, +}); +let browser; +try { + await server.listen(); + browser = await chromium.launch({ headless: true }); + mkdirSync("/tmp/pulse-health-ordering", { recursive: true }); + let cases = 0; + for (const width of [1440, 900, 390]) { + for (const [old, newer] of [ + ["healthy", "degraded"], + ["degraded", "healthy"], + ["error", "healthy"], + ["healthy", "error"], + ]) { + const page = await browser.newPage({ viewport: { width, height: 900 } }); + await page.goto("http://127.0.0.1:5197/qualification"); + await page.waitForFunction(() => window.count?.() === 1); + await page + .getByRole("button", { name: "Configuration Retry", exact: true }) + .click(); + await page.waitForFunction(() => window.count() === 2); + await page.evaluate((s) => window.finish(1, s), newer); + await page.waitForFunction( + () => document.querySelector("output").textContent === "false", + ); + const before = await page.locator("main").innerText(); + await page.evaluate((s) => window.finish(0, s), old); + await page.evaluate( + () => + new Promise((r) => + requestAnimationFrame(() => requestAnimationFrame(r)), + ), + ); + assert.equal(await page.locator("main").innerText(), before); + assert.equal( + await page.getByRole("alert").count(), + ["degraded", "error"].includes(newer) ? 1 : 0, + ); + await page.screenshot({ + path: `/tmp/pulse-health-ordering/${width}-${old}-${newer}.png`, + }); + await page.close(); + cases++; + } + for (const action of [ + "Retry retained deliveries", + "Dismiss retained failures", + ]) { + const page = await browser.newPage({ viewport: { width, height: 900 } }); + await page.goto("http://127.0.0.1:5197/qualification"); + await page.waitForFunction(() => window.count?.() === 1); + await page.evaluate(() => window.finish(0, "degraded")); + await page.getByRole("alert").waitFor(); + await page + .getByRole("button", { name: "Configuration Retry", exact: true }) + .click(); + await page.getByRole("button", { name: action, exact: true }).click(); + await page.waitForFunction(() => window.count() === 3); + await page.evaluate(() => window.finish(1, "healthy")); + assert.equal(await page.getByTestId("loading").textContent(), "true"); + await page.evaluate(() => window.finish(2, "healthy")); + await page.waitForFunction( + () => document.querySelector("output").textContent === "false", + ); + assert.equal(await page.getByRole("alert").count(), 0); + await page.close(); + cases++; + } + } + console.log( + JSON.stringify({ + result: "passed", + cases, + viewports: [1440, 900, 390], + scope: + "real Chromium, real caller/hook/card, scripted API; not installed application or notification receipt", + }), + ); +} finally { + await browser?.close(); + await server.close(); +} diff --git a/scripts/check-overview-health-refresh.mjs b/scripts/check-overview-health-refresh.mjs new file mode 100644 index 000000000..4c184162b --- /dev/null +++ b/scripts/check-overview-health-refresh.mjs @@ -0,0 +1,143 @@ +// Isolated real-browser component qualification; no installed backend or delivery claim. +import { createServer } from "../frontend-modern/node_modules/vite/dist/node/index.js"; +import solid from "../frontend-modern/node_modules/vite-plugin-solid/dist/esm/index.mjs"; +import { chromium } from "@playwright/test"; +import { resolve } from "node:path"; +import { mkdirSync } from "node:fs"; +import assert from "node:assert/strict"; +const root = resolve("frontend-modern"); +process.chdir(root); +const fixture = ` +import { render } from 'solid-js/web'; +import { Router, Route } from '@solidjs/router'; +import { NotificationsAPI } from '/src/api/notifications'; +import { AlertsAPI } from '/src/api/alerts'; +import { OverviewTab } from '/src/features/alerts/OverviewTab'; +import '/src/index.css'; +const pending = []; +NotificationsAPI.getHealth = () => new Promise((resolve, reject) => pending.push({resolve, reject})); +AlertsAPI.getDeliveryDiagnoses = async () => []; +window.actions = 0; +NotificationsAPI.retryTerminalFailures = NotificationsAPI.dismissTerminalFailures = async () => { window.actions++; return {affected: 1}; }; +window.confirm = () => true; +window.finish = (i, status) => status === 'error' ? pending[i].reject(new Error('scripted offline')) : pending[i].resolve({queue:{status, failed:0, deadLetter:status === 'healthy' ? 0 : 1, attentionRequired:status === 'healthy' ? 0 : 1}}); +window.count = () => pending.length; +function Fixture() { +return
{}} showQuickTip={()=>false} dismissQuickTip={()=>{}} showAcknowledged={()=>true} setShowAcknowledged={()=>{}} alertsDisabled={()=>false}/>
; +} +render(() => , document.getElementById('root')); +`; +const server = await createServer({ + root, + configFile: false, + optimizeDeps: { + noDiscovery: true, + entries: [], + esbuildOptions: { target: "esnext" }, + }, + esbuild: { target: "esnext" }, + plugins: [ + solid(), + { + name: "ordering-fixture", + configureServer(s) { + s.middlewares.use((req, res, next) => { + if (req.url === "/qualification") { + res.setHeader("Content-Type", "text/html"); + res.end( + '
', + ); + } else next(); + }); + }, + resolveId(id) { + if (id === "/ordering-fixture.tsx") return id; + }, + load(id) { + if (id === "/ordering-fixture.tsx") return fixture; + }, + }, + ], + resolve: { alias: { "@": resolve(root, "src") } }, + server: { host: "127.0.0.1", port: 5197, strictPort: true }, +}); +let browser; +try { + await server.listen(); + browser = await chromium.launch({ headless: true }); + mkdirSync("/tmp/pulse-overview-refresh", { recursive: true }); + let cases = 0; + for (const theme of ["light", "dark"]) { + for (const width of [1440, 900, 390]) { + for (const action of [ + "Retry retained deliveries", + "Dismiss retained failures", + ]) { + const page = await browser.newPage({ + viewport: { width, height: 900 }, + }); + await page.goto("http://127.0.0.1:5197/qualification"); + await page.evaluate( + (theme) => + document.documentElement.classList.toggle("dark", theme === "dark"), + theme, + ); + await page.waitForFunction(() => window.count?.() === 1); + await page.evaluate(() => window.finish(0, "degraded")); + await page.getByRole("button", { name: action, exact: true }).click(); + await page.waitForFunction(() => window.count() === 2); + await page.evaluate(() => window.finish(1, "error")); + const refresh = page.getByRole("button", { + name: "Refresh delivery status", + exact: true, + }); + await refresh.waitFor(); + assert.match( + await page.getByRole("alert").innerText(), + /status is unavailable/, + ); + const geometry = await page.getByRole("alert").evaluate((el) => { + const heading = el.querySelector("h3"); + const bounds = el.getBoundingClientRect(); + return { + titleFits: heading.scrollWidth <= heading.clientWidth + 1, + actionsFit: [...el.querySelectorAll("button, a")].every( + (control) => { + const rect = control.getBoundingClientRect(); + return rect.left >= bounds.left && rect.right <= bounds.right; + }, + ), + }; + }); + assert.equal( + geometry.titleFits, + true, + "warning heading must not overflow beneath actions", + ); + assert.equal( + geometry.actionsFit, + true, + "warning actions must stay inside the card", + ); + await page.screenshot({ + path: `/tmp/pulse-overview-refresh/${theme}-${width}-${cases}-unavailable.png`, + }); + await refresh.click(); + await page.waitForFunction(() => window.count() === 3); + assert.equal(await refresh.isDisabled(), true); + await page.evaluate(() => window.finish(2, "healthy")); + await page.getByRole("alert").waitFor({ state: "detached" }); + assert.equal(await page.evaluate(() => window.actions), 1); + await page.screenshot({ + path: `/tmp/pulse-overview-refresh/${theme}-${width}-${cases}-healthy.png`, + }); + cases++; + await page.close(); + } + } + } + console.log(`${cases} overview action/outage/refresh browser cases passed`); +} finally { + await browser?.close(); + await server.close(); +} diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 51239852b..769d3b6c6 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2119,7 +2119,8 @@ func TestDeploymentDefaultsPinVersionedImagesAndHelmDocsChecksum(t *testing.T) { `helm repo index "${index_work}"`, `git -C gh-pages push origin HEAD:gh-pages`, `grep -q "version: ${VERSION}"`, - `helm show chart pulse-public/pulse --version "${VERSION}"`, + `helm pull pulse-public/pulse --version "${VERSION}" --destination "${public_work}"`, + `cmp -s "${qualified_chart}" "${public_work}/pulse-${VERSION}.tgz"`, } for _, needle := range required { if !strings.Contains(helmPages, needle) { @@ -3660,6 +3661,26 @@ func TestReleasePipelinePromotesOneImmutableCandidate(t *testing.T) { } } +func TestReleaseTrainCITriggersIncludeBuildAndE2E(t *testing.T) { + for _, workflow := range []string{"build-and-test.yml", "test-e2e.yml"} { + content, err := os.ReadFile(repoFile(".github", "workflows", workflow)) + if err != nil { + t.Fatal(err) + } + for _, event := range []string{"push", "pull_request"} { + t.Run(workflow+"/"+event, func(t *testing.T) { + // Limit the assertion to this event's branch list, not another + // trigger or a job comment mentioning the same pattern. + expression := regexp.MustCompile(`(?m)^ ` + event + `:\n branches:\n((?: - [^\n]+\n)+)`) + match := expression.FindStringSubmatch(string(content)) + if len(match) != 2 || !strings.Contains(match[1], " - 'release/v*'\n") || !strings.Contains(match[1], " - main\n") { + t.Fatal("CI must admit main and release/v* branches for this event") + } + }) + } + } +} + func TestFrontendDependencySecurityAuditsAreRequired(t *testing.T) { workflowPath := repoFile(".github", "workflows", "build-and-test.yml") assertFileContainsAll(t, workflowPath, diff --git a/scripts/release_control/helm_pages_retry_test.py b/scripts/release_control/helm_pages_retry_test.py index 4ccfb78f0..2625cf3fa 100644 --- a/scripts/release_control/helm_pages_retry_test.py +++ b/scripts/release_control/helm_pages_retry_test.py @@ -106,5 +106,56 @@ class HelmPagesRetryTests(unittest.TestCase): self.assertIn("--latest=false", calls[-1]) +class PublicReceiptTests(unittest.TestCase): + def receipt(self, mode): + workflow = (ROOT / '.github/workflows/helm-pages.yml').read_text() + step = workflow.split(' - name: Verify public Pages chart\n', 1)[1] + script = textwrap.dedent(step.split(' run: |\n', 1)[1]) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / 'dist').mkdir() + (root / 'dist/pulse-6.4.3.tgz').write_bytes(b'qualified') + programs = { + 'curl': '''#!/usr/bin/env python3 +import pathlib, sys +pathlib.Path(sys.argv[sys.argv.index('-o')+1]).write_text('version: 6.4.3\\n') +''', + 'helm': '''#!/usr/bin/env python3 +import os, pathlib, sys +args = sys.argv[1:] +if args[0] == 'pull': + mode = os.environ['MODE'] + if mode == 'unavailable': sys.exit(1) + if mode != 'missing': + dest = pathlib.Path(args[args.index('--destination')+1]) + (dest / 'pulse-6.4.3.tgz').write_bytes(b'qualified' if mode == 'match' else b'wrong') +''', + 'sleep': '#!/bin/sh\nexit 0\n', + } + for name, body in programs.items(): + path = root / name + path.write_text(body) + path.chmod(0o755) + return subprocess.run(['bash', '-c', script], cwd=root, + env={'PATH': f"{root}:{os.environ['PATH']}", + 'VERSION': '6.4.3', 'MODE': mode}, + capture_output=True, text=True) + + def test_matching_public_package_passes(self): + result = self.receipt('match') + self.assertEqual(result.returncode, 0, result.stderr) + + def test_same_version_wrong_bytes_fail(self): + result = self.receipt('wrong') + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('[OK]', result.stdout) + + def test_missing_download_fails(self): + self.assertNotEqual(self.receipt('missing').returncode, 0) + + def test_unavailable_download_fails(self): + self.assertNotEqual(self.receipt('unavailable').returncode, 0) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/release_control/helm_publish_version_test.py b/scripts/release_control/helm_publish_version_test.py new file mode 100644 index 000000000..dc1b4f6f6 --- /dev/null +++ b/scripts/release_control/helm_publish_version_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Exercise hosted Helm version resolution without packaging or publication.""" +from pathlib import Path +import os +import subprocess +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +class HelmPublishVersionTests(unittest.TestCase): + def resolve(self, chart="", app="", tag=""): + workflow = (ROOT / ".github/workflows/publish-helm-chart.yml").read_text() + step = workflow.split(" - name: Determine chart version\n", 1)[1] + script = textwrap.dedent(step.split(" run: |\n", 1)[1].split( + " - name:", 1)[0]) + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "output" + result = subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], cwd=ROOT, + env={"PATH": os.environ["PATH"], "INPUT_CHART_VERSION": chart, + "INPUT_APP_VERSION": app, "RELEASE_TAG_NAME": tag, + "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True) + return result, output.read_text() if output.exists() else "" + + def test_matching_and_default_application_versions(self): + for version in ("6.4.3", "6.4.3-rc.2", "6.5.0-beta.1", "6.5.0-alpha.1"): + for app in ("", version): + with self.subTest(version=version, app=app): + result, output = self.resolve(version, app) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("\n" + version + "\n", output) + self.assertIn("is_prerelease=" + ("true" if "-" in version else "false"), output) + + def test_release_event_uses_tag(self): + result, output = self.resolve(tag="v6.4.3-rc.2") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("\n6.4.3-rc.2\n", output) + + def test_mismatched_application_rejected_before_outputs(self): + for chart, app in (("6.4.3", "6.4.2"), ("6.4.3", "6.4.3-rc.2"), + ("6.4.3-rc.2", "6.4.3"), ("6.4.3", "latest")): + with self.subTest(chart=chart, app=app): + result, output = self.resolve(chart, app) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output, "") + + def test_missing_version_rejected(self): + result, output = self.resolve() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release_control/reconcile_release_convergence.py b/scripts/release_control/reconcile_release_convergence.py index 553deb1e6..727d69606 100644 --- a/scripts/release_control/reconcile_release_convergence.py +++ b/scripts/release_control/reconcile_release_convergence.py @@ -130,14 +130,8 @@ def validate_marker( return int(owner) -def latest_failed_runs( - releases: Iterable[object], - runs: Iterable[object], - *, - workflow_id: int, - default_branch: str, -) -> list[int]: - """Return the latest failed convergence for each current immutable channel.""" +def current_channel_heads(releases: Iterable[object]) -> dict[bool, dict[str, Any]]: + """Select published server heads before applying retry eligibility.""" latest_release: dict[bool, tuple[datetime, dict[str, Any]]] = {} for index, value in enumerate(releases): if not isinstance(value, dict) or value.get("draft") is not False: @@ -154,12 +148,23 @@ def latest_failed_runs( if prerelease not in latest_release or published > latest_release[prerelease][0]: latest_release[prerelease] = (published, value) + return {channel: value for channel, (_, value) in latest_release.items()} + + +def latest_failed_runs( + releases: Iterable[object], + runs: Iterable[object], + *, + workflow_id: int, + default_branch: str, +) -> list[int]: + """Return the latest failed convergence for each current immutable channel.""" # Never fall back to an older release when the advertised channel head is # mutable. That is continuity debt requiring a replacement, not a target # whose aliases should be promoted again. current_tags = { str(release["tag_name"]) - for _, release in latest_release.values() + for release in current_channel_heads(releases).values() if release.get("immutable") is True } newest: dict[str, tuple[datetime, dict[str, Any]]] = {} @@ -679,6 +684,14 @@ def discover(github: GitHub) -> list[int]: releases = flatten_pages( github.pages(f"repos/{repository}/releases?per_page=100") ) + for prerelease, release in current_channel_heads(releases).items(): + if release.get("immutable") is not True: + channel = "preview" if prerelease else "stable" + report_decision( + f"Current {channel} head {release['tag_name']} has no confirmed " + "immutable activation commit; continuity debt remains. " + "No retry or fallback to an older release is authorised by this check." + ) runs = flatten_pages( github.pages( f"repos/{repository}/actions/workflows/release-convergence.yml/runs" diff --git a/scripts/release_control/reconcile_release_convergence_test.py b/scripts/release_control/reconcile_release_convergence_test.py index 25cb288c0..85502a133 100644 --- a/scripts/release_control/reconcile_release_convergence_test.py +++ b/scripts/release_control/reconcile_release_convergence_test.py @@ -290,6 +290,63 @@ class DecisionSummaryTests(unittest.TestCase): self.assertIn("No failed run selected for retry", message) self.assertIn("missing convergence runs are not qualified", message) + def discover_with_releases(self, releases): + github = FakeGitHub() + github.pages = lambda endpoint: ( + [releases] if "/releases?" in endpoint + else [{"workflow_runs": github.runs}] + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "summary.md" + with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(path)}), \ + contextlib.redirect_stdout(io.StringIO()) as output: + selected = subject.discover(github) + summary = path.read_text() if path.exists() else "" + self.assertEqual([], github.posts) + return selected, output.getvalue(), summary + + def test_mutable_stable_debt_visible_even_when_preview_retry_selected(self): + selected, output, summary = self.discover_with_releases([ + release("v6.4.1", "2026-09-01T00:00:00Z", + prerelease=False, immutable=False), + release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True), + ]) + self.assertEqual([100], selected) + for text in (output, summary): + self.assertIn("Current stable head v6.4.1", text) + self.assertIn("continuity debt remains", text) + self.assertIn("No retry or fallback", text) + + def test_unknown_immutability_reports_debt_without_older_fallback(self): + for unknown in (None, "true", 1, False): + with self.subTest(immutable=unknown): + head = release("v6.5.0-rc.2", "2026-09-03T00:00:00Z", + prerelease=True) + head["immutable"] = unknown + selected, output, summary = self.discover_with_releases([ + release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True), + head, + ]) + self.assertEqual([], selected) + self.assertIn("Current preview head v6.5.0-rc.2", output) + self.assertIn("no confirmed immutable activation commit", summary) + + def test_old_mutable_draft_and_companion_do_not_report_current_debt(self): + draft = release("v6.6.0", "2026-09-04T00:00:00Z", + prerelease=False, immutable=False) + draft["draft"] = True + selected, output, summary = self.discover_with_releases([ + release("v6.4.0-rc.1", "2026-09-01T00:00:00Z", + prerelease=True, immutable=False), + release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True), + release("helm-chart-6.5.0", "2026-09-03T00:00:00Z", + prerelease=False, immutable=False), + draft, + ]) + self.assertEqual([100], selected) + self.assertEqual("", output) + self.assertEqual("", summary) + def test_credential_hold_is_visible_without_retry(self): github = FakeGitHub(current_controls=True) github.unchanged_credential_containment_block = lambda run_id: True diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index 9509cd3b2..f7ba48efc 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -2424,7 +2424,8 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn('helm repo index "${index_work}"', helm_pages) self.assertIn('git -C gh-pages push origin HEAD:gh-pages', helm_pages) self.assertIn('grep -q "version: ${VERSION}"', helm_pages) - self.assertIn('helm show chart pulse-public/pulse --version "${VERSION}"', helm_pages) + self.assertIn('helm pull pulse-public/pulse --version "${VERSION}"', helm_pages) + self.assertIn('cmp -s "${qualified_chart}" "${public_work}/pulse-${VERSION}.tgz"', helm_pages) self.assertNotIn("helm status pulse || true", helm_pages) self.assertNotIn("kubectl describe pods", helm_pages) self.assertIn("release-convergence.yml/dispatches", release_workflow) @@ -2434,6 +2435,13 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("sync_chart_release_metadata.py", helm) self.assertNotIn("sync_chart_release_metadata.py", helm_pages) self.assertIn("--chart deploy/helm/pulse/Chart.yaml", helm) + version_guard = 'if [ "$APP_VERSION" != "$CHART_VERSION" ]; then' + self.assertIn(version_guard, helm) + self.assertLess(helm.index(version_guard), helm.index(" - name: Package chart")) + self.assertIn( + "python3 scripts/release_control/helm_publish_version_test.py", + (Path(__file__).resolve().parents[2] / ".github/workflows/canonical-governance.yml").read_text(), + ) self.assertIn('git checkout --detach "refs/tags/${RELEASE_TAG}"', helm) self.assertIn("Verify public GHCR chart identity and provenance", helm) self.assertIn("helm registry logout ghcr.io || true", helm) diff --git a/scripts/tests/test_release_train_ci_contract.py b/scripts/tests/test_release_train_ci_contract.py new file mode 100644 index 000000000..33115e857 --- /dev/null +++ b/scripts/tests/test_release_train_ci_contract.py @@ -0,0 +1,36 @@ +"""Release trains must receive the same CI triggers as main.""" +import fnmatch +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +class ReleaseTrainCIContractTest(unittest.TestCase): + def test_build_and_e2e_cover_release_train_pushes_and_proposals(self): + for workflow in ("build-and-test.yml", "test-e2e.yml"): + source = (ROOT / ".github/workflows" / workflow).read_text() + for event in ("push", "pull_request"): + with self.subTest(workflow=workflow, event=event): + block = re.search( + rf"^ {event}:\n(.*?)(?=^ \w|^\S|\Z)", + source, re.MULTILINE | re.DOTALL, + ) + self.assertIsNotNone(block) + branches = re.search( + r"^ branches:\n((?: - .*\n)+)", + block.group(1), re.MULTILINE, + ) + self.assertIsNotNone(branches) + patterns = [line.strip()[2:].strip('\"\'') + for line in branches.group(1).splitlines()] + for branch in ("main", "release/v6.4", "release/v7.0"): + self.assertTrue( + any(fnmatch.fnmatchcase(branch, p) for p in patterns), + f"{workflow} {event} excludes {branch}: {patterns}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/tests/96-navigation-socket-recovery.spec.ts b/tests/integration/tests/96-navigation-socket-recovery.spec.ts index 71eefa039..685a028a7 100644 --- a/tests/integration/tests/96-navigation-socket-recovery.spec.ts +++ b/tests/integration/tests/96-navigation-socket-recovery.spec.ts @@ -1,5 +1,5 @@ import { expect, test, type Page, type WebSocketRoute } from '@playwright/test'; -import { ensureAuthenticated } from './helpers'; +import { apiRequest, ensureAuthenticated } from './helpers'; // Opt-in: uses synthetic inventory in an isolated real backend. Never interrupts // a shared runtime or substitutes the application's websocket store. @@ -208,3 +208,108 @@ for (const admissionFailure of [false, true]) { }); } } + +// Mutations go to the owned backend, not a fulfilled REST fixture or a store hook. +// Keep this after the inventory matrix: global detection is briefly paused while +// clearing incidents, then its original configuration is restored. +for (const width of [1440, 320]) { + test(`changed backend incidents recover without reload at ${width}px`, async ({ page, browser }, testInfo) => { + test.skip(!enabled, 'Requires isolated mock backend and explicit qualification opt-in'); + test.setTimeout(120_000); + await page.setViewportSize({ width, height: 900 }); + let blocked = false; + const sockets: WebSocketRoute[] = []; + await page.routeWebSocket('**/ws*', async socket => { + if (blocked) return socket.close({ code: 1013, reason: 'qualification interruption' }); + socket.connectToServer(); + sockets.push(socket); + }); + // Prevent periodic REST reads from hiding a missing reconnect refresh. + // APIRequestContext mutations/reads below do not pass through page routing. + await page.route('**/api/alerts/active', route => blocked ? route.abort() : route.continue()); + await ensureAuthenticated(page); + await page.goto('/alerts'); + await expect(page.getByRole('status', { name: healthy })).toBeVisible(); + const readActive = async () => { + const response = await apiRequest(page, '/api/alerts/active'); + expect(response.ok()).toBe(true); + return await response.json() as Array<{ id: string; resourceName: string; type: string; acknowledged: boolean }>; + }; + // Service names vary with the generated estate. Select an identified service + // health incident, not a name from a previous run or a cold offline host. + await expect.poll(async () => (await readActive()).some(alert => + alert.type === 'docker-service-health')).toBe(true); + const initial = await readActive(); + const target = initial.find(alert => alert.type === 'docker-service-health'); + expect(target).toBeTruthy(); + if (target!.acknowledged) { + const reset = await apiRequest(page, '/api/alerts/unacknowledge', { + method: 'POST', data: { alertIdentifier: target!.id }, + }); + expect(reset.ok()).toBe(true); + } + const card = page.locator(`[id=${JSON.stringify('alert-' + target!.id)}]`); + await expect(card.getByRole('button', { name: 'Acknowledge', exact: true })).toBeVisible(); + const documentIdentity = await page.evaluate(() => performance.timeOrigin); + const disconnect = async () => { + blocked = true; + for (const socket of sockets.splice(0)) await socket.close({ code: 1013, reason: 'qualification interruption' }); + await expect(page.getByRole('status', { name: 'Backend is healthy. Live updates are reconnecting.' })).toBeVisible(); + }; + const reconnect = async () => { + const response = page.waitForResponse(r => new URL(r.url()).pathname === '/api/alerts/active' && r.ok()); + blocked = false; + const snapshot = await (await response).json(); + await expect(page.getByRole('status', { name: healthy })).toBeVisible({ timeout: 45_000 }); + expect(await page.evaluate(() => performance.timeOrigin)).toBe(documentIdentity); + return snapshot; + }; + await disconnect(); + const ack = await apiRequest(page, '/api/alerts/acknowledge', { + method: 'POST', data: { alertIdentifier: target!.id, user: 'reconnect-qualification' }, + }); + expect(ack.ok()).toBe(true); + await expect.poll(async () => (await readActive()).find(a => a.id === target!.id)?.acknowledged).toBe(true); + // It must still be the old rendered value before restoring either transport. + await expect(card.getByRole('button', { name: 'Acknowledge', exact: true })).toBeVisible(); + const acknowledgedSnapshot = await reconnect(); + expect(acknowledgedSnapshot.find((a: { id: string }) => a.id === target!.id)?.acknowledged).toBe(true); + // Acknowledged incidents sort last. The production list windows large + // estates, so a correct update can remove this card from the current DOM. + await page.getByRole('contentinfo').scrollIntoViewIfNeeded(); + await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible(); + await card.scrollIntoViewIfNeeded(); + await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' }); + + const configResponse = await apiRequest(page, '/api/alerts/config'); + expect(configResponse.ok()).toBe(true); + const config = await configResponse.json(); + try { + await disconnect(); + const paused = await apiRequest(page, '/api/alerts/config', { method: 'PUT', data: { ...config, enabled: false } }); + expect(paused.ok()).toBe(true); + const active = await readActive(); + if (active.length) { + const cleared = await apiRequest(page, '/api/alerts/bulk/clear', { + method: 'POST', data: { alertIdentifiers: active.map(a => a.id) }, + }); + expect(cleared.ok()).toBe(true); + } + await expect.poll(readActive).toEqual([]); + await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible(); + const emptySnapshot = await reconnect(); + expect(emptySnapshot).toEqual([]); + await expect(page.locator('[id^="alert-"]').filter({ has: page.getByRole('button', { name: /^(Unacknowledge|Acknowledge)$/ }) })).toHaveCount(0); + await expect(card).toHaveCount(0); + await testInfo.attach('empty-active-response', { + body: JSON.stringify({ browser: browser.version(), width, target, acknowledgedSnapshot: acknowledgedSnapshot.filter((a: { id: string }) => a.id === target!.id), emptySnapshot, documentIdentity }), + contentType: 'application/json', + }); + await testInfo.attach('cleared-incidents', { body: await page.screenshot(), contentType: 'image/png' }); + } finally { + blocked = false; + const restored = await apiRequest(page, '/api/alerts/config', { method: 'PUT', data: config }); + expect(restored.ok()).toBe(true); + } + }); +} diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/README.md b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/README.md new file mode 100644 index 000000000..ea8d61d23 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/README.md @@ -0,0 +1,122 @@ +# Changed incident recovery — 5 September 2026 + +## Purpose and evidence boundary + +The previous integrated run only established unchanged navigation and socket +recovery. The admission continuation explicitly asked for changed-data and empty +active-response evidence. Fresh reading of the [community comparison thread](https://www.reddit.com/r/Proxmox/comments/1lblkk8/anyone_else_switch_to_pulse_from_netdata_or_any/) +on 5 September still supports protecting the quick-glance overview, including +phone use; one user separates this from historical monitoring and alerting in +CheckMK. These are historical self-reports, not new demand counts or current +performance measurements. No new product surface or ledger bet is introduced. + +The test uses an owned, source-built core backend with synthetic inventory and +production embedded frontend assets. It never fulfils active-alert REST with a +fixture, injects a store event, or replaces the application's WebSocket store. +Real sockets close with 1013; browser active-alert requests are aborted while +blocked, preventing periodic REST polling from hiding missing recovery. A +separate authenticated API client changes an identified incident on the backend and +verifies the changed server value while the rendered value remains old. + +After reconnect, the browser must receive a successful active-alert response, +render the same incident as acknowledged, and retain document time origin. +A second interruption pauses detection and bulk-clears backend incidents; the +backend and browser must both return `[]`, and the old incident cards must go. +Original alert configuration is restored in `finally`. This is deliberate +administrative clearing, **not** a claim that lost telemetry or disabling +alerts represents monitored-resource recovery. No notification delivery is exercised. + +## Results and test development + +Final full matrix: **10 passed, no retries**, 2.4 minutes starting +2026-09-05T21:41:31Z. Chromium 141.0.7390.37. Eight navigation cases cover +1440/1100/390/320px, each with/without admission HTTP failure; table-access opt-in +ran at 390/320px. Two additional cases cover changed acknowledgement and genuine +empty-response clearing at 1440/320px (height 900). A preceding cold-runtime +focused invocation passed both changed-data cases in 30 seconds. + +`final/` retains unmodified JUnit, extracted report timings, both +`narrow-table-access` JSON attachments, environment attachments, actual identified +selection/acknowledged/empty response receipts, and four screenshots. `focused/` +retains the preceding two-case pass. Inspected desktop acknowledged and phone +cleared screenshots from the focused run show the changed service card and +zero active/acknowledged counts with an **Alerting is paused** state respectively. +This is intentionally not an “all infrastructure healthy” claim. + +Earlier test-development failures are retained rather than counted as product +regressions: + +| Receipt | Local draft | Result and diagnosis | +| --- | --- | --- | +| `initial/` | `d82f55cb23` | Eight navigation cases passed; two new cases received acknowledged REST but failed to find the card. Initial grouping hypothesis was incomplete. | +| `standalone/` | `2a3dcc3378` | Eight passed; standalone host also moved outside the DOM window after acknowledgement. Second case inherited the acknowledged fixture. Source inspection established acknowledged-last sorting and list windowing. | +| `cold-fixture/` | `e934385ddc` | Two filtered cases failed before mutation: offline-host incident had not appeared in the fresh runtime. | +| `named-service/` | `75aa1b6b0a` | Two filtered cases failed before mutation: service names differ between generated estates. | + +The final test selects a current `docker-service-health` incident by canonical +ID, resets acknowledgement if needed and scrolls the windowed list to its last +group after acknowledgement. No UI hook or artificial renderer setting is used. +Test-development commits were consolidated locally into `1abaea81c3`; no remote +history changed. Passing runs executed draft `0cccef0dd4`, whose exact test blob +`685a028a71c8e3a13a881c4ae66d7988bfa08ec9` is unchanged in the consolidated commit. +Application source was unchanged throughout. + +In response attachments, `target` is the selection-time snapshot, before the +optional reset. It is already acknowledged in the final 320px case because the +previous case's acknowledgement can survive fixture regeneration. The test then +uses the real unacknowledge endpoint and requires its rendered Acknowledge button +before disconnecting and acknowledging again. Do not read `target.acknowledged` +as the post-reset baseline. The retained `acknowledgedSnapshot` and +`emptySnapshot` are actual browser recovery responses. + +## Invocation + +Locked dependencies installed with `npm ci --no-audit --no-fund` in repository +root, frontend-modern and tests/integration. The two full development attempts and final full matrix used exactly: + +```sh +pulse-heavy-run -- env \ + PULSE_E2E_USE_LOCAL_BACKEND=1 PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL=1 \ + PULSE_MOCK_MODE=true PULSE_E2E_NAVIGATION_RECOVERY=1 \ + PULSE_E2E_TABLE_ACCESS=1 PULSE_E2E_LOCAL_BACKEND_PORT=18765 \ + npm --prefix tests/integration test -- \ + tests/96-navigation-socket-recovery.spec.ts --project=chromium +``` + +The three filtered development attempts and successful focused run appended +`--grep 'changed backend'` to that command. All builds/browser runs were +serialized through `pulse-heavy-run`; the final rerun reused the same binary. + +Application source: `adeecd4079185cf9358415e2b7a55fc371f408b7` (the supplied +integrated main snapshot, not the newer remote head in the admission packet). +Only tests differ at the tested `0cccef0dd4` and consolidated `1abaea81c3`. Local executable SHA-256: +`891fc3827d1e6522f21011ef62052b73f29f6407f498131e7e0cbcf454f7c677`. +Built with Go 1.26.8 on Linux amd64, embedded Vite production assets and locked +Playwright 1.56.1. This binary has no embedded VCS build fields; the source +snapshot and content hash, not the displayed development version, bind this run. + +## Limits and next trigger + +No application repair, released-candidate qualification, reporter retest, +acknowledgement persistence across process restart, history durability, installed +alert receipt or off-host delivery is claimed. REST and stream are both restored, +so this does not establish which path wins their race. Malformed snapshots are +not covered. Global detection is paused only inside the owned test runtime. + +Prefer changed integration/candidate or installed evidence next, not another +unchanged synthetic matrix. Release judgment remains independently held on +publication continuity and candidate issue disposition; these checks do not +clear those boundaries. No timed revisit is justified by elapsed time alone. + +## Mechanical admission repair + +The first admission rejected eight trailing spaces inside failed-run JUnit +CDATA. Failed-run `junit.xml` files in `initial/`, `standalone/`, +`cold-fixture/` and `named-service/` now have trailing line whitespace removed. +Their original bytes are preserved in adjacent deterministic `junit.xml.gz` +archives, with original and normalised SHA-256 values in +`receipt-normalisation.json`. Decompression was checked byte-for-byte. The +passing `focused/` and `final/` JUnit files remain unmodified. No test result, +application source or test code changed; browser checks were not rerun for this +receipt-only repair. Evidence commits were consolidated locally so intermediate +commits no longer introduce the rejected whitespace. diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/cold-fixture/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/cold-fixture/junit.xml new file mode 100644 index 000000000..803f92a86 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/cold-fixture/junit.xml @@ -0,0 +1,88 @@ + + + + + 240 | await expect.poll(async () => (await readActive()).some(alert => + | ^ + 241 | alert.resourceName === 'edge-proxy-01')).toBe(true); + 242 | const initial = await readActive(); + 243 | const target = initial.find(alert => alert.resourceName === 'edge-proxy-01'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md +]]> + + + + + + + + 240 | await expect.poll(async () => (await readActive()).some(alert => + | ^ + 241 | alert.resourceName === 'edge-proxy-01')).toBe(true); + 242 | const initial = await readActive(); + 243 | const target = initial.find(alert => alert.resourceName === 'edge-proxy-01'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md +]]> + + + + + + + \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/cold-fixture/junit.xml.gz b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/cold-fixture/junit.xml.gz new file mode 100644 index 000000000..8e844e47a Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/cold-fixture/junit.xml.gz differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/01-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/01-environment.json new file mode 100644 index 000000000..5d06cdbb6 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/01-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":1440,"height":900,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":true,"tabs":[{"label":"Proxmox","text":"Proxmox","icons":1,"visible":true},{"label":"Docker","text":"Docker","icons":1,"visible":true},{"label":"Kubernetes","text":"Kubernetes","icons":1,"visible":true},{"label":"TrueNAS","text":"TrueNAS","icons":1,"visible":true},{"label":"vSphere","text":"vSphere","icons":1,"visible":true},{"label":"Machines","text":"Machines","icons":1,"visible":true},{"label":"Alerts","text":"Alerts","icons":1,"visible":true},{"label":"Patrol","text":"Patrol","icons":1,"visible":true},{"label":"Actions","text":"Actions","icons":1,"visible":true},{"label":"Settings","text":"Settings","icons":1,"visible":true}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/02-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/02-environment.json new file mode 100644 index 000000000..f5420646b --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/02-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":1100,"height":900,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/03-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/03-environment.json new file mode 100644 index 000000000..03149a9c6 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/03-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":390,"height":844,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/03-narrow-table-access.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/03-narrow-table-access.json new file mode 100644 index 000000000..33207f6c7 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/03-narrow-table-access.json @@ -0,0 +1,164 @@ +{ + "initial": { + "width": 362, + "scrollWidth": 362, + "left": 0, + "overflowX": "clip", + "tabIndex": -1, + "cells": [ + { + "text": "loki", + "x": 9, + "width": 108.59375, + "titles": [ + "Expand details for loki", + "Healthy", + "loki" + ] + }, + { + "text": "running", + "x": 117.59375, + "width": 48.4375, + "titles": [ + "running" + ] + }, + { + "text": "12%", + "x": 166.03125, + "width": 55.890625, + "titles": [] + }, + { + "text": "74%", + "x": 221.921875, + "width": 63.34375, + "titles": [] + }, + { + "text": "2", + "x": 285.265625, + "width": 37.25, + "titles": [] + }, + { + "text": "—", + "x": 322.515625, + "width": 48.4375, + "titles": [ + "No image update check reported" + ] + } + ] + }, + "pointer": { + "width": 362, + "scrollWidth": 362, + "left": 0, + "overflowX": "clip", + "tabIndex": -1, + "cells": [ + { + "text": "loki", + "x": 9, + "width": 108.59375, + "titles": [ + "Expand details for loki", + "Healthy", + "loki" + ] + }, + { + "text": "running", + "x": 117.59375, + "width": 48.4375, + "titles": [ + "running" + ] + }, + { + "text": "12%", + "x": 166.03125, + "width": 55.890625, + "titles": [] + }, + { + "text": "74%", + "x": 221.921875, + "width": 63.34375, + "titles": [] + }, + { + "text": "2", + "x": 285.265625, + "width": 37.25, + "titles": [] + }, + { + "text": "—", + "x": 322.515625, + "width": 48.4375, + "titles": [ + "No image update check reported" + ] + } + ] + }, + "keyboard": { + "width": 362, + "scrollWidth": 362, + "left": 0, + "overflowX": "clip", + "tabIndex": -1, + "cells": [ + { + "text": "loki", + "x": 9, + "width": 108.59375, + "titles": [ + "Expand details for loki", + "Healthy", + "loki" + ] + }, + { + "text": "running", + "x": 117.59375, + "width": 48.4375, + "titles": [ + "running" + ] + }, + { + "text": "12%", + "x": 166.03125, + "width": 55.890625, + "titles": [] + }, + { + "text": "74%", + "x": 221.921875, + "width": 63.34375, + "titles": [] + }, + { + "text": "2", + "x": 285.265625, + "width": 37.25, + "titles": [] + }, + { + "text": "—", + "x": 322.515625, + "width": 48.4375, + "titles": [ + "No image update check reported" + ] + } + ] + }, + "accessibleRow": "- row \"Expand details for notifications-worker notifications-worker running 12% 48% 1 —\":\n - cell \"Expand details for notifications-worker notifications-worker\":\n - button \"Expand details for notifications-worker\"\n - text: notifications-worker\n - cell \"running\"\n - cell \"12%\"\n - cell \"48%\"\n - cell \"1\"\n - cell \"—\"", + "expanded": "true", + "detailText": "notifications-worker\nContainer\nDocker / Podman\nOverview\nHistory\nManage\nRUNTIME CONTEXT\nObserved state\t\nRunning\n\nUptime\t\n6d 6h\n\nLast seen\t\n17s ago\n\nCONTAINER\nImage\t\nghcr.io/pulse-demo/notifications-worker:2026.04\n\nRestarts\t\n1\n\nCreated\t\n18 days ago\n\nStarted\t\n6 days ago\n\nCompose project\t\norion-2\n\nCompose service\t\npostgres\n\nLabels\t\ncom.docker.compose.project: orion-2\ncom.docker.compose.service: postgres\ncom.pulse.demo: true\ncom.pulse.role.production: true\ncom.pulse.role.queue: true\ncom.pulse.role.worker: true\n\nIDENTITY\nHostname\t\nnotifications-worker\n\nPrimary ID\t\napp-container:orion-2-26f93461b215\n\nParent\t\nagent-9dbedd74104d1527\n\nDiscovery\t\napp-container:notifications-worker\n\nMetrics Target\t\napp-container:orion-2-26f93461b215\n\nAliases\t\napp-container:orion-2-26f93461b215\norion-2-26f93461b215\norion-2-mock\napp-platform-01\n+4\nRelationship map\n\n1 canonical relationship\n\nCANONICAL RELATIONSHIPS\nnotifications-worker\n→\ndocker-network-0ab79ab291cd98fa\nAttached To\n100% confidence · Docker Adapter · last seen just now" +} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/04-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/04-environment.json new file mode 100644 index 000000000..f2c24d8f9 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/04-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":320,"height":844,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/04-narrow-table-access.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/04-narrow-table-access.json new file mode 100644 index 000000000..db0d08939 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/04-narrow-table-access.json @@ -0,0 +1,146 @@ +{ + "initial": { + "width": 292, + "scrollWidth": 292, + "left": 0, + "overflowX": "clip", + "tabIndex": -1, + "cells": [ + { + "text": "loki", + "x": 9, + "width": 116.796875, + "titles": [ + "Expand details for loki", + "Healthy", + "loki" + ] + }, + { + "text": "running", + "x": 125.796875, + "width": 43.796875, + "titles": [ + "running" + ] + }, + { + "text": "12%", + "x": 169.59375, + "width": 43.796875, + "titles": [] + }, + { + "text": "74%", + "x": 213.390625, + "width": 49.625, + "titles": [] + }, + { + "text": "—", + "x": 263.015625, + "width": 37.953125, + "titles": [ + "No image update check reported" + ] + } + ] + }, + "pointer": { + "width": 292, + "scrollWidth": 292, + "left": 0, + "overflowX": "clip", + "tabIndex": -1, + "cells": [ + { + "text": "loki", + "x": 9, + "width": 116.796875, + "titles": [ + "Expand details for loki", + "Healthy", + "loki" + ] + }, + { + "text": "running", + "x": 125.796875, + "width": 43.796875, + "titles": [ + "running" + ] + }, + { + "text": "12%", + "x": 169.59375, + "width": 43.796875, + "titles": [] + }, + { + "text": "74%", + "x": 213.390625, + "width": 49.625, + "titles": [] + }, + { + "text": "—", + "x": 263.015625, + "width": 37.953125, + "titles": [ + "No image update check reported" + ] + } + ] + }, + "keyboard": { + "width": 292, + "scrollWidth": 292, + "left": 0, + "overflowX": "clip", + "tabIndex": -1, + "cells": [ + { + "text": "loki", + "x": 9, + "width": 116.796875, + "titles": [ + "Expand details for loki", + "Healthy", + "loki" + ] + }, + { + "text": "running", + "x": 125.796875, + "width": 43.796875, + "titles": [ + "running" + ] + }, + { + "text": "12%", + "x": 169.59375, + "width": 43.796875, + "titles": [] + }, + { + "text": "74%", + "x": 213.390625, + "width": 49.625, + "titles": [] + }, + { + "text": "—", + "x": 263.015625, + "width": 37.953125, + "titles": [ + "No image update check reported" + ] + } + ] + }, + "accessibleRow": "- row \"Expand details for notifications-worker notifications-worker running 12% 48% —\":\n - cell \"Expand details for notifications-worker notifications-worker\":\n - button \"Expand details for notifications-worker\"\n - text: notifications-worker\n - cell \"running\"\n - cell \"12%\"\n - cell \"48%\"\n - cell \"—\"", + "expanded": "true", + "detailText": "notifications-worker\nContainer\nDocker / Podman\nOverview\nHistory\nManage\nRUNTIME CONTEXT\nObserved state\t\nRunning\n\nUptime\t\n6d 6h\n\nLast seen\t\n15s ago\n\nCONTAINER\nImage\t\nghcr.io/pulse-demo/notifications-worker:2026.04\n\nRestarts\t\n1\n\nCreated\t\n18 days ago\n\nStarted\t\n6 days ago\n\nCompose project\t\norion-2\n\nCompose service\t\npostgres\n\nLabels\t\ncom.docker.compose.project: orion-2\ncom.docker.compose.service: postgres\ncom.pulse.demo: true\ncom.pulse.role.production: true\ncom.pulse.role.queue: true\ncom.pulse.role.worker: true\n\nIDENTITY\nHostname\t\nnotifications-worker\n\nPrimary ID\t\napp-container:orion-2-26f93461b215\n\nParent\t\nagent-9dbedd74104d1527\n\nDiscovery\t\napp-container:notifications-worker\n\nMetrics Target\t\napp-container:orion-2-26f93461b215\n\nAliases\t\napp-container:orion-2-26f93461b215\norion-2-26f93461b215\norion-2-mock\napp-platform-01\n+4\nRelationship map\n\n1 canonical relationship\n\nCANONICAL RELATIONSHIPS\nnotifications-worker\n→\ndocker-network-0ab79ab291cd98fa\nAttached To\n100% confidence · Docker Adapter · last seen just now" +} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/05-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/05-environment.json new file mode 100644 index 000000000..99172d691 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/05-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":1440,"height":900,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":true,"tabs":[{"label":"Proxmox","text":"Proxmox","icons":1,"visible":true},{"label":"Docker","text":"Docker","icons":1,"visible":true},{"label":"Kubernetes","text":"Kubernetes","icons":1,"visible":true},{"label":"TrueNAS","text":"TrueNAS","icons":1,"visible":true},{"label":"vSphere","text":"vSphere","icons":1,"visible":true},{"label":"Machines","text":"Machines","icons":1,"visible":true},{"label":"Alerts","text":"Alerts","icons":1,"visible":true},{"label":"Patrol","text":"Patrol","icons":1,"visible":true},{"label":"Actions","text":"Actions","icons":1,"visible":true},{"label":"Settings","text":"Settings","icons":1,"visible":true}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/06-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/06-environment.json new file mode 100644 index 000000000..19c839eb4 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/06-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":1100,"height":900,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/07-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/07-environment.json new file mode 100644 index 000000000..c923f3999 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/07-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":390,"height":844,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/08-environment.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/08-environment.json new file mode 100644 index 000000000..dd138da42 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/08-environment.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":320,"height":844,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-changed-incident.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-changed-incident.png new file mode 100644 index 000000000..af72e800e Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-changed-incident.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-cleared-incidents.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-cleared-incidents.png new file mode 100644 index 000000000..d6c520212 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-cleared-incidents.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-empty-active-response.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-empty-active-response.json new file mode 100644 index 000000000..67a272436 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/09-empty-active-response.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":1440,"target":{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-5","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","resourceName":"backend-openvpn","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","value":100,"threshold":50,"startTime":"2026-09-05T22:41:34.197151243+01:00","lastSeen":"2026-09-05T22:43:23.333177338+01:00","acknowledged":false,"metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-5-service-gap","completedTasks":1,"desiredTasks":1,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":1,"mode":"global","percentMissing":100,"resourceType":"docker-service","runningTasks":0,"serviceId":"svc-backend-5","serviceName":"backend-openvpn","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-5","state":"open","severity":"critical","firstObservedAt":"2026-09-05T22:41:34.197151243+01:00","lastObservedAt":"2026-09-05T22:43:23.333177338+01:00","stateChangedAt":"2026-09-05T22:41:34.197151243+01:00","evidenceIds":["evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65"],"causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_5b14e593678afc3451d2d8aa723acc2c","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"observing","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},"transitions":[{"id":"transition_5b14e593678afc3451d2d8aa723acc2c","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"observing","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_3173ff805ea0f70df7e2d7b9a74fde65"]}],"evidence":[{"id":"evidence_b036708e91550bd1ce1db6d89664cb39","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:23.333177338+01:00","ingestedAt":"2026-09-05T22:43:23.333181775+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"f409601886fb7c30389b363ed06d93ae6f94bf0d054d8a05e7524ec6abaa1416"}},{"id":"evidence_de24759570440798eed0deb991980d1a","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:12.421010915+01:00","ingestedAt":"2026-09-05T22:43:12.421014556+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"822254ef5934612297c28c13fcd7d02b7d40f5862afed86043a3ea9f1630f37e"}},{"id":"evidence_718e6ae111d9635704d3a4f5e052bd37","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:02.923398013+01:00","ingestedAt":"2026-09-05T22:43:02.923401625+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"ef54111c5dafcf1eb44cceca1d6ea2586ec02dec6b7a267f2a16bae84ebf8f43"}},{"id":"evidence_9fceeaeb5e1d31dda41593adfea49ff4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:53.831738484+01:00","ingestedAt":"2026-09-05T22:42:53.831742041+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"0d90d93ea8f717a75a5d994b858781e9c3e49d1a9165b964ae928b9696fe9b8a"}},{"id":"evidence_80f27bbd84d7f17055a6d7fe2b531e9e","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:43.106212419+01:00","ingestedAt":"2026-09-05T22:42:43.106216109+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"b8ca83a28388d79058d6faadc3ee3437c2741f75a8b03751acee24b40245b745"}},{"id":"evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:33.038148736+01:00","ingestedAt":"2026-09-05T22:42:33.038152096+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"d2bb9c044d4510254f0fdb8110c7482c6f1a3c93e777457e9723a36e6e90448a"}},{"id":"evidence_d5aee13a350f427770577c9a765b16f1","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:23.28971349+01:00","ingestedAt":"2026-09-05T22:42:23.289718688+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"8a95bc5f22400ee7d7c0fac6321e791bf88e7131573cd65e8c65cd700472250b"}},{"id":"evidence_2992601e622e5491e8b970a646c3a526","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:12.407817999+01:00","ingestedAt":"2026-09-05T22:42:12.407821327+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"6cacd519483a28e3ab91726fe7c215ce0b6d814695967e8cb4e73b7ddfd33267"}},{"id":"evidence_5e2bece2756a483093e1533158075633","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:04.130724464+01:00","ingestedAt":"2026-09-05T22:42:04.13072807+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"4f5297b9389a2373ccaf719cbfef65838cf29821de64bce5b1769a613b61c700"}},{"id":"evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:53.146210156+01:00","ingestedAt":"2026-09-05T22:41:53.146213929+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"1c39dace49df8fa68ea005af1dbef372993a66bdcb6a16b8232611c19999af2d"}},{"id":"evidence_45535b8e2b278a9cc538a92775bd2107","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:45.356965029+01:00","ingestedAt":"2026-09-05T22:41:45.356970149+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"3a5ff0bdcbc1219a84dc34a6b47040e2ee796f28efb62fb8e237bb997efd52d6"}},{"id":"evidence_3173ff805ea0f70df7e2d7b9a74fde65","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:34.197151243+01:00","ingestedAt":"2026-09-05T22:41:34.19716108+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"35bad74f40ba802ac7930ef9a101b2e8b7b49272ea9b915f07917e34a297ada0"}}]},"acknowledgedSnapshot":[{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-5","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","resourceName":"backend-openvpn","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","value":100,"threshold":50,"startTime":"2026-09-05T22:41:34.197151243+01:00","lastSeen":"2026-09-05T22:43:23.333177338+01:00","acknowledged":true,"ackTime":"2026-09-05T22:43:29.153821982+01:00","ackUser":"admin","metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-5-service-gap","completedTasks":1,"desiredTasks":1,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":1,"mode":"global","percentMissing":100,"resourceType":"docker-service","runningTasks":0,"serviceId":"svc-backend-5","serviceName":"backend-openvpn","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-5","state":"acknowledged","severity":"critical","firstObservedAt":"2026-09-05T22:41:34.197151243+01:00","lastObservedAt":"2026-09-05T22:43:23.333177338+01:00","stateChangedAt":"2026-09-05T22:43:29.153821982+01:00","acknowledgement":{"at":"2026-09-05T22:43:29.153821982+01:00","by":"admin"},"evidenceIds":["evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65"],"causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_92d2e9178212ad6bb5a54a89f8336b38","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:29.153821982+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},"transitions":[{"id":"transition_5b14e593678afc3451d2d8aa723acc2c","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"observing","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},{"id":"transition_92d2e9178212ad6bb5a54a89f8336b38","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:29.153821982+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65"]}],"evidence":[{"id":"evidence_b036708e91550bd1ce1db6d89664cb39","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:23.333177338+01:00","ingestedAt":"2026-09-05T22:43:23.333181775+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"f409601886fb7c30389b363ed06d93ae6f94bf0d054d8a05e7524ec6abaa1416"}},{"id":"evidence_de24759570440798eed0deb991980d1a","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:12.421010915+01:00","ingestedAt":"2026-09-05T22:43:12.421014556+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"822254ef5934612297c28c13fcd7d02b7d40f5862afed86043a3ea9f1630f37e"}},{"id":"evidence_718e6ae111d9635704d3a4f5e052bd37","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:02.923398013+01:00","ingestedAt":"2026-09-05T22:43:02.923401625+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"ef54111c5dafcf1eb44cceca1d6ea2586ec02dec6b7a267f2a16bae84ebf8f43"}},{"id":"evidence_9fceeaeb5e1d31dda41593adfea49ff4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:53.831738484+01:00","ingestedAt":"2026-09-05T22:42:53.831742041+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"0d90d93ea8f717a75a5d994b858781e9c3e49d1a9165b964ae928b9696fe9b8a"}},{"id":"evidence_80f27bbd84d7f17055a6d7fe2b531e9e","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:43.106212419+01:00","ingestedAt":"2026-09-05T22:42:43.106216109+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"b8ca83a28388d79058d6faadc3ee3437c2741f75a8b03751acee24b40245b745"}},{"id":"evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:33.038148736+01:00","ingestedAt":"2026-09-05T22:42:33.038152096+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"d2bb9c044d4510254f0fdb8110c7482c6f1a3c93e777457e9723a36e6e90448a"}},{"id":"evidence_d5aee13a350f427770577c9a765b16f1","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:23.28971349+01:00","ingestedAt":"2026-09-05T22:42:23.289718688+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"8a95bc5f22400ee7d7c0fac6321e791bf88e7131573cd65e8c65cd700472250b"}},{"id":"evidence_2992601e622e5491e8b970a646c3a526","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:12.407817999+01:00","ingestedAt":"2026-09-05T22:42:12.407821327+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"6cacd519483a28e3ab91726fe7c215ce0b6d814695967e8cb4e73b7ddfd33267"}},{"id":"evidence_5e2bece2756a483093e1533158075633","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:04.130724464+01:00","ingestedAt":"2026-09-05T22:42:04.13072807+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"4f5297b9389a2373ccaf719cbfef65838cf29821de64bce5b1769a613b61c700"}},{"id":"evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:53.146210156+01:00","ingestedAt":"2026-09-05T22:41:53.146213929+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"1c39dace49df8fa68ea005af1dbef372993a66bdcb6a16b8232611c19999af2d"}},{"id":"evidence_45535b8e2b278a9cc538a92775bd2107","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:45.356965029+01:00","ingestedAt":"2026-09-05T22:41:45.356970149+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"3a5ff0bdcbc1219a84dc34a6b47040e2ee796f28efb62fb8e237bb997efd52d6"}},{"id":"evidence_3173ff805ea0f70df7e2d7b9a74fde65","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:34.197151243+01:00","ingestedAt":"2026-09-05T22:41:34.19716108+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"35bad74f40ba802ac7930ef9a101b2e8b7b49272ea9b915f07917e34a297ada0"}}]}],"emptySnapshot":[],"documentIdentity":1788644603995.2} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-changed-incident.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-changed-incident.png new file mode 100644 index 000000000..89b607bd9 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-changed-incident.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-cleared-incidents.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-cleared-incidents.png new file mode 100644 index 000000000..c4e00ec91 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-cleared-incidents.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-empty-active-response.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-empty-active-response.json new file mode 100644 index 000000000..558b6c03c --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/10-empty-active-response.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":320,"target":{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-5","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","resourceName":"backend-openvpn","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","value":100,"threshold":50,"startTime":"2026-09-05T22:41:34.197151243+01:00","lastSeen":"2026-09-05T22:43:44.353363072+01:00","acknowledged":true,"ackTime":"2026-09-05T22:43:29.153821982+01:00","ackUser":"admin","metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-5-service-gap","completedTasks":1,"desiredTasks":1,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":1,"mode":"global","percentMissing":100,"resourceType":"docker-service","runningTasks":0,"serviceId":"svc-backend-5","serviceName":"backend-openvpn","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-5","state":"acknowledged","severity":"critical","firstObservedAt":"2026-09-05T22:41:34.197151243+01:00","lastObservedAt":"2026-09-05T22:43:44.353363072+01:00","stateChangedAt":"2026-09-05T22:43:44.355042617+01:00","acknowledgement":{"at":"2026-09-05T22:43:29.153821982+01:00","by":"admin"},"evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"],"causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_a84e40d06425ea137cf28264c8b80466","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:44.355042617+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]},"transitions":[{"id":"transition_5b14e593678afc3451d2d8aa723acc2c","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"observing","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},{"id":"transition_92d2e9178212ad6bb5a54a89f8336b38","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:29.153821982+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},{"id":"transition_dcc9e25331b81b154d04955812e51fe9","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"acknowledged","to":"resolved","at":"2026-09-05T22:43:34.055389984+01:00","cause":"recovery_evidence","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_8c5fd67c981dd7a76762dfe38d0f9b59"]},{"id":"transition_6d53a284fa8d79413b7e6708d94d8106","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"resolved","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]},{"id":"transition_a84e40d06425ea137cf28264c8b80466","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:44.355042617+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]}],"evidence":[{"id":"evidence_1e50a5cbd3ae8626275bbb0c977778e7","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:33.019791626+01:00","ingestedAt":"2026-09-05T22:43:33.019798426+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"119c1d81ef982e77fae3bb92fe3dfd07bed0b657eda2c8b1e099f28d330820e9"}},{"id":"evidence_b036708e91550bd1ce1db6d89664cb39","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:23.333177338+01:00","ingestedAt":"2026-09-05T22:43:23.333181775+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"f409601886fb7c30389b363ed06d93ae6f94bf0d054d8a05e7524ec6abaa1416"}},{"id":"evidence_de24759570440798eed0deb991980d1a","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:12.421010915+01:00","ingestedAt":"2026-09-05T22:43:12.421014556+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"822254ef5934612297c28c13fcd7d02b7d40f5862afed86043a3ea9f1630f37e"}},{"id":"evidence_718e6ae111d9635704d3a4f5e052bd37","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:02.923398013+01:00","ingestedAt":"2026-09-05T22:43:02.923401625+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"ef54111c5dafcf1eb44cceca1d6ea2586ec02dec6b7a267f2a16bae84ebf8f43"}},{"id":"evidence_9fceeaeb5e1d31dda41593adfea49ff4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:53.831738484+01:00","ingestedAt":"2026-09-05T22:42:53.831742041+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"0d90d93ea8f717a75a5d994b858781e9c3e49d1a9165b964ae928b9696fe9b8a"}},{"id":"evidence_80f27bbd84d7f17055a6d7fe2b531e9e","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:43.106212419+01:00","ingestedAt":"2026-09-05T22:42:43.106216109+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"b8ca83a28388d79058d6faadc3ee3437c2741f75a8b03751acee24b40245b745"}},{"id":"evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:33.038148736+01:00","ingestedAt":"2026-09-05T22:42:33.038152096+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"d2bb9c044d4510254f0fdb8110c7482c6f1a3c93e777457e9723a36e6e90448a"}},{"id":"evidence_d5aee13a350f427770577c9a765b16f1","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:23.28971349+01:00","ingestedAt":"2026-09-05T22:42:23.289718688+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"8a95bc5f22400ee7d7c0fac6321e791bf88e7131573cd65e8c65cd700472250b"}},{"id":"evidence_2992601e622e5491e8b970a646c3a526","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:12.407817999+01:00","ingestedAt":"2026-09-05T22:42:12.407821327+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"6cacd519483a28e3ab91726fe7c215ce0b6d814695967e8cb4e73b7ddfd33267"}},{"id":"evidence_5e2bece2756a483093e1533158075633","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:04.130724464+01:00","ingestedAt":"2026-09-05T22:42:04.13072807+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"4f5297b9389a2373ccaf719cbfef65838cf29821de64bce5b1769a613b61c700"}},{"id":"evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:53.146210156+01:00","ingestedAt":"2026-09-05T22:41:53.146213929+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"1c39dace49df8fa68ea005af1dbef372993a66bdcb6a16b8232611c19999af2d"}},{"id":"evidence_45535b8e2b278a9cc538a92775bd2107","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:45.356965029+01:00","ingestedAt":"2026-09-05T22:41:45.356970149+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"3a5ff0bdcbc1219a84dc34a6b47040e2ee796f28efb62fb8e237bb997efd52d6"}},{"id":"evidence_3173ff805ea0f70df7e2d7b9a74fde65","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:34.197151243+01:00","ingestedAt":"2026-09-05T22:41:34.19716108+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"35bad74f40ba802ac7930ef9a101b2e8b7b49272ea9b915f07917e34a297ada0"}},{"id":"evidence_8c5fd67c981dd7a76762dfe38d0f9b59","source":{"provider":"pulse","collector":"legacy-alert-recovery-adapter","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:34.055389984+01:00","ingestedAt":"2026-09-05T22:43:34.055389984+01:00","completeness":"partial","confidence":"unknown","reason":{"code":"legacy_alert_recovery_projection"},"permissions":"unknown"},{"id":"evidence_ffa25ffcd4ba827d54ac49f5afbe96ee","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:44.353363072+01:00","ingestedAt":"2026-09-05T22:43:44.35337056+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"16e774cb2be5e9228fccb4aca94684d0d9f2cb925ef655d54f0cfc7b3022c8b2"}}]},"acknowledgedSnapshot":[{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-5","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","resourceName":"backend-openvpn","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","value":100,"threshold":50,"startTime":"2026-09-05T22:41:34.197151243+01:00","lastSeen":"2026-09-05T22:43:44.353363072+01:00","acknowledged":true,"ackTime":"2026-09-05T22:43:46.140456919+01:00","ackUser":"admin","metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-5-service-gap","completedTasks":1,"desiredTasks":1,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":1,"mode":"global","percentMissing":100,"resourceType":"docker-service","runningTasks":0,"serviceId":"svc-backend-5","serviceName":"backend-openvpn","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-5-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-5","state":"acknowledged","severity":"critical","firstObservedAt":"2026-09-05T22:41:34.197151243+01:00","lastObservedAt":"2026-09-05T22:43:44.353363072+01:00","stateChangedAt":"2026-09-05T22:43:46.140456919+01:00","acknowledgement":{"at":"2026-09-05T22:43:46.140456919+01:00","by":"admin"},"evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"],"causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-openvpn' is running 0 of 1 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_5618a33e9206876bd3200ef9b01a49ad","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:46.140456919+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]},"transitions":[{"id":"transition_5b14e593678afc3451d2d8aa723acc2c","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"observing","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},{"id":"transition_92d2e9178212ad6bb5a54a89f8336b38","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:29.153821982+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65"]},{"id":"transition_dcc9e25331b81b154d04955812e51fe9","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"acknowledged","to":"resolved","at":"2026-09-05T22:43:34.055389984+01:00","cause":"recovery_evidence","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_8c5fd67c981dd7a76762dfe38d0f9b59"]},{"id":"transition_6d53a284fa8d79413b7e6708d94d8106","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"resolved","to":"open","at":"2026-09-05T22:41:34.197151243+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]},{"id":"transition_a84e40d06425ea137cf28264c8b80466","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:44.355042617+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]},{"id":"transition_bef3f2e6aa74f24b0303f36e6e990b51","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"acknowledged","to":"open","at":"2026-09-05T22:43:44.581847651+01:00","cause":"unacknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]},{"id":"transition_5618a33e9206876bd3200ef9b01a49ad","operationalRecordId":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:43:46.140456919+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-5::docker:orion-2-mock/service/svc-backend-5-service-gap","evidenceIds":["evidence_1e50a5cbd3ae8626275bbb0c977778e7","evidence_b036708e91550bd1ce1db6d89664cb39","evidence_de24759570440798eed0deb991980d1a","evidence_718e6ae111d9635704d3a4f5e052bd37","evidence_9fceeaeb5e1d31dda41593adfea49ff4","evidence_80f27bbd84d7f17055a6d7fe2b531e9e","evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","evidence_d5aee13a350f427770577c9a765b16f1","evidence_2992601e622e5491e8b970a646c3a526","evidence_5e2bece2756a483093e1533158075633","evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","evidence_45535b8e2b278a9cc538a92775bd2107","evidence_3173ff805ea0f70df7e2d7b9a74fde65","evidence_8c5fd67c981dd7a76762dfe38d0f9b59","evidence_ffa25ffcd4ba827d54ac49f5afbe96ee"]}],"evidence":[{"id":"evidence_1e50a5cbd3ae8626275bbb0c977778e7","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:33.019791626+01:00","ingestedAt":"2026-09-05T22:43:33.019798426+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"119c1d81ef982e77fae3bb92fe3dfd07bed0b657eda2c8b1e099f28d330820e9"}},{"id":"evidence_b036708e91550bd1ce1db6d89664cb39","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:23.333177338+01:00","ingestedAt":"2026-09-05T22:43:23.333181775+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"f409601886fb7c30389b363ed06d93ae6f94bf0d054d8a05e7524ec6abaa1416"}},{"id":"evidence_de24759570440798eed0deb991980d1a","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:12.421010915+01:00","ingestedAt":"2026-09-05T22:43:12.421014556+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"822254ef5934612297c28c13fcd7d02b7d40f5862afed86043a3ea9f1630f37e"}},{"id":"evidence_718e6ae111d9635704d3a4f5e052bd37","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:02.923398013+01:00","ingestedAt":"2026-09-05T22:43:02.923401625+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"ef54111c5dafcf1eb44cceca1d6ea2586ec02dec6b7a267f2a16bae84ebf8f43"}},{"id":"evidence_9fceeaeb5e1d31dda41593adfea49ff4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:53.831738484+01:00","ingestedAt":"2026-09-05T22:42:53.831742041+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"0d90d93ea8f717a75a5d994b858781e9c3e49d1a9165b964ae928b9696fe9b8a"}},{"id":"evidence_80f27bbd84d7f17055a6d7fe2b531e9e","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:43.106212419+01:00","ingestedAt":"2026-09-05T22:42:43.106216109+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"b8ca83a28388d79058d6faadc3ee3437c2741f75a8b03751acee24b40245b745"}},{"id":"evidence_0b78a5fc7f47b74b87bbf42ce6bc8500","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:33.038148736+01:00","ingestedAt":"2026-09-05T22:42:33.038152096+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"d2bb9c044d4510254f0fdb8110c7482c6f1a3c93e777457e9723a36e6e90448a"}},{"id":"evidence_d5aee13a350f427770577c9a765b16f1","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:23.28971349+01:00","ingestedAt":"2026-09-05T22:42:23.289718688+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"8a95bc5f22400ee7d7c0fac6321e791bf88e7131573cd65e8c65cd700472250b"}},{"id":"evidence_2992601e622e5491e8b970a646c3a526","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:12.407817999+01:00","ingestedAt":"2026-09-05T22:42:12.407821327+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"6cacd519483a28e3ab91726fe7c215ce0b6d814695967e8cb4e73b7ddfd33267"}},{"id":"evidence_5e2bece2756a483093e1533158075633","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:42:04.130724464+01:00","ingestedAt":"2026-09-05T22:42:04.13072807+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"4f5297b9389a2373ccaf719cbfef65838cf29821de64bce5b1769a613b61c700"}},{"id":"evidence_ada61722c4ffc5e4d11ba6e8ccf50bbd","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:53.146210156+01:00","ingestedAt":"2026-09-05T22:41:53.146213929+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"1c39dace49df8fa68ea005af1dbef372993a66bdcb6a16b8232611c19999af2d"}},{"id":"evidence_45535b8e2b278a9cc538a92775bd2107","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:45.356965029+01:00","ingestedAt":"2026-09-05T22:41:45.356970149+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"3a5ff0bdcbc1219a84dc34a6b47040e2ee796f28efb62fb8e237bb997efd52d6"}},{"id":"evidence_3173ff805ea0f70df7e2d7b9a74fde65","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:41:34.197151243+01:00","ingestedAt":"2026-09-05T22:41:34.19716108+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"35bad74f40ba802ac7930ef9a101b2e8b7b49272ea9b915f07917e34a297ada0"}},{"id":"evidence_8c5fd67c981dd7a76762dfe38d0f9b59","source":{"provider":"pulse","collector":"legacy-alert-recovery-adapter","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:34.055389984+01:00","ingestedAt":"2026-09-05T22:43:34.055389984+01:00","completeness":"partial","confidence":"unknown","reason":{"code":"legacy_alert_recovery_projection"},"permissions":"unknown"},{"id":"evidence_ffa25ffcd4ba827d54ac49f5afbe96ee","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-5"},"observedAt":"2026-09-05T22:43:44.353363072+01:00","ingestedAt":"2026-09-05T22:43:44.35337056+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"16e774cb2be5e9228fccb4aca94684d0d9f2cb925ef655d54f0cfc7b3022c8b2"}}]}],"emptySnapshot":[],"documentIdentity":1788644617857.4} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/junit.xml new file mode 100644 index 000000000..e95f689be --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/junit.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/results.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/results.json new file mode 100644 index 000000000..2baf0086d --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/final/results.json @@ -0,0 +1,72 @@ +[ + { + "title": "populated navigation survives socket loss at 1440px (admission failure: false)", + "outcome": "expected", + "duration": 14988, + "startTime": "2026-09-05T21:41:31.466Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 1100px (admission failure: false)", + "outcome": "expected", + "duration": 8345, + "startTime": "2026-09-05T21:41:46.567Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 390px (admission failure: false)", + "outcome": "expected", + "duration": 20390, + "startTime": "2026-09-05T21:41:54.929Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 320px (admission failure: false)", + "outcome": "expected", + "duration": 19410, + "startTime": "2026-09-05T21:42:15.330Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 1440px (admission failure: true)", + "outcome": "expected", + "duration": 8215, + "startTime": "2026-09-05T21:42:34.752Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 1100px (admission failure: true)", + "outcome": "expected", + "duration": 7265, + "startTime": "2026-09-05T21:42:42.978Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 390px (admission failure: true)", + "outcome": "expected", + "duration": 17578, + "startTime": "2026-09-05T21:42:50.252Z", + "retry": 0 + }, + { + "title": "populated navigation survives socket loss at 320px (admission failure: true)", + "outcome": "expected", + "duration": 15298, + "startTime": "2026-09-05T21:43:07.838Z", + "retry": 0 + }, + { + "title": "changed backend incidents recover without reload at 1440px", + "outcome": "expected", + "duration": 13297, + "startTime": "2026-09-05T21:43:23.147Z", + "retry": 0 + }, + { + "title": "changed backend incidents recover without reload at 320px", + "outcome": "expected", + "duration": 15969, + "startTime": "2026-09-05T21:43:36.453Z", + "retry": 0 + } +] diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-changed-incident.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-changed-incident.png new file mode 100644 index 000000000..36f77e374 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-changed-incident.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-cleared-incidents.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-cleared-incidents.png new file mode 100644 index 000000000..48ca5324a Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-cleared-incidents.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-empty-active-response.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-empty-active-response.json new file mode 100644 index 000000000..ba90566f1 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/1440-empty-active-response.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":1440,"target":{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-9","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","resourceName":"backend-gitlab","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","value":75,"threshold":50,"startTime":"2026-09-05T22:40:19.48795396+01:00","lastSeen":"2026-09-05T22:40:19.48795396+01:00","acknowledged":false,"metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-9-service-gap","completedTasks":0,"desiredTasks":4,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":3,"mode":"global","percentMissing":75,"resourceType":"docker-service","runningTasks":1,"serviceId":"svc-backend-9","serviceName":"backend-gitlab","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-9","state":"open","severity":"critical","firstObservedAt":"2026-09-05T22:40:19.48795396+01:00","lastObservedAt":"2026-09-05T22:40:19.48795396+01:00","stateChangedAt":"2026-09-05T22:40:19.48795396+01:00","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"],"causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_8d96a1794faa083fa42bb46ff51f7613","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"observing","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},"transitions":[{"id":"transition_8d96a1794faa083fa42bb46ff51f7613","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"observing","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]}],"evidence":[{"id":"evidence_cfda55db482d46a131d98e537f10f9f4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:19.48795396+01:00","ingestedAt":"2026-09-05T22:40:19.487973216+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"5ad5520f30b274dfafde69e1f7908a1605f2b2b237208df35e88b886ea0cea86"}}]},"acknowledgedSnapshot":[{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-9","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","resourceName":"backend-gitlab","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","value":75,"threshold":50,"startTime":"2026-09-05T22:40:19.48795396+01:00","lastSeen":"2026-09-05T22:40:19.48795396+01:00","acknowledged":true,"ackTime":"2026-09-05T22:40:24.610694532+01:00","ackUser":"admin","metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-9-service-gap","completedTasks":0,"desiredTasks":4,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":3,"mode":"global","percentMissing":75,"resourceType":"docker-service","runningTasks":1,"serviceId":"svc-backend-9","serviceName":"backend-gitlab","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-9","state":"acknowledged","severity":"critical","firstObservedAt":"2026-09-05T22:40:19.48795396+01:00","lastObservedAt":"2026-09-05T22:40:19.48795396+01:00","stateChangedAt":"2026-09-05T22:40:24.610694532+01:00","acknowledgement":{"at":"2026-09-05T22:40:24.610694532+01:00","by":"admin"},"evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"],"causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_cd90d8f8a1388fca82a51a61fa8bf448","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:24.610694532+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},"transitions":[{"id":"transition_8d96a1794faa083fa42bb46ff51f7613","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"observing","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},{"id":"transition_cd90d8f8a1388fca82a51a61fa8bf448","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:24.610694532+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]}],"evidence":[{"id":"evidence_cfda55db482d46a131d98e537f10f9f4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:19.48795396+01:00","ingestedAt":"2026-09-05T22:40:19.487973216+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"5ad5520f30b274dfafde69e1f7908a1605f2b2b237208df35e88b886ea0cea86"}}]}],"emptySnapshot":[],"documentIdentity":1788644419667.2} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-changed-incident.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-changed-incident.png new file mode 100644 index 000000000..011ab6172 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-changed-incident.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-cleared-incidents.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-cleared-incidents.png new file mode 100644 index 000000000..49374b715 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-cleared-incidents.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-empty-active-response.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-empty-active-response.json new file mode 100644 index 000000000..4a4efafcf --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/320-empty-active-response.json @@ -0,0 +1 @@ +{"browser":"141.0.7390.37","width":320,"target":{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-9","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","resourceName":"backend-gitlab","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","value":75,"threshold":50,"startTime":"2026-09-05T22:40:19.48795396+01:00","lastSeen":"2026-09-05T22:40:39.195907137+01:00","acknowledged":true,"ackTime":"2026-09-05T22:40:24.610694532+01:00","ackUser":"admin","metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-9-service-gap","completedTasks":0,"desiredTasks":4,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":3,"mode":"global","percentMissing":75,"resourceType":"docker-service","runningTasks":1,"serviceId":"svc-backend-9","serviceName":"backend-gitlab","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-9","state":"acknowledged","severity":"critical","firstObservedAt":"2026-09-05T22:40:19.48795396+01:00","lastObservedAt":"2026-09-05T22:40:39.195907137+01:00","stateChangedAt":"2026-09-05T22:40:39.196942405+01:00","acknowledgement":{"at":"2026-09-05T22:40:24.610694532+01:00","by":"admin"},"evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"],"causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_4de6db4ee25b0b3733760c7cfeaf2839","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:39.196942405+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]},"transitions":[{"id":"transition_8d96a1794faa083fa42bb46ff51f7613","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"observing","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},{"id":"transition_cd90d8f8a1388fca82a51a61fa8bf448","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:24.610694532+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},{"id":"transition_b558097ee371c4586fd25e1be4cbced3","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"acknowledged","to":"resolved","at":"2026-09-05T22:40:29.11704271+01:00","cause":"recovery_evidence","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_c99188c0b070873fc08334719829440a"]},{"id":"transition_ef706414d1d27d3518ddb44c2af59627","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"resolved","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]},{"id":"transition_4de6db4ee25b0b3733760c7cfeaf2839","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:39.196942405+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]}],"evidence":[{"id":"evidence_cfda55db482d46a131d98e537f10f9f4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:19.48795396+01:00","ingestedAt":"2026-09-05T22:40:19.487973216+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"5ad5520f30b274dfafde69e1f7908a1605f2b2b237208df35e88b886ea0cea86"}},{"id":"evidence_c99188c0b070873fc08334719829440a","source":{"provider":"pulse","collector":"legacy-alert-recovery-adapter","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:29.11704271+01:00","ingestedAt":"2026-09-05T22:40:29.11704271+01:00","completeness":"partial","confidence":"unknown","reason":{"code":"legacy_alert_recovery_projection"},"permissions":"unknown"},{"id":"evidence_5b477f2f37a7057e5d5d06f47e8a0ff4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:39.195907137+01:00","ingestedAt":"2026-09-05T22:40:39.195912881+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"a703581dc9101028af4744d7dc243531f82ddbb86962ce3d07dae4dc1125e267"}}]},"acknowledgedSnapshot":[{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","type":"docker-service-health","level":"critical","resourceId":"docker:orion-2-mock/service/svc-backend-9","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalKind":"service-gap","canonicalState":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","resourceName":"backend-gitlab","node":"app-platform-01","instance":"Docker:App Platform 01","message":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","value":75,"threshold":50,"startTime":"2026-09-05T22:40:19.48795396+01:00","lastSeen":"2026-09-05T22:40:39.195907137+01:00","acknowledged":true,"ackTime":"2026-09-05T22:40:41.094991617+01:00","ackUser":"admin","metadata":{"canonicalAlertKind":"service-gap","canonicalSpecID":"docker:orion-2-mock/service/svc-backend-9-service-gap","completedTasks":0,"desiredTasks":4,"hostHostname":"app-platform-01","hostId":"orion-2-mock","hostName":"App Platform 01","missingTasks":3,"mode":"global","percentMissing":75,"resourceType":"docker-service","runningTasks":1,"serviceId":"svc-backend-9","serviceName":"backend-gitlab","stack":"","tags":["com.docker.stack.namespace:backend"],"updateMessage":"Service replicas below desired threshold","updateState":"rollback_started"},"operationalRecord":{"id":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","canonicalSpecId":"docker:orion-2-mock/service/svc-backend-9-service-gap","subjectResourceId":"docker:orion-2-mock/service/svc-backend-9","state":"acknowledged","severity":"critical","firstObservedAt":"2026-09-05T22:40:19.48795396+01:00","lastObservedAt":"2026-09-05T22:40:39.195907137+01:00","stateChangedAt":"2026-09-05T22:40:41.094991617+01:00","acknowledgement":{"at":"2026-09-05T22:40:41.094991617+01:00","by":"admin"},"evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"],"causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","relatedResourceIds":null,"impactSummary":"Docker service 'backend-gitlab' is running 1 of 4 desired tasks","recommendedNextStep":"Open the affected resource and verify its current state before making changes."},"latestTransition":{"id":"transition_f0d2fe35dc941ecff190a4a930e8bb90","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:41.094991617+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]},"transitions":[{"id":"transition_8d96a1794faa083fa42bb46ff51f7613","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"observing","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},{"id":"transition_cd90d8f8a1388fca82a51a61fa8bf448","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:24.610694532+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4"]},{"id":"transition_b558097ee371c4586fd25e1be4cbced3","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"acknowledged","to":"resolved","at":"2026-09-05T22:40:29.11704271+01:00","cause":"recovery_evidence","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_c99188c0b070873fc08334719829440a"]},{"id":"transition_ef706414d1d27d3518ddb44c2af59627","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"resolved","to":"open","at":"2026-09-05T22:40:19.48795396+01:00","cause":"detector_decision","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]},{"id":"transition_4de6db4ee25b0b3733760c7cfeaf2839","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:39.196942405+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]},{"id":"transition_b6237a56686dd8afd13a773414509086","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"acknowledged","to":"open","at":"2026-09-05T22:40:39.99779271+01:00","cause":"unacknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]},{"id":"transition_f0d2fe35dc941ecff190a4a930e8bb90","operationalRecordId":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","from":"open","to":"acknowledged","at":"2026-09-05T22:40:41.094991617+01:00","cause":"acknowledgement","causeKey":"docker:orion-2-mock/service/svc-backend-9::docker:orion-2-mock/service/svc-backend-9-service-gap","evidenceIds":["evidence_cfda55db482d46a131d98e537f10f9f4","evidence_c99188c0b070873fc08334719829440a","evidence_5b477f2f37a7057e5d5d06f47e8a0ff4"]}],"evidence":[{"id":"evidence_cfda55db482d46a131d98e537f10f9f4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:19.48795396+01:00","ingestedAt":"2026-09-05T22:40:19.487973216+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"5ad5520f30b274dfafde69e1f7908a1605f2b2b237208df35e88b886ea0cea86"}},{"id":"evidence_c99188c0b070873fc08334719829440a","source":{"provider":"pulse","collector":"legacy-alert-recovery-adapter","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:29.11704271+01:00","ingestedAt":"2026-09-05T22:40:29.11704271+01:00","completeness":"partial","confidence":"unknown","reason":{"code":"legacy_alert_recovery_projection"},"permissions":"unknown"},{"id":"evidence_5b477f2f37a7057e5d5d06f47e8a0ff4","source":{"provider":"pulse","collector":"canonical-alert:service-gap","instance":"Docker:App Platform 01"},"subject":{"resourceId":"docker:orion-2-mock/service/svc-backend-9"},"observedAt":"2026-09-05T22:40:39.195907137+01:00","ingestedAt":"2026-09-05T22:40:39.195912881+01:00","completeness":"complete","confidence":"confirmed","permissions":"sufficient","payloadRef":{"kind":"alert-evidence-digest","id":"a703581dc9101028af4744d7dc243531f82ddbb86962ce3d07dae4dc1125e267"}}]}],"emptySnapshot":[],"documentIdentity":1788644431772.6} \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/junit.xml new file mode 100644 index 000000000..67c789a0a --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/focused/junit.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/grouped-target.png b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/grouped-target.png new file mode 100644 index 000000000..3c3a4cf36 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/grouped-target.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/junit.xml new file mode 100644 index 000000000..0643f4bd8 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/junit.xml @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + a.id === target!.id)?.acknowledged).toBe(true); + > 267 | await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible(); + | ^ + 268 | await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' }); + 269 | + 270 | const configResponse = await apiRequest(page, '/api/alerts/config'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:267:84 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md +]]> + + + + + + + + a.id === target!.id)?.acknowledged).toBe(true); + > 267 | await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible(); + | ^ + 268 | await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' }); + 269 | + 270 | const configResponse = await apiRequest(page, '/api/alerts/config'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:267:84 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md +]]> + + + + + + + \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/junit.xml.gz b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/junit.xml.gz new file mode 100644 index 000000000..1e39d0348 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/initial/junit.xml.gz differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/named-service/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/named-service/junit.xml new file mode 100644 index 000000000..bfcec10ae --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/named-service/junit.xml @@ -0,0 +1,88 @@ + + + + + 240 | await expect.poll(async () => (await readActive()).some(alert => + | ^ + 241 | alert.resourceName === 'infra-docker')).toBe(true); + 242 | const initial = await readActive(); + 243 | const target = initial.find(alert => alert.resourceName === 'infra-docker'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md +]]> + + + + + + + + 240 | await expect.poll(async () => (await readActive()).some(alert => + | ^ + 241 | alert.resourceName === 'infra-docker')).toBe(true); + 242 | const initial = await readActive(); + 243 | const target = initial.find(alert => alert.resourceName === 'infra-docker'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md +]]> + + + + + + + \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/named-service/junit.xml.gz b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/named-service/junit.xml.gz new file mode 100644 index 000000000..637413f17 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/named-service/junit.xml.gz differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/receipt-normalisation.json b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/receipt-normalisation.json new file mode 100644 index 000000000..5ba732c97 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/receipt-normalisation.json @@ -0,0 +1,26 @@ +[ + { + "path": "cold-fixture/junit.xml", + "original_gzip": "cold-fixture/junit.xml.gz", + "original_sha256": "1f2e930cd0e768b8ebc50e3d8cf0b359f0dad8859961a952f0739201efc663bc", + "normalised_sha256": "d8add4a54e01a3c3989a222cc4b6d48e58f2c55be49ada4108849a576f27fdf8" + }, + { + "path": "initial/junit.xml", + "original_gzip": "initial/junit.xml.gz", + "original_sha256": "00a92c8c2100abe19663ea2a90809d78b468d920148a315db2b2b4b440b10220", + "normalised_sha256": "07e4a08eee694e38068ee6fd025e4c4f3752438c7629ac6c329c4df3fb5b73c4" + }, + { + "path": "named-service/junit.xml", + "original_gzip": "named-service/junit.xml.gz", + "original_sha256": "f325c144f617d6031b835d2d34b1e25d86c8181fb1fe73ffa8e043089e65a876", + "normalised_sha256": "99377de17b633459b3c5d1f30723d6ba73a43cc58dd5ec9cbec9ff53ef7bf5e1" + }, + { + "path": "standalone/junit.xml", + "original_gzip": "standalone/junit.xml.gz", + "original_sha256": "27e74939a4451199306ba701f8a73c823aa68cae29af23f620c00838f462ca85", + "normalised_sha256": "ad130acf7faecd348093162a0bec613d43c81ddb01a7e238db40c30638b3ddd5" + } +] diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/standalone/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/standalone/junit.xml new file mode 100644 index 000000000..7726f852c --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/standalone/junit.xml @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + a.id === target!.id)?.acknowledged).toBe(true); + > 271 | await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible(); + | ^ + 272 | await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' }); + 273 | + 274 | const configResponse = await apiRequest(page, '/api/alerts/config'); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:271:84 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md +]]> + + + + + + + + 240 | await expect.poll(async () => (await readActive()).some(alert => + | ^ + 241 | alert.resourceName === 'edge-proxy-01' && !alert.acknowledged)).toBe(true); + 242 | const initial = await readActive(); + 243 | const target = initial.find(alert => alert.resourceName === 'edge-proxy-01' && !alert.acknowledged); + at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5 + + attachment #1: screenshot (image/png) ────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png + ──────────────────────────────────────────────────────────────────────────────────────────────── + + attachment #2: video (video/webm) ────────────────────────────────────────────────────────────── + ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm + ──────────────────────────────────────────────────────────────────────────────────────────────── + + Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md +]]> + + + + + + + \ No newline at end of file diff --git a/tests/qualification/navigation-reconnect/2026-09-05-changed-data/standalone/junit.xml.gz b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/standalone/junit.xml.gz new file mode 100644 index 000000000..1f5dde9b6 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-changed-data/standalone/junit.xml.gz differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-integrated/320px-inventory.png b/tests/qualification/navigation-reconnect/2026-09-05-integrated/320px-inventory.png new file mode 100644 index 000000000..cdfcd8e70 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-integrated/320px-inventory.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-integrated/README.md b/tests/qualification/navigation-reconnect/2026-09-05-integrated/README.md new file mode 100644 index 000000000..654d15c32 --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-integrated/README.md @@ -0,0 +1,88 @@ +# Integrated navigation recovery check — 5 September 2026 + +## Result + +At source `c8931787adb7bc3152f060a829ebe2a1aa2a2b9c`, all eight existing +`96-navigation-socket-recovery.spec.ts` Chromium cases passed (132 seconds, +starting 20:43:55 UTC). No application or test changes were made for this run. +The backend was built locally with embedded production frontend assets and +synthetic inventory; it was stopped by the runner afterwards. + +Local executable SHA-256: +`a5aaf32fdba70744d1dfa4e0749f080dd8960dcd19d381ab6ec2fdc286d36b27`. +This identifies this source-built test executable, not a published artifact. +Locked dependencies were installed in root, frontend-modern and tests/integration; +the runner used Playwright 1.56.1 and the Chromium project. + +Widths 1440 and 1100 (height 900), and 390 and 320 (height 844), each passed +with and without injected admission HTTP failure. The checks exercise socket +closure with 1013, retained navigation/inventory, recovery without document +reload, and incident-control access at desktop/mobile widths. Narrow cases +also exercise platform switching and More navigation. Docker table keyboard +access and clipped-text assertions require the separate `PULSE_E2E_TABLE_ACCESS=1` +opt-in. The retained JUnit records eight successful cases but does not record +that flag, so it cannot independently establish those conditional assertions +ran. Admission-failure cases require an observed failed request. + +`junit.xml` is the unmodified runner result. Two inspected screenshots retain +representative desktop incident access during reconnect and 320px inventory. +The narrow screenshot alone is not proof of connection state; the test asserts +that state separately. Small update cells still wrap heavily at 320px, despite +the earlier reported no-clipped-text result. That conditional result is not +independently established by this retained JUnit. No readability redesign is implied. + +Additional focused checks: 53 tests passed across websocket-resilience and +websocket-unified; `TestAlertCharacterizationGetActiveAlertsExportsCanonicalIdentity` +passed with `-count=1` in internal/alerts. No full repository suite was run. + +## Repeat + +From repository root after installing locked dependencies: + +```sh +pulse-heavy-run -- env \ + PULSE_E2E_USE_LOCAL_BACKEND=1 PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL=1 \ + PULSE_MOCK_MODE=true PULSE_E2E_NAVIGATION_RECOVERY=1 \ + PULSE_E2E_TABLE_ACCESS=1 \ + PULSE_E2E_LOCAL_BACKEND_PORT=18765 \ + npm --prefix tests/integration test -- \ + tests/96-navigation-socket-recovery.spec.ts --project=chromium +``` + +## Decision and limits + +Fresh retrieval of the [community comparison thread](https://www.reddit.com/r/Proxmox/comments/1lblkk8/anyone_else_switch_to_pulse_from_netdata_or_any/) +on 5 September reinforced protecting quick-glance, low-overhead monitoring. +This is historical, anecdotal feedback, not a new independent demand count or +benchmark. The latest integrated source warranted checking the existing repairs +together rather than creating another surface or repeating a withdrawn repair. + +This result narrows the integration regression uncertainty. It does **not** +qualify acknowledgement persistence, off-host delivery, restart history, +malformed alert snapshots, a reporter installation, or a release candidate. +The REST recovery code still skips entries without usable IDs. The server's +normal read model exports typed alerts and canonical identity has focused test +coverage, but that is not proof every malformed-response path is impossible. +Do not count that previously withdrawn repair as delivered. Revisit it with +browser fault injection, genuine empty-response clearing, and the required +same-content performance/contracts qualification if pursued. + +Next useful evidence is exact-candidate recovery and installed alert receipt, +not another equivalent source-only navigation run without changed inputs. +No release readiness or backport eligibility judgment is made here. + +## Receipt audit — 5 September 2026 + +The repeat command above now explicitly enables table-access assertions. This +is a documentation correction, not a fresh browser run or a claim that the +original run omitted the flag. Retain the invocation flags and the +`narrow-table-access` attachment on the next qualification run. + +The recovery assertions in this spec establish reconnection, retained navigation +and unchanged document identity. They do not compare a changed incident or +resource value before and after the interruption. The managed-runtime recovery +spec likewise checks connection status and HTTP health, not changed incident +content. A future data-recovery qualification should change an identified +incident during disconnection and verify its rendered state after reconnect, +including a genuine empty active-alert response. Existing store-level snapshot +tests are narrower evidence; no missing-data browser regression is claimed here. diff --git a/tests/qualification/navigation-reconnect/2026-09-05-integrated/desktop-incidents.png b/tests/qualification/navigation-reconnect/2026-09-05-integrated/desktop-incidents.png new file mode 100644 index 000000000..e94acb3c5 Binary files /dev/null and b/tests/qualification/navigation-reconnect/2026-09-05-integrated/desktop-incidents.png differ diff --git a/tests/qualification/navigation-reconnect/2026-09-05-integrated/junit.xml b/tests/qualification/navigation-reconnect/2026-09-05-integrated/junit.xml new file mode 100644 index 000000000..5b835214f --- /dev/null +++ b/tests/qualification/navigation-reconnect/2026-09-05-integrated/junit.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file