Merge current main and verify diagnostic workflows
Integrate the latest alert, delivery and action-result changes with the Patrol evidence conversation. Replace the conflicted browser receipt with current source-bound qualification and fix shared warning-card wrapping exposed by the intermediate-width check. Real diagnostic and autonomous action outcome qualification stays open.
@@ -5,10 +5,12 @@ on:
|
||||
branches:
|
||||
- main
|
||||
- pulse/v6-release
|
||||
- 'release/v*'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- pulse/v6-release
|
||||
- 'release/v*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -191,6 +191,9 @@ jobs:
|
||||
- name: Run Helm Pages publication retry tests
|
||||
run: python3 scripts/release_control/helm_pages_retry_test.py
|
||||
|
||||
- name: Helm publication version binding
|
||||
run: python3 scripts/release_control/helm_publish_version_test.py
|
||||
|
||||
- name: Run status audit unit tests
|
||||
run: python3 scripts/release_control/status_audit_test.py
|
||||
|
||||
|
||||
@@ -275,6 +275,10 @@ jobs:
|
||||
set -euo pipefail
|
||||
public_repo="https://rcourtman.github.io/Pulse"
|
||||
public_ready=false
|
||||
public_work="$(mktemp -d)"
|
||||
trap 'rm -rf "${public_work}"' EXIT
|
||||
qualified_chart="dist/pulse-${VERSION}.tgz"
|
||||
test -f "${qualified_chart}"
|
||||
for attempt in $(seq 1 30); do
|
||||
public_index="$(mktemp)"
|
||||
if curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors \
|
||||
@@ -283,7 +287,17 @@ jobs:
|
||||
helm repo remove pulse-public >/dev/null 2>&1 || true
|
||||
helm repo add pulse-public "${public_repo}" --force-update
|
||||
helm repo update pulse-public
|
||||
if helm show chart pulse-public/pulse --version "${VERSION}" >/dev/null; then
|
||||
# Metadata readability is not an exact-artifact receipt. Pull via
|
||||
# the consumer index and compare with the OCI-qualified package,
|
||||
# not the OCI manifest digest (which hashes a different object).
|
||||
rm -f "${public_work}/pulse-${VERSION}.tgz"
|
||||
if helm pull pulse-public/pulse --version "${VERSION}" --destination "${public_work}" && \
|
||||
helm show chart "${public_work}/pulse-${VERSION}.tgz" >/dev/null; then
|
||||
if ! cmp -s "${qualified_chart}" "${public_work}/pulse-${VERSION}.tgz"; then
|
||||
echo "::error::Public Helm chart bytes differ from the qualified package; refusing a successful convergence receipt."
|
||||
rm -f "${public_index}"
|
||||
exit 1
|
||||
fi
|
||||
public_ready=true
|
||||
rm -f "${public_index}"
|
||||
break
|
||||
@@ -297,4 +311,4 @@ jobs:
|
||||
echo "::error::Public Helm repository did not expose chart ${VERSION}."
|
||||
exit 1
|
||||
fi
|
||||
echo "[OK] Public Helm repository serves pulse ${VERSION}."
|
||||
echo "[OK] Public Helm repository serves the exact qualified pulse ${VERSION} package."
|
||||
|
||||
@@ -22,7 +22,7 @@ on:
|
||||
required: true
|
||||
type: string
|
||||
app_version:
|
||||
description: "Application version to embed (defaults to chart version)."
|
||||
description: "Application version (must equal chart version; defaults to it)."
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
@@ -36,7 +36,7 @@ on:
|
||||
description: "Chart version (required when running manually, use format 4.24.0)"
|
||||
required: true
|
||||
app_version:
|
||||
description: "Application version to embed (defaults to chart version)"
|
||||
description: "Application version (must equal chart version; defaults to it)"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
@@ -90,6 +90,13 @@ jobs:
|
||||
APP_VERSION="$CHART_VERSION"
|
||||
fi
|
||||
|
||||
# Pulse's server and agent image defaults use Chart.AppVersion.
|
||||
# Source provenance alone cannot detect a mismatched image override.
|
||||
if [ "$APP_VERSION" != "$CHART_VERSION" ]; then
|
||||
echo "::error::Release chart app_version must equal chart_version."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IS_PRERELEASE="false"
|
||||
if [[ "$APP_VERSION" =~ -rc\.[0-9]+$ ]] || [[ "$APP_VERSION" =~ -alpha\.[0-9]+$ ]] || [[ "$APP_VERSION" =~ -beta\.[0-9]+$ ]]; then
|
||||
IS_PRERELEASE="true"
|
||||
|
||||
@@ -5,6 +5,7 @@ on:
|
||||
branches:
|
||||
- main
|
||||
- pulse/v6-release
|
||||
- 'release/v*'
|
||||
paths:
|
||||
- 'frontend-modern/**'
|
||||
- 'internal/**'
|
||||
@@ -16,6 +17,7 @@ on:
|
||||
- main
|
||||
- master
|
||||
- pulse/v6-release
|
||||
- 'release/v*'
|
||||
paths:
|
||||
- 'frontend-modern/**'
|
||||
- 'internal/**'
|
||||
|
||||
@@ -242,6 +242,7 @@ scripts/release_control/*
|
||||
!scripts/release_control/governance_stage_guard.py
|
||||
!scripts/release_control/governance_stage_guard_test.py
|
||||
!scripts/release_control/helm_pages_retry_test.py
|
||||
!scripts/release_control/helm_publish_version_test.py
|
||||
!scripts/release_control/live_runtime_proof.py
|
||||
!scripts/release_control/live_runtime_proof_test.py
|
||||
!scripts/release_control/mobile_relay_auth_approvals_proof.py
|
||||
|
||||
@@ -867,3 +867,24 @@ The final full tools package passes in 59.413s and focused file-read race proof
|
||||
in 1.033s, covering successful reads alongside the failed-read controls.
|
||||
Private worker logs are `file-read-full.log` and `file-read-race.log` under
|
||||
`/opt/pulse-release-worker/`.
|
||||
|
||||
## Integration qualification, 2026-09-06
|
||||
|
||||
The detection and failed-read slice is committed as `61607333cc9e`. Integration
|
||||
with main `3f74c0c27304` required a fresh browser receipt. Intermediate-width
|
||||
inspection exposed the shared delivery-health card squeezing its explanation
|
||||
under the action buttons. The shared card now wraps by available width and uses
|
||||
an opaque semantic heading colour. Permanent fixtures include 900 pixels and
|
||||
check control bounds and heading overflow.
|
||||
|
||||
Final worker proof passes 18 delivery-ordering cases and 12 Overview refresh
|
||||
cases at 1440, 900 and 390 pixels, with Overview checked in light and dark themes.
|
||||
Unavailable health, retained retry/dismiss actions, pending refresh and healthy
|
||||
recovery were exercised. Final card pixels were inspected at all three widths.
|
||||
Frontend lint, type checking and three affected test files pass. Focused merged
|
||||
Patrol/API/adapter/Docker-result proof passes. The final local scripted
|
||||
`/patrol` and `/alerts` journey passes at 1440x1000, 900x1000 and 390x1000,
|
||||
including nested failed-read evidence and the linked Assistant. This qualifies
|
||||
the integrated presentation and context contract only. Exact staged hook and
|
||||
remote landing are subsequent delivery checks. Real diagnosis and autonomous
|
||||
action outcomes remain unqualified, and the provider refusal remains enforced.
|
||||
|
||||
|
After Width: | Height: | Size: 35 KiB |
|
After Width: | Height: | Size: 50 KiB |
@@ -0,0 +1,28 @@
|
||||
# Delivery health request ordering — 5 September 2026
|
||||
|
||||
Run from repository root after root and frontend `npm ci --ignore-scripts`:
|
||||
|
||||
pulse-heavy-run -- node scripts/check-delivery-health-ordering.mjs
|
||||
|
||||
Twelve real Chromium cases passed at 1440×900 and 390×900. The fixture imports
|
||||
the production destinations caller, health hook, warning card and CSS; only API
|
||||
responses, configuration props and confirmation acceptance are scripted.
|
||||
It does not serve the complete application or exercise an installed backend.
|
||||
The temporary Vite fixture is never part of the product bundle.
|
||||
|
||||
Four success/error orderings per width preserve latest attention and rendered
|
||||
state. Two action cases per width exercise retry/dismissal overlap: an older
|
||||
completion cannot clear loading while the post-action read remains pending,
|
||||
and the latest healthy response clears attention. All 19 focused hook, caller
|
||||
and card tests pass; reverting only the runtime hook produces nine failures in
|
||||
the 16 hook/caller tests. TypeScript and changed-file ESLint pass.
|
||||
|
||||
The first browser attempt failed before UI verification because the isolated
|
||||
Vite configuration omitted the repository's esnext dependency target. The
|
||||
retained runner corrects that and uses the frontend working directory for CSS.
|
||||
`browser-result.json` is the successful rerun's output; hashes bind the runner
|
||||
and runtime source. Representative screenshots show actual fixture rendering.
|
||||
Warning titles appear pale in this isolated CSS context; this receipt qualifies
|
||||
ordering, not whole-app contrast or accessibility. Full-shell visual review,
|
||||
installed delivery, release-line applicability and exact-candidate integration
|
||||
remain separate qualification work. No public release is claimed.
|
||||
@@ -0,0 +1 @@
|
||||
{"result":"passed","cases":12,"viewports":[1440,390],"scope":"real Chromium, real caller/hook/card, scripted API; not installed application or notification receipt"}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"scripts/check-delivery-health-ordering.mjs": "69f2c34c0fb0ecf1e67f5e5c85051f63d943314d975d6eadb8f37363622dd9c9",
|
||||
"frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts": "1eadf6df30b4f1130f868b8f139561ea60fdb87e1823deeeaaf2f8fbc699a00c"
|
||||
}
|
||||
@@ -635,6 +635,20 @@ installer download and the agent's subsequent Pulse TLS connection.
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
### Container update receipt and independent observation
|
||||
|
||||
The server's independent Docker-update verification must compare the daemon
|
||||
state/running observation with the replacement state recorded by the agent,
|
||||
not only its container ID. A matching running replacement needs healthy or
|
||||
no-healthcheck evidence; intentionally stopped replacements remain stopped.
|
||||
Missing agent readback cannot become independent confirmation. This server-side
|
||||
classification does not amend the agent's mutation receipt, trigger another
|
||||
update, change runner permissions, or reinterpret compensation as execution.
|
||||
`TestDockerContainerUpdateIndependentObservationMustMatchState` in
|
||||
`internal/api/docker_container_action_result_test.go` verifies these boundaries;
|
||||
existing callback-loss reconciliation must continue without redispatch.
|
||||
|
||||
|
||||
The shared `PBSInstance.NodeMetricsUnavailable` field belongs exclusively to
|
||||
provider polling and alert evaluation. It is retained by in-process state copies
|
||||
but excluded from JSON; it neither grants nor revokes host-agent identity,
|
||||
@@ -2880,6 +2894,22 @@ traverse an agent.
|
||||
|
||||
## Completion Obligations
|
||||
|
||||
### Docker update readback is not mutation history
|
||||
|
||||
The agent's post-update inspect can observe a running replacement whose health
|
||||
has not settled or has deteriorated after module success. The API must retain
|
||||
the completed mutation as execution history without treating replacement
|
||||
identity alone as confirmed running health. Agent-attested verification requires
|
||||
running state and healthy or explicit no-healthcheck evidence for running
|
||||
replacements; unknown health remains inconclusive. This does not trigger an
|
||||
automatic resend or rollback and preserves deliberately stopped updates.
|
||||
|
||||
Verification: `TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim`
|
||||
asserts both verification outcomes and unchanged successful execution.
|
||||
`TestDockerContainerActionExecutorDispatchesTypedUpdate` retains typed-only
|
||||
dispatch with explicit no-healthcheck readback evidence.
|
||||
|
||||
|
||||
Command-capable agent completion must prove more than fresh telemetry. The
|
||||
dedicated agent listener must admit the full bootstrap/report/WebSocket
|
||||
lifecycle, command sessions must be keyed by organization plus canonical bound
|
||||
|
||||
@@ -15,6 +15,19 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
The shared delivery-health card wraps action groups according to available
|
||||
space, retaining readable explanation width when Review, Retry, Dismiss and
|
||||
Refresh appear together. Its heading uses the opaque semantic foreground,
|
||||
not the translucent palette shades reserved for status backgrounds. Verify
|
||||
light/dark layouts at desktop, intermediate and narrow widths, including
|
||||
unavailable health, pending refresh and recovery.
|
||||
|
||||
The alerts overview offers the existing delivery-status refresh control when
|
||||
health is unavailable, including after a successful retained-queue action whose
|
||||
follow-up health read fails. The warning remains until a verified healthy read;
|
||||
a successful queue action alone is not evidence of delivery health. Normal
|
||||
degraded summary presentation continues to omit refresh.
|
||||
|
||||
Confirmed canonical metric recovery publishes the clearing evaluation's value,
|
||||
observation time, and resolved metric wording in the snapshot consumed by
|
||||
recent-resolution reads and notification callbacks. It must not reuse the last
|
||||
@@ -71,6 +84,22 @@ their updated thresholds and may apply explicit resource-disable policies, but
|
||||
it must not treat provider-owned incidents as missing thresholds. Unrelated
|
||||
configuration saves preserve those incidents and their acknowledgement state
|
||||
until their provider evaluator supplies recovery evidence.
|
||||
Storage configuration re-evaluation must use the same ordered resource ID and
|
||||
alias override lookup as polling. Static and forecast capacity alerts retain
|
||||
storage policy aliases in durable metadata, including through JSON and SQLite
|
||||
restore; a configuration reload must not fabricate recovery by substituting
|
||||
global defaults for a still-applicable datastore override. Older PBS snapshots
|
||||
without alias metadata may reconstruct the canonical datastore alias only when
|
||||
the recorded PBS instance, datastore name and complete legacy resource ID agree.
|
||||
Hyphenated names must not be split heuristically, and a same-named datastore on
|
||||
another instance must not inherit the override. Explicit policy changes retain
|
||||
normal resolution semantics.
|
||||
`TestPBSDatastoreOverrideLifecycleAcrossRestart` and
|
||||
`TestStoragePolicyAliasesLegacyIdentity` in
|
||||
`internal/alerts/canonical_stateful_test.go` pin restored incident identity,
|
||||
hysteresis, confirmed recovery, refiring, event counts and instance isolation
|
||||
for both persisted-alias and legacy snapshots.
|
||||
|
||||
When a VM or container stops, guest evaluation resolves only metric-threshold
|
||||
alerts whose observations are no longer meaningful. Backup-age and snapshot
|
||||
posture remain owned by their posture evaluator and may stay active while the
|
||||
@@ -2579,3 +2608,19 @@ reasoning and real remediation in
|
||||
`docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md`. The repeatable browser
|
||||
proof is `scripts/check-patrol-assistant-journey.mjs`. A passing scripted
|
||||
response does not establish a useful customer outcome or model qualification.
|
||||
|
||||
### Delivery health requests have latest-started ownership
|
||||
|
||||
Overlapping mount, configuration Retry and post-queue-action reads must not
|
||||
allow an older completion to replace newer delivery health. Success, failure,
|
||||
first-load completion and the refreshing flag belong only to the most recently
|
||||
started health request. A stale healthy response cannot hide degraded attention;
|
||||
a stale error cannot invent unavailability after recovery. Queue actions still
|
||||
refresh from the server, not from their affected count. This changes no provider
|
||||
acceptance, incident lifecycle or delivery guarantee.
|
||||
|
||||
The hook and destinations caller regressions in
|
||||
`useNotificationDeliveryHealth.test.tsx` and
|
||||
`useAlertDestinationsTabState.test.tsx` pin ordering and loading ownership.
|
||||
`scripts/check-delivery-health-ordering.mjs` exercises the real caller and card
|
||||
in Chromium with scripted API completions; it is not installed delivery proof.
|
||||
|
||||
@@ -461,6 +461,21 @@ enums locally.
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
### Independent Docker update readback
|
||||
|
||||
`dockerContainerUpdateExecutionResult` must not promote replacement-ID equality
|
||||
alone to independent confirmation. The daemon observation must match the
|
||||
agent readback's state and running flag; running replacements additionally
|
||||
require state `running` and health `healthy` or `none`. A deliberately stopped
|
||||
replacement remains confirmable without being started. Missing agent readback
|
||||
or its state leaves independent verification inconclusive. Contradictory
|
||||
readback changes verification, not the recorded execution or compensation.
|
||||
No wire schema or mutation authority changes. Verification:
|
||||
`TestDockerContainerUpdateIndependentObservationMustMatchState` in
|
||||
`internal/api/docker_container_action_result_test.go` covers running, stopped,
|
||||
restarting, unhealthy, unknown health and absent readback cases.
|
||||
|
||||
|
||||
GET /api/security/status includes currentUsername in authenticated and privileged responses, derived from the validated authentication snapshot rather than configured administrator identity. Public login discovery omits it. The field is identity only: it does not grant instance-settings authority or change token scopes. Scoped local sessions retain this identity when privileged authUsername is withheld.
|
||||
|
||||
Commercial migration payloads are a shared API/cloud-paid contract. The
|
||||
@@ -3592,6 +3607,22 @@ counters exist to measure.
|
||||
|
||||
## Completion Obligations
|
||||
|
||||
### Docker update agent-attested running verification
|
||||
|
||||
In `internal/api/docker_container_action_result.go`, a matching replacement
|
||||
container ID is insufficient to confirm a running agent readback. A running
|
||||
replacement must report state `running` and health `healthy` or explicit
|
||||
`none`. Known unsuitable health or lifecycle states contradict verification;
|
||||
missing or unrecognised running health yields `inconclusive` with
|
||||
`container_health_unknown`. This does not change execution or compensation
|
||||
history, introduce a wire field, or require stopped replacements to run.
|
||||
|
||||
Verification: `TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim`
|
||||
in `internal/api/docker_container_action_result_test.go` covers healthy,
|
||||
no-healthcheck, stopped, unhealthy, starting, restarting and unknown-health
|
||||
readbacks. The typed-dispatch fixture supplies explicit no-healthcheck evidence.
|
||||
|
||||
|
||||
The public connection ledger and action APIs must project telemetry liveness
|
||||
and command admission independently. An agent may be adapter-healthy while
|
||||
remote control is `disconnected`; in that state command policy is blocked and
|
||||
|
||||
@@ -15,6 +15,22 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
### Public Helm exact-package receipt
|
||||
|
||||
The post-activation public Pages verification in `.github/workflows/helm-pages.yml`
|
||||
must pull the requested chart version through the consumer repository and compare
|
||||
its archive bytes with the package already recovered through OCI qualification.
|
||||
Readable chart metadata alone is not a successful convergence receipt. The OCI
|
||||
manifest digest hashes a different object and must not be compared directly with
|
||||
the archive. A byte mismatch fails immediately; unavailable downloads retain the
|
||||
bounded retry, with previous downloaded files removed before each attempt.
|
||||
Activation bindings, publication authority and containment remain prerequisites.
|
||||
The executed-shell fixtures in `scripts/release_control/helm_pages_retry_test.py`
|
||||
cover matching, mismatched, missing and unavailable public packages without
|
||||
network or publication writes. They are not installed Helm qualification or proof
|
||||
that any historical published chart was wrong or has been repaired.
|
||||
|
||||
|
||||
The shell-owned multi-tenant integration suite uses a dedicated desktop-only
|
||||
Playwright configuration selecting the seven multi-tenant scenarios. It must
|
||||
reject any non-empty E2E tier identity rather than impersonating stable or
|
||||
@@ -4073,6 +4089,13 @@ are part of the same governed bootstrap input even when the package manifest
|
||||
range already permits the newer version; the lockfile must identify the
|
||||
resolved package version and integrity that the release build will actually
|
||||
consume.
|
||||
Build and Test and Core E2E must admit both pushes and pull requests for
|
||||
`release/v*` trains as well as main. A release proposal must not appear
|
||||
qualified merely because documentation and boundary checks passed while the
|
||||
branch filters excluded build, dependency-security and applicable E2E checks.
|
||||
Existing path filters and per-job requirements remain in force; branch admission
|
||||
is not evidence that those jobs executed or passed.
|
||||
|
||||
Frontend dependency-security changes use their own proof route rather than
|
||||
borrowing the local dev-runtime orchestration tests. The canonical
|
||||
`.github/workflows/build-and-test.yml` frontend job must run the complete
|
||||
@@ -5168,3 +5191,20 @@ passed. A read-only probe of job 101235205647 retained the expected failure text
|
||||
without ESC bytes. Private containment classification and successful scheduled
|
||||
reconciliation still require post-integration evidence; this is not customer
|
||||
convergence or release qualification.
|
||||
|
||||
### 2026-09-05 — Bind hosted chart application version to release version
|
||||
|
||||
The hosted Helm publisher rejects a supplied application version that differs
|
||||
from the chart version before emitting version outputs or packaging. Both server
|
||||
and agent image defaults use Chart.AppVersion, so an exact source SHA and chart
|
||||
digest alone do not prevent an override from selecting another release's images.
|
||||
The normal release caller already supplies equal versions; default and release
|
||||
event paths remain unchanged. This does not remove users' image value overrides.
|
||||
|
||||
Verification: helm_publish_version_test.py executes the actual version-resolution
|
||||
shell. Four mismatches failed assertions before the fix and are rejected after it;
|
||||
stable/alpha/beta/RC equal and default versions and release-event defaults pass.
|
||||
All 4 tests, 7 Helm Pages retry tests and 47 promotion-policy tests pass locally.
|
||||
No hosted publication or installed-image qualification is claimed.
|
||||
External reference retrieved 2026-09-05: https://helm.sh/docs/topics/charts/#the-appversion-field
|
||||
explains that application version is separate from chart version.
|
||||
|
||||
@@ -20,6 +20,19 @@
|
||||
|
||||
## Purpose
|
||||
|
||||
The shared delivery-health card wraps action groups according to available
|
||||
space, retaining readable explanation width when Review, Retry, Dismiss and
|
||||
Refresh appear together. Its heading uses the opaque semantic foreground,
|
||||
not the translucent palette shades reserved for status backgrounds. Verify
|
||||
light/dark layouts at desktop, intermediate and narrow widths, including
|
||||
unavailable health, pending refresh and recovery.
|
||||
|
||||
The alerts overview offers the existing delivery-status refresh control when
|
||||
health is unavailable, including after a successful retained-queue action whose
|
||||
follow-up health read fails. The warning remains until a verified healthy read;
|
||||
a successful queue action alone is not evidence of delivery health. Normal
|
||||
degraded summary presentation continues to omit refresh.
|
||||
|
||||
Proxmox backup presentation treats every manifestless PBS artifact as
|
||||
non-recoverable. It renders the artifact as `Running` when current writer
|
||||
visibility is absent or a matching writer is active, and as danger-tone
|
||||
@@ -7117,3 +7130,15 @@ reasoning and real remediation in
|
||||
`docs/qualification/PATROL_ASSISTANT_CUSTOMER_JOURNEY.md`. The repeatable browser
|
||||
proof is `scripts/check-patrol-assistant-journey.mjs`. A passing scripted
|
||||
response does not establish a useful customer outcome or model qualification.
|
||||
|
||||
### Alert health attention preserves asynchronous ownership
|
||||
|
||||
The existing delivery-health card and shared buttons consume only the latest
|
||||
started health read's state. Configuration Retry can overlap a disabled card
|
||||
refresh; disabling that button is not a concurrency guard. Older completions
|
||||
must neither clear the latest request's busy flag nor replace its attention or
|
||||
unavailable presentation. Existing danger tone, accessible alert role, labels,
|
||||
confirmation and wrapping controls remain unchanged; no new primitive is added.
|
||||
The focused hook/caller tests and `scripts/check-delivery-health-ordering.mjs`
|
||||
cover this dependency at desktop and narrow widths using scripted health and
|
||||
queue-action responses, without claiming backend notification delivery.
|
||||
|
||||
@@ -571,11 +571,59 @@ every destination the decision webhook delivery already made for HTTP 4xx in
|
||||
`isRetryableWebhookError`.
|
||||
|
||||
Dead-lettering early must not lose the notification: `RetryTerminalFailures`
|
||||
remains the operator's recovery path, returning retained terminal failures to
|
||||
the queue with a fresh budget once the credentials or configuration are fixed.
|
||||
remains the operator's recovery path, returning eligible retained terminal
|
||||
failures to the queue with a fresh budget once the credentials or configuration
|
||||
are fixed. Resolution removes obsolete firing entries from that eligibility;
|
||||
retry must not resurrect an incident which has already recovered.
|
||||
A dead-letter row records `failureClass` and a `deadLetterReason` of
|
||||
`failure_class_not_retryable` or `max_retries_exhausted` so the two are
|
||||
distinguishable in local logs.
|
||||
|
||||
`internal/notifications/failure_class_test.go` pins the retryable split and
|
||||
that a deterministic failure dead-letters on its first attempt.
|
||||
|
||||
|
||||
### Resolution remains final across terminal retries and restart
|
||||
|
||||
Resolution cancellation covers pending, sending, failed, and dead-lettered
|
||||
firing rows. A wholly obsolete row becomes cancelled; a grouped row retains
|
||||
only unrelated firing alerts and their operational links. Recovery jobs are
|
||||
not cancelled by this operation. Only removed pending entries contribute to
|
||||
the pending-suppression return count; terminal or interrupted sends must not
|
||||
be counted as proof that firing was never delivered.
|
||||
|
||||
The cancellation verdict persists across queue reopen and bulk operator
|
||||
retry. Per-item retry also rejects cancelled and already-sent rows atomically,
|
||||
so a stale retry request cannot bypass resolution or duplicate a completed
|
||||
delivery. Pending, sending, failed, and dead-lettered rows remain eligible for
|
||||
the existing retry scheduler.
|
||||
|
||||
Cancelling a row retains its failed-attempt audit history and announces the
|
||||
changed queue-health verdict only after releasing both the database mutex and
|
||||
per-alert delivery gates. Clearing obsolete retained failures does not prove
|
||||
that a destination has been repaired. Nor does this operation retrospectively
|
||||
identify obsolete rows whose resolution happened before this behaviour was
|
||||
installed; historical backlogs still require incident reconciliation.
|
||||
|
||||
`internal/notifications/queue_resolution_retry_test.go` proves failed and
|
||||
dead-lettered cancellation across durable reopen and actual queue processing,
|
||||
preservation of unrelated grouped firing and recovery jobs, retained failed
|
||||
attempts, callback lock release and committed-state visibility, and the
|
||||
per-item retry eligibility matrix. These are component proofs, not installed
|
||||
receiver receipts or exactly-once delivery guarantees.
|
||||
|
||||
### Disabled delivery is cancellation, not a receipt
|
||||
|
||||
At processing time, globally disabled delivery or a disabled/removed destination
|
||||
returns `ErrNotificationDeliverySkipped`. The queue persists that job as
|
||||
cancelled with the policy reason and cancelled operational links. It does not
|
||||
write a provider-attempt audit, a successful receipt, or a delivery failure, and
|
||||
operator retry does not replay the cancelled job. Existing attempt history is
|
||||
retained. Queue health reconciliation runs after releasing the database mutex
|
||||
and alert delivery gates.
|
||||
|
||||
`queue_disabled_delivery_test.go` exercises the real manager/queue boundary for
|
||||
email, webhook and Apprise, firing and recovery, and global versus destination
|
||||
disablement. This corrects false successful queue/audit records; it does not
|
||||
establish maintenance-window expiry, stop an already-started provider request,
|
||||
or repair historical false-success records.
|
||||
|
||||
@@ -271,6 +271,22 @@ command-capable profile.
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
### Shared Docker-update verification boundary
|
||||
|
||||
The shared API result converter classifies independent Docker update readback
|
||||
using replacement identity, state/running agreement and running health, rather
|
||||
than identity alone. Both immediate execution and durable receipt reconciliation
|
||||
use this converter. Contradictory observations affect verification only;
|
||||
missing agent readback is inconclusive. Execution and compensation records
|
||||
remain unchanged: container backup-rename compensation is not a storage backup,
|
||||
recovery point, or independently verified restore. No storage selection,
|
||||
retention or recovery authority is added. The focused
|
||||
`TestDockerContainerUpdateIndependentObservationMustMatchState` in
|
||||
`internal/api/docker_container_action_result_test.go` asserts that independent
|
||||
verification changes never rewrite successful execution history, including
|
||||
stopped replacements and missing readback.
|
||||
|
||||
|
||||
Recovery consumers of security status must distinguish currentUsername (the validated caller identity) from privileged authUsername (administrator configuration). Presence of currentUsername in a scoped response does not authorise export, import, recovery or unprotected backup transfer; existing settings capabilities and backend enforcement still gate those operations.
|
||||
|
||||
The Patrol action broker and shared policy-writer wiring under `internal/api/`
|
||||
@@ -2115,6 +2131,22 @@ take a correctness dependency on its contents.
|
||||
|
||||
## Completion Obligations
|
||||
|
||||
### Recovered Docker update evidence preserves health uncertainty
|
||||
|
||||
The shared API action-result boundary must not turn a matching replacement ID
|
||||
into confirmed running health when the agent readback reports unhealthy,
|
||||
health-check starting or restarting. Unknown running health remains
|
||||
inconclusive, while the recorded completed execution and compensation facts
|
||||
remain unchanged. This rule does not settle missing receipts, prove intended
|
||||
stopped state, or authorise redispatch of a recovered update.
|
||||
|
||||
Verification: `TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim`
|
||||
in `internal/api/docker_container_action_result_test.go` pins these result
|
||||
distinctions. Existing `TestIssue1649StrandedDockerUpdateReconcilesWithoutRedispatch`
|
||||
and `TestRouterRecoverExecutingDockerUpdateAfterCapabilityDisappears` exercise
|
||||
reconciliation without treating the verification correction as a new mutation.
|
||||
|
||||
|
||||
Coverage rows reconcile replacement recovery-model snapshots by their logical
|
||||
`key` before windowing. Refreshed names must render without discarding expanded
|
||||
restore evidence or keyboard focus on a surviving row. Verify with
|
||||
|
||||
@@ -1,19 +1,36 @@
|
||||
{
|
||||
"version": 1,
|
||||
"base_sha": "c5d2f56dda42946ebafe09266c8e2e52319ba24b",
|
||||
"verified_at": "2026-09-06T01:21:13.188Z",
|
||||
"base_sha": "61607333cc9e2fe1c1712b94ce458ac45abdabd1",
|
||||
"verified_at": "2026-09-06T03:17:28.710286Z",
|
||||
"result": "passed",
|
||||
"changed_paths": [
|
||||
"frontend-modern/src/components/AI/FindingsPanel.tsx"
|
||||
"frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx",
|
||||
"frontend-modern/src/features/alerts/OverviewTab.tsx",
|
||||
"frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts"
|
||||
],
|
||||
"content_sha256": {
|
||||
"frontend-modern/src/components/AI/FindingsPanel.tsx": "aead071f9431e5c2284014cd8e4af8fc5cde69951f3f850821808dcf885c5226"
|
||||
"frontend-modern/src/features/alerts/AlertDeliveryHealthCard.tsx": "dad60838804e575c0314398b304bda6d2fbd842cb2f693b3f33d168a4bce5dc1",
|
||||
"frontend-modern/src/features/alerts/OverviewTab.tsx": "8f7fdc04bd0546f86152c8bb392ff3e3fb755f1f0a1c28d1ee85da3e9fe23582",
|
||||
"frontend-modern/src/features/alerts/useNotificationDeliveryHealth.ts": "1eadf6df30b4f1130f868b8f139561ea60fdb87e1823deeeaaf2f8fbc699a00c"
|
||||
},
|
||||
"routes": [
|
||||
"/qualification (actual delivery-health and Overview components in isolated Solid Router fixtures, scripted API promises)",
|
||||
"/patrol",
|
||||
"/alerts"
|
||||
],
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"width": 900,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"width": 390,
|
||||
"height": 900
|
||||
},
|
||||
{
|
||||
"width": 1440,
|
||||
"height": 1000
|
||||
@@ -28,12 +45,13 @@
|
||||
}
|
||||
],
|
||||
"states": [
|
||||
"Ordinary and alert-mirrored findings retain the same review selection. Completed investigation remains Needs attention with unknown cause and failed access evidence.",
|
||||
"Original failed read and uncertain conclusion remain visible in the nested investigation transcript and linked Assistant request. Scripted responses establish presentation and context preservation, not diagnostic competence."
|
||||
"Delivery ordering: 18 scripted cases at three widths. Overview refresh: 12 scripted cases across light/dark themes and three widths. Unavailable health, retained Retry/Dismiss actions, pending disabled Refresh and healthy recovery. These fixtures do not qualify installed backend delivery or recipient outcomes.",
|
||||
"Final card pixels inspected at desktop, intermediate and narrow widths in both themes. Heading is readable and actions wrap within the card. This is a focused card check, not a full dark-theme shell audit.",
|
||||
"Ordinary and alert-mirrored Patrol findings preserve unknown cause and failed-read evidence in the investigation transcript and linked Assistant. Scripted responses qualify presentation and context preservation, not diagnostic competence."
|
||||
],
|
||||
"interactions": [
|
||||
"Activity, Finding options and history, mirrored disclosure, keyboard Review, investigation thread expansion/collapse, review close at 1440/900/390 widths. Inspect final pixels for readable content and reachable controls.",
|
||||
"Inbox selected issue, evidence expansion, keyboard Explain, same canonical finding with failed-read evidence and uncertainty in Assistant handoff, Escape, draft preservation, reload without resubmission.",
|
||||
"Existing adjacent alert menu Escape/outside dismissal and provider failure/retry checked. No infrastructure mutations or autonomous provider calls. Private source-bound proof: tmp/patrol-assistant-journey/result.json."
|
||||
"Retry and Dismiss followed by failed health read, manual Refresh while pending, healthy recovery and overlapping old/new response ordering. Geometry assertions check heading overflow and control bounds.",
|
||||
"Selected issue, evidence expansion, keyboard Review and Explain, nested transcript expansion/collapse, Assistant streaming, preserved draft, reload, error/retry, menu Escape and outside dismissal at 1440/900/390 widths.",
|
||||
"Private receipts: /Volumes/Development/pulse/tmp/patrol-merge-browser/final-health-ordering and final-overview-refresh, and tmp/patrol-assistant-journey/result.json. No infrastructure mutations or autonomous provider calls."
|
||||
]
|
||||
}
|
||||
|
||||
@@ -103,6 +103,26 @@ describe('AlertDeliveryHealthCard', () => {
|
||||
expect(screen.getByRole('button', { name: 'Refresh delivery status' })).toBeDisabled();
|
||||
});
|
||||
|
||||
it('allows unavailable summary health to be rechecked without a queue mutation', () => {
|
||||
const onRefresh = vi.fn();
|
||||
render(() => (
|
||||
<AlertDeliveryHealthCard
|
||||
health={null}
|
||||
unavailable
|
||||
refreshing={false}
|
||||
onRefresh={onRefresh}
|
||||
detailLevel="summary"
|
||||
showRefresh
|
||||
/>
|
||||
));
|
||||
expect(screen.getByRole('alert')).toHaveTextContent(
|
||||
'Notification delivery status is unavailable',
|
||||
);
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Refresh delivery status' }));
|
||||
expect(onRefresh).toHaveBeenCalledOnce();
|
||||
expect(screen.getByRole('alert')).toBeTruthy();
|
||||
});
|
||||
|
||||
it('keeps the overview treatment concise and points directly to delivery evidence', () => {
|
||||
render(() => (
|
||||
<Router>
|
||||
|
||||
@@ -52,20 +52,20 @@ export function AlertDeliveryHealthCard(props: AlertDeliveryHealthCardProps) {
|
||||
|
||||
return (
|
||||
<Card tone="danger" padding="sm" class="border-red-200 dark:border-red-800 sm:p-4" role="alert">
|
||||
<div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
||||
<div class="flex min-w-0 items-start gap-3">
|
||||
<div class="flex flex-wrap items-start justify-between gap-3">
|
||||
<div class="flex min-w-0 flex-1 basis-72 items-start gap-3">
|
||||
<AlertTriangleIcon
|
||||
class="mt-0.5 h-4 w-4 flex-shrink-0 text-red-700 dark:text-red-300"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
<div class="min-w-0">
|
||||
<h3 class="text-sm font-semibold text-red-900 dark:text-red-100">
|
||||
<h3 class="text-sm font-semibold text-base-content">
|
||||
{getAlertDestinationsDeliveryHealthTitle(status())}
|
||||
</h3>
|
||||
<p class="mt-1 text-sm leading-6 text-red-800 dark:text-red-200">{description()}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex flex-shrink-0 flex-wrap items-center gap-2">
|
||||
<div class="flex max-w-full flex-wrap items-center gap-2">
|
||||
{props.detailsHref ? (
|
||||
<ButtonLink variant="secondary" size="sm" href={props.detailsHref}>
|
||||
{getAlertDestinationsDeliveryReviewLabel()}
|
||||
|
||||
@@ -90,7 +90,7 @@ export function OverviewTab(props: {
|
||||
onDismissFailures={() => void deliveryHealthState.dismissTerminalFailures()}
|
||||
detailsHref="/alerts/notifications#notification-delivery-activity"
|
||||
detailLevel="summary"
|
||||
showRefresh={false}
|
||||
showRefresh={deliveryHealthState.deliveryHealthUnavailable()}
|
||||
/>
|
||||
</Show>
|
||||
<AlertOverviewStatsCards state={overviewState} />
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { cleanup, render, screen, waitFor } from '@solidjs/testing-library';
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library';
|
||||
import { DEFAULT_LOCALE, setActiveLocale } from '@/i18n';
|
||||
import type { Alert } from '@/types/api';
|
||||
import type { NotificationHealth } from '@/api/notifications';
|
||||
@@ -49,6 +49,8 @@ vi.mock('@/components/Alerts/InvestigateAlertButton', () => ({
|
||||
InvestigateAlertButton: () => null,
|
||||
}));
|
||||
|
||||
import { notificationStore } from '@/stores/notifications';
|
||||
|
||||
import { OverviewTab } from '../OverviewTab';
|
||||
|
||||
function degradedHealth(): NotificationHealth {
|
||||
@@ -107,10 +109,15 @@ describe('OverviewTab delivery health actions', () => {
|
||||
getDeliveryDiagnoses.mockReset();
|
||||
getDeliveryDiagnoses.mockResolvedValue([]);
|
||||
getHealth.mockReset();
|
||||
retryTerminalFailures.mockReset();
|
||||
dismissTerminalFailures.mockReset();
|
||||
vi.mocked(notificationStore.success).mockClear();
|
||||
vi.mocked(notificationStore.error).mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
setActiveLocale(DEFAULT_LOCALE);
|
||||
});
|
||||
|
||||
@@ -127,4 +134,110 @@ describe('OverviewTab delivery health actions', () => {
|
||||
expect(screen.queryByRole('button', { name: 'Refresh delivery status' })).toBeNull();
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('Most recent failures: connectivity (1).');
|
||||
});
|
||||
for (const action of [
|
||||
{ name: 'Retry retained deliveries', api: retryTerminalFailures },
|
||||
{ name: 'Dismiss retained failures', api: dismissTerminalFailures },
|
||||
]) {
|
||||
it(`does not mutate or refresh when ${action.name} is cancelled`, async () => {
|
||||
getHealth.mockResolvedValue(degradedHealth());
|
||||
const confirmation = vi.spyOn(window, 'confirm').mockReturnValue(false);
|
||||
render(() => <OverviewTab {...defaultProps()} />);
|
||||
fireEvent.click(await screen.findByRole('button', { name: action.name }));
|
||||
|
||||
expect(confirmation).toHaveBeenCalledOnce();
|
||||
expect(action.api).not.toHaveBeenCalled();
|
||||
expect(getHealth).toHaveBeenCalledTimes(1);
|
||||
expect(screen.getByRole('alert')).toBeTruthy();
|
||||
});
|
||||
|
||||
it(`retains attention and enables another attempt when ${action.name} fails`, async () => {
|
||||
getHealth.mockResolvedValue(degradedHealth());
|
||||
vi.spyOn(window, 'confirm').mockReturnValue(true);
|
||||
action.api.mockRejectedValue(new Error('queue action unavailable'));
|
||||
render(() => <OverviewTab {...defaultProps()} />);
|
||||
fireEvent.click(await screen.findByRole('button', { name: action.name }));
|
||||
|
||||
await waitFor(() => expect(notificationStore.error).toHaveBeenCalledOnce());
|
||||
expect(notificationStore.success).not.toHaveBeenCalled();
|
||||
expect(getHealth).toHaveBeenCalledTimes(1);
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('connectivity (1)');
|
||||
expect(screen.getByRole('button', { name: action.name })).not.toBeDisabled();
|
||||
});
|
||||
|
||||
it(`keeps health visibly unknown after ${action.name} succeeds but refresh fails`, async () => {
|
||||
getHealth
|
||||
.mockResolvedValueOnce(degradedHealth())
|
||||
.mockRejectedValueOnce(new Error('health unavailable'));
|
||||
action.api.mockResolvedValue({ affected: 85 });
|
||||
vi.spyOn(window, 'confirm').mockReturnValue(true);
|
||||
render(() => <OverviewTab {...defaultProps()} />);
|
||||
fireEvent.click(await screen.findByRole('button', { name: action.name }));
|
||||
|
||||
const refresh = await screen.findByRole('button', { name: 'Refresh delivery status' });
|
||||
expect(screen.getByRole('alert')).toBeTruthy();
|
||||
expect(notificationStore.success).toHaveBeenCalledOnce();
|
||||
expect(notificationStore.error).not.toHaveBeenCalled();
|
||||
|
||||
const healthy = degradedHealth();
|
||||
healthy.overallHealthy = true;
|
||||
healthy.queue = {
|
||||
...healthy.queue,
|
||||
status: 'healthy',
|
||||
healthy: true,
|
||||
attentionRequired: 0,
|
||||
deadLetter: 0,
|
||||
};
|
||||
getHealth.mockResolvedValueOnce(healthy);
|
||||
await waitFor(() => expect(refresh).not.toBeDisabled());
|
||||
fireEvent.click(refresh);
|
||||
await waitFor(() => expect(screen.queryByRole('alert')).toBeNull());
|
||||
expect(getHealth).toHaveBeenCalledTimes(3);
|
||||
expect(action.api).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it(`keeps both actions disabled until ${action.name} and its health refresh finish`, async () => {
|
||||
const healthy = degradedHealth();
|
||||
healthy.overallHealthy = true;
|
||||
healthy.queue = {
|
||||
...healthy.queue,
|
||||
status: 'healthy',
|
||||
healthy: true,
|
||||
attentionRequired: 0,
|
||||
deadLetter: 0,
|
||||
};
|
||||
let completeAction!: (value: { affected: number }) => void;
|
||||
let completeHealth!: (value: NotificationHealth) => void;
|
||||
action.api.mockReturnValue(
|
||||
new Promise((resolve) => {
|
||||
completeAction = resolve;
|
||||
}),
|
||||
);
|
||||
getHealth.mockResolvedValueOnce(degradedHealth()).mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
completeHealth = resolve;
|
||||
}),
|
||||
);
|
||||
vi.spyOn(window, 'confirm').mockReturnValue(true);
|
||||
render(() => <OverviewTab {...defaultProps()} />);
|
||||
fireEvent.click(await screen.findByRole('button', { name: action.name }));
|
||||
|
||||
const expectActionsDisabled = () => {
|
||||
const buttons = screen.getByRole('alert').querySelectorAll('button');
|
||||
expect(buttons.length).toBe(2);
|
||||
for (const button of buttons) expect(button).toBeDisabled();
|
||||
};
|
||||
expectActionsDisabled();
|
||||
expect(getHealth).toHaveBeenCalledTimes(1);
|
||||
completeAction({ affected: 85 });
|
||||
await waitFor(() => expect(getHealth).toHaveBeenCalledTimes(2));
|
||||
expectActionsDisabled();
|
||||
expect(screen.getByRole('alert')).toBeTruthy();
|
||||
|
||||
completeHealth(healthy);
|
||||
await waitFor(() => expect(screen.queryByRole('alert')).toBeNull());
|
||||
expect(action.api).toHaveBeenCalledOnce();
|
||||
expect(notificationStore.success).toHaveBeenCalledOnce();
|
||||
expect(notificationStore.error).not.toHaveBeenCalled();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -81,6 +81,40 @@ const buildAppriseConfig = (): UIAppriseConfig => ({
|
||||
});
|
||||
|
||||
describe('useAlertDestinationsTabState', () => {
|
||||
it('preserves degraded configuration Retry health when the older mount request finishes', async () => {
|
||||
let finishMount!: (health: Awaited<ReturnType<typeof NotificationsAPI.getHealth>>) => void;
|
||||
const healthy = { queue: { status: 'healthy', attentionRequired: 0 } } as Awaited<
|
||||
ReturnType<typeof NotificationsAPI.getHealth>
|
||||
>;
|
||||
vi.mocked(NotificationsAPI.getWebhooks).mockResolvedValue([]);
|
||||
vi.mocked(NotificationsAPI.getDeliveryLog).mockResolvedValue({ entries: [] } as never);
|
||||
vi.mocked(NotificationsAPI.getHealth)
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
finishMount = resolve;
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce({ ...healthy, queue: { ...healthy.queue, status: 'degraded' } });
|
||||
const [appriseConfig, setAppriseConfig] = createSignal(buildAppriseConfig());
|
||||
const { result } = renderHook(() =>
|
||||
useAlertDestinationsTabState({
|
||||
appriseConfig,
|
||||
setAppriseConfig,
|
||||
configLoadError: () => null,
|
||||
emailConfig: () => buildEmailConfig(),
|
||||
isLoadingDestinations: () => false,
|
||||
isRetrying: () => false,
|
||||
onRetryLoad: vi.fn(),
|
||||
}),
|
||||
);
|
||||
result.handleRetry();
|
||||
await waitFor(() => expect(result.deliveryNeedsAttention()).toBe(true));
|
||||
finishMount(healthy);
|
||||
await Promise.resolve();
|
||||
expect(result.deliveryHealth()?.queue.status).toBe('degraded');
|
||||
expect(result.deliveryNeedsAttention()).toBe(true);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(AlertsAPI.getEvents).mockReset();
|
||||
vi.mocked(AlertsAPI.getEvents).mockResolvedValue([]);
|
||||
|
||||
@@ -6,7 +6,11 @@ import { NotificationsAPI } from '@/api/notifications';
|
||||
import { useNotificationDeliveryHealth } from '../useNotificationDeliveryHealth';
|
||||
|
||||
vi.mock('@/api/notifications', () => ({
|
||||
NotificationsAPI: { getHealth: vi.fn() },
|
||||
NotificationsAPI: {
|
||||
getHealth: vi.fn(),
|
||||
dismissTerminalFailures: vi.fn(),
|
||||
retryTerminalFailures: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const healthWith = (status: string) => ({ queue: { status, failed: 3, deadLetter: 1 } }) as never;
|
||||
@@ -60,4 +64,107 @@ describe('useNotificationDeliveryHealth', () => {
|
||||
expect(state.deliveryNeedsAttention()).toBe(true);
|
||||
dispose();
|
||||
}));
|
||||
it.each([
|
||||
['healthy', 'degraded'],
|
||||
['degraded', 'healthy'],
|
||||
['error', 'healthy'],
|
||||
['healthy', 'error'],
|
||||
])('ignores older %s completion after newer %s result', (older, newer) =>
|
||||
createRoot(async (dispose) => {
|
||||
let resolve!: (value: Awaited<ReturnType<typeof NotificationsAPI.getHealth>>) => void;
|
||||
let reject!: (error: Error) => void;
|
||||
vi.mocked(NotificationsAPI.getHealth).mockReturnValueOnce(
|
||||
new Promise((yes, no) => {
|
||||
resolve = yes;
|
||||
reject = no;
|
||||
}),
|
||||
);
|
||||
const state = useNotificationDeliveryHealth();
|
||||
const pending = state.loadDeliveryHealth();
|
||||
if (newer === 'error') {
|
||||
vi.mocked(NotificationsAPI.getHealth).mockRejectedValueOnce(new Error('new failure'));
|
||||
} else {
|
||||
vi.mocked(NotificationsAPI.getHealth).mockResolvedValueOnce(healthWith(newer));
|
||||
}
|
||||
await state.loadDeliveryHealth();
|
||||
if (older === 'error') reject(new Error('old failure'));
|
||||
else resolve(healthWith(older));
|
||||
await pending;
|
||||
expect(state.deliveryHealth()?.queue.status).toBe(newer === 'error' ? undefined : newer);
|
||||
expect(state.deliveryHealthUnavailable()).toBe(newer === 'error');
|
||||
expect(state.deliveryNeedsAttention()).toBe(newer !== 'healthy');
|
||||
expect(state.refreshingDeliveryHealth()).toBe(false);
|
||||
dispose();
|
||||
}),
|
||||
);
|
||||
|
||||
it.each(['healthy', 'error'])(
|
||||
'keeps loading and first-load silence when older %s finishes first',
|
||||
(older) =>
|
||||
createRoot(async (dispose) => {
|
||||
let finishOld!: () => void;
|
||||
let finishNew!: () => void;
|
||||
vi.mocked(NotificationsAPI.getHealth)
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve, reject) => {
|
||||
finishOld = () =>
|
||||
older === 'error'
|
||||
? reject(new Error('old failure'))
|
||||
: resolve(healthWith('healthy'));
|
||||
}),
|
||||
)
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
finishNew = () => resolve(healthWith('degraded'));
|
||||
}),
|
||||
);
|
||||
const state = useNotificationDeliveryHealth();
|
||||
const oldRequest = state.loadDeliveryHealth();
|
||||
const newRequest = state.loadDeliveryHealth();
|
||||
finishOld();
|
||||
await oldRequest;
|
||||
expect(state.refreshingDeliveryHealth()).toBe(true);
|
||||
expect(state.deliveryHealth()).toBeNull();
|
||||
expect(state.deliveryNeedsAttention()).toBe(false);
|
||||
finishNew();
|
||||
await newRequest;
|
||||
expect(state.refreshingDeliveryHealth()).toBe(false);
|
||||
expect(state.deliveryNeedsAttention()).toBe(true);
|
||||
dispose();
|
||||
}),
|
||||
);
|
||||
|
||||
it.each(['dismissTerminalFailures', 'retryTerminalFailures'] as const)(
|
||||
'keeps post-%s health when a pre-action request finishes late',
|
||||
(action) =>
|
||||
createRoot(async (dispose) => {
|
||||
const confirmSpy = vi.spyOn(window, 'confirm').mockReturnValue(true);
|
||||
try {
|
||||
const state = useNotificationDeliveryHealth();
|
||||
vi.mocked(NotificationsAPI.getHealth).mockResolvedValueOnce({
|
||||
queue: { status: 'degraded', attentionRequired: 2 },
|
||||
} as never);
|
||||
await state.loadDeliveryHealth();
|
||||
let finishOld!: (health: Awaited<ReturnType<typeof NotificationsAPI.getHealth>>) => void;
|
||||
vi.mocked(NotificationsAPI.getHealth)
|
||||
.mockReturnValueOnce(
|
||||
new Promise((resolve) => {
|
||||
finishOld = resolve;
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(healthWith('healthy'));
|
||||
const pending = state.loadDeliveryHealth();
|
||||
vi.mocked(NotificationsAPI[action]).mockResolvedValueOnce({ affected: 2 } as never);
|
||||
await state[action]();
|
||||
expect(state.deliveryNeedsAttention()).toBe(false);
|
||||
finishOld(healthWith('degraded'));
|
||||
await pending;
|
||||
expect(state.deliveryHealth()?.queue.status).toBe('healthy');
|
||||
expect(state.deliveryNeedsAttention()).toBe(false);
|
||||
} finally {
|
||||
confirmSpy.mockRestore();
|
||||
dispose();
|
||||
}
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -22,19 +22,27 @@ export function useNotificationDeliveryHealth(options?: {
|
||||
const [refreshingDeliveryHealth, setRefreshingDeliveryHealth] = createSignal(false);
|
||||
const [loadedOnce, setLoadedOnce] = createSignal(false);
|
||||
|
||||
// Mount, configuration retry and queue actions can overlap. Only the latest
|
||||
// requested snapshot owns health and loading state, regardless of completion order.
|
||||
let latestHealthRequest = 0;
|
||||
const loadDeliveryHealth = async () => {
|
||||
const request = ++latestHealthRequest;
|
||||
setRefreshingDeliveryHealth(true);
|
||||
try {
|
||||
const health = await NotificationsAPI.getHealth();
|
||||
if (request !== latestHealthRequest) return;
|
||||
setDeliveryHealth(health);
|
||||
setDeliveryHealthUnavailable(health.queue.status === 'unavailable');
|
||||
} catch (error) {
|
||||
if (request !== latestHealthRequest) return;
|
||||
logger.error('Failed to load notification delivery health', error);
|
||||
setDeliveryHealth(null);
|
||||
setDeliveryHealthUnavailable(true);
|
||||
} finally {
|
||||
setLoadedOnce(true);
|
||||
setRefreshingDeliveryHealth(false);
|
||||
if (request === latestHealthRequest) {
|
||||
setLoadedOnce(true);
|
||||
setRefreshingDeliveryHealth(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
)
|
||||
@@ -165,3 +167,115 @@ func TestStatefulAlertReFireCooldown(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A UI-keyed datastore override must govern actual incidents, not just the
|
||||
// threshold resolver. Exercise hysteresis and recurrence across SQLite reopen,
|
||||
// including a same-named datastore on another PBS instance.
|
||||
func TestPBSDatastoreOverrideLifecycleAcrossRestart(t *testing.T) {
|
||||
for _, legacySnapshot := range []bool{false, true} {
|
||||
name := "persisted aliases"
|
||||
if legacySnapshot {
|
||||
name = "legacy snapshot"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
start := func() *Manager {
|
||||
m := NewManagerWithDataDir(dir)
|
||||
t.Cleanup(m.Stop)
|
||||
m.EnableEventLog()
|
||||
if !m.activeStateAuthoritative.Load() {
|
||||
t.Fatal("SQLite active state is not authoritative")
|
||||
}
|
||||
m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive,
|
||||
StorageDefault: HysteresisThreshold{Trigger: 95, Clear: 90},
|
||||
Overrides: map[string]ThresholdConfig{"pbs-primary/backups": {Usage: &HysteresisThreshold{Trigger: 80, Clear: 70}}},
|
||||
})
|
||||
disableTestTimeThresholds(m)
|
||||
return m
|
||||
}
|
||||
storage := func(instance string, usage float64) models.Storage {
|
||||
return models.Storage{ID: instance + "-backups", AliasIDs: []string{instance + "/backups"}, Name: "backups", Instance: instance, Type: "pbs", Status: "online", Total: 1000, Used: int64(usage * 10), Free: int64(1000 - usage*10), Usage: usage}
|
||||
}
|
||||
observe := func(m *Manager, usage float64) {
|
||||
t.Helper()
|
||||
for range 5 {
|
||||
m.CheckStorage(storage("pbs-primary", usage))
|
||||
m.CheckStorage(storage("pbs-secondary", usage))
|
||||
}
|
||||
if testHasActiveAlert(t, m, canonicalMetricStateID("pbs-secondary-backups", "usage")) {
|
||||
t.Fatal("override leaked to another PBS instance")
|
||||
}
|
||||
}
|
||||
events := func(m *Manager, fired, resolved int) {
|
||||
t.Helper()
|
||||
for kind, want := range map[string]int{eventlog.TypeFired: fired, eventlog.TypeResolved: resolved} {
|
||||
if got := len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{kind}})); got != want {
|
||||
t.Fatalf("%s events = %d, want %d", kind, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
id := canonicalMetricStateID("pbs-primary-backups", "usage")
|
||||
m := start()
|
||||
observe(m, 85)
|
||||
original := *testRequireActiveAlert(t, m, id)
|
||||
events(m, 1, 0)
|
||||
if legacySnapshot {
|
||||
m.mu.Lock()
|
||||
delete(m.activeAlerts[id].Metadata, storagePolicyAliasesKey)
|
||||
m.mu.Unlock()
|
||||
if err := m.SaveActiveAlerts(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
m.Stop()
|
||||
|
||||
m = start()
|
||||
observe(m, 75) // Below trigger, but not below the override's clear threshold.
|
||||
if got := testRequireActiveAlert(t, m, id); !got.StartTime.Equal(original.StartTime) {
|
||||
t.Fatal("restart replaced the firing incident")
|
||||
}
|
||||
events(m, 1, 0)
|
||||
observe(m, 65)
|
||||
if testHasActiveAlert(t, m, id) {
|
||||
t.Fatal("override recovery did not clear incident")
|
||||
}
|
||||
events(m, 1, 1)
|
||||
m.Stop()
|
||||
|
||||
m = start()
|
||||
observe(m, 75)
|
||||
if testHasActiveAlert(t, m, id) {
|
||||
t.Fatal("resolved incident resurrected inside hysteresis band")
|
||||
}
|
||||
events(m, 1, 1)
|
||||
observe(m, 85)
|
||||
if got := testRequireActiveAlert(t, m, id); !got.StartTime.After(original.StartTime) {
|
||||
t.Fatal("refire reused original incident start")
|
||||
}
|
||||
events(m, 2, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoragePolicyAliasesLegacyIdentity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, instance, resource, datastore string
|
||||
want bool
|
||||
}{
|
||||
{"hyphenated names", "pbs-backup-east", "pbs-backup-east-daily-store", "daily-store", true},
|
||||
{"different instance", "pbs-backup-west", "pbs-backup-east-daily-store", "daily-store", false},
|
||||
{"not PBS", "backup-east", "backup-east-daily-store", "daily-store", false},
|
||||
{"missing datastore", "pbs-backup-east", "pbs-backup-east-", "", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := storagePolicyAliases(&Alert{Instance: tc.instance, ResourceID: tc.resource, ResourceName: tc.datastore})
|
||||
if tc.want {
|
||||
if len(got) != 1 || got[0] != tc.instance+"/"+tc.datastore {
|
||||
t.Fatalf("aliases = %v", got)
|
||||
}
|
||||
} else if len(got) != 0 {
|
||||
t.Fatalf("invented alias: %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -215,12 +215,13 @@ var capacityForecastMetadataKeys = []string{
|
||||
|
||||
func (m *Manager) evaluateStorageCapacity(storage models.Storage, thresholds ThresholdConfig, trend CapacityTrendObservation) {
|
||||
input := &UnifiedResourceInput{
|
||||
ID: storage.ID,
|
||||
Type: "storage",
|
||||
Name: storage.Name,
|
||||
Node: storage.Node,
|
||||
Instance: storage.Instance,
|
||||
Disk: &UnifiedResourceMetric{Percent: storage.Usage},
|
||||
ID: storage.ID,
|
||||
StorageAliases: storage.AliasIDs,
|
||||
Type: "storage",
|
||||
Name: storage.Name,
|
||||
Node: storage.Node,
|
||||
Instance: storage.Instance,
|
||||
Disk: &UnifiedResourceMetric{Percent: storage.Usage},
|
||||
}
|
||||
m.evaluateUnifiedCapacity(input, thresholds, trend, func() bool {
|
||||
if !m.config.Enabled {
|
||||
@@ -377,6 +378,9 @@ func (m *Manager) evaluateCapacityForecast(input *UnifiedResourceInput, threshol
|
||||
"forecastBucketCount": trend.BucketCount,
|
||||
"forecastCoverageSeconds": int64(trend.CoverageSpan / time.Second),
|
||||
}
|
||||
if len(input.StorageAliases) > 0 {
|
||||
metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...)
|
||||
}
|
||||
_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
|
||||
Spec: spec,
|
||||
Evidence: alertspecs.AlertEvidence{
|
||||
|
||||
@@ -503,7 +503,7 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
continue
|
||||
}
|
||||
thresholds := m.resolveResourceThresholds("storage", resourceID)
|
||||
thresholds := m.effectiveAlertPolicyNoLock(alertPolicyQuery{TypeKey: "storage", ResourceID: resourceID, StorageAliases: storagePolicyAliases(alert)}).Thresholds
|
||||
if thresholds.Disabled {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
||||
// Unlike an orderly restart, process exit must not get a shutdown checkpoint
|
||||
// or store Close to rescue a lifecycle transition. Keep the recovery mirror
|
||||
// deliberately stale to prove SQLite, not the mirror, preserves the incident.
|
||||
// This tests process interruption, not power loss or installed delivery.
|
||||
func TestStorageLifecycleAcrossProcessExit(t *testing.T) {
|
||||
const helperEnv = "PULSE_TEST_STORAGE_CRASH_PHASE"
|
||||
storage := models.Storage{ID: "pbs-crash-backups", Name: "backups", Type: "pbs", Status: "online", Total: 1000, Used: 850, Free: 150, Usage: 85}
|
||||
id := canonicalMetricStateID(storage.ID, "usage")
|
||||
start := func(t *testing.T, dir string) *Manager {
|
||||
t.Helper()
|
||||
m := NewManagerWithDataDir(dir)
|
||||
m.EnableEventLog()
|
||||
if !m.activeStateAuthoritative.Load() {
|
||||
t.Fatal("SQLite authority not enabled")
|
||||
}
|
||||
m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive,
|
||||
TimeThresholds: map[string]int{"storage": 0}, StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 70}})
|
||||
return m
|
||||
}
|
||||
observe := func(m *Manager, s models.Storage) {
|
||||
for range 5 {
|
||||
m.CheckStorage(s)
|
||||
}
|
||||
}
|
||||
if phase := os.Getenv(helperEnv); phase != "" {
|
||||
if phase != "fired" && phase != "resolved" {
|
||||
t.Fatal("invalid child phase")
|
||||
}
|
||||
m := start(t, os.Getenv("PULSE_TEST_STORAGE_CRASH_DIR"))
|
||||
if phase == "resolved" {
|
||||
observe(m, storage)
|
||||
testRequireActiveAlert(t, m, id)
|
||||
}
|
||||
if err := m.SaveActiveAlerts(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Freeze checkpoints, leaving either an empty or a firing JSON mirror.
|
||||
// Lifecycle commits must remain independent of periodic/async saves.
|
||||
m.saveMu.Lock()
|
||||
if phase == "resolved" {
|
||||
storage.Used, storage.Free, storage.Usage = 0, 1000, 0
|
||||
}
|
||||
observe(m, storage)
|
||||
if testHasActiveAlert(t, m, id) != (phase == "fired") {
|
||||
t.Fatal("child did not reach expected lifecycle state")
|
||||
}
|
||||
os.Exit(0) // Intentionally bypass all cleanups, Stop and store Close.
|
||||
}
|
||||
|
||||
for _, phase := range []string{"fired", "resolved"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStorageLifecycleAcrossProcessExit$")
|
||||
cmd.Env = append(os.Environ(), helperEnv+"="+phase, "PULSE_TEST_STORAGE_CRASH_DIR="+dir)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("child failed: %v\n%s", err, output)
|
||||
}
|
||||
mirror, err := os.ReadFile(filepath.Join(dir, "alerts", "active-alerts.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stale []*Alert
|
||||
if err := json.Unmarshal(mirror, &stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantStale := 0
|
||||
if phase == "resolved" {
|
||||
wantStale = 1
|
||||
}
|
||||
if len(stale) != wantStale {
|
||||
t.Fatalf("recovery mirror was not stale: %s", mirror)
|
||||
}
|
||||
m := start(t, dir)
|
||||
t.Cleanup(m.Stop)
|
||||
if testHasActiveAlert(t, m, id) != (phase == "fired") {
|
||||
t.Fatal("process exit lost firing incident or resurrected resolved incident")
|
||||
}
|
||||
events, err := m.AlertEvents(eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 1
|
||||
if phase == "resolved" {
|
||||
want = 2
|
||||
}
|
||||
counts := make(map[string]int)
|
||||
for _, event := range events {
|
||||
counts[event.Type]++
|
||||
}
|
||||
if counts[eventlog.TypeFired] != 1 || counts[eventlog.TypeResolved] != want-1 {
|
||||
t.Fatalf("unexpected lifecycle history: %v", counts)
|
||||
}
|
||||
if len(events) != want {
|
||||
t.Fatalf("durable lifecycle events = %d, want %d", len(events), want)
|
||||
}
|
||||
missing := storage
|
||||
missing.Total, missing.Used, missing.Free, missing.Usage = 0, 0, 0, 0
|
||||
observe(m, missing)
|
||||
if testHasActiveAlert(t, m, id) != (phase == "fired") {
|
||||
t.Fatal("missing post-restart counters changed the incident state")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -67,3 +67,32 @@ func (m *Manager) resolveStorageThresholdOverride(base ThresholdConfig, resource
|
||||
func (m *Manager) resolveStorageThresholdsNoLock(storage models.Storage) ThresholdConfig {
|
||||
return m.resolveStorageThresholdOverride(m.defaultThresholdsForResourceType("storage"), storage.ID, storage.AliasIDs)
|
||||
}
|
||||
|
||||
// Keep the polling identity through JSON/SQLite restore so configuration saves
|
||||
// cannot silently replace a datastore override with global storage defaults.
|
||||
const storagePolicyAliasesKey = "storagePolicyAliases"
|
||||
|
||||
func storagePolicyAliases(alert *Alert) []string {
|
||||
if alert == nil {
|
||||
return nil
|
||||
}
|
||||
switch aliases := alert.Metadata[storagePolicyAliasesKey].(type) {
|
||||
case []string:
|
||||
return append([]string(nil), aliases...)
|
||||
case []interface{}:
|
||||
result := make([]string, 0, len(aliases))
|
||||
for _, value := range aliases {
|
||||
if alias, ok := value.(string); ok {
|
||||
result = append(result, alias)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
// Older PBS snapshots predate alias metadata. The poller records the
|
||||
// instance and datastore name separately: only reconstruct an alias when
|
||||
// the complete legacy ID agrees, never split hyphenated names heuristically.
|
||||
if strings.HasPrefix(alert.Instance, "pbs-") && alert.ResourceName != "" && alert.ResourceID == alert.Instance+"-"+alert.ResourceName {
|
||||
return []string{alert.Instance + "/" + alert.ResourceName}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,7 +8,8 @@ import (
|
||||
)
|
||||
|
||||
func TestGetTimeThresholdMappings(t *testing.T) {
|
||||
manager := NewManager()
|
||||
manager := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
manager.mu.Lock()
|
||||
manager.config.TimeThresholds = map[string]int{
|
||||
@@ -42,7 +43,8 @@ func TestGetTimeThresholdMappings(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetTimeThresholdMetricOverrides(t *testing.T) {
|
||||
manager := NewManager()
|
||||
manager := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
manager.mu.Lock()
|
||||
manager.config.TimeThresholds = map[string]int{
|
||||
@@ -139,7 +141,8 @@ func TestCheckMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckMetricUsesPendingStartTime(t *testing.T) {
|
||||
manager := NewManager()
|
||||
manager := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
manager.mu.Lock()
|
||||
manager.config.TimeThresholds["guest"] = 2
|
||||
|
||||
@@ -55,6 +55,8 @@ type UnifiedResourceInput struct {
|
||||
NetworkIn *UnifiedResourceMetric
|
||||
NetworkOut *UnifiedResourceMetric
|
||||
Temperature *UnifiedResourceMetric
|
||||
|
||||
StorageAliases []string // Durable policy lookup identities for storage alerts.
|
||||
}
|
||||
|
||||
type unifiedMetricCandidate struct {
|
||||
@@ -259,6 +261,19 @@ func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds
|
||||
}
|
||||
|
||||
opts = metricOptionsWithTags(opts, input.Tags)
|
||||
if len(input.StorageAliases) > 0 {
|
||||
merged := metricOptions{}
|
||||
if opts != nil {
|
||||
merged = *opts
|
||||
}
|
||||
metadata := make(map[string]interface{}, len(merged.Metadata)+1)
|
||||
for k, v := range merged.Metadata {
|
||||
metadata[k] = v
|
||||
}
|
||||
metadata[storagePolicyAliasesKey] = append([]string(nil), input.StorageAliases...)
|
||||
merged.Metadata = metadata
|
||||
opts = &merged
|
||||
}
|
||||
for _, candidate := range buildUnifiedMetricCandidates(input, thresholds) {
|
||||
m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)
|
||||
}
|
||||
|
||||
@@ -120,7 +120,7 @@ func (f *fakeDockerActionAgentCommander) ExecuteDockerContainerUpdate(_ context.
|
||||
OldImageDigest: "sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
NewImageDigest: "sha256:2222222222222222222222222222222222222222222222222222222222222222",
|
||||
BackupCreated: true, BackupContainer: "api_pulse_backup_20260714_000000",
|
||||
After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: newID, State: "running", Running: true, StartedAt: now, ObservedAt: now},
|
||||
After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: newID, State: "running", Running: true, Health: agentexec.DockerContainerHealthNone, StartedAt: now, ObservedAt: now},
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -115,7 +115,15 @@ func dockerContainerUpdateExecutionResult(resourceID, agentID string, facts agen
|
||||
}
|
||||
status := unified.ActionVerificationContradicted
|
||||
reason := "postcondition_contradicted"
|
||||
if dockerUpdateFactsMatch(facts, facts.After.ContainerID) {
|
||||
// A running replacement must also have a usable running/health
|
||||
// readback; identity alone must not confirm an unhealthy update.
|
||||
// Preserve updates of containers intentionally left stopped.
|
||||
if facts.After.Running && !agentexec.IsDockerContainerHealth(facts.After.Health) {
|
||||
status = unified.ActionVerificationInconclusive
|
||||
reason = "container_health_unknown"
|
||||
} else if dockerUpdateFactsMatch(facts, facts.After.ContainerID) &&
|
||||
(!facts.After.Running || (facts.After.State == "running" &&
|
||||
agentexec.DockerContainerHealthAllowsVerifiedRunningState(facts.After.Health))) {
|
||||
status = unified.ActionVerificationConfirmed
|
||||
reason = ""
|
||||
}
|
||||
@@ -145,7 +153,18 @@ func dockerContainerUpdateExecutionResult(resourceID, agentID string, facts agen
|
||||
}
|
||||
status := unified.ActionVerificationContradicted
|
||||
reason := "postcondition_contradicted"
|
||||
if dockerUpdateFactsMatch(facts, independent.Snapshot.ContainerID) {
|
||||
// Identity is necessary but not sufficient: a replacement can exit or
|
||||
// become unhealthy after the agent reported success. Compare against
|
||||
// its readback rather than requiring running unconditionally, because
|
||||
// updates deliberately preserve stopped containers.
|
||||
if !facts.ReadbackRan || facts.After.State == "" {
|
||||
status = unified.ActionVerificationInconclusive
|
||||
reason = "agent_readback_unavailable"
|
||||
} else if dockerUpdateFactsMatch(facts, independent.Snapshot.ContainerID) &&
|
||||
independent.Snapshot.State == facts.After.State &&
|
||||
independent.Snapshot.Running == facts.After.Running &&
|
||||
(!independent.Snapshot.Running || (independent.Snapshot.State == "running" &&
|
||||
agentexec.DockerContainerHealthAllowsVerifiedRunningState(independent.Snapshot.Health))) {
|
||||
status = unified.ActionVerificationConfirmed
|
||||
reason = ""
|
||||
}
|
||||
@@ -254,8 +273,8 @@ func dockerActionEvidenceTimes(observedAt, receivedAt time.Time) (time.Time, tim
|
||||
}
|
||||
|
||||
// dockerUpdateFactsMatch confirms the observed container is the replacement
|
||||
// the agent claims to have created. A stopped original is recreated without
|
||||
// being started, so run-state is not part of the postcondition; identity is.
|
||||
// the agent claims to have created. Independent verification additionally
|
||||
// compares its state with the agent readback, preserving stopped updates.
|
||||
func dockerUpdateFactsMatch(facts agentexec.DockerContainerUpdateResultPayload, observedContainerID string) bool {
|
||||
if facts.NewContainerID == "" || observedContainerID == "" {
|
||||
return false
|
||||
|
||||
@@ -106,7 +106,7 @@ func TestDockerContainerUpdateExecutionResultClampsBoundedPositiveAgentClockSkew
|
||||
facts := agentexec.DockerContainerUpdateResultPayload{
|
||||
Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
|
||||
MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID,
|
||||
After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, ObservedAt: observedAt},
|
||||
After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, Health: agentexec.DockerContainerHealthHealthy, ObservedAt: observedAt},
|
||||
}
|
||||
result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, receivedAt)
|
||||
if err != nil {
|
||||
@@ -198,3 +198,84 @@ func dockerResultFacts(now time.Time, started, completed, readback, matches bool
|
||||
}
|
||||
|
||||
const dockerLifecycleTestID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
// A replacement identity alone is not proof that its reported running state
|
||||
// survived until the independent daemon readback.
|
||||
func TestDockerContainerUpdateIndependentObservationMustMatchState(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
for _, tc := range []struct {
|
||||
name, baseline, state, health string
|
||||
running bool
|
||||
want unified.ActionVerificationStatus
|
||||
}{
|
||||
{"running", "running", "running", "healthy", true, unified.ActionVerificationConfirmed},
|
||||
{"no healthcheck", "running", "running", "none", true, unified.ActionVerificationConfirmed},
|
||||
{"stopped original", "created", "created", "none", false, unified.ActionVerificationConfirmed},
|
||||
{"stopped after update", "running", "exited", "", false, unified.ActionVerificationContradicted},
|
||||
{"restarting", "running", "restarting", "", true, unified.ActionVerificationContradicted},
|
||||
{"unhealthy", "running", "running", "unhealthy", true, unified.ActionVerificationContradicted},
|
||||
{"starting healthcheck", "running", "running", "starting", true, unified.ActionVerificationContradicted},
|
||||
{"unknown health", "running", "running", "", true, unified.ActionVerificationContradicted},
|
||||
{"missing agent readback", "", "running", "healthy", true, unified.ActionVerificationInconclusive},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
facts := agentexec.DockerContainerUpdateResultPayload{
|
||||
Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
|
||||
MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID,
|
||||
After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, ObservedAt: now},
|
||||
}
|
||||
facts.After.State = tc.baseline
|
||||
facts.After.Running = tc.baseline == "running"
|
||||
facts.ReadbackRan = tc.baseline != ""
|
||||
observation := &dockerContainerPostconditionObservation{
|
||||
ObserverID: "daemon-1", TrustDomain: "daemon:1", Method: "daemon_inspect", ReceivedAt: now,
|
||||
Snapshot: agentexec.DockerContainerObservationSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now},
|
||||
}
|
||||
result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, observation, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := result.ActionResultV2.Verification; got.Status != tc.want || got.EvidenceClass != unified.ActionEvidenceIndependent {
|
||||
t.Fatalf("verification = %+v, want %s / independent", got, tc.want)
|
||||
}
|
||||
if result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded {
|
||||
t.Fatal("readback changed execution history")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
for _, tc := range []struct {
|
||||
name, state, health string
|
||||
running bool
|
||||
want unified.ActionVerificationStatus
|
||||
}{
|
||||
{"healthy", "running", "healthy", true, unified.ActionVerificationConfirmed},
|
||||
{"no healthcheck", "running", "none", true, unified.ActionVerificationConfirmed},
|
||||
{"stopped preserved", "created", "none", false, unified.ActionVerificationConfirmed},
|
||||
{"unhealthy", "running", "unhealthy", true, unified.ActionVerificationContradicted},
|
||||
{"starting", "running", "starting", true, unified.ActionVerificationContradicted},
|
||||
{"restarting", "restarting", "healthy", true, unified.ActionVerificationContradicted},
|
||||
{"unknown health", "running", "", true, unified.ActionVerificationInconclusive},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
facts := agentexec.DockerContainerUpdateResultPayload{
|
||||
Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
|
||||
MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID,
|
||||
After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now},
|
||||
}
|
||||
result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := result.ActionResultV2.Verification; got.Status != tc.want || got.EvidenceClass != unified.ActionEvidenceAgentAttested {
|
||||
t.Fatalf("verification = %+v, want %s / agent_attested", got, tc.want)
|
||||
}
|
||||
if result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded {
|
||||
t.Fatal("readback changed execution history")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3868,7 +3868,7 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Str("type", baseType).
|
||||
Str("event", string(event)).
|
||||
Msg("skipping queued email notification because email delivery is disabled")
|
||||
return nil
|
||||
return ErrNotificationDeliverySkipped
|
||||
}
|
||||
deliveredJob = notificationDeliveryJob{
|
||||
Type: "email",
|
||||
@@ -3891,7 +3891,7 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Str("event", string(event)).
|
||||
Str("webhookID", webhookConfig.ID).
|
||||
Msg("skipping queued webhook notification because delivery is disabled")
|
||||
return nil
|
||||
return ErrNotificationDeliverySkipped
|
||||
}
|
||||
deliveredJob = notificationDeliveryJob{
|
||||
Type: "webhook",
|
||||
@@ -3914,7 +3914,7 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Str("type", baseType).
|
||||
Str("event", string(event)).
|
||||
Msg("skipping queued Apprise notification because delivery is disabled")
|
||||
return nil
|
||||
return ErrNotificationDeliverySkipped
|
||||
}
|
||||
deliveredJob = notificationDeliveryJob{
|
||||
Type: "apprise",
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -3367,8 +3368,8 @@ func TestProcessQueuedNotification_SkipsDisabledEmailDelivery(t *testing.T) {
|
||||
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
||||
}
|
||||
|
||||
if err := nm.ProcessQueuedNotification(notif); err != nil {
|
||||
t.Fatalf("expected queued email notification to be skipped without error, got %v", err)
|
||||
if err := nm.ProcessQueuedNotification(notif); !errors.Is(err, ErrNotificationDeliverySkipped) {
|
||||
t.Fatalf("expected queued email notification to report a policy skip, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3414,8 +3415,8 @@ func TestProcessQueuedNotification_SkipsWhenNotificationsDisabled(t *testing.T)
|
||||
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
||||
}
|
||||
|
||||
if err := nm.ProcessQueuedNotification(notif); err != nil {
|
||||
t.Fatalf("expected queued webhook notification to be skipped without error, got %v", err)
|
||||
if err := nm.ProcessQueuedNotification(notif); !errors.Is(err, ErrNotificationDeliverySkipped) {
|
||||
t.Fatalf("expected queued webhook notification to report a policy skip, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
)
|
||||
|
||||
// Provider rejection must preserve the firing receipt needed for recovery,
|
||||
// including when an operator retries the recovery after reopening SQLite.
|
||||
func TestQueuedNtfyRecoveryAfterProviderOutageAndRestart(t *testing.T) {
|
||||
var unavailable atomic.Bool
|
||||
var accepted atomic.Int32
|
||||
server := newIPv4HTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
t.Errorf("read body: %v", err)
|
||||
}
|
||||
if unavailable.Load() {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if accepted.Add(1) == 2 {
|
||||
if r.Header.Get("Priority") != "default" || r.Header.Get("Title") != "RESOLVED: database" ||
|
||||
!strings.Contains(string(body), "is now healthy") {
|
||||
t.Errorf("incorrect recovery: headers=%v body=%q", r.Header, body)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}))
|
||||
defer server.Close()
|
||||
dir := t.TempDir()
|
||||
webhook := WebhookConfig{ID: "ops", Name: "ops", URL: server.URL + "/topic", Enabled: true, Service: "ntfy"}
|
||||
open := func() *NotificationManager {
|
||||
m := NewNotificationManagerWithDataDir("", dir)
|
||||
m.webhookClient = server.Client()
|
||||
if err := m.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddWebhook(webhook)
|
||||
t.Cleanup(m.Stop)
|
||||
return m
|
||||
}
|
||||
m := open()
|
||||
config, err := json.Marshal(webhook)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
alert := &alerts.Alert{ID: "cpu", ResourceName: "database", Type: "cpu", Level: alerts.AlertLevelCritical,
|
||||
Message: "CPU above threshold", StartTime: time.Now().Add(-time.Minute)}
|
||||
resolved := notificationDeliveryJob{Type: "webhook", Event: eventResolved, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook}
|
||||
wait := func(m *NotificationManager, id string, want NotificationQueueStatus, wantAudits int) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for {
|
||||
var status string
|
||||
var audits int
|
||||
if err := m.queue.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", id).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = ?", id).Scan(&audits); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status == string(want) && audits == wantAudits {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s: status=%s audits=%d, want %s with audit", id, status, audits, want)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
enqueue := func(id, kind string, payload *alerts.Alert) {
|
||||
t.Helper()
|
||||
if err := m.queue.Enqueue(&QueuedNotification{ID: id, Type: kind, Status: QueueStatusPending,
|
||||
Config: config, Alerts: []*alerts.Alert{payload}, MaxAttempts: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
enqueue("firing", "webhook", alert.Clone())
|
||||
wait(m, "firing", QueueStatusSent, 1)
|
||||
unavailable.Store(true)
|
||||
recovery := alert.Clone()
|
||||
annotateResolvedMetadata(recovery, time.Now())
|
||||
enqueue("recovery", "webhook_resolved", recovery)
|
||||
wait(m, "recovery", QueueStatusDLQ, 1)
|
||||
if got := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 1 {
|
||||
t.Fatal("failed recovery consumed the firing receipt")
|
||||
}
|
||||
m.Stop()
|
||||
m = open()
|
||||
if got := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 1 {
|
||||
t.Fatal("restart lost the firing receipt")
|
||||
}
|
||||
unavailable.Store(false)
|
||||
if count, err := m.queue.RetryTerminalFailures(); err != nil || count != 1 {
|
||||
t.Fatalf("retry = %d, %v; want one recovery", count, err)
|
||||
}
|
||||
wait(m, "recovery", QueueStatusSent, 2)
|
||||
if got := m.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 0 {
|
||||
t.Fatal("successful recovery did not consume the firing receipt")
|
||||
}
|
||||
if got := accepted.Load(); got != 2 {
|
||||
t.Fatalf("accepted HTTP requests = %d, want firing and recovery only", got)
|
||||
}
|
||||
var failures, successes int
|
||||
if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = 'recovery' AND success = 0").Scan(&failures); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.queue.db.QueryRow("SELECT count(*) FROM notification_audit WHERE notification_id = 'recovery' AND success = 1").Scan(&successes); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failures != 1 || successes != 1 {
|
||||
t.Fatalf("recovery audit failures=%d successes=%d, want 1 each", failures, successes)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
@@ -14,6 +15,16 @@ import (
|
||||
// Exercise the shared destination across transitions: generated firing headers
|
||||
// must not leak into the stored configuration and override recovery metadata.
|
||||
func TestNtfySeverityRecoveryTransition(t *testing.T) {
|
||||
for _, queued := range []bool{false, true} {
|
||||
name := "direct"
|
||||
if queued {
|
||||
name = "queued"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) { testNtfySeverityRecoveryTransition(t, queued) })
|
||||
}
|
||||
}
|
||||
|
||||
func testNtfySeverityRecoveryTransition(t *testing.T, queued bool) {
|
||||
type receipt struct {
|
||||
header http.Header
|
||||
body string
|
||||
@@ -28,7 +39,7 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) {
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}))
|
||||
defer server.Close()
|
||||
manager := NewNotificationManager("")
|
||||
manager := NewNotificationManagerWithDataDir("", t.TempDir())
|
||||
defer manager.Stop()
|
||||
manager.webhookClient = server.Client()
|
||||
if err := manager.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
|
||||
@@ -36,6 +47,7 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) {
|
||||
}
|
||||
webhook := WebhookConfig{Name: "transition", URL: server.URL + "/topic", Enabled: true, Service: "ntfy",
|
||||
Headers: map[string]string{"X-Static": "preserved"}}
|
||||
manager.AddWebhook(webhook)
|
||||
originalHeaders := map[string]string{"X-Static": "preserved"}
|
||||
alert := &alerts.Alert{ID: "transition", Type: "cpu", ResourceID: "vm-1", ResourceName: "database", Node: "node-a",
|
||||
Message: "CPU above threshold", Value: 99, Threshold: 90, StartTime: time.Now().Add(-time.Minute)}
|
||||
@@ -54,7 +66,25 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) {
|
||||
alert.Level = step.level
|
||||
before := *alert
|
||||
var err error
|
||||
if step.resolved {
|
||||
if queued {
|
||||
config, marshalErr := json.Marshal(webhook)
|
||||
if marshalErr != nil {
|
||||
t.Fatal(marshalErr)
|
||||
}
|
||||
kind := "webhook"
|
||||
payload := alert.Clone()
|
||||
if step.resolved {
|
||||
kind += "_resolved"
|
||||
annotateResolvedMetadata(payload, time.Now())
|
||||
}
|
||||
if manager.queue == nil {
|
||||
t.Fatal("notification queue unavailable")
|
||||
}
|
||||
err = manager.queue.Enqueue(&QueuedNotification{
|
||||
ID: step.name, Type: kind, Status: QueueStatusPending,
|
||||
Config: config, Alerts: []*alerts.Alert{payload}, MaxAttempts: 1,
|
||||
})
|
||||
} else if step.resolved {
|
||||
err = manager.sendResolvedWebhook(webhook, []*alerts.Alert{alert}, time.Now())
|
||||
} else {
|
||||
err = manager.sendGroupedWebhook(webhook, []*alerts.Alert{alert})
|
||||
@@ -86,6 +116,24 @@ func TestNtfySeverityRecoveryTransition(t *testing.T) {
|
||||
if !reflect.DeepEqual(*alert, before) {
|
||||
t.Error("source alert mutated")
|
||||
}
|
||||
if queued {
|
||||
// Receipt precedes the queue commit. Wait for completion before
|
||||
// advancing the same alert identity to its next lifecycle state.
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
var status string
|
||||
if err := manager.queue.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", step.name).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status == string(QueueStatusSent) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("received HTTP request but queue status remained %s", status)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package notifications
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -19,6 +20,10 @@ import (
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// ErrNotificationDeliverySkipped distinguishes a policy cancellation from a
|
||||
// successful provider delivery. Queue processors must not report skips as nil.
|
||||
var ErrNotificationDeliverySkipped = errors.New("notification delivery disabled")
|
||||
|
||||
// defaultQueueMaxAttempts is the default number of delivery attempts
|
||||
// before a notification is moved to the dead-letter queue. With the
|
||||
// exponential backoff schedule (1s doubling, capped at 60s) eight attempts
|
||||
@@ -1205,7 +1210,8 @@ func (nq *NotificationQueue) scanNotification(rows *sql.Rows) (*QueuedNotificati
|
||||
return ¬if, nil
|
||||
}
|
||||
|
||||
// ScheduleRetry schedules a notification for retry with exponential backoff
|
||||
// ScheduleRetry schedules a notification for retry with exponential backoff.
|
||||
// Cancelled and delivered rows are final, even for stale operator requests.
|
||||
func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {
|
||||
backoff := calculateBackoff(attempt)
|
||||
nextRetry := time.Now().Add(backoff)
|
||||
@@ -1231,10 +1237,10 @@ func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {
|
||||
UPDATE notification_queue
|
||||
SET status = 'pending', next_retry_at = ?, last_attempt = ?,
|
||||
operational_links = ?, completed_at = NULL, last_error = NULL
|
||||
WHERE id = ?
|
||||
WHERE id = ? AND status IN ('pending', 'sending', 'failed', 'dlq')
|
||||
`
|
||||
|
||||
_, err = nq.db.Exec(
|
||||
result, err := nq.db.Exec(
|
||||
query,
|
||||
nextRetry.Unix(),
|
||||
time.Now().Unix(),
|
||||
@@ -1245,7 +1251,14 @@ func (nq *NotificationQueue) ScheduleRetry(id string, attempt int) error {
|
||||
nq.mu.Unlock()
|
||||
return fmt.Errorf("failed to schedule retry: %w", err)
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
nq.mu.Unlock()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read scheduled retry result: %w", err)
|
||||
}
|
||||
if affected == 0 {
|
||||
return fmt.Errorf("notification %s is no longer eligible for retry", id)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("id", id).
|
||||
@@ -1729,7 +1742,13 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
|
||||
return
|
||||
}
|
||||
releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiersFromAlerts(notif.Alerts), false)
|
||||
defer releaseDeliveryGates()
|
||||
healthChanged := false
|
||||
defer func() {
|
||||
releaseDeliveryGates()
|
||||
if healthChanged {
|
||||
nq.notifyDeliveryHealthChanged()
|
||||
}
|
||||
}()
|
||||
|
||||
// Atomically claim the pending row. A concurrent resolution may have
|
||||
// cancelled it while it was waiting for its per-alert delivery gate.
|
||||
@@ -1765,6 +1784,21 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
|
||||
|
||||
err = processor(notif)
|
||||
|
||||
if errors.Is(err, ErrNotificationDeliverySkipped) {
|
||||
// No provider attempt occurred. Preserve the cancellation and its reason,
|
||||
// but do not manufacture a successful (or failed) delivery audit entry.
|
||||
// Reconcile health only after releasing the per-alert gate.
|
||||
nq.mu.Lock()
|
||||
cancelErr := nq.updateNotificationStatusNoLock(notif.ID, QueueStatusCancelled, err.Error(), time.Now())
|
||||
nq.mu.Unlock()
|
||||
if cancelErr != nil {
|
||||
log.Error().Err(cancelErr).Str("id", notif.ID).Msg("Failed to cancel skipped notification")
|
||||
} else {
|
||||
healthChanged = true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
success := err == nil
|
||||
errorMsg := ""
|
||||
failureClass := NotificationFailureClass("")
|
||||
@@ -2074,22 +2108,29 @@ func calculateBackoff(attempt int) time.Duration {
|
||||
// returns the number of matched firing-alert entries removed from rows that
|
||||
// were still waiting for delivery ('pending'). Entries in rows already
|
||||
// mid-send ('sending') are cancelled best-effort but not counted, because
|
||||
// their delivery may still complete.
|
||||
// their delivery may still complete. Failed/dead-lettered firing entries are
|
||||
// also suppressed so operator retry cannot resurrect a resolved incident;
|
||||
// these do not contribute to the pending-only return count.
|
||||
func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string) (int, error) {
|
||||
alertIdentifiers = normalizeAlertIdentifiers(alertIdentifiers)
|
||||
if len(alertIdentifiers) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiers, true)
|
||||
defer releaseDeliveryGates()
|
||||
|
||||
healthChanged := false
|
||||
nq.mu.Lock()
|
||||
defer nq.mu.Unlock()
|
||||
defer func() {
|
||||
nq.mu.Unlock()
|
||||
releaseDeliveryGates()
|
||||
if healthChanged {
|
||||
nq.notifyDeliveryHealthChanged()
|
||||
}
|
||||
}()
|
||||
|
||||
query := `
|
||||
SELECT id, type, status, alerts, operational_links
|
||||
FROM notification_queue
|
||||
WHERE status IN ('pending', 'sending')
|
||||
WHERE status IN ('pending', 'sending', 'failed', 'dlq')
|
||||
`
|
||||
|
||||
rows, err := nq.db.Query(query)
|
||||
@@ -2248,7 +2289,9 @@ func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string)
|
||||
Str("action", "cancel_mark_notification").
|
||||
Str("notifID", notifID).
|
||||
Msg("Failed to mark notification as cancelled")
|
||||
return 0, fmt.Errorf("cancel resolved notification %s: %w", notifID, err)
|
||||
}
|
||||
healthChanged = true
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
|
||||
)
|
||||
|
||||
// Policy skips are neither provider receipts nor delivery failures. Exercise
|
||||
// the real queue processor: checking only its return value hid false success.
|
||||
func TestQueueDisabledDeliveryIsCancelledNotSent(t *testing.T) {
|
||||
for _, kind := range []string{"email", "webhook", "apprise"} {
|
||||
for _, suffix := range []string{"", "_resolved"} {
|
||||
for _, enabled := range []bool{false, true} {
|
||||
name := kind + suffix + "/global-disabled"
|
||||
if enabled {
|
||||
name = kind + suffix + "/destination-disabled"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
q, err := NewNotificationQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer q.Stop()
|
||||
nm := &NotificationManager{enabled: enabled}
|
||||
if !enabled {
|
||||
nm.emailConfig.Enabled = true
|
||||
nm.appriseConfig.Enabled = true
|
||||
nm.webhooks = []WebhookConfig{{ID: "ops", Enabled: true}}
|
||||
}
|
||||
n := &QueuedNotification{ID: "disabled", Type: kind + suffix, Status: QueueStatusPending,
|
||||
Links: []operationaltrust.NotificationLink{{DestinationID: "ops", OperationalRecordID: "incident", TransitionID: "firing", LifecycleState: operationaltrust.OperationalOpen, CauseKey: "cpu"}},
|
||||
Config: []byte(`{"enabled":true,"id":"ops"}`), MaxAttempts: 3, Alerts: []*alerts.Alert{{ID: "incident"}}}
|
||||
if err := q.Enqueue(n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Queue construction starts workers. Configure via the locked setter
|
||||
// and wait for reconciliation even if a worker wins the claim.
|
||||
var callbacks atomic.Int32
|
||||
reconciled := make(chan struct{}, 2)
|
||||
q.SetDeliveryHealthChangedCallback(func() {
|
||||
callbacks.Add(1)
|
||||
release := q.acquireAlertDeliveryGates([]string{"incident"}, true)
|
||||
defer release()
|
||||
if _, err := q.GetQueueStats(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
reconciled <- struct{}{}
|
||||
})
|
||||
q.SetProcessor(nm.ProcessQueuedNotification)
|
||||
q.processNotification(n)
|
||||
select {
|
||||
case <-reconciled:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("disabled delivery did not reconcile")
|
||||
}
|
||||
var status string
|
||||
var completed *int64
|
||||
if err := q.db.QueryRow(`SELECT status, completed_at FROM notification_queue WHERE id = ?`, n.ID).Scan(&status, &completed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != string(QueueStatusCancelled) || completed == nil {
|
||||
t.Errorf("status=%s completed=%v; want terminal cancellation", status, completed)
|
||||
}
|
||||
links, err := q.getNotificationLinks(n.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(links) != 1 || links[0].DeliveryState != operationaltrust.NotificationCancelled {
|
||||
t.Errorf("links = %+v, want cancelled", links)
|
||||
}
|
||||
for _, table := range []string{"notification_audit", "notification_delivery_receipts"} {
|
||||
var count int
|
||||
if err := q.db.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("%s has %d rows for a policy skip", table, count)
|
||||
}
|
||||
}
|
||||
if count := callbacks.Load(); count != 1 {
|
||||
t.Errorf("health callbacks = %d, want 1", count)
|
||||
}
|
||||
if count, err := q.RetryTerminalFailures(); err != nil || count != 0 {
|
||||
t.Errorf("retry = %d, %v; want no replay", count, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
)
|
||||
|
||||
// Exit without Stop or database Close: graceful reopen tests cannot establish
|
||||
// that resolution rewrites and cancellations survive a process exit.
|
||||
// This is queue durability evidence, not an installed provider receipt or a
|
||||
// power-loss simulation; an interrupted provider send remains at-least-once.
|
||||
func TestQueueResolutionSurvivesAbruptProcessExit(t *testing.T) {
|
||||
const helperEnv = "PULSE_TEST_QUEUE_RESOLUTION_EXIT_DIR"
|
||||
const exitCode = 23
|
||||
states := []NotificationQueueStatus{QueueStatusPending, QueueStatusSending, QueueStatusFailed, QueueStatusDLQ}
|
||||
if dir := os.Getenv(helperEnv); dir != "" {
|
||||
q, err := NewNotificationQueue(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enqueue := func(id, kind string, members ...string) *QueuedNotification {
|
||||
t.Helper()
|
||||
n := &QueuedNotification{ID: id, Type: kind, Status: QueueStatusPending, Config: []byte(`{}`), MaxAttempts: 3}
|
||||
for _, member := range members {
|
||||
n.Alerts = append(n.Alerts, &alerts.Alert{ID: member})
|
||||
}
|
||||
if err := q.Enqueue(n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
for _, status := range states {
|
||||
for _, grouped := range []bool{false, true} {
|
||||
id := string(status)
|
||||
members := []string{"recovered"}
|
||||
if grouped {
|
||||
id += "-group"
|
||||
members = append(members, "still-firing")
|
||||
}
|
||||
enqueue(id, "webhook", members...)
|
||||
if err := q.UpdateStatus(id, status, "destination unavailable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
enqueue("recovery", "webhook_resolved", "recovered")
|
||||
if count, err := q.CancelByAlertIdentifiers([]string{"recovered"}); err != nil || count != 2 {
|
||||
t.Fatalf("resolution = %d, %v; want two pending members", count, err)
|
||||
}
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestQueueResolutionSurvivesAbruptProcessExit$")
|
||||
cmd.Env = append(os.Environ(), helperEnv+"="+dir)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("child timed out: %v\n%s", ctx.Err(), output)
|
||||
}
|
||||
if exited, ok := err.(*exec.ExitError); !ok || exited.ExitCode() != exitCode {
|
||||
t.Fatalf("child did not reach deliberate exit: %v\n%s", err, output)
|
||||
}
|
||||
q, err := NewNotificationQueue(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer q.Stop()
|
||||
if count, err := q.RetryTerminalFailures(); err != nil || count != 2 {
|
||||
t.Fatalf("terminal retry = %d, %v; want only failed/dlq surviving groups", count, err)
|
||||
}
|
||||
var mu sync.Mutex
|
||||
delivered := make(map[string][]string)
|
||||
// Exercise ordinary dispatch as well as repeated batch selection.
|
||||
q.SetProcessor(func(n *QueuedNotification) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, a := range n.Alerts {
|
||||
delivered[n.ID] = append(delivered[n.ID], a.ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
q.processBatch()
|
||||
q.processBatch()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
stats, err := q.GetQueueStats()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stats["pending"]+stats["sending"] == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("restarted queue did not drain")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(delivered) != 5 {
|
||||
t.Fatalf("delivered = %v; want four surviving groups and recovery", delivered)
|
||||
}
|
||||
for _, id := range []string{"pending-group", "sending-group", "failed-group", "dlq-group", "recovery"} {
|
||||
want := "still-firing"
|
||||
if id == "recovery" {
|
||||
want = "recovered"
|
||||
}
|
||||
if got := delivered[id]; len(got) != 1 || got[0] != want {
|
||||
t.Errorf("%s delivered %v, want [%s] exactly once in this run", id, got, want)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"pending", "sending", "failed", "dlq"} {
|
||||
var status string
|
||||
if err := q.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", id).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != string(QueueStatusCancelled) {
|
||||
t.Errorf("%s status = %s, want cancelled", id, status)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
)
|
||||
|
||||
// A repaired destination must not replay obsolete firing alerts. Exercise the
|
||||
// operator retry after reopening the database, not just the cancellation query.
|
||||
func TestResolvedTerminalFiringIsNotReplayedAfterRestart(t *testing.T) {
|
||||
for _, terminal := range []NotificationQueueStatus{QueueStatusFailed, QueueStatusDLQ} {
|
||||
t.Run(string(terminal), func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
q, err := NewNotificationQueue(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = q.Stop() }()
|
||||
for _, n := range []*QueuedNotification{
|
||||
{ID: "obsolete", Type: "webhook", Alerts: []*alerts.Alert{{ID: "healthy"}}},
|
||||
{ID: "group", Type: "webhook", Alerts: []*alerts.Alert{{ID: "healthy"}, {ID: "still-firing"}}},
|
||||
{ID: "recovery", Type: "webhook_resolved", Alerts: []*alerts.Alert{{ID: "healthy"}}},
|
||||
} {
|
||||
n.Config = []byte("{}")
|
||||
n.Status = QueueStatusPending
|
||||
n.MaxAttempts = 3
|
||||
if err := q.Enqueue(n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.UpdateStatus(n.ID, terminal, "destination unavailable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.RecordAudit(n, false, "destination unavailable"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
healthCallbacks := 0
|
||||
q.SetDeliveryHealthChangedCallback(func() {
|
||||
healthCallbacks++
|
||||
// Callback must run after both the DB mutex and alert gate
|
||||
// are released, and see the committed cancellation.
|
||||
release := q.acquireAlertDeliveryGates([]string{"healthy"}, true)
|
||||
defer release()
|
||||
stats, err := q.GetQueueStats()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if stats[string(terminal)] != 2 {
|
||||
t.Errorf("remaining terminal rows = %d, want 2", stats[string(terminal)])
|
||||
}
|
||||
})
|
||||
count, err := q.CancelByAlertIdentifiers([]string{"healthy"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q.SetDeliveryHealthChangedCallback(nil)
|
||||
if healthCallbacks != 1 {
|
||||
t.Errorf("health callbacks = %d, want 1", healthCallbacks)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("pending suppression count = %d, want 0 for terminal rows", count)
|
||||
}
|
||||
if err := q.Stop(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err = NewNotificationQueue(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retried, err := q.RetryTerminalFailures()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retried != 2 {
|
||||
t.Errorf("retried %d rows, want only surviving group and recovery", retried)
|
||||
}
|
||||
// Alert IDs identify a resource/condition and can recur. Resolution
|
||||
// must suppress the old queue row, not permanently mute that ID.
|
||||
if err := q.Enqueue(&QueuedNotification{
|
||||
ID: "new-incident", Type: "webhook", Status: QueueStatusPending,
|
||||
Config: []byte("{}"), MaxAttempts: 3,
|
||||
Alerts: []*alerts.Alert{{ID: "healthy"}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivered := map[string][]string{}
|
||||
var mu sync.Mutex
|
||||
q.SetProcessor(func(n *QueuedNotification) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
for _, a := range n.Alerts {
|
||||
delivered[n.ID] = append(delivered[n.ID], a.ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
q.processBatch()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
var pending int
|
||||
if err := q.db.QueryRow("SELECT count(*) FROM notification_queue WHERE status IN ('pending', 'sending')").Scan(&pending); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("queue did not drain")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
mu.Lock()
|
||||
if len(delivered) != 3 || len(delivered["new-incident"]) != 1 ||
|
||||
delivered["new-incident"][0] != "healthy" || len(delivered["group"]) != 1 ||
|
||||
delivered["group"][0] != "still-firing" ||
|
||||
len(delivered["recovery"]) != 1 || delivered["recovery"][0] != "healthy" {
|
||||
t.Errorf("replayed payloads = %v, want still-firing, genuine recovery and the new incident", delivered)
|
||||
}
|
||||
mu.Unlock()
|
||||
var failures int
|
||||
if err := q.db.QueryRow("SELECT count(*) FROM notification_audit WHERE success = 0").Scan(&failures); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failures != 3 {
|
||||
t.Errorf("retained failed attempts = %d, want 3", failures)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A stale per-item retry request must not bypass resolution or resend a
|
||||
// successful notification. The same scheduler handles transient send failures.
|
||||
func TestScheduleRetryRejectsCancelledAndSent(t *testing.T) {
|
||||
q, err := NewNotificationQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = q.Stop() }()
|
||||
for _, status := range []NotificationQueueStatus{
|
||||
QueueStatusCancelled, QueueStatusSent, QueueStatusPending,
|
||||
QueueStatusSending, QueueStatusFailed, QueueStatusDLQ,
|
||||
} {
|
||||
t.Run(string(status), func(t *testing.T) {
|
||||
id := string(status)
|
||||
n := &QueuedNotification{ID: id, Type: "webhook", Status: QueueStatusPending, Config: []byte("{}")}
|
||||
if err := q.Enqueue(n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := q.UpdateStatus(id, status, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := q.ScheduleRetry(id, 0)
|
||||
blocked := status == QueueStatusCancelled || status == QueueStatusSent
|
||||
if (err != nil) != blocked {
|
||||
t.Fatalf("retry error = %v, blocked = %v", err, blocked)
|
||||
}
|
||||
want := QueueStatusPending
|
||||
if blocked {
|
||||
want = status
|
||||
}
|
||||
var got NotificationQueueStatus
|
||||
if err := q.db.QueryRow("SELECT status FROM notification_queue WHERE id = ?", id).Scan(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("status = %s, want %s", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
@@ -52,7 +54,7 @@ func TestManagedRuntimeRelayRegistrationReconnectDrain(t *testing.T) {
|
||||
pulseRoot, pulseProRelayDir := managedRelayWorkspaceRoots(t)
|
||||
relayBinary := buildManagedRelayBinary(t, pulseProRelayDir)
|
||||
const revocationFeedToken = "managed-runtime-revocation-feed-token"
|
||||
revocationFeed := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
revocationFeed := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/revocations" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
@@ -69,6 +71,16 @@ func TestManagedRuntimeRelayRegistrationReconnectDrain(t *testing.T) {
|
||||
})
|
||||
}))
|
||||
defer revocationFeed.Close()
|
||||
revocationCertificate, err := x509.ParseCertificate(revocationFeed.TLS.Certificates[0].Certificate[0])
|
||||
if err != nil {
|
||||
t.Fatalf("parse revocation feed certificate: %v", err)
|
||||
}
|
||||
revocationCAPath := filepath.Join(t.TempDir(), "revocation-feed-ca.pem")
|
||||
revocationCAPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: revocationCertificate.Raw})
|
||||
if err := os.WriteFile(revocationCAPath, revocationCAPEM, 0o600); err != nil {
|
||||
t.Fatalf("write revocation feed CA: %v", err)
|
||||
}
|
||||
t.Setenv("SSL_CERT_FILE", revocationCAPath)
|
||||
t.Setenv("PULSE_RELAY_LICENSE_SERVER_URL", revocationFeed.URL)
|
||||
t.Setenv("PULSE_RELAY_REVOCATION_FEED_TOKEN", revocationFeedToken)
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// Isolated real-browser component qualification; no installed backend or delivery claim.
|
||||
import { createServer } from "../frontend-modern/node_modules/vite/dist/node/index.js";
|
||||
import solid from "../frontend-modern/node_modules/vite-plugin-solid/dist/esm/index.mjs";
|
||||
import { chromium } from "@playwright/test";
|
||||
import { resolve } from "node:path";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import assert from "node:assert/strict";
|
||||
const root = resolve("frontend-modern");
|
||||
process.chdir(root);
|
||||
const fixture = `
|
||||
import { render } from 'solid-js/web';
|
||||
import { Show } from 'solid-js';
|
||||
import { NotificationsAPI } from '/src/api/notifications';
|
||||
import { AlertsAPI } from '/src/api/alerts';
|
||||
import { useAlertDestinationsTabState } from '/src/features/alerts/useAlertDestinationsTabState';
|
||||
import { AlertDeliveryHealthCard } from '/src/features/alerts/AlertDeliveryHealthCard';
|
||||
import '/src/index.css';
|
||||
const pending = [];
|
||||
NotificationsAPI.getHealth = () => new Promise((resolve, reject) => pending.push({resolve, reject}));
|
||||
NotificationsAPI.getDeliveryLog = async () => [];
|
||||
AlertsAPI.getEvents = async () => [];
|
||||
NotificationsAPI.retryTerminalFailures = NotificationsAPI.dismissTerminalFailures = async () => ({affected: 1});
|
||||
window.confirm = () => true;
|
||||
window.finish = (i, status) => status === 'error' ? pending[i].reject(new Error('scripted offline')) : pending[i].resolve({queue:{status, failed:1, deadLetter:0, attentionRequired:1}});
|
||||
window.count = () => pending.length;
|
||||
function Fixture() {
|
||||
const s = useAlertDestinationsTabState({emailConfig:()=>({}), appriseConfig:()=>({}), setAppriseConfig:()=>{}, configLoadError:()=> 'scripted config unavailable', isRetrying:()=>false, isLoadingDestinations:()=>false, onRetryLoad:()=>{}, webhooks:()=>[]});
|
||||
return <main class="p-4"><h1>Delivery health ordering fixture</h1><button onClick={s.handleRetry}>Configuration Retry</button><output data-testid="loading">{String(s.refreshingDeliveryHealth())}</output><Show when={s.deliveryNeedsAttention()}><AlertDeliveryHealthCard health={s.deliveryHealth()?.queue ?? null} unavailable={s.deliveryHealthUnavailable()} refreshing={s.refreshingDeliveryHealth()} onRefresh={s.loadDeliveryHealth} onRetryFailures={s.retryTerminalFailures} onDismissFailures={s.dismissTerminalFailures}/></Show></main>;
|
||||
}
|
||||
render(() => <Fixture/>, document.getElementById('root'));
|
||||
`;
|
||||
const server = await createServer({
|
||||
root,
|
||||
configFile: false,
|
||||
optimizeDeps: {
|
||||
noDiscovery: true,
|
||||
entries: [],
|
||||
esbuildOptions: { target: "esnext" },
|
||||
},
|
||||
esbuild: { target: "esnext" },
|
||||
plugins: [
|
||||
solid(),
|
||||
{
|
||||
name: "ordering-fixture",
|
||||
configureServer(s) {
|
||||
s.middlewares.use((req, res, next) => {
|
||||
if (req.url === "/qualification") {
|
||||
res.setHeader("Content-Type", "text/html");
|
||||
res.end(
|
||||
'<div id="root"></div><script type="module" src="/ordering-fixture.tsx"></script>',
|
||||
);
|
||||
} else next();
|
||||
});
|
||||
},
|
||||
resolveId(id) {
|
||||
if (id === "/ordering-fixture.tsx") return id;
|
||||
},
|
||||
load(id) {
|
||||
if (id === "/ordering-fixture.tsx") return fixture;
|
||||
},
|
||||
},
|
||||
],
|
||||
resolve: { alias: { "@": resolve(root, "src") } },
|
||||
server: { host: "127.0.0.1", port: 5197, strictPort: true },
|
||||
});
|
||||
let browser;
|
||||
try {
|
||||
await server.listen();
|
||||
browser = await chromium.launch({ headless: true });
|
||||
mkdirSync("/tmp/pulse-health-ordering", { recursive: true });
|
||||
let cases = 0;
|
||||
for (const width of [1440, 900, 390]) {
|
||||
for (const [old, newer] of [
|
||||
["healthy", "degraded"],
|
||||
["degraded", "healthy"],
|
||||
["error", "healthy"],
|
||||
["healthy", "error"],
|
||||
]) {
|
||||
const page = await browser.newPage({ viewport: { width, height: 900 } });
|
||||
await page.goto("http://127.0.0.1:5197/qualification");
|
||||
await page.waitForFunction(() => window.count?.() === 1);
|
||||
await page
|
||||
.getByRole("button", { name: "Configuration Retry", exact: true })
|
||||
.click();
|
||||
await page.waitForFunction(() => window.count() === 2);
|
||||
await page.evaluate((s) => window.finish(1, s), newer);
|
||||
await page.waitForFunction(
|
||||
() => document.querySelector("output").textContent === "false",
|
||||
);
|
||||
const before = await page.locator("main").innerText();
|
||||
await page.evaluate((s) => window.finish(0, s), old);
|
||||
await page.evaluate(
|
||||
() =>
|
||||
new Promise((r) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(r)),
|
||||
),
|
||||
);
|
||||
assert.equal(await page.locator("main").innerText(), before);
|
||||
assert.equal(
|
||||
await page.getByRole("alert").count(),
|
||||
["degraded", "error"].includes(newer) ? 1 : 0,
|
||||
);
|
||||
await page.screenshot({
|
||||
path: `/tmp/pulse-health-ordering/${width}-${old}-${newer}.png`,
|
||||
});
|
||||
await page.close();
|
||||
cases++;
|
||||
}
|
||||
for (const action of [
|
||||
"Retry retained deliveries",
|
||||
"Dismiss retained failures",
|
||||
]) {
|
||||
const page = await browser.newPage({ viewport: { width, height: 900 } });
|
||||
await page.goto("http://127.0.0.1:5197/qualification");
|
||||
await page.waitForFunction(() => window.count?.() === 1);
|
||||
await page.evaluate(() => window.finish(0, "degraded"));
|
||||
await page.getByRole("alert").waitFor();
|
||||
await page
|
||||
.getByRole("button", { name: "Configuration Retry", exact: true })
|
||||
.click();
|
||||
await page.getByRole("button", { name: action, exact: true }).click();
|
||||
await page.waitForFunction(() => window.count() === 3);
|
||||
await page.evaluate(() => window.finish(1, "healthy"));
|
||||
assert.equal(await page.getByTestId("loading").textContent(), "true");
|
||||
await page.evaluate(() => window.finish(2, "healthy"));
|
||||
await page.waitForFunction(
|
||||
() => document.querySelector("output").textContent === "false",
|
||||
);
|
||||
assert.equal(await page.getByRole("alert").count(), 0);
|
||||
await page.close();
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
result: "passed",
|
||||
cases,
|
||||
viewports: [1440, 900, 390],
|
||||
scope:
|
||||
"real Chromium, real caller/hook/card, scripted API; not installed application or notification receipt",
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
await browser?.close();
|
||||
await server.close();
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Isolated real-browser component qualification; no installed backend or delivery claim.
|
||||
import { createServer } from "../frontend-modern/node_modules/vite/dist/node/index.js";
|
||||
import solid from "../frontend-modern/node_modules/vite-plugin-solid/dist/esm/index.mjs";
|
||||
import { chromium } from "@playwright/test";
|
||||
import { resolve } from "node:path";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import assert from "node:assert/strict";
|
||||
const root = resolve("frontend-modern");
|
||||
process.chdir(root);
|
||||
const fixture = `
|
||||
import { render } from 'solid-js/web';
|
||||
import { Router, Route } from '@solidjs/router';
|
||||
import { NotificationsAPI } from '/src/api/notifications';
|
||||
import { AlertsAPI } from '/src/api/alerts';
|
||||
import { OverviewTab } from '/src/features/alerts/OverviewTab';
|
||||
import '/src/index.css';
|
||||
const pending = [];
|
||||
NotificationsAPI.getHealth = () => new Promise((resolve, reject) => pending.push({resolve, reject}));
|
||||
AlertsAPI.getDeliveryDiagnoses = async () => [];
|
||||
window.actions = 0;
|
||||
NotificationsAPI.retryTerminalFailures = NotificationsAPI.dismissTerminalFailures = async () => { window.actions++; return {affected: 1}; };
|
||||
window.confirm = () => true;
|
||||
window.finish = (i, status) => status === 'error' ? pending[i].reject(new Error('scripted offline')) : pending[i].resolve({queue:{status, failed:0, deadLetter:status === 'healthy' ? 0 : 1, attentionRequired:status === 'healthy' ? 0 : 1}});
|
||||
window.count = () => pending.length;
|
||||
function Fixture() {
|
||||
return <main class="p-4"><OverviewTab overrides={[]} activeAlerts={{}} updateAlert={()=>{}} showQuickTip={()=>false} dismissQuickTip={()=>{}} showAcknowledged={()=>true} setShowAcknowledged={()=>{}} alertsDisabled={()=>false}/></main>;
|
||||
}
|
||||
render(() => <Router><Route path="/qualification" component={Fixture}/></Router>, document.getElementById('root'));
|
||||
`;
|
||||
const server = await createServer({
|
||||
root,
|
||||
configFile: false,
|
||||
optimizeDeps: {
|
||||
noDiscovery: true,
|
||||
entries: [],
|
||||
esbuildOptions: { target: "esnext" },
|
||||
},
|
||||
esbuild: { target: "esnext" },
|
||||
plugins: [
|
||||
solid(),
|
||||
{
|
||||
name: "ordering-fixture",
|
||||
configureServer(s) {
|
||||
s.middlewares.use((req, res, next) => {
|
||||
if (req.url === "/qualification") {
|
||||
res.setHeader("Content-Type", "text/html");
|
||||
res.end(
|
||||
'<div id="root"></div><script type="module" src="/ordering-fixture.tsx"></script>',
|
||||
);
|
||||
} else next();
|
||||
});
|
||||
},
|
||||
resolveId(id) {
|
||||
if (id === "/ordering-fixture.tsx") return id;
|
||||
},
|
||||
load(id) {
|
||||
if (id === "/ordering-fixture.tsx") return fixture;
|
||||
},
|
||||
},
|
||||
],
|
||||
resolve: { alias: { "@": resolve(root, "src") } },
|
||||
server: { host: "127.0.0.1", port: 5197, strictPort: true },
|
||||
});
|
||||
let browser;
|
||||
try {
|
||||
await server.listen();
|
||||
browser = await chromium.launch({ headless: true });
|
||||
mkdirSync("/tmp/pulse-overview-refresh", { recursive: true });
|
||||
let cases = 0;
|
||||
for (const theme of ["light", "dark"]) {
|
||||
for (const width of [1440, 900, 390]) {
|
||||
for (const action of [
|
||||
"Retry retained deliveries",
|
||||
"Dismiss retained failures",
|
||||
]) {
|
||||
const page = await browser.newPage({
|
||||
viewport: { width, height: 900 },
|
||||
});
|
||||
await page.goto("http://127.0.0.1:5197/qualification");
|
||||
await page.evaluate(
|
||||
(theme) =>
|
||||
document.documentElement.classList.toggle("dark", theme === "dark"),
|
||||
theme,
|
||||
);
|
||||
await page.waitForFunction(() => window.count?.() === 1);
|
||||
await page.evaluate(() => window.finish(0, "degraded"));
|
||||
await page.getByRole("button", { name: action, exact: true }).click();
|
||||
await page.waitForFunction(() => window.count() === 2);
|
||||
await page.evaluate(() => window.finish(1, "error"));
|
||||
const refresh = page.getByRole("button", {
|
||||
name: "Refresh delivery status",
|
||||
exact: true,
|
||||
});
|
||||
await refresh.waitFor();
|
||||
assert.match(
|
||||
await page.getByRole("alert").innerText(),
|
||||
/status is unavailable/,
|
||||
);
|
||||
const geometry = await page.getByRole("alert").evaluate((el) => {
|
||||
const heading = el.querySelector("h3");
|
||||
const bounds = el.getBoundingClientRect();
|
||||
return {
|
||||
titleFits: heading.scrollWidth <= heading.clientWidth + 1,
|
||||
actionsFit: [...el.querySelectorAll("button, a")].every(
|
||||
(control) => {
|
||||
const rect = control.getBoundingClientRect();
|
||||
return rect.left >= bounds.left && rect.right <= bounds.right;
|
||||
},
|
||||
),
|
||||
};
|
||||
});
|
||||
assert.equal(
|
||||
geometry.titleFits,
|
||||
true,
|
||||
"warning heading must not overflow beneath actions",
|
||||
);
|
||||
assert.equal(
|
||||
geometry.actionsFit,
|
||||
true,
|
||||
"warning actions must stay inside the card",
|
||||
);
|
||||
await page.screenshot({
|
||||
path: `/tmp/pulse-overview-refresh/${theme}-${width}-${cases}-unavailable.png`,
|
||||
});
|
||||
await refresh.click();
|
||||
await page.waitForFunction(() => window.count() === 3);
|
||||
assert.equal(await refresh.isDisabled(), true);
|
||||
await page.evaluate(() => window.finish(2, "healthy"));
|
||||
await page.getByRole("alert").waitFor({ state: "detached" });
|
||||
assert.equal(await page.evaluate(() => window.actions), 1);
|
||||
await page.screenshot({
|
||||
path: `/tmp/pulse-overview-refresh/${theme}-${width}-${cases}-healthy.png`,
|
||||
});
|
||||
cases++;
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`${cases} overview action/outage/refresh browser cases passed`);
|
||||
} finally {
|
||||
await browser?.close();
|
||||
await server.close();
|
||||
}
|
||||
@@ -2119,7 +2119,8 @@ func TestDeploymentDefaultsPinVersionedImagesAndHelmDocsChecksum(t *testing.T) {
|
||||
`helm repo index "${index_work}"`,
|
||||
`git -C gh-pages push origin HEAD:gh-pages`,
|
||||
`grep -q "version: ${VERSION}"`,
|
||||
`helm show chart pulse-public/pulse --version "${VERSION}"`,
|
||||
`helm pull pulse-public/pulse --version "${VERSION}" --destination "${public_work}"`,
|
||||
`cmp -s "${qualified_chart}" "${public_work}/pulse-${VERSION}.tgz"`,
|
||||
}
|
||||
for _, needle := range required {
|
||||
if !strings.Contains(helmPages, needle) {
|
||||
@@ -3660,6 +3661,26 @@ func TestReleasePipelinePromotesOneImmutableCandidate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseTrainCITriggersIncludeBuildAndE2E(t *testing.T) {
|
||||
for _, workflow := range []string{"build-and-test.yml", "test-e2e.yml"} {
|
||||
content, err := os.ReadFile(repoFile(".github", "workflows", workflow))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, event := range []string{"push", "pull_request"} {
|
||||
t.Run(workflow+"/"+event, func(t *testing.T) {
|
||||
// Limit the assertion to this event's branch list, not another
|
||||
// trigger or a job comment mentioning the same pattern.
|
||||
expression := regexp.MustCompile(`(?m)^ ` + event + `:\n branches:\n((?: - [^\n]+\n)+)`)
|
||||
match := expression.FindStringSubmatch(string(content))
|
||||
if len(match) != 2 || !strings.Contains(match[1], " - 'release/v*'\n") || !strings.Contains(match[1], " - main\n") {
|
||||
t.Fatal("CI must admit main and release/v* branches for this event")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontendDependencySecurityAuditsAreRequired(t *testing.T) {
|
||||
workflowPath := repoFile(".github", "workflows", "build-and-test.yml")
|
||||
assertFileContainsAll(t, workflowPath,
|
||||
|
||||
@@ -106,5 +106,56 @@ class HelmPagesRetryTests(unittest.TestCase):
|
||||
self.assertIn("--latest=false", calls[-1])
|
||||
|
||||
|
||||
class PublicReceiptTests(unittest.TestCase):
|
||||
def receipt(self, mode):
|
||||
workflow = (ROOT / '.github/workflows/helm-pages.yml').read_text()
|
||||
step = workflow.split(' - name: Verify public Pages chart\n', 1)[1]
|
||||
script = textwrap.dedent(step.split(' run: |\n', 1)[1])
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
(root / 'dist').mkdir()
|
||||
(root / 'dist/pulse-6.4.3.tgz').write_bytes(b'qualified')
|
||||
programs = {
|
||||
'curl': '''#!/usr/bin/env python3
|
||||
import pathlib, sys
|
||||
pathlib.Path(sys.argv[sys.argv.index('-o')+1]).write_text('version: 6.4.3\\n')
|
||||
''',
|
||||
'helm': '''#!/usr/bin/env python3
|
||||
import os, pathlib, sys
|
||||
args = sys.argv[1:]
|
||||
if args[0] == 'pull':
|
||||
mode = os.environ['MODE']
|
||||
if mode == 'unavailable': sys.exit(1)
|
||||
if mode != 'missing':
|
||||
dest = pathlib.Path(args[args.index('--destination')+1])
|
||||
(dest / 'pulse-6.4.3.tgz').write_bytes(b'qualified' if mode == 'match' else b'wrong')
|
||||
''',
|
||||
'sleep': '#!/bin/sh\nexit 0\n',
|
||||
}
|
||||
for name, body in programs.items():
|
||||
path = root / name
|
||||
path.write_text(body)
|
||||
path.chmod(0o755)
|
||||
return subprocess.run(['bash', '-c', script], cwd=root,
|
||||
env={'PATH': f"{root}:{os.environ['PATH']}",
|
||||
'VERSION': '6.4.3', 'MODE': mode},
|
||||
capture_output=True, text=True)
|
||||
|
||||
def test_matching_public_package_passes(self):
|
||||
result = self.receipt('match')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_same_version_wrong_bytes_fail(self):
|
||||
result = self.receipt('wrong')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn('[OK]', result.stdout)
|
||||
|
||||
def test_missing_download_fails(self):
|
||||
self.assertNotEqual(self.receipt('missing').returncode, 0)
|
||||
|
||||
def test_unavailable_download_fails(self):
|
||||
self.assertNotEqual(self.receipt('unavailable').returncode, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise hosted Helm version resolution without packaging or publication."""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class HelmPublishVersionTests(unittest.TestCase):
|
||||
def resolve(self, chart="", app="", tag=""):
|
||||
workflow = (ROOT / ".github/workflows/publish-helm-chart.yml").read_text()
|
||||
step = workflow.split(" - name: Determine chart version\n", 1)[1]
|
||||
script = textwrap.dedent(step.split(" run: |\n", 1)[1].split(
|
||||
" - name:", 1)[0])
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output = Path(tmp) / "output"
|
||||
result = subprocess.run(
|
||||
["bash", "-euo", "pipefail", "-c", script], cwd=ROOT,
|
||||
env={"PATH": os.environ["PATH"], "INPUT_CHART_VERSION": chart,
|
||||
"INPUT_APP_VERSION": app, "RELEASE_TAG_NAME": tag,
|
||||
"GITHUB_OUTPUT": str(output)},
|
||||
capture_output=True, text=True)
|
||||
return result, output.read_text() if output.exists() else ""
|
||||
|
||||
def test_matching_and_default_application_versions(self):
|
||||
for version in ("6.4.3", "6.4.3-rc.2", "6.5.0-beta.1", "6.5.0-alpha.1"):
|
||||
for app in ("", version):
|
||||
with self.subTest(version=version, app=app):
|
||||
result, output = self.resolve(version, app)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("\n" + version + "\n", output)
|
||||
self.assertIn("is_prerelease=" + ("true" if "-" in version else "false"), output)
|
||||
|
||||
def test_release_event_uses_tag(self):
|
||||
result, output = self.resolve(tag="v6.4.3-rc.2")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("\n6.4.3-rc.2\n", output)
|
||||
|
||||
def test_mismatched_application_rejected_before_outputs(self):
|
||||
for chart, app in (("6.4.3", "6.4.2"), ("6.4.3", "6.4.3-rc.2"),
|
||||
("6.4.3-rc.2", "6.4.3"), ("6.4.3", "latest")):
|
||||
with self.subTest(chart=chart, app=app):
|
||||
result, output = self.resolve(chart, app)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(output, "")
|
||||
|
||||
def test_missing_version_rejected(self):
|
||||
result, output = self.resolve()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(output, "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -130,14 +130,8 @@ def validate_marker(
|
||||
return int(owner)
|
||||
|
||||
|
||||
def latest_failed_runs(
|
||||
releases: Iterable[object],
|
||||
runs: Iterable[object],
|
||||
*,
|
||||
workflow_id: int,
|
||||
default_branch: str,
|
||||
) -> list[int]:
|
||||
"""Return the latest failed convergence for each current immutable channel."""
|
||||
def current_channel_heads(releases: Iterable[object]) -> dict[bool, dict[str, Any]]:
|
||||
"""Select published server heads before applying retry eligibility."""
|
||||
latest_release: dict[bool, tuple[datetime, dict[str, Any]]] = {}
|
||||
for index, value in enumerate(releases):
|
||||
if not isinstance(value, dict) or value.get("draft") is not False:
|
||||
@@ -154,12 +148,23 @@ def latest_failed_runs(
|
||||
if prerelease not in latest_release or published > latest_release[prerelease][0]:
|
||||
latest_release[prerelease] = (published, value)
|
||||
|
||||
return {channel: value for channel, (_, value) in latest_release.items()}
|
||||
|
||||
|
||||
def latest_failed_runs(
|
||||
releases: Iterable[object],
|
||||
runs: Iterable[object],
|
||||
*,
|
||||
workflow_id: int,
|
||||
default_branch: str,
|
||||
) -> list[int]:
|
||||
"""Return the latest failed convergence for each current immutable channel."""
|
||||
# Never fall back to an older release when the advertised channel head is
|
||||
# mutable. That is continuity debt requiring a replacement, not a target
|
||||
# whose aliases should be promoted again.
|
||||
current_tags = {
|
||||
str(release["tag_name"])
|
||||
for _, release in latest_release.values()
|
||||
for release in current_channel_heads(releases).values()
|
||||
if release.get("immutable") is True
|
||||
}
|
||||
newest: dict[str, tuple[datetime, dict[str, Any]]] = {}
|
||||
@@ -679,6 +684,14 @@ def discover(github: GitHub) -> list[int]:
|
||||
releases = flatten_pages(
|
||||
github.pages(f"repos/{repository}/releases?per_page=100")
|
||||
)
|
||||
for prerelease, release in current_channel_heads(releases).items():
|
||||
if release.get("immutable") is not True:
|
||||
channel = "preview" if prerelease else "stable"
|
||||
report_decision(
|
||||
f"Current {channel} head {release['tag_name']} has no confirmed "
|
||||
"immutable activation commit; continuity debt remains. "
|
||||
"No retry or fallback to an older release is authorised by this check."
|
||||
)
|
||||
runs = flatten_pages(
|
||||
github.pages(
|
||||
f"repos/{repository}/actions/workflows/release-convergence.yml/runs"
|
||||
|
||||
@@ -290,6 +290,63 @@ class DecisionSummaryTests(unittest.TestCase):
|
||||
self.assertIn("No failed run selected for retry", message)
|
||||
self.assertIn("missing convergence runs are not qualified", message)
|
||||
|
||||
def discover_with_releases(self, releases):
|
||||
github = FakeGitHub()
|
||||
github.pages = lambda endpoint: (
|
||||
[releases] if "/releases?" in endpoint
|
||||
else [{"workflow_runs": github.runs}]
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "summary.md"
|
||||
with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(path)}), \
|
||||
contextlib.redirect_stdout(io.StringIO()) as output:
|
||||
selected = subject.discover(github)
|
||||
summary = path.read_text() if path.exists() else ""
|
||||
self.assertEqual([], github.posts)
|
||||
return selected, output.getvalue(), summary
|
||||
|
||||
def test_mutable_stable_debt_visible_even_when_preview_retry_selected(self):
|
||||
selected, output, summary = self.discover_with_releases([
|
||||
release("v6.4.1", "2026-09-01T00:00:00Z",
|
||||
prerelease=False, immutable=False),
|
||||
release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
|
||||
])
|
||||
self.assertEqual([100], selected)
|
||||
for text in (output, summary):
|
||||
self.assertIn("Current stable head v6.4.1", text)
|
||||
self.assertIn("continuity debt remains", text)
|
||||
self.assertIn("No retry or fallback", text)
|
||||
|
||||
def test_unknown_immutability_reports_debt_without_older_fallback(self):
|
||||
for unknown in (None, "true", 1, False):
|
||||
with self.subTest(immutable=unknown):
|
||||
head = release("v6.5.0-rc.2", "2026-09-03T00:00:00Z",
|
||||
prerelease=True)
|
||||
head["immutable"] = unknown
|
||||
selected, output, summary = self.discover_with_releases([
|
||||
release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
|
||||
head,
|
||||
])
|
||||
self.assertEqual([], selected)
|
||||
self.assertIn("Current preview head v6.5.0-rc.2", output)
|
||||
self.assertIn("no confirmed immutable activation commit", summary)
|
||||
|
||||
def test_old_mutable_draft_and_companion_do_not_report_current_debt(self):
|
||||
draft = release("v6.6.0", "2026-09-04T00:00:00Z",
|
||||
prerelease=False, immutable=False)
|
||||
draft["draft"] = True
|
||||
selected, output, summary = self.discover_with_releases([
|
||||
release("v6.4.0-rc.1", "2026-09-01T00:00:00Z",
|
||||
prerelease=True, immutable=False),
|
||||
release("v6.5.0-rc.1", "2026-09-02T00:00:00Z", prerelease=True),
|
||||
release("helm-chart-6.5.0", "2026-09-03T00:00:00Z",
|
||||
prerelease=False, immutable=False),
|
||||
draft,
|
||||
])
|
||||
self.assertEqual([100], selected)
|
||||
self.assertEqual("", output)
|
||||
self.assertEqual("", summary)
|
||||
|
||||
def test_credential_hold_is_visible_without_retry(self):
|
||||
github = FakeGitHub(current_controls=True)
|
||||
github.unchanged_credential_containment_block = lambda run_id: True
|
||||
|
||||
@@ -2424,7 +2424,8 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
|
||||
self.assertIn('helm repo index "${index_work}"', helm_pages)
|
||||
self.assertIn('git -C gh-pages push origin HEAD:gh-pages', helm_pages)
|
||||
self.assertIn('grep -q "version: ${VERSION}"', helm_pages)
|
||||
self.assertIn('helm show chart pulse-public/pulse --version "${VERSION}"', helm_pages)
|
||||
self.assertIn('helm pull pulse-public/pulse --version "${VERSION}"', helm_pages)
|
||||
self.assertIn('cmp -s "${qualified_chart}" "${public_work}/pulse-${VERSION}.tgz"', helm_pages)
|
||||
self.assertNotIn("helm status pulse || true", helm_pages)
|
||||
self.assertNotIn("kubectl describe pods", helm_pages)
|
||||
self.assertIn("release-convergence.yml/dispatches", release_workflow)
|
||||
@@ -2434,6 +2435,13 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
|
||||
self.assertIn("sync_chart_release_metadata.py", helm)
|
||||
self.assertNotIn("sync_chart_release_metadata.py", helm_pages)
|
||||
self.assertIn("--chart deploy/helm/pulse/Chart.yaml", helm)
|
||||
version_guard = 'if [ "$APP_VERSION" != "$CHART_VERSION" ]; then'
|
||||
self.assertIn(version_guard, helm)
|
||||
self.assertLess(helm.index(version_guard), helm.index(" - name: Package chart"))
|
||||
self.assertIn(
|
||||
"python3 scripts/release_control/helm_publish_version_test.py",
|
||||
(Path(__file__).resolve().parents[2] / ".github/workflows/canonical-governance.yml").read_text(),
|
||||
)
|
||||
self.assertIn('git checkout --detach "refs/tags/${RELEASE_TAG}"', helm)
|
||||
self.assertIn("Verify public GHCR chart identity and provenance", helm)
|
||||
self.assertIn("helm registry logout ghcr.io || true", helm)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Release trains must receive the same CI triggers as main."""
|
||||
import fnmatch
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class ReleaseTrainCIContractTest(unittest.TestCase):
|
||||
def test_build_and_e2e_cover_release_train_pushes_and_proposals(self):
|
||||
for workflow in ("build-and-test.yml", "test-e2e.yml"):
|
||||
source = (ROOT / ".github/workflows" / workflow).read_text()
|
||||
for event in ("push", "pull_request"):
|
||||
with self.subTest(workflow=workflow, event=event):
|
||||
block = re.search(
|
||||
rf"^ {event}:\n(.*?)(?=^ \w|^\S|\Z)",
|
||||
source, re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
self.assertIsNotNone(block)
|
||||
branches = re.search(
|
||||
r"^ branches:\n((?: - .*\n)+)",
|
||||
block.group(1), re.MULTILINE,
|
||||
)
|
||||
self.assertIsNotNone(branches)
|
||||
patterns = [line.strip()[2:].strip('\"\'')
|
||||
for line in branches.group(1).splitlines()]
|
||||
for branch in ("main", "release/v6.4", "release/v7.0"):
|
||||
self.assertTrue(
|
||||
any(fnmatch.fnmatchcase(branch, p) for p in patterns),
|
||||
f"{workflow} {event} excludes {branch}: {patterns}",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, test, type Page, type WebSocketRoute } from '@playwright/test';
|
||||
import { ensureAuthenticated } from './helpers';
|
||||
import { apiRequest, ensureAuthenticated } from './helpers';
|
||||
|
||||
// Opt-in: uses synthetic inventory in an isolated real backend. Never interrupts
|
||||
// a shared runtime or substitutes the application's websocket store.
|
||||
@@ -208,3 +208,108 @@ for (const admissionFailure of [false, true]) {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Mutations go to the owned backend, not a fulfilled REST fixture or a store hook.
|
||||
// Keep this after the inventory matrix: global detection is briefly paused while
|
||||
// clearing incidents, then its original configuration is restored.
|
||||
for (const width of [1440, 320]) {
|
||||
test(`changed backend incidents recover without reload at ${width}px`, async ({ page, browser }, testInfo) => {
|
||||
test.skip(!enabled, 'Requires isolated mock backend and explicit qualification opt-in');
|
||||
test.setTimeout(120_000);
|
||||
await page.setViewportSize({ width, height: 900 });
|
||||
let blocked = false;
|
||||
const sockets: WebSocketRoute[] = [];
|
||||
await page.routeWebSocket('**/ws*', async socket => {
|
||||
if (blocked) return socket.close({ code: 1013, reason: 'qualification interruption' });
|
||||
socket.connectToServer();
|
||||
sockets.push(socket);
|
||||
});
|
||||
// Prevent periodic REST reads from hiding a missing reconnect refresh.
|
||||
// APIRequestContext mutations/reads below do not pass through page routing.
|
||||
await page.route('**/api/alerts/active', route => blocked ? route.abort() : route.continue());
|
||||
await ensureAuthenticated(page);
|
||||
await page.goto('/alerts');
|
||||
await expect(page.getByRole('status', { name: healthy })).toBeVisible();
|
||||
const readActive = async () => {
|
||||
const response = await apiRequest(page, '/api/alerts/active');
|
||||
expect(response.ok()).toBe(true);
|
||||
return await response.json() as Array<{ id: string; resourceName: string; type: string; acknowledged: boolean }>;
|
||||
};
|
||||
// Service names vary with the generated estate. Select an identified service
|
||||
// health incident, not a name from a previous run or a cold offline host.
|
||||
await expect.poll(async () => (await readActive()).some(alert =>
|
||||
alert.type === 'docker-service-health')).toBe(true);
|
||||
const initial = await readActive();
|
||||
const target = initial.find(alert => alert.type === 'docker-service-health');
|
||||
expect(target).toBeTruthy();
|
||||
if (target!.acknowledged) {
|
||||
const reset = await apiRequest(page, '/api/alerts/unacknowledge', {
|
||||
method: 'POST', data: { alertIdentifier: target!.id },
|
||||
});
|
||||
expect(reset.ok()).toBe(true);
|
||||
}
|
||||
const card = page.locator(`[id=${JSON.stringify('alert-' + target!.id)}]`);
|
||||
await expect(card.getByRole('button', { name: 'Acknowledge', exact: true })).toBeVisible();
|
||||
const documentIdentity = await page.evaluate(() => performance.timeOrigin);
|
||||
const disconnect = async () => {
|
||||
blocked = true;
|
||||
for (const socket of sockets.splice(0)) await socket.close({ code: 1013, reason: 'qualification interruption' });
|
||||
await expect(page.getByRole('status', { name: 'Backend is healthy. Live updates are reconnecting.' })).toBeVisible();
|
||||
};
|
||||
const reconnect = async () => {
|
||||
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/api/alerts/active' && r.ok());
|
||||
blocked = false;
|
||||
const snapshot = await (await response).json();
|
||||
await expect(page.getByRole('status', { name: healthy })).toBeVisible({ timeout: 45_000 });
|
||||
expect(await page.evaluate(() => performance.timeOrigin)).toBe(documentIdentity);
|
||||
return snapshot;
|
||||
};
|
||||
await disconnect();
|
||||
const ack = await apiRequest(page, '/api/alerts/acknowledge', {
|
||||
method: 'POST', data: { alertIdentifier: target!.id, user: 'reconnect-qualification' },
|
||||
});
|
||||
expect(ack.ok()).toBe(true);
|
||||
await expect.poll(async () => (await readActive()).find(a => a.id === target!.id)?.acknowledged).toBe(true);
|
||||
// It must still be the old rendered value before restoring either transport.
|
||||
await expect(card.getByRole('button', { name: 'Acknowledge', exact: true })).toBeVisible();
|
||||
const acknowledgedSnapshot = await reconnect();
|
||||
expect(acknowledgedSnapshot.find((a: { id: string }) => a.id === target!.id)?.acknowledged).toBe(true);
|
||||
// Acknowledged incidents sort last. The production list windows large
|
||||
// estates, so a correct update can remove this card from the current DOM.
|
||||
await page.getByRole('contentinfo').scrollIntoViewIfNeeded();
|
||||
await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible();
|
||||
await card.scrollIntoViewIfNeeded();
|
||||
await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' });
|
||||
|
||||
const configResponse = await apiRequest(page, '/api/alerts/config');
|
||||
expect(configResponse.ok()).toBe(true);
|
||||
const config = await configResponse.json();
|
||||
try {
|
||||
await disconnect();
|
||||
const paused = await apiRequest(page, '/api/alerts/config', { method: 'PUT', data: { ...config, enabled: false } });
|
||||
expect(paused.ok()).toBe(true);
|
||||
const active = await readActive();
|
||||
if (active.length) {
|
||||
const cleared = await apiRequest(page, '/api/alerts/bulk/clear', {
|
||||
method: 'POST', data: { alertIdentifiers: active.map(a => a.id) },
|
||||
});
|
||||
expect(cleared.ok()).toBe(true);
|
||||
}
|
||||
await expect.poll(readActive).toEqual([]);
|
||||
await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible();
|
||||
const emptySnapshot = await reconnect();
|
||||
expect(emptySnapshot).toEqual([]);
|
||||
await expect(page.locator('[id^="alert-"]').filter({ has: page.getByRole('button', { name: /^(Unacknowledge|Acknowledge)$/ }) })).toHaveCount(0);
|
||||
await expect(card).toHaveCount(0);
|
||||
await testInfo.attach('empty-active-response', {
|
||||
body: JSON.stringify({ browser: browser.version(), width, target, acknowledgedSnapshot: acknowledgedSnapshot.filter((a: { id: string }) => a.id === target!.id), emptySnapshot, documentIdentity }),
|
||||
contentType: 'application/json',
|
||||
});
|
||||
await testInfo.attach('cleared-incidents', { body: await page.screenshot(), contentType: 'image/png' });
|
||||
} finally {
|
||||
blocked = false;
|
||||
const restored = await apiRequest(page, '/api/alerts/config', { method: 'PUT', data: config });
|
||||
expect(restored.ok()).toBe(true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
# Changed incident recovery — 5 September 2026
|
||||
|
||||
## Purpose and evidence boundary
|
||||
|
||||
The previous integrated run only established unchanged navigation and socket
|
||||
recovery. The admission continuation explicitly asked for changed-data and empty
|
||||
active-response evidence. Fresh reading of the [community comparison thread](https://www.reddit.com/r/Proxmox/comments/1lblkk8/anyone_else_switch_to_pulse_from_netdata_or_any/)
|
||||
on 5 September still supports protecting the quick-glance overview, including
|
||||
phone use; one user separates this from historical monitoring and alerting in
|
||||
CheckMK. These are historical self-reports, not new demand counts or current
|
||||
performance measurements. No new product surface or ledger bet is introduced.
|
||||
|
||||
The test uses an owned, source-built core backend with synthetic inventory and
|
||||
production embedded frontend assets. It never fulfils active-alert REST with a
|
||||
fixture, injects a store event, or replaces the application's WebSocket store.
|
||||
Real sockets close with 1013; browser active-alert requests are aborted while
|
||||
blocked, preventing periodic REST polling from hiding missing recovery. A
|
||||
separate authenticated API client changes an identified incident on the backend and
|
||||
verifies the changed server value while the rendered value remains old.
|
||||
|
||||
After reconnect, the browser must receive a successful active-alert response,
|
||||
render the same incident as acknowledged, and retain document time origin.
|
||||
A second interruption pauses detection and bulk-clears backend incidents; the
|
||||
backend and browser must both return `[]`, and the old incident cards must go.
|
||||
Original alert configuration is restored in `finally`. This is deliberate
|
||||
administrative clearing, **not** a claim that lost telemetry or disabling
|
||||
alerts represents monitored-resource recovery. No notification delivery is exercised.
|
||||
|
||||
## Results and test development
|
||||
|
||||
Final full matrix: **10 passed, no retries**, 2.4 minutes starting
|
||||
2026-09-05T21:41:31Z. Chromium 141.0.7390.37. Eight navigation cases cover
|
||||
1440/1100/390/320px, each with/without admission HTTP failure; table-access opt-in
|
||||
ran at 390/320px. Two additional cases cover changed acknowledgement and genuine
|
||||
empty-response clearing at 1440/320px (height 900). A preceding cold-runtime
|
||||
focused invocation passed both changed-data cases in 30 seconds.
|
||||
|
||||
`final/` retains unmodified JUnit, extracted report timings, both
|
||||
`narrow-table-access` JSON attachments, environment attachments, actual identified
|
||||
selection/acknowledged/empty response receipts, and four screenshots. `focused/`
|
||||
retains the preceding two-case pass. Inspected desktop acknowledged and phone
|
||||
cleared screenshots from the focused run show the changed service card and
|
||||
zero active/acknowledged counts with an **Alerting is paused** state respectively.
|
||||
This is intentionally not an “all infrastructure healthy” claim.
|
||||
|
||||
Earlier test-development failures are retained rather than counted as product
|
||||
regressions:
|
||||
|
||||
| Receipt | Local draft | Result and diagnosis |
|
||||
| --- | --- | --- |
|
||||
| `initial/` | `d82f55cb23` | Eight navigation cases passed; two new cases received acknowledged REST but failed to find the card. Initial grouping hypothesis was incomplete. |
|
||||
| `standalone/` | `2a3dcc3378` | Eight passed; standalone host also moved outside the DOM window after acknowledgement. Second case inherited the acknowledged fixture. Source inspection established acknowledged-last sorting and list windowing. |
|
||||
| `cold-fixture/` | `e934385ddc` | Two filtered cases failed before mutation: offline-host incident had not appeared in the fresh runtime. |
|
||||
| `named-service/` | `75aa1b6b0a` | Two filtered cases failed before mutation: service names differ between generated estates. |
|
||||
|
||||
The final test selects a current `docker-service-health` incident by canonical
|
||||
ID, resets acknowledgement if needed and scrolls the windowed list to its last
|
||||
group after acknowledgement. No UI hook or artificial renderer setting is used.
|
||||
Test-development commits were consolidated locally into `1abaea81c3`; no remote
|
||||
history changed. Passing runs executed draft `0cccef0dd4`, whose exact test blob
|
||||
`685a028a71c8e3a13a881c4ae66d7988bfa08ec9` is unchanged in the consolidated commit.
|
||||
Application source was unchanged throughout.
|
||||
|
||||
In response attachments, `target` is the selection-time snapshot, before the
|
||||
optional reset. It is already acknowledged in the final 320px case because the
|
||||
previous case's acknowledgement can survive fixture regeneration. The test then
|
||||
uses the real unacknowledge endpoint and requires its rendered Acknowledge button
|
||||
before disconnecting and acknowledging again. Do not read `target.acknowledged`
|
||||
as the post-reset baseline. The retained `acknowledgedSnapshot` and
|
||||
`emptySnapshot` are actual browser recovery responses.
|
||||
|
||||
## Invocation
|
||||
|
||||
Locked dependencies installed with `npm ci --no-audit --no-fund` in repository
|
||||
root, frontend-modern and tests/integration. The two full development attempts and final full matrix used exactly:
|
||||
|
||||
```sh
|
||||
pulse-heavy-run -- env \
|
||||
PULSE_E2E_USE_LOCAL_BACKEND=1 PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL=1 \
|
||||
PULSE_MOCK_MODE=true PULSE_E2E_NAVIGATION_RECOVERY=1 \
|
||||
PULSE_E2E_TABLE_ACCESS=1 PULSE_E2E_LOCAL_BACKEND_PORT=18765 \
|
||||
npm --prefix tests/integration test -- \
|
||||
tests/96-navigation-socket-recovery.spec.ts --project=chromium
|
||||
```
|
||||
|
||||
The three filtered development attempts and successful focused run appended
|
||||
`--grep 'changed backend'` to that command. All builds/browser runs were
|
||||
serialized through `pulse-heavy-run`; the final rerun reused the same binary.
|
||||
|
||||
Application source: `adeecd4079185cf9358415e2b7a55fc371f408b7` (the supplied
|
||||
integrated main snapshot, not the newer remote head in the admission packet).
|
||||
Only tests differ at the tested `0cccef0dd4` and consolidated `1abaea81c3`. Local executable SHA-256:
|
||||
`891fc3827d1e6522f21011ef62052b73f29f6407f498131e7e0cbcf454f7c677`.
|
||||
Built with Go 1.26.8 on Linux amd64, embedded Vite production assets and locked
|
||||
Playwright 1.56.1. This binary has no embedded VCS build fields; the source
|
||||
snapshot and content hash, not the displayed development version, bind this run.
|
||||
|
||||
## Limits and next trigger
|
||||
|
||||
No application repair, released-candidate qualification, reporter retest,
|
||||
acknowledgement persistence across process restart, history durability, installed
|
||||
alert receipt or off-host delivery is claimed. REST and stream are both restored,
|
||||
so this does not establish which path wins their race. Malformed snapshots are
|
||||
not covered. Global detection is paused only inside the owned test runtime.
|
||||
|
||||
Prefer changed integration/candidate or installed evidence next, not another
|
||||
unchanged synthetic matrix. Release judgment remains independently held on
|
||||
publication continuity and candidate issue disposition; these checks do not
|
||||
clear those boundaries. No timed revisit is justified by elapsed time alone.
|
||||
|
||||
## Mechanical admission repair
|
||||
|
||||
The first admission rejected eight trailing spaces inside failed-run JUnit
|
||||
CDATA. Failed-run `junit.xml` files in `initial/`, `standalone/`,
|
||||
`cold-fixture/` and `named-service/` now have trailing line whitespace removed.
|
||||
Their original bytes are preserved in adjacent deterministic `junit.xml.gz`
|
||||
archives, with original and normalised SHA-256 values in
|
||||
`receipt-normalisation.json`. Decompression was checked byte-for-byte. The
|
||||
passing `focused/` and `final/` JUnit files remain unmodified. No test result,
|
||||
application source or test code changed; browser checks were not rerun for this
|
||||
receipt-only repair. Evidence commits were consolidated locally so intermediate
|
||||
commits no longer introduce the rejected whitespace.
|
||||
@@ -0,0 +1,88 @@
|
||||
<testsuites id="" name="" tests="2" failures="2" skipped="0" errors="0" time="25.182378">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T21:36:53.866Z" hostname="chromium" tests="2" failures="2" skipped="0" time="23.719" errors="0">
|
||||
<testcase name="changed backend incidents recover without reload at 1440px" classname="96-navigation-socket-recovery.spec.ts" time="13.001">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 1440px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 1440px
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: true
|
||||
Received: false
|
||||
|
||||
Call Log:
|
||||
- Timeout 10000ms exceeded while waiting on the predicate
|
||||
|
||||
238 | // This standalone fixture avoids acknowledgement moving a grouped storage
|
||||
239 | // incident into a collapsed related-incidents panel.
|
||||
> 240 | await expect.poll(async () => (await readActive()).some(alert =>
|
||||
| ^
|
||||
241 | alert.resourceName === 'edge-proxy-01')).toBe(true);
|
||||
242 | const initial = await readActive();
|
||||
243 | const target = initial.find(alert => alert.resourceName === 'edge-proxy-01');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 320px" classname="96-navigation-socket-recovery.spec.ts" time="10.718">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 320px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 320px
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: true
|
||||
Received: false
|
||||
|
||||
Call Log:
|
||||
- Timeout 10000ms exceeded while waiting on the predicate
|
||||
|
||||
238 | // This standalone fixture avoids acknowledgement moving a grouped storage
|
||||
239 | // incident into a collapsed related-incidents panel.
|
||||
> 240 | await expect.poll(async () => (await readActive()).some(alert =>
|
||||
| ^
|
||||
241 | alert.resourceName === 'edge-proxy-01')).toBe(true);
|
||||
242 | const initial = await readActive();
|
||||
243 | const target = initial.find(alert => alert.resourceName === 'edge-proxy-01');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":1440,"height":900,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":true,"tabs":[{"label":"Proxmox","text":"Proxmox","icons":1,"visible":true},{"label":"Docker","text":"Docker","icons":1,"visible":true},{"label":"Kubernetes","text":"Kubernetes","icons":1,"visible":true},{"label":"TrueNAS","text":"TrueNAS","icons":1,"visible":true},{"label":"vSphere","text":"vSphere","icons":1,"visible":true},{"label":"Machines","text":"Machines","icons":1,"visible":true},{"label":"Alerts","text":"Alerts","icons":1,"visible":true},{"label":"Patrol","text":"Patrol","icons":1,"visible":true},{"label":"Actions","text":"Actions","icons":1,"visible":true},{"label":"Settings","text":"Settings","icons":1,"visible":true}]}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":1100,"height":900,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":390,"height":844,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]}
|
||||
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"initial": {
|
||||
"width": 362,
|
||||
"scrollWidth": 362,
|
||||
"left": 0,
|
||||
"overflowX": "clip",
|
||||
"tabIndex": -1,
|
||||
"cells": [
|
||||
{
|
||||
"text": "loki",
|
||||
"x": 9,
|
||||
"width": 108.59375,
|
||||
"titles": [
|
||||
"Expand details for loki",
|
||||
"Healthy",
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "running",
|
||||
"x": 117.59375,
|
||||
"width": 48.4375,
|
||||
"titles": [
|
||||
"running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "12%",
|
||||
"x": 166.03125,
|
||||
"width": 55.890625,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "74%",
|
||||
"x": 221.921875,
|
||||
"width": 63.34375,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "2",
|
||||
"x": 285.265625,
|
||||
"width": 37.25,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "—",
|
||||
"x": 322.515625,
|
||||
"width": 48.4375,
|
||||
"titles": [
|
||||
"No image update check reported"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"pointer": {
|
||||
"width": 362,
|
||||
"scrollWidth": 362,
|
||||
"left": 0,
|
||||
"overflowX": "clip",
|
||||
"tabIndex": -1,
|
||||
"cells": [
|
||||
{
|
||||
"text": "loki",
|
||||
"x": 9,
|
||||
"width": 108.59375,
|
||||
"titles": [
|
||||
"Expand details for loki",
|
||||
"Healthy",
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "running",
|
||||
"x": 117.59375,
|
||||
"width": 48.4375,
|
||||
"titles": [
|
||||
"running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "12%",
|
||||
"x": 166.03125,
|
||||
"width": 55.890625,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "74%",
|
||||
"x": 221.921875,
|
||||
"width": 63.34375,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "2",
|
||||
"x": 285.265625,
|
||||
"width": 37.25,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "—",
|
||||
"x": 322.515625,
|
||||
"width": 48.4375,
|
||||
"titles": [
|
||||
"No image update check reported"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"keyboard": {
|
||||
"width": 362,
|
||||
"scrollWidth": 362,
|
||||
"left": 0,
|
||||
"overflowX": "clip",
|
||||
"tabIndex": -1,
|
||||
"cells": [
|
||||
{
|
||||
"text": "loki",
|
||||
"x": 9,
|
||||
"width": 108.59375,
|
||||
"titles": [
|
||||
"Expand details for loki",
|
||||
"Healthy",
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "running",
|
||||
"x": 117.59375,
|
||||
"width": 48.4375,
|
||||
"titles": [
|
||||
"running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "12%",
|
||||
"x": 166.03125,
|
||||
"width": 55.890625,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "74%",
|
||||
"x": 221.921875,
|
||||
"width": 63.34375,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "2",
|
||||
"x": 285.265625,
|
||||
"width": 37.25,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "—",
|
||||
"x": 322.515625,
|
||||
"width": 48.4375,
|
||||
"titles": [
|
||||
"No image update check reported"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"accessibleRow": "- row \"Expand details for notifications-worker notifications-worker running 12% 48% 1 —\":\n - cell \"Expand details for notifications-worker notifications-worker\":\n - button \"Expand details for notifications-worker\"\n - text: notifications-worker\n - cell \"running\"\n - cell \"12%\"\n - cell \"48%\"\n - cell \"1\"\n - cell \"—\"",
|
||||
"expanded": "true",
|
||||
"detailText": "notifications-worker\nContainer\nDocker / Podman\nOverview\nHistory\nManage\nRUNTIME CONTEXT\nObserved state\t\nRunning\n\nUptime\t\n6d 6h\n\nLast seen\t\n17s ago\n\nCONTAINER\nImage\t\nghcr.io/pulse-demo/notifications-worker:2026.04\n\nRestarts\t\n1\n\nCreated\t\n18 days ago\n\nStarted\t\n6 days ago\n\nCompose project\t\norion-2\n\nCompose service\t\npostgres\n\nLabels\t\ncom.docker.compose.project: orion-2\ncom.docker.compose.service: postgres\ncom.pulse.demo: true\ncom.pulse.role.production: true\ncom.pulse.role.queue: true\ncom.pulse.role.worker: true\n\nIDENTITY\nHostname\t\nnotifications-worker\n\nPrimary ID\t\napp-container:orion-2-26f93461b215\n\nParent\t\nagent-9dbedd74104d1527\n\nDiscovery\t\napp-container:notifications-worker\n\nMetrics Target\t\napp-container:orion-2-26f93461b215\n\nAliases\t\napp-container:orion-2-26f93461b215\norion-2-26f93461b215\norion-2-mock\napp-platform-01\n+4\nRelationship map\n\n1 canonical relationship\n\nCANONICAL RELATIONSHIPS\nnotifications-worker\n→\ndocker-network-0ab79ab291cd98fa\nAttached To\n100% confidence · Docker Adapter · last seen just now"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":320,"height":844,"zoom":1,"admissionFailure":false,"failedAdmissions":0,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]}
|
||||
@@ -0,0 +1,146 @@
|
||||
{
|
||||
"initial": {
|
||||
"width": 292,
|
||||
"scrollWidth": 292,
|
||||
"left": 0,
|
||||
"overflowX": "clip",
|
||||
"tabIndex": -1,
|
||||
"cells": [
|
||||
{
|
||||
"text": "loki",
|
||||
"x": 9,
|
||||
"width": 116.796875,
|
||||
"titles": [
|
||||
"Expand details for loki",
|
||||
"Healthy",
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "running",
|
||||
"x": 125.796875,
|
||||
"width": 43.796875,
|
||||
"titles": [
|
||||
"running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "12%",
|
||||
"x": 169.59375,
|
||||
"width": 43.796875,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "74%",
|
||||
"x": 213.390625,
|
||||
"width": 49.625,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "—",
|
||||
"x": 263.015625,
|
||||
"width": 37.953125,
|
||||
"titles": [
|
||||
"No image update check reported"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"pointer": {
|
||||
"width": 292,
|
||||
"scrollWidth": 292,
|
||||
"left": 0,
|
||||
"overflowX": "clip",
|
||||
"tabIndex": -1,
|
||||
"cells": [
|
||||
{
|
||||
"text": "loki",
|
||||
"x": 9,
|
||||
"width": 116.796875,
|
||||
"titles": [
|
||||
"Expand details for loki",
|
||||
"Healthy",
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "running",
|
||||
"x": 125.796875,
|
||||
"width": 43.796875,
|
||||
"titles": [
|
||||
"running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "12%",
|
||||
"x": 169.59375,
|
||||
"width": 43.796875,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "74%",
|
||||
"x": 213.390625,
|
||||
"width": 49.625,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "—",
|
||||
"x": 263.015625,
|
||||
"width": 37.953125,
|
||||
"titles": [
|
||||
"No image update check reported"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"keyboard": {
|
||||
"width": 292,
|
||||
"scrollWidth": 292,
|
||||
"left": 0,
|
||||
"overflowX": "clip",
|
||||
"tabIndex": -1,
|
||||
"cells": [
|
||||
{
|
||||
"text": "loki",
|
||||
"x": 9,
|
||||
"width": 116.796875,
|
||||
"titles": [
|
||||
"Expand details for loki",
|
||||
"Healthy",
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "running",
|
||||
"x": 125.796875,
|
||||
"width": 43.796875,
|
||||
"titles": [
|
||||
"running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"text": "12%",
|
||||
"x": 169.59375,
|
||||
"width": 43.796875,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "74%",
|
||||
"x": 213.390625,
|
||||
"width": 49.625,
|
||||
"titles": []
|
||||
},
|
||||
{
|
||||
"text": "—",
|
||||
"x": 263.015625,
|
||||
"width": 37.953125,
|
||||
"titles": [
|
||||
"No image update check reported"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"accessibleRow": "- row \"Expand details for notifications-worker notifications-worker running 12% 48% —\":\n - cell \"Expand details for notifications-worker notifications-worker\":\n - button \"Expand details for notifications-worker\"\n - text: notifications-worker\n - cell \"running\"\n - cell \"12%\"\n - cell \"48%\"\n - cell \"—\"",
|
||||
"expanded": "true",
|
||||
"detailText": "notifications-worker\nContainer\nDocker / Podman\nOverview\nHistory\nManage\nRUNTIME CONTEXT\nObserved state\t\nRunning\n\nUptime\t\n6d 6h\n\nLast seen\t\n15s ago\n\nCONTAINER\nImage\t\nghcr.io/pulse-demo/notifications-worker:2026.04\n\nRestarts\t\n1\n\nCreated\t\n18 days ago\n\nStarted\t\n6 days ago\n\nCompose project\t\norion-2\n\nCompose service\t\npostgres\n\nLabels\t\ncom.docker.compose.project: orion-2\ncom.docker.compose.service: postgres\ncom.pulse.demo: true\ncom.pulse.role.production: true\ncom.pulse.role.queue: true\ncom.pulse.role.worker: true\n\nIDENTITY\nHostname\t\nnotifications-worker\n\nPrimary ID\t\napp-container:orion-2-26f93461b215\n\nParent\t\nagent-9dbedd74104d1527\n\nDiscovery\t\napp-container:notifications-worker\n\nMetrics Target\t\napp-container:orion-2-26f93461b215\n\nAliases\t\napp-container:orion-2-26f93461b215\norion-2-26f93461b215\norion-2-mock\napp-platform-01\n+4\nRelationship map\n\n1 canonical relationship\n\nCANONICAL RELATIONSHIPS\nnotifications-worker\n→\ndocker-network-0ab79ab291cd98fa\nAttached To\n100% confidence · Docker Adapter · last seen just now"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":1440,"height":900,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":true,"tabs":[{"label":"Proxmox","text":"Proxmox","icons":1,"visible":true},{"label":"Docker","text":"Docker","icons":1,"visible":true},{"label":"Kubernetes","text":"Kubernetes","icons":1,"visible":true},{"label":"TrueNAS","text":"TrueNAS","icons":1,"visible":true},{"label":"vSphere","text":"vSphere","icons":1,"visible":true},{"label":"Machines","text":"Machines","icons":1,"visible":true},{"label":"Alerts","text":"Alerts","icons":1,"visible":true},{"label":"Patrol","text":"Patrol","icons":1,"visible":true},{"label":"Actions","text":"Actions","icons":1,"visible":true},{"label":"Settings","text":"Settings","icons":1,"visible":true}]}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":1100,"height":900,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":390,"height":844,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"browser":"141.0.7390.37","width":320,"height":844,"zoom":1,"admissionFailure":true,"failedAdmissions":1,"before":[{"visible":false,"tabs":[{"label":"Proxmox","text":"ProxmoxProxmoxP","icons":1,"visible":false},{"label":"Docker","text":"DockerDockerD","icons":1,"visible":false},{"label":"Kubernetes","text":"KubernetesKubernetesK","icons":1,"visible":false},{"label":"TrueNAS","text":"TrueNASTrueNAST","icons":1,"visible":false},{"label":"vSphere","text":"vSpherev","icons":1,"visible":false},{"label":"Machines","text":"MachinesM","icons":1,"visible":false},{"label":"Alerts","text":"AlertsA","icons":1,"visible":false},{"label":"Patrol","text":"Pulse PatrolPatrolP","icons":1,"visible":false},{"label":"Actions","text":"ActionsA","icons":1,"visible":false},{"label":"Settings","text":"SettingsS","icons":1,"visible":false}]}]}
|
||||
|
After Width: | Height: | Size: 125 KiB |
|
After Width: | Height: | Size: 76 KiB |
|
After Width: | Height: | Size: 62 KiB |
|
After Width: | Height: | Size: 45 KiB |
@@ -0,0 +1,24 @@
|
||||
<testsuites id="" name="" tests="10" failures="0" skipped="0" errors="0" time="141.702948">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T21:41:30.828Z" hostname="chromium" tests="10" failures="0" skipped="0" time="140.755" errors="0">
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="14.988">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="8.345">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="20.39">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="19.41">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="8.215">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="7.265">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="17.578">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="15.298">
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 1440px" classname="96-navigation-socket-recovery.spec.ts" time="13.297">
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 320px" classname="96-navigation-socket-recovery.spec.ts" time="15.969">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -0,0 +1,72 @@
|
||||
[
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 1440px (admission failure: false)",
|
||||
"outcome": "expected",
|
||||
"duration": 14988,
|
||||
"startTime": "2026-09-05T21:41:31.466Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 1100px (admission failure: false)",
|
||||
"outcome": "expected",
|
||||
"duration": 8345,
|
||||
"startTime": "2026-09-05T21:41:46.567Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 390px (admission failure: false)",
|
||||
"outcome": "expected",
|
||||
"duration": 20390,
|
||||
"startTime": "2026-09-05T21:41:54.929Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 320px (admission failure: false)",
|
||||
"outcome": "expected",
|
||||
"duration": 19410,
|
||||
"startTime": "2026-09-05T21:42:15.330Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 1440px (admission failure: true)",
|
||||
"outcome": "expected",
|
||||
"duration": 8215,
|
||||
"startTime": "2026-09-05T21:42:34.752Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 1100px (admission failure: true)",
|
||||
"outcome": "expected",
|
||||
"duration": 7265,
|
||||
"startTime": "2026-09-05T21:42:42.978Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 390px (admission failure: true)",
|
||||
"outcome": "expected",
|
||||
"duration": 17578,
|
||||
"startTime": "2026-09-05T21:42:50.252Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "populated navigation survives socket loss at 320px (admission failure: true)",
|
||||
"outcome": "expected",
|
||||
"duration": 15298,
|
||||
"startTime": "2026-09-05T21:43:07.838Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "changed backend incidents recover without reload at 1440px",
|
||||
"outcome": "expected",
|
||||
"duration": 13297,
|
||||
"startTime": "2026-09-05T21:43:23.147Z",
|
||||
"retry": 0
|
||||
},
|
||||
{
|
||||
"title": "changed backend incidents recover without reload at 320px",
|
||||
"outcome": "expected",
|
||||
"duration": 15969,
|
||||
"startTime": "2026-09-05T21:43:36.453Z",
|
||||
"retry": 0
|
||||
}
|
||||
]
|
||||
|
After Width: | Height: | Size: 121 KiB |
|
After Width: | Height: | Size: 76 KiB |
|
After Width: | Height: | Size: 62 KiB |
|
After Width: | Height: | Size: 45 KiB |
@@ -0,0 +1,8 @@
|
||||
<testsuites id="" name="" tests="2" failures="0" skipped="0" errors="0" time="29.969079">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T21:40:17.066Z" hostname="chromium" tests="2" failures="0" skipped="0" time="29.07" errors="0">
|
||||
<testcase name="changed backend incidents recover without reload at 1440px" classname="96-navigation-socket-recovery.spec.ts" time="13.01">
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 320px" classname="96-navigation-socket-recovery.spec.ts" time="16.06">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
|
After Width: | Height: | Size: 138 KiB |
@@ -0,0 +1,112 @@
|
||||
<testsuites id="" name="" tests="10" failures="2" skipped="0" errors="0" time="140.39786600000002">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T21:24:36.804Z" hostname="chromium" tests="10" failures="2" skipped="0" time="138.52" errors="0">
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="7.971">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="8.415">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="17.518">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="18.745">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="7.864">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="9.074">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="17.632">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="16.043">
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 1440px" classname="96-navigation-socket-recovery.spec.ts" time="17.6">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 1440px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 1440px
|
||||
|
||||
Error: expect(locator).toBeVisible() failed
|
||||
|
||||
Locator: locator('[id="alert-mock-cluster-3-pve16-local-zfs/zfs-pool:local-zfs/device:sdb2::mock-cluster-3-pve16-local-zfs/zfs-pool:local-zfs/device:sdb2-health"]').getByRole('button', { name: 'Unacknowledge', exact: true })
|
||||
Expected: visible
|
||||
Timeout: 10000ms
|
||||
Error: element(s) not found
|
||||
|
||||
Call log:
|
||||
- Expect "toBeVisible" with timeout 10000ms
|
||||
- waiting for locator('[id="alert-mock-cluster-3-pve16-local-zfs/zfs-pool:local-zfs/device:sdb2::mock-cluster-3-pve16-local-zfs/zfs-pool:local-zfs/device:sdb2-health"]').getByRole('button', { name: 'Unacknowledge', exact: true })
|
||||
|
||||
|
||||
265 | const acknowledgedSnapshot = await reconnect();
|
||||
266 | expect(acknowledgedSnapshot.find((a: { id: string }) => a.id === target!.id)?.acknowledged).toBe(true);
|
||||
> 267 | await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible();
|
||||
| ^
|
||||
268 | await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' });
|
||||
269 |
|
||||
270 | const configResponse = await apiRequest(page, '/api/alerts/config');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:267:84
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 320px" classname="96-navigation-socket-recovery.spec.ts" time="17.658">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 320px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 320px
|
||||
|
||||
Error: expect(locator).toBeVisible() failed
|
||||
|
||||
Locator: locator('[id="alert-mock-cluster-8-pve44-local-zfs/zfs-pool:local-zfs/device:sdb2::mock-cluster-8-pve44-local-zfs/zfs-pool:local-zfs/device:sdb2-health"]').getByRole('button', { name: 'Unacknowledge', exact: true })
|
||||
Expected: visible
|
||||
Timeout: 10000ms
|
||||
Error: element(s) not found
|
||||
|
||||
Call log:
|
||||
- Expect "toBeVisible" with timeout 10000ms
|
||||
- waiting for locator('[id="alert-mock-cluster-8-pve44-local-zfs/zfs-pool:local-zfs/device:sdb2::mock-cluster-8-pve44-local-zfs/zfs-pool:local-zfs/device:sdb2-health"]').getByRole('button', { name: 'Unacknowledge', exact: true })
|
||||
|
||||
|
||||
265 | const acknowledgedSnapshot = await reconnect();
|
||||
266 | expect(acknowledgedSnapshot.find((a: { id: string }) => a.id === target!.id)?.acknowledged).toBe(true);
|
||||
> 267 | await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible();
|
||||
| ^
|
||||
268 | await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' });
|
||||
269 |
|
||||
270 | const configResponse = await apiRequest(page, '/api/alerts/config');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:267:84
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -0,0 +1,88 @@
|
||||
<testsuites id="" name="" tests="2" failures="2" skipped="0" errors="0" time="24.149623">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T21:38:22.149Z" hostname="chromium" tests="2" failures="2" skipped="0" time="22.578" errors="0">
|
||||
<testcase name="changed backend incidents recover without reload at 1440px" classname="96-navigation-socket-recovery.spec.ts" time="11.928">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 1440px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 1440px
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: true
|
||||
Received: false
|
||||
|
||||
Call Log:
|
||||
- Timeout 10000ms exceeded while waiting on the predicate
|
||||
|
||||
238 | // This standalone service fixture needs no host-offline confirmation delay
|
||||
239 | // and avoids moving grouped storage incidents into a collapsed panel.
|
||||
> 240 | await expect.poll(async () => (await readActive()).some(alert =>
|
||||
| ^
|
||||
241 | alert.resourceName === 'infra-docker')).toBe(true);
|
||||
242 | const initial = await readActive();
|
||||
243 | const target = initial.find(alert => alert.resourceName === 'infra-docker');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 320px" classname="96-navigation-socket-recovery.spec.ts" time="10.65">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 320px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 320px
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: true
|
||||
Received: false
|
||||
|
||||
Call Log:
|
||||
- Timeout 10000ms exceeded while waiting on the predicate
|
||||
|
||||
238 | // This standalone service fixture needs no host-offline confirmation delay
|
||||
239 | // and avoids moving grouped storage incidents into a collapsed panel.
|
||||
> 240 | await expect.poll(async () => (await readActive()).some(alert =>
|
||||
| ^
|
||||
241 | alert.resourceName === 'infra-docker')).toBe(true);
|
||||
242 | const initial = await readActive();
|
||||
243 | const target = initial.find(alert => alert.resourceName === 'infra-docker');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -0,0 +1,26 @@
|
||||
[
|
||||
{
|
||||
"path": "cold-fixture/junit.xml",
|
||||
"original_gzip": "cold-fixture/junit.xml.gz",
|
||||
"original_sha256": "1f2e930cd0e768b8ebc50e3d8cf0b359f0dad8859961a952f0739201efc663bc",
|
||||
"normalised_sha256": "d8add4a54e01a3c3989a222cc4b6d48e58f2c55be49ada4108849a576f27fdf8"
|
||||
},
|
||||
{
|
||||
"path": "initial/junit.xml",
|
||||
"original_gzip": "initial/junit.xml.gz",
|
||||
"original_sha256": "00a92c8c2100abe19663ea2a90809d78b468d920148a315db2b2b4b440b10220",
|
||||
"normalised_sha256": "07e4a08eee694e38068ee6fd025e4c4f3752438c7629ac6c329c4df3fb5b73c4"
|
||||
},
|
||||
{
|
||||
"path": "named-service/junit.xml",
|
||||
"original_gzip": "named-service/junit.xml.gz",
|
||||
"original_sha256": "f325c144f617d6031b835d2d34b1e25d86c8181fb1fe73ffa8e043089e65a876",
|
||||
"normalised_sha256": "99377de17b633459b3c5d1f30723d6ba73a43cc58dd5ec9cbec9ff53ef7bf5e1"
|
||||
},
|
||||
{
|
||||
"path": "standalone/junit.xml",
|
||||
"original_gzip": "standalone/junit.xml.gz",
|
||||
"original_sha256": "27e74939a4451199306ba701f8a73c823aa68cae29af23f620c00838f462ca85",
|
||||
"normalised_sha256": "ad130acf7faecd348093162a0bec613d43c81ddb01a7e238db40c30638b3ddd5"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,108 @@
|
||||
<testsuites id="" name="" tests="10" failures="2" skipped="0" errors="0" time="131.86015300000003">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T21:32:42.779Z" hostname="chromium" tests="10" failures="2" skipped="0" time="130.05" errors="0">
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="9.693">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="7.787">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="19.151">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="17.869">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="6.753">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="9.026">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="16.656">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="14.237">
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 1440px" classname="96-navigation-socket-recovery.spec.ts" time="16.536">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 1440px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 1440px
|
||||
|
||||
Error: expect(locator).toBeVisible() failed
|
||||
|
||||
Locator: locator('[id="alert-docker:proxmox-lxc-docker:Production West:pve1:112::docker:proxmox-lxc-docker:Production West:pve1:112-connectivity"]').getByRole('button', { name: 'Unacknowledge', exact: true })
|
||||
Expected: visible
|
||||
Timeout: 10000ms
|
||||
Error: element(s) not found
|
||||
|
||||
Call log:
|
||||
- Expect "toBeVisible" with timeout 10000ms
|
||||
- waiting for locator('[id="alert-docker:proxmox-lxc-docker:Production West:pve1:112::docker:proxmox-lxc-docker:Production West:pve1:112-connectivity"]').getByRole('button', { name: 'Unacknowledge', exact: true })
|
||||
|
||||
|
||||
269 | const acknowledgedSnapshot = await reconnect();
|
||||
270 | expect(acknowledgedSnapshot.find((a: { id: string }) => a.id === target!.id)?.acknowledged).toBe(true);
|
||||
> 271 | await expect(card.getByRole('button', { name: 'Unacknowledge', exact: true })).toBeVisible();
|
||||
| ^
|
||||
272 | await testInfo.attach('changed-incident', { body: await page.screenshot(), contentType: 'image/png' });
|
||||
273 |
|
||||
274 | const configResponse = await apiRequest(page, '/api/alerts/config');
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:271:84
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-2bfb7-er-without-reload-at-1440px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
<testcase name="changed backend incidents recover without reload at 320px" classname="96-navigation-socket-recovery.spec.ts" time="12.342">
|
||||
<failure message="96-navigation-socket-recovery.spec.ts:301:3 changed backend incidents recover without reload at 320px" type="FAILURE">
|
||||
<![CDATA[ [chromium] › 96-navigation-socket-recovery.spec.ts:301:3 › changed backend incidents recover without reload at 320px
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: true
|
||||
Received: false
|
||||
|
||||
Call Log:
|
||||
- Timeout 10000ms exceeded while waiting on the predicate
|
||||
|
||||
238 | // This standalone fixture avoids acknowledgement moving a grouped storage
|
||||
239 | // incident into a collapsed related-incidents panel.
|
||||
> 240 | await expect.poll(async () => (await readActive()).some(alert =>
|
||||
| ^
|
||||
241 | alert.resourceName === 'edge-proxy-01' && !alert.acknowledged)).toBe(true);
|
||||
242 | const initial = await readActive();
|
||||
243 | const target = initial.find(alert => alert.resourceName === 'edge-proxy-01' && !alert.acknowledged);
|
||||
at /var/lib/pulse-maintainer/team-worktrees/20260905T211528Z-web-product/pulse/tests/integration/tests/96-navigation-socket-recovery.spec.ts:240:5
|
||||
|
||||
attachment #1: screenshot (image/png) ──────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
attachment #2: video (video/webm) ──────────────────────────────────────────────────────────────
|
||||
../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error Context: ../test-results/96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/test-failed-1.png]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/video.webm]]
|
||||
|
||||
[[ATTACHMENT|96-navigation-socket-recov-cb0ef-ver-without-reload-at-320px-chromium/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
|
After Width: | Height: | Size: 59 KiB |
@@ -0,0 +1,88 @@
|
||||
# Integrated navigation recovery check — 5 September 2026
|
||||
|
||||
## Result
|
||||
|
||||
At source `c8931787adb7bc3152f060a829ebe2a1aa2a2b9c`, all eight existing
|
||||
`96-navigation-socket-recovery.spec.ts` Chromium cases passed (132 seconds,
|
||||
starting 20:43:55 UTC). No application or test changes were made for this run.
|
||||
The backend was built locally with embedded production frontend assets and
|
||||
synthetic inventory; it was stopped by the runner afterwards.
|
||||
|
||||
Local executable SHA-256:
|
||||
`a5aaf32fdba70744d1dfa4e0749f080dd8960dcd19d381ab6ec2fdc286d36b27`.
|
||||
This identifies this source-built test executable, not a published artifact.
|
||||
Locked dependencies were installed in root, frontend-modern and tests/integration;
|
||||
the runner used Playwright 1.56.1 and the Chromium project.
|
||||
|
||||
Widths 1440 and 1100 (height 900), and 390 and 320 (height 844), each passed
|
||||
with and without injected admission HTTP failure. The checks exercise socket
|
||||
closure with 1013, retained navigation/inventory, recovery without document
|
||||
reload, and incident-control access at desktop/mobile widths. Narrow cases
|
||||
also exercise platform switching and More navigation. Docker table keyboard
|
||||
access and clipped-text assertions require the separate `PULSE_E2E_TABLE_ACCESS=1`
|
||||
opt-in. The retained JUnit records eight successful cases but does not record
|
||||
that flag, so it cannot independently establish those conditional assertions
|
||||
ran. Admission-failure cases require an observed failed request.
|
||||
|
||||
`junit.xml` is the unmodified runner result. Two inspected screenshots retain
|
||||
representative desktop incident access during reconnect and 320px inventory.
|
||||
The narrow screenshot alone is not proof of connection state; the test asserts
|
||||
that state separately. Small update cells still wrap heavily at 320px, despite
|
||||
the earlier reported no-clipped-text result. That conditional result is not
|
||||
independently established by this retained JUnit. No readability redesign is implied.
|
||||
|
||||
Additional focused checks: 53 tests passed across websocket-resilience and
|
||||
websocket-unified; `TestAlertCharacterizationGetActiveAlertsExportsCanonicalIdentity`
|
||||
passed with `-count=1` in internal/alerts. No full repository suite was run.
|
||||
|
||||
## Repeat
|
||||
|
||||
From repository root after installing locked dependencies:
|
||||
|
||||
```sh
|
||||
pulse-heavy-run -- env \
|
||||
PULSE_E2E_USE_LOCAL_BACKEND=1 PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL=1 \
|
||||
PULSE_MOCK_MODE=true PULSE_E2E_NAVIGATION_RECOVERY=1 \
|
||||
PULSE_E2E_TABLE_ACCESS=1 \
|
||||
PULSE_E2E_LOCAL_BACKEND_PORT=18765 \
|
||||
npm --prefix tests/integration test -- \
|
||||
tests/96-navigation-socket-recovery.spec.ts --project=chromium
|
||||
```
|
||||
|
||||
## Decision and limits
|
||||
|
||||
Fresh retrieval of the [community comparison thread](https://www.reddit.com/r/Proxmox/comments/1lblkk8/anyone_else_switch_to_pulse_from_netdata_or_any/)
|
||||
on 5 September reinforced protecting quick-glance, low-overhead monitoring.
|
||||
This is historical, anecdotal feedback, not a new independent demand count or
|
||||
benchmark. The latest integrated source warranted checking the existing repairs
|
||||
together rather than creating another surface or repeating a withdrawn repair.
|
||||
|
||||
This result narrows the integration regression uncertainty. It does **not**
|
||||
qualify acknowledgement persistence, off-host delivery, restart history,
|
||||
malformed alert snapshots, a reporter installation, or a release candidate.
|
||||
The REST recovery code still skips entries without usable IDs. The server's
|
||||
normal read model exports typed alerts and canonical identity has focused test
|
||||
coverage, but that is not proof every malformed-response path is impossible.
|
||||
Do not count that previously withdrawn repair as delivered. Revisit it with
|
||||
browser fault injection, genuine empty-response clearing, and the required
|
||||
same-content performance/contracts qualification if pursued.
|
||||
|
||||
Next useful evidence is exact-candidate recovery and installed alert receipt,
|
||||
not another equivalent source-only navigation run without changed inputs.
|
||||
No release readiness or backport eligibility judgment is made here.
|
||||
|
||||
## Receipt audit — 5 September 2026
|
||||
|
||||
The repeat command above now explicitly enables table-access assertions. This
|
||||
is a documentation correction, not a fresh browser run or a claim that the
|
||||
original run omitted the flag. Retain the invocation flags and the
|
||||
`narrow-table-access` attachment on the next qualification run.
|
||||
|
||||
The recovery assertions in this spec establish reconnection, retained navigation
|
||||
and unchanged document identity. They do not compare a changed incident or
|
||||
resource value before and after the interruption. The managed-runtime recovery
|
||||
spec likewise checks connection status and HTTP health, not changed incident
|
||||
content. A future data-recovery qualification should change an identified
|
||||
incident during disconnection and verify its rendered state after reconnect,
|
||||
including a genuine empty active-alert response. Existing store-level snapshot
|
||||
tests are narrower evidence; no missing-data browser regression is claimed here.
|
||||
|
After Width: | Height: | Size: 139 KiB |
@@ -0,0 +1,20 @@
|
||||
<testsuites id="" name="" tests="8" failures="0" skipped="0" errors="0" time="132.169269">
|
||||
<testsuite name="96-navigation-socket-recovery.spec.ts" timestamp="2026-09-05T20:43:55.759Z" hostname="chromium" tests="8" failures="0" skipped="0" time="131.012" errors="0">
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="9.231">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="7.962">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="18.039">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: false)" classname="96-navigation-socket-recovery.spec.ts" time="16.376">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1440px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="11.781">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 1100px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="24.992">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 390px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="23.496">
|
||||
</testcase>
|
||||
<testcase name="populated navigation survives socket loss at 320px (admission failure: true)" classname="96-navigation-socket-recovery.spec.ts" time="19.135">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||