diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 6322c5759..da4722b53 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -6392,3 +6392,14 @@ has been applied; it cannot select an agent, add an agent-routed tool, establish a command session, or bypass capability, feasibility, policy, approval, dispatch, receipt, and verification gates. Unknown types retain the existing governed Patrol profile rather than guessing at agent authority. + +### Notification delivery-log routes do not touch agent lifecycle + +The shared `internal/api/` surface now includes the notifications-owned +read-only `GET /api/notifications/delivery-log` route and a `deliveryPaused` +field on successful test-send responses. Both are notification delivery +evidence only: they read retained notification audit rows and the manager's +enabled gate, and they do not enroll or select an agent, open or alter a +command session, change fleet policy or update state, or add any +agent-routed capability. Agent lifecycle obligations over `internal/api/` +are unchanged by this surface. diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 733b70c7d..f382b1968 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1722,3 +1722,21 @@ cannot recreate them while suppression remains active. Existing prefix, tag, and `pulse-no-alerts` bulk rules remain compatible inputs and do not create a second per-resource state store. `internal/alerts/intent_policy_test.go` pins factory operator-state evaluation, writer gating, and active reconciliation. + +### Destinations tab carries delivery evidence + +The alert destinations tab is where the belief "delivery works" is formed, so +it must carry delivery evidence, not only configuration. It renders the +notifications-owned delivery log (`AlertDeliveryLogCard` fed by +`useNotificationDeliveryLog` over `GET /api/notifications/delivery-log`) as a +newest-first record of real alert delivery attempts with plain-language +outcome labels, destination names resolved from the loaded webhook configs, +failure classes, and secret-redacted error text. The card names its retention +window and says that test sends skip the queue, and an unreadable log renders +as unavailable rather than empty. Test-send results that report +`deliveryPaused` surface as a warning toast, never plain success, across the +email, Apprise, and webhook test actions in `useAlertDestinationsTabState` +and `useAlertWebhookDestinationsState`. Delivery evidence remains +notification truth: the delivery log card must not resolve, suppress, or +re-evaluate alerts, and it does not alter the `AlertConfig.enabled` versus +`activationState` ownership boundary above. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 9a3a586e3..13146634c 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -4321,6 +4321,22 @@ it, and an update that echoes the masked placeholder must preserve the stored secret rather than overwriting it, matching the header and custom-field secret-handling rules above. +The notifications boundary additionally owns the delivery-log and +test-send-honesty payload contracts. `GET /api/notifications/delivery-log` +(settings-read scope) returns retained per-attempt delivery outcomes newest +first — `entries` carrying `notificationId`, `type`, `destinationId`, +`outcome` (`sent`/`retry`/`failed`/`dead_letter`/`cancelled`), `alertIds`, +`attempts`, `success`, `failureClass`, redacted `errorMessage`, and RFC 3339 +`timestamp` — alongside `window_days` and retention metadata, because the +counts are retention-bounded and must not read as lifetime history; webhook +secrets are redacted from error text before the payload leaves the API. The +frontend transport in `frontend-modern/src/api/notifications.ts` validates +entry shape and drops malformed rows rather than rendering them. Successful +test-send responses from `POST /api/notifications/test` and +`POST /api/notifications/webhooks/test` must include `deliveryPaused: true` +whenever real alert delivery is gated off, so a passing test cannot be +presented as proof that live alerts flow. + Proxmox setup bootstrap now includes a non-destructive Audit/Repair path in the generated PVE script and existing-source setup guide. That path audits token presence, token expiry, Pulse-managed token drift, and expected ACLs, then diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index ae2d40d2d..f277e8101 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -6247,3 +6247,15 @@ Proofs live in `frontend-modern/src/components/shared/__tests__/MobileNavBar.tes `frontend-modern/src/components/__tests__/GitHubStarBanner.test.tsx`, and `frontend-modern/src/__tests__/App.architecture.test.ts`, which fails if any runtime source reintroduces a literal bar height. + +### Alert delivery log presentation + +The destinations-tab delivery log renders through the shared `Card` primitive +in the feature-owned `AlertDeliveryLogCard`. Outcome badges use the +plain-language labels from `alertDestinationsPresentation` rather than queue +vocabulary ("Failed, retries exhausted", never "dead letter"), failure detail +lines use the shared red emphasis tokens in both themes, and entry rows wrap +without horizontal overflow at mobile widths. The unavailable state is a +`role="alert"` message distinct from the empty state, because "cannot read +the log" and "no attempts" mean opposite things to someone deciding whether +to trust their alerting. diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md index 0bf04c6e9..e3763edbd 100644 --- a/docs/release-control/v6/internal/subsystems/notifications.md +++ b/docs/release-control/v6/internal/subsystems/notifications.md @@ -325,6 +325,34 @@ counts cannot be read. Pending retries do not degrade health. The response must expose fixed reason codes and retention metadata rather than raw queue errors or notification content. +### User-facing delivery log and honest test sends + +The queue owner exposes its retained per-attempt audit rows to the local +notification-management API as a bounded, newest-first delivery log +(`GetDeliveryLog` in `internal/notifications/delivery_log.go`). Each entry +carries the attempt outcome (`sent`, `retry`, `failed`, `dead_letter`, +`cancelled`), derived by the same single rule that feeds the delivery outcome +metric in `RecordAudit`, plus alert identifiers, normalized destination +identity, attempt count, failure class, error text, and timestamp. Audit rows +persist destination identity in a dedicated `destination_id` column; rows +written before that column existed resolve destination identity from their +retained operational links. `GET /api/notifications/delivery-log` +(settings-read scope) serves the log as local operator evidence: webhook +secrets are redacted from error text at the API boundary, the payload names +its retention window instead of presenting itself as lifetime history, and an +unreadable queue is an error, never an empty log. This per-attempt surface is +deliberately distinct from the content-free telemetry aggregate above, which +remains identity-free. Test sends bypass the queue and must not appear in the +delivery log, and the destinations UI says so where the log renders. + +Because test sends also bypass the alert activation gate, a bare success +result is exactly how installs come to believe delivery works while every +real alert is suppressed. Successful responses from +`POST /api/notifications/test` and `POST /api/notifications/webhooks/test` +must therefore report `deliveryPaused: true` whenever the notification +manager is disabled, and the destinations UI must surface that as a warning +instead of a plain success toast. + ### Occurrence-bound delivery receipts The notification owner records successful firing delivery by exact alert ID, diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index f1df569c6..701465240 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -5330,3 +5330,13 @@ outside the already-projected Patrol profile and grants no plan, approval, execution, retention, cleanup, restore, or verification authority. An unknown resource type keeps the normal governed profile instead of inferring storage authority from prompt text. + +### Notification delivery-log routes do not confer storage authority + +The shared `internal/api/` surface now includes the notifications-owned +read-only `GET /api/notifications/delivery-log` route and a `deliveryPaused` +field on successful test-send responses. They expose retained notification +delivery outcomes and the delivery gate state only; they add no storage, +backup, snapshot, restore, retention, cleanup, or recovery capability, write +nothing, and leave storage and recovery state, evidence freshness, +persistence, and admission contracts unchanged. diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index db35c4b33..c74469d66 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,16 +1,28 @@ { "version": 1, - "base_sha": "fd35c547730a62ad7118de6cf898408f308d8317", - "verified_at": "2026-08-20T16:29:56Z", + "base_sha": "a5f1567172314003ee1b96b93090ffcbbe18e8f6", + "verified_at": "2026-08-20T16:20:02Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/i18n/messages.de.ts" + "frontend-modern/src/api/notifications.ts", + "frontend-modern/src/features/alerts/AlertDeliveryLogCard.tsx", + "frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx", + "frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts", + "frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts", + "frontend-modern/src/features/alerts/useNotificationDeliveryLog.ts", + "frontend-modern/src/utils/alertDestinationsPresentation.ts" ], "content_sha256": { - "frontend-modern/src/i18n/messages.de.ts": "0adaabff61c0ed7fb9064ea05f9d83595ab673958249d6cb5b0e6bc212a3ca46" + "frontend-modern/src/api/notifications.ts": "5bd796b6ec9222d6973a4a26b892b4fcfde11b2d9dd2e2ef705ad34531af0071", + "frontend-modern/src/features/alerts/AlertDeliveryLogCard.tsx": "7e7e4c5da2a4ea8c267c546da0064806949573bc57a51cf7c5b5f5f8576826ef", + "frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx": "0411429355fd78e75948df8ede39da75f9f9e3600840d9b98f7342d2adbfad7b", + "frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts": "c88336576ec08b69271244947f7892cac0b77e9ec68fdf33a12fe0f130e4e29d", + "frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts": "b9c959388a06ce146f3706cbc2b191449d4dd5380198d711ed15fdf06fdc48b9", + "frontend-modern/src/features/alerts/useNotificationDeliveryLog.ts": "48ca901ec9435d94a06de5e55bf05dbf254cfd23114cd2b31383909d1585237b", + "frontend-modern/src/utils/alertDestinationsPresentation.ts": "8a2e9446dfbdd77d46ef6a678ff2aa403741e3d296ac8ef67ba708c8940a9b2a" }, "routes": [ - "/settings/system-general" + "/alerts/notifications" ], "viewports": [ { @@ -23,12 +35,19 @@ } ], "states": [ - "German locale (pulseLocalePreference=de): Docker in Proxmox-LXCs card on the general settings panel renders the guestDocker toggle description with the corrected sentence break 'werden uebersprungen. Pulse erfasst nur Containerliste und Auslastung' in place of the semicolon the copy-style audit rejects", - "Mobile viewport: the same description renders with no horizontal overflow (documentElement.scrollWidth == 375)", - "Receipt re-pinned from base 23304bc01 to fd35c5477 during rebase: the staged messages.de.ts blob is byte-identical to the exercised content, and the new parent does not touch messages.de.ts" + "Fresh-install destinations tab (activation pending): AlertDeliveryPausedCard at top, Recent delivery activity card at bottom with honest empty state naming the 7-day window and the test-sends-skip-the-queue caveat", + "Delivery log with seeded audit rows served by the sidecar-built backend at GET /api/notifications/delivery-log: newest-first entries with Failed-retries-exhausted (red), Retrying (amber), Delivered (green) outcome badges", + "Webhook destination id (webhook:) resolved to the configured webhook name 'Ops Sink'; email entry labeled 'Email' with its opaque destination hash", + "Failure detail line renders 'Authentication failure: HTTP 401 Unauthorized from https://hooks.example.test/notify?token=REDACTED' proving API-layer webhook secret redaction (seeded value was token=supersecret)", + "Mobile viewport: card and badges wrap with no horizontal overflow (documentElement.scrollWidth == 375)", + "No console errors on the destinations tab after load, test send, and delivery log refresh", + "Receipt re-pinned during rebase onto a5f156717 (copy-fix replayed onto fd35c5477 canonical search work): every staged blob hash above is byte-identical to the exercised content, and neither upstream commit touches any of these files" ], "interactions": [ - "Set localStorage pulseLocalePreference=de on the sidecar-built stack (backend :7811, vite :5487), navigated to /settings/system-general, and located the rendered description text via a DOM text walk at desktop width", - "Resized to 375x812 and re-located the corrected text on the same panel" + "Logged into the isolated sidecar-built stack (backend :7811, vite :5487), opened Alerts > Notifications", + "Added webhook 'Ops Sink' -> http://127.0.0.1:18712/hook (allowlisted 127.0.0.1/32) and clicked Test: local sink logged the POST 200 and the UI raised the warning toast 'Test sent, but notification delivery is paused: real alerts are not being sent' instead of plain success, because the install's activation gate is pending", + "Saved the webhook via Add Webhook and confirmed GET /api/notifications/webhooks returns it", + "Seeded notification_audit rows (sent email, retry webhook, dead-letter webhook with token-bearing error text), clicked Refresh delivery status, and verified the three entries rendered newest-first with redacted error text", + "Resized to 375x812 and re-exercised the delivery log card" ] } diff --git a/frontend-modern/src/api/__tests__/notifications.test.ts b/frontend-modern/src/api/__tests__/notifications.test.ts index a6d333306..e669be2da 100644 --- a/frontend-modern/src/api/__tests__/notifications.test.ts +++ b/frontend-modern/src/api/__tests__/notifications.test.ts @@ -291,4 +291,87 @@ describe('NotificationsAPI', () => { }), ]); }); + + it('normalizes the delivery log payload and drops malformed entries', async () => { + apiFetchJSONMock.mockResolvedValueOnce({ + entries: [ + { + notificationId: 'webhook-1', + type: 'webhook', + destinationId: 'webhook:wh-ops', + outcome: 'dead_letter', + alertIds: ['vm-offline-101'], + alertCount: 1, + attempts: 3, + success: false, + errorMessage: 'HTTP 401 Unauthorized', + failureClass: 'authentication', + timestamp: '2026-08-20T12:00:00Z', + }, + // Malformed rows must be dropped, not rendered: unknown outcome, + // missing timestamp, and a non-object entry. + { + notificationId: 'bad-outcome', + type: 'email', + outcome: 'exploded', + timestamp: '2026-08-20T12:00:00Z', + }, + { notificationId: 'no-timestamp', type: 'email', outcome: 'sent' }, + 'not-an-object', + { + notificationId: 'email-1', + type: 'email', + outcome: 'sent', + alertIds: ['disk-critical-1'], + alertCount: 1, + attempts: 1, + success: true, + // An unrecognized failure class is omitted rather than passed through. + failureClass: 'made-up', + timestamp: '2026-08-20T11:00:00Z', + }, + ], + window_days: 7, + } as any); + + const log = await NotificationsAPI.getDeliveryLog(25); + + expect(apiFetchJSONMock).toHaveBeenCalledWith('/api/notifications/delivery-log?limit=25'); + expect(log.windowDays).toBe(7); + expect(log.entries).toHaveLength(2); + expect(log.entries[0]).toEqual( + expect.objectContaining({ + notificationId: 'webhook-1', + outcome: 'dead_letter', + destinationId: 'webhook:wh-ops', + failureClass: 'authentication', + errorMessage: 'HTTP 401 Unauthorized', + }), + ); + expect(log.entries[1].failureClass).toBeUndefined(); + }); + + it('requests the delivery log without a limit query when none is given', async () => { + apiFetchJSONMock.mockResolvedValueOnce({ entries: [], window_days: 0 } as any); + + const log = await NotificationsAPI.getDeliveryLog(); + + expect(apiFetchJSONMock).toHaveBeenCalledWith('/api/notifications/delivery-log'); + expect(log.entries).toEqual([]); + // A missing or nonsensical window falls back to the seven-day default. + expect(log.windowDays).toBe(7); + }); + + it('passes the deliveryPaused flag through from test-send responses', async () => { + apiFetchJSONMock.mockResolvedValueOnce({ + status: 'success', + message: 'Test notification sent, but alert delivery is paused', + deliveryPaused: true, + } as any); + + const result = await NotificationsAPI.testNotification({ type: 'email' }); + + expect(result.deliveryPaused).toBe(true); + expect(result.status).toBe('success'); + }); }); diff --git a/frontend-modern/src/api/notifications.ts b/frontend-modern/src/api/notifications.ts index 5705ef495..056959209 100644 --- a/frontend-modern/src/api/notifications.ts +++ b/frontend-modern/src/api/notifications.ts @@ -129,6 +129,37 @@ export interface NotificationHealth { queue: NotificationQueueHealth; } +export type NotificationDeliveryOutcome = 'sent' | 'retry' | 'failed' | 'dead_letter' | 'cancelled'; + +export interface NotificationDeliveryLogEntry { + notificationId: string; + type: string; + method?: string; + destinationId?: string; + outcome: NotificationDeliveryOutcome; + alertIds: string[]; + alertCount: number; + attempts: number; + success: boolean; + errorMessage?: string; + failureClass?: NotificationFailureClass; + timestamp: string; +} + +export interface NotificationDeliveryLog { + entries: NotificationDeliveryLogEntry[]; + windowDays: number; +} + +export interface NotificationTestResult { + status?: string; + message?: string; + // True when the test was delivered but real alert delivery is paused by the + // activation gate, so a successful test must not be read as proof that live + // alerts are getting through. + deliveryPaused?: boolean; +} + function apiRecord(value: unknown): Record { return value && typeof value === 'object' ? (value as Record) : {}; } @@ -154,6 +185,49 @@ const notificationFailureClasses: NotificationFailureClass[] = [ 'unknown', ]; +const notificationDeliveryOutcomes: NotificationDeliveryOutcome[] = [ + 'sent', + 'retry', + 'failed', + 'dead_letter', + 'cancelled', +]; + +function normalizeDeliveryLogEntry(value: unknown): NotificationDeliveryLogEntry | undefined { + const record = apiRecord(value); + const notificationId = strictString(record.notificationId); + const outcome = record.outcome; + const timestamp = strictString(record.timestamp); + if ( + !notificationId || + !timestamp || + !notificationDeliveryOutcomes.includes(outcome as NotificationDeliveryOutcome) + ) { + return undefined; + } + const failureClass = record.failureClass; + const entry: NotificationDeliveryLogEntry = { + notificationId, + type: strictString(record.type), + outcome: outcome as NotificationDeliveryOutcome, + alertIds: stringArray(record.alertIds), + alertCount: nonNegativeCount(record.alertCount) ?? 0, + attempts: nonNegativeCount(record.attempts) ?? 0, + success: strictBoolean(record.success), + timestamp, + }; + const method = strictString(record.method); + if (method) entry.method = method; + const destinationId = strictString(record.destinationId); + if (destinationId) entry.destinationId = destinationId; + const errorMessage = strictString(record.errorMessage); + if (errorMessage) entry.errorMessage = errorMessage; + if (notificationFailureClasses.includes(failureClass as NotificationFailureClass)) { + entry.failureClass = failureClass as NotificationFailureClass; + } + return entry; +} + function normalizeFailureClassCounts(value: unknown): NotificationFailureClassCounts | undefined { const record = apiRecord(value); const counts = {} as NotificationFailureClassCounts; @@ -260,6 +334,24 @@ export class NotificationsAPI { }; } + static async getDeliveryLog(limit?: number): Promise { + const query = limit && limit > 0 ? `?limit=${limit}` : ''; + const payload = await apiFetchJSON>( + `${this.baseUrl}/delivery-log${query}`, + ); + const rawEntries = Array.isArray(payload.entries) ? payload.entries : []; + const entries: NotificationDeliveryLogEntry[] = []; + for (const rawEntry of rawEntries) { + const entry = normalizeDeliveryLogEntry(rawEntry); + if (entry) entries.push(entry); + } + const windowDays = nonNegativeCount(payload.window_days); + return { + entries, + windowDays: windowDays && windowDays > 0 ? windowDays : 7, + }; + } + static async updateAppriseConfig(config: AppriseConfig): Promise { return apiFetchJSON(`${this.baseUrl}/apprise`, { method: 'PUT', @@ -363,9 +455,7 @@ export class NotificationsAPI { } // Testing - static async testNotification( - request: NotificationTestRequest, - ): Promise<{ success: boolean; message?: string }> { + static async testNotification(request: NotificationTestRequest): Promise { const body: { method: string; config?: Record | AppriseConfig; @@ -392,7 +482,7 @@ export class NotificationsAPI { static async testWebhook( webhook: Omit, - ): Promise<{ success: boolean; message?: string }> { + ): Promise<{ success?: boolean; message?: string; deliveryPaused?: boolean }> { return apiFetchJSON(`${this.baseUrl}/webhooks/test`, { method: 'POST', body: JSON.stringify(webhook), diff --git a/frontend-modern/src/features/alerts/AlertDeliveryLogCard.test.tsx b/frontend-modern/src/features/alerts/AlertDeliveryLogCard.test.tsx new file mode 100644 index 000000000..3559a2590 --- /dev/null +++ b/frontend-modern/src/features/alerts/AlertDeliveryLogCard.test.tsx @@ -0,0 +1,114 @@ +import { cleanup, render, screen } from '@solidjs/testing-library'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { NotificationDeliveryLog, Webhook } from '@/api/notifications'; + +import { AlertDeliveryLogCard } from './AlertDeliveryLogCard'; + +const webhooks: Webhook[] = [ + { + id: 'wh-ops', + name: 'Ops Discord', + url: 'https://discord.example.test/hook', + method: 'POST', + headers: {}, + enabled: true, + }, +]; + +const log: NotificationDeliveryLog = { + entries: [ + { + notificationId: 'webhook-1', + type: 'webhook', + destinationId: 'webhook:wh-ops', + outcome: 'dead_letter', + alertIds: ['vm-offline-101', 'vm-offline-102'], + alertCount: 2, + attempts: 3, + success: false, + errorMessage: 'HTTP 401 Unauthorized', + failureClass: 'authentication', + timestamp: new Date(Date.now() - 60_000).toISOString(), + }, + { + notificationId: 'email-1', + type: 'email', + destinationId: 'destination:abcd', + outcome: 'sent', + alertIds: ['disk-critical-1'], + alertCount: 1, + attempts: 1, + success: true, + timestamp: new Date(Date.now() - 120_000).toISOString(), + }, + ], + windowDays: 7, +}; + +describe('AlertDeliveryLogCard', () => { + afterEach(() => cleanup()); + + it('shows each attempt with a plain-language outcome and the resolved destination name', () => { + render(() => ( + + )); + + expect(screen.getByText('Ops Discord')).toBeInTheDocument(); + expect(screen.getByText('Email')).toBeInTheDocument(); + expect(screen.getByText('Failed, retries exhausted')).toBeInTheDocument(); + expect(screen.getByText('Delivered')).toBeInTheDocument(); + expect(screen.getByText('vm-offline-101 +1 more')).toBeInTheDocument(); + expect(screen.getByText(/Authentication failure/)).toBeInTheDocument(); + expect(screen.getByText(/HTTP 401 Unauthorized/)).toBeInTheDocument(); + }); + + it('says test sends are not listed so their absence is not read as failure', () => { + render(() => ( + + )); + + expect(screen.getByText(/Test sends skip the queue/)).toBeInTheDocument(); + }); + + it('renders an honest empty state when no deliveries were attempted', () => { + render(() => ( + + )); + + expect(screen.getByText(/No alert deliveries were attempted/)).toBeInTheDocument(); + }); + + it('reports an unreadable log as unavailable instead of empty', () => { + render(() => ( + + )); + + expect(screen.getByRole('alert')).toHaveTextContent(/could not read the delivery log/); + expect(screen.queryByText(/No alert deliveries were attempted/)).not.toBeInTheDocument(); + }); +}); diff --git a/frontend-modern/src/features/alerts/AlertDeliveryLogCard.tsx b/frontend-modern/src/features/alerts/AlertDeliveryLogCard.tsx new file mode 100644 index 000000000..c32e2e2c6 --- /dev/null +++ b/frontend-modern/src/features/alerts/AlertDeliveryLogCard.tsx @@ -0,0 +1,155 @@ +import { For, Show } from 'solid-js'; +import RefreshCwIcon from 'lucide-solid/icons/refresh-cw'; + +import type { + NotificationDeliveryLog, + NotificationDeliveryLogEntry, + Webhook, +} from '@/api/notifications'; +import { Card } from '@/components/shared/Card'; +import { + getAlertDeliveryLogFailureClassLabel, + getAlertDeliveryLogOutcomeLabel, + getAlertDestinationsDeliveryLogDescription, + getAlertDestinationsDeliveryLogEmpty, + getAlertDestinationsDeliveryLogTitle, + getAlertDestinationsDeliveryLogUnavailable, + getAlertDestinationsDeliveryRefreshLabel, +} from '@/utils/alertDestinationsPresentation'; +import { formatRelativeTime } from '@/utils/format'; + +interface AlertDeliveryLogCardProps { + log: NotificationDeliveryLog | null; + unavailable: boolean; + refreshing: boolean; + onRefresh: () => void; + webhooks: Webhook[]; +} + +const WEBHOOK_DESTINATION_PREFIX = 'webhook:'; + +const outcomeBadgeClasses: Record = { + sent: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300', + retry: 'bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300', + failed: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300', + dead_letter: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300', + cancelled: 'bg-gray-100 text-gray-600 dark:bg-gray-700/60 dark:text-gray-300', +}; + +export function AlertDeliveryLogCard(props: AlertDeliveryLogCardProps) { + // Destination ids are opaque on purpose (hashes for email/apprise, config + // ids for webhooks); only webhook ids can be resolved to a configured name. + const destinationLabel = (entry: NotificationDeliveryLogEntry): string => { + if (entry.destinationId?.startsWith(WEBHOOK_DESTINATION_PREFIX)) { + const webhookId = entry.destinationId.slice(WEBHOOK_DESTINATION_PREFIX.length); + const webhook = props.webhooks.find((candidate) => candidate.id === webhookId); + if (webhook?.name) return webhook.name; + } + switch (entry.type) { + case 'email': + return 'Email'; + case 'apprise': + return 'Apprise'; + case 'webhook': + return 'Webhook'; + default: + return entry.type || 'Destination'; + } + }; + + const alertSummary = (entry: NotificationDeliveryLogEntry): string => { + if (entry.alertIds.length === 0) { + return entry.alertCount === 1 ? '1 alert' : `${entry.alertCount} alerts`; + } + const [first, ...rest] = entry.alertIds; + return rest.length > 0 ? `${first} +${rest.length} more` : first; + }; + + const entries = () => props.log?.entries ?? []; + const windowDays = () => props.log?.windowDays ?? 7; + + return ( + +
+
+
+

+ {getAlertDestinationsDeliveryLogTitle()} +

+

+ {getAlertDestinationsDeliveryLogDescription(windowDays())} +

+
+ +
+ + + {getAlertDestinationsDeliveryLogUnavailable()} +

+ } + > + 0} + fallback={ +

+ {getAlertDestinationsDeliveryLogEmpty()} +

+ } + > +
    + + {(entry) => ( +
  • +
    + + {getAlertDeliveryLogOutcomeLabel(entry.outcome)} + + + {destinationLabel(entry)} + + + {alertSummary(entry)} + + + {formatRelativeTime(entry.timestamp)} + +
    + +

    + + {(failureClass) => ( + + {getAlertDeliveryLogFailureClassLabel(failureClass())} + {entry.errorMessage ? ': ' : ''} + + )} + + {entry.errorMessage} +

    +     +
    • + )} +     +
+ + +
+ + ); +} diff --git a/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx b/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx index c369e0d28..0f5f3bf84 100644 --- a/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx @@ -13,6 +13,7 @@ vi.mock('@/api/notifications', () => ({ NotificationsAPI: { createWebhook: vi.fn(), deleteWebhook: vi.fn(), + getDeliveryLog: vi.fn(), getHealth: vi.fn(), getWebhooks: vi.fn(), testNotification: vi.fn(), @@ -25,6 +26,7 @@ vi.mock('@/stores/notifications', () => ({ notificationStore: { error: vi.fn(), success: vi.fn(), + warning: vi.fn(), }, })); @@ -72,6 +74,7 @@ describe('useAlertDestinationsTabState', () => { beforeEach(() => { vi.mocked(NotificationsAPI.createWebhook).mockReset(); vi.mocked(NotificationsAPI.deleteWebhook).mockReset(); + vi.mocked(NotificationsAPI.getDeliveryLog).mockReset(); vi.mocked(NotificationsAPI.getHealth).mockReset(); vi.mocked(NotificationsAPI.getWebhooks).mockReset(); vi.mocked(NotificationsAPI.testNotification).mockReset(); @@ -79,6 +82,7 @@ describe('useAlertDestinationsTabState', () => { vi.mocked(NotificationsAPI.updateWebhook).mockReset(); vi.mocked(notificationStore.error).mockReset(); vi.mocked(notificationStore.success).mockReset(); + vi.mocked(notificationStore.warning).mockReset(); vi.mocked(showErrorWithDetail).mockReset(); }); @@ -130,7 +134,22 @@ describe('useAlertDestinationsTabState', () => { failureClassWindowDays: 7, }, }); - vi.mocked(NotificationsAPI.testNotification).mockResolvedValue({ success: true } as never); + vi.mocked(NotificationsAPI.getDeliveryLog).mockResolvedValue({ + entries: [ + { + notificationId: 'email-1', + type: 'email', + outcome: 'sent', + alertIds: ['disk-critical-1'], + alertCount: 1, + attempts: 1, + success: true, + timestamp: '2026-08-20T12:00:00Z', + }, + ], + windowDays: 7, + }); + vi.mocked(NotificationsAPI.testNotification).mockResolvedValue({ status: 'success' } as never); vi.mocked(NotificationsAPI.testWebhook).mockResolvedValue({ success: true } as never); const { result } = renderHook(() => @@ -147,7 +166,10 @@ describe('useAlertDestinationsTabState', () => { await waitFor(() => expect(NotificationsAPI.getWebhooks).toHaveBeenCalledTimes(1)); await waitFor(() => expect(NotificationsAPI.getHealth).toHaveBeenCalledTimes(1)); + await waitFor(() => expect(NotificationsAPI.getDeliveryLog).toHaveBeenCalledTimes(1)); expect(result.deliveryHealth()?.queue.status).toBe('healthy'); + expect(result.deliveryLog()?.entries).toHaveLength(1); + expect(result.deliveryLogUnavailable()).toBe(false); expect(result.webhooks()).toEqual([ expect.objectContaining({ id: 'hook-1', service: 'generic' }), ]); @@ -181,7 +203,49 @@ describe('useAlertDestinationsTabState', () => { expect(onRetryLoad).toHaveBeenCalledTimes(1); await waitFor(() => expect(NotificationsAPI.getWebhooks).toHaveBeenCalledTimes(2)); await waitFor(() => expect(NotificationsAPI.getHealth).toHaveBeenCalledTimes(2)); + await waitFor(() => expect(NotificationsAPI.getDeliveryLog).toHaveBeenCalledTimes(2)); expect(notificationStore.success).toHaveBeenCalledTimes(2); + expect(notificationStore.warning).not.toHaveBeenCalled(); expect(showErrorWithDetail).not.toHaveBeenCalled(); }); + + it('warns instead of celebrating when a test send reports delivery is paused', async () => { + const [emailConfig] = createSignal(buildEmailConfig()); + const [appriseConfig, setAppriseConfig] = createSignal(buildAppriseConfig()); + const [configLoadError] = createSignal(null); + const [isRetrying] = createSignal(false); + const [isLoadingDestinations] = createSignal(false); + + vi.mocked(NotificationsAPI.getWebhooks).mockResolvedValue([]); + vi.mocked(NotificationsAPI.getHealth).mockRejectedValue(new Error('offline')); + vi.mocked(NotificationsAPI.getDeliveryLog).mockResolvedValue({ entries: [], windowDays: 7 }); + // The backend reports the test went out while the activation gate keeps + // real alerts suppressed; plain success here is the postmortem trap. + vi.mocked(NotificationsAPI.testNotification).mockResolvedValue({ + status: 'success', + deliveryPaused: true, + } as never); + + const { result } = renderHook(() => + useAlertDestinationsTabState({ + appriseConfig, + configLoadError, + emailConfig, + isLoadingDestinations, + isRetrying, + onRetryLoad: vi.fn(), + setAppriseConfig, + }), + ); + + await result.testEmailConfig(); + expect(notificationStore.warning).toHaveBeenCalledWith( + expect.stringContaining('delivery is paused'), + ); + expect(notificationStore.success).not.toHaveBeenCalled(); + + await result.testApprise(); + expect(notificationStore.warning).toHaveBeenCalledTimes(2); + expect(notificationStore.success).not.toHaveBeenCalled(); + }); }); diff --git a/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryLog.test.tsx b/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryLog.test.tsx new file mode 100644 index 000000000..e97033227 --- /dev/null +++ b/frontend-modern/src/features/alerts/__tests__/useNotificationDeliveryLog.test.tsx @@ -0,0 +1,69 @@ +import { createRoot } from 'solid-js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { NotificationsAPI } from '@/api/notifications'; + +import { useNotificationDeliveryLog } from '../useNotificationDeliveryLog'; + +vi.mock('@/api/notifications', () => ({ + NotificationsAPI: { getDeliveryLog: vi.fn() }, +})); + +describe('useNotificationDeliveryLog', () => { + beforeEach(() => { + vi.mocked(NotificationsAPI.getDeliveryLog).mockReset(); + }); + + it('exposes the loaded log and clears the unavailable flag', () => + createRoot(async (dispose) => { + vi.mocked(NotificationsAPI.getDeliveryLog).mockResolvedValue({ + entries: [ + { + notificationId: 'email-1', + type: 'email', + outcome: 'sent', + alertIds: ['disk-critical-1'], + alertCount: 1, + attempts: 1, + success: true, + timestamp: '2026-08-20T12:00:00Z', + }, + ], + windowDays: 7, + }); + const state = useNotificationDeliveryLog(); + + await state.loadDeliveryLog(); + expect(state.deliveryLog()?.entries).toHaveLength(1); + expect(state.deliveryLogUnavailable()).toBe(false); + dispose(); + })); + + it('reports an unreadable log as unavailable, never as empty', () => + createRoot(async (dispose) => { + vi.mocked(NotificationsAPI.getDeliveryLog).mockRejectedValue(new Error('network down')); + const state = useNotificationDeliveryLog(); + + await state.loadDeliveryLog(); + expect(state.deliveryLogUnavailable()).toBe(true); + expect(state.deliveryLog()).toBeNull(); + dispose(); + })); + + it('recovers the unavailable flag once a later load succeeds', () => + createRoot(async (dispose) => { + vi.mocked(NotificationsAPI.getDeliveryLog).mockRejectedValueOnce(new Error('network down')); + vi.mocked(NotificationsAPI.getDeliveryLog).mockResolvedValue({ + entries: [], + windowDays: 7, + }); + const state = useNotificationDeliveryLog(); + + await state.loadDeliveryLog(); + expect(state.deliveryLogUnavailable()).toBe(true); + await state.loadDeliveryLog(); + expect(state.deliveryLogUnavailable()).toBe(false); + expect(state.deliveryLog()?.entries).toHaveLength(0); + dispose(); + })); +}); diff --git a/frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx b/frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx index 70f80c3e9..2b89fcea0 100644 --- a/frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx +++ b/frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx @@ -7,6 +7,7 @@ import { logger } from '@/utils/logger'; import type { AlertDestinationsDeliveryPausedReason } from '@/utils/alertDestinationsPresentation'; import { AlertAppriseDestinationsSection } from '../AlertAppriseDestinationsSection'; import { AlertDeliveryHealthCard } from '../AlertDeliveryHealthCard'; +import { AlertDeliveryLogCard } from '../AlertDeliveryLogCard'; import { AlertDeliveryPausedCard } from '../AlertDeliveryPausedCard'; import { AlertDestinationsLoadErrorCard } from '../AlertDestinationsLoadErrorCard'; import { AlertDestinationsLoadingState } from '../AlertDestinationsLoadingState'; @@ -120,6 +121,14 @@ export function DestinationsTab(props: DestinationsTabProps) { showUpgradePrompts={!presentationPolicyHidesUpgradePrompts()} upgradeDestination={getUpgradeActionDestination('relay')} /> + + void state.loadDeliveryLog()} + webhooks={state.webhooks()} + /> ); diff --git a/frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts b/frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts index c20d8f11a..692a7ed3d 100644 --- a/frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts +++ b/frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts @@ -10,11 +10,13 @@ import { getAlertDestinationsAppriseValidationError, getAlertDestinationsEmailTestFailure, getAlertDestinationsEmailTestSuccess, + getAlertDestinationsTestPausedWarning, } from '@/utils/alertDestinationsPresentation'; import { parseAppriseTargets } from './helpers'; import type { UIAppriseConfig, UIEmailConfig } from './types'; import { useNotificationDeliveryHealth } from './useNotificationDeliveryHealth'; +import { useNotificationDeliveryLog } from './useNotificationDeliveryLog'; import { useAlertWebhookDestinationsState } from './useAlertWebhookDestinationsState'; export interface AlertDestinationsTabStateProps { @@ -37,6 +39,8 @@ export function useAlertDestinationsTabState(props: AlertDestinationsTabStatePro deliveryNeedsAttention, loadDeliveryHealth, } = useNotificationDeliveryHealth(); + const { deliveryLog, deliveryLogUnavailable, refreshingDeliveryLog, loadDeliveryLog } = + useNotificationDeliveryLog(); const webhookState = useAlertWebhookDestinationsState(); const isLoading = createMemo( @@ -70,11 +74,15 @@ export function useAlertDestinationsTabState(props: AlertDestinationsTabStatePro const testEmailConfig = async () => { setTestingEmail(true); try { - await NotificationsAPI.testNotification({ + const result = await NotificationsAPI.testNotification({ type: 'email', config: { ...props.emailConfig() } as Record, }); - notificationStore.success(getAlertDestinationsEmailTestSuccess()); + if (result.deliveryPaused) { + notificationStore.warning(getAlertDestinationsTestPausedWarning()); + } else { + notificationStore.success(getAlertDestinationsEmailTestSuccess()); + } } catch (error) { logger.error(getAlertDestinationsEmailTestFailure(), error); const message = @@ -103,11 +111,15 @@ export function useAlertDestinationsTabState(props: AlertDestinationsTabStatePro throw new Error(getAlertDestinationsAppriseValidationError('missingServerUrl')); } - await NotificationsAPI.testNotification({ + const result = await NotificationsAPI.testNotification({ type: 'apprise', config, }); - notificationStore.success(getAlertDestinationsAppriseTestSuccess()); + if (result.deliveryPaused) { + notificationStore.warning(getAlertDestinationsTestPausedWarning()); + } else { + notificationStore.success(getAlertDestinationsAppriseTestSuccess()); + } } catch (error) { logger.error(getAlertDestinationsAppriseTestFailure(), error); const message = @@ -123,22 +135,28 @@ export function useAlertDestinationsTabState(props: AlertDestinationsTabStatePro props.onRetryLoad(); void webhookState.loadWebhooks(); void loadDeliveryHealth(); + void loadDeliveryLog(); }; onMount(() => { void loadDeliveryHealth(); + void loadDeliveryLog(); }); return { appriseState, deliveryHealth, deliveryHealthUnavailable, + deliveryLog, + deliveryLogUnavailable, deliveryNeedsAttention, handleRetry, hasLoadError, isLoading, loadDeliveryHealth, + loadDeliveryLog, refreshingDeliveryHealth, + refreshingDeliveryLog, testApprise, testEmailConfig, testingApprise, diff --git a/frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts b/frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts index 5064c687b..f95163b68 100644 --- a/frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts +++ b/frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts @@ -4,7 +4,10 @@ import { NotificationsAPI, type Webhook } from '@/api/notifications'; import { notificationStore } from '@/stores/notifications'; import { logger } from '@/utils/logger'; import { showErrorWithDetail } from '@/utils/toast'; -import { getAlertDestinationsWebhookLoadError } from '@/utils/alertDestinationsPresentation'; +import { + getAlertDestinationsTestPausedWarning, + getAlertDestinationsWebhookLoadError, +} from '@/utils/alertDestinationsPresentation'; import { getAlertWebhookMutationFailure, getAlertWebhookMutationSuccess, @@ -44,12 +47,14 @@ export function useAlertWebhookDestinationsState() { const testWebhook = async (webhookId: string, webhookData?: Omit) => { setTestingWebhook(webhookId); try { - if (webhookData) { - await NotificationsAPI.testWebhook(webhookData); + const result = webhookData + ? await NotificationsAPI.testWebhook(webhookData) + : await NotificationsAPI.testNotification({ type: 'webhook', webhookId }); + if (result.deliveryPaused) { + notificationStore.warning(getAlertDestinationsTestPausedWarning()); } else { - await NotificationsAPI.testNotification({ type: 'webhook', webhookId }); + notificationStore.success(getAlertWebhookTestSuccess()); } - notificationStore.success(getAlertWebhookTestSuccess()); } catch (error) { const message = error instanceof Error ? error.message : getAlertWebhookTestFailure(); const detail = (error as Error & { detail?: string })?.detail; diff --git a/frontend-modern/src/features/alerts/useNotificationDeliveryLog.ts b/frontend-modern/src/features/alerts/useNotificationDeliveryLog.ts new file mode 100644 index 000000000..980ca7239 --- /dev/null +++ b/frontend-modern/src/features/alerts/useNotificationDeliveryLog.ts @@ -0,0 +1,37 @@ +import { createSignal } from 'solid-js'; + +import { NotificationsAPI, type NotificationDeliveryLog } from '@/api/notifications'; +import { logger } from '@/utils/logger'; + +// The delivery log is the positive half of delivery evidence: health warns +// when something is wrong, the log shows what actually fired and where it +// went. A log that cannot be read is reported as unavailable, never as empty, +// because "no attempts" and "cannot tell" mean opposite things to someone +// deciding whether to trust their alerting. +export function useNotificationDeliveryLog() { + const [deliveryLog, setDeliveryLog] = createSignal(null); + const [deliveryLogUnavailable, setDeliveryLogUnavailable] = createSignal(false); + const [refreshingDeliveryLog, setRefreshingDeliveryLog] = createSignal(false); + + const loadDeliveryLog = async () => { + setRefreshingDeliveryLog(true); + try { + const log = await NotificationsAPI.getDeliveryLog(); + setDeliveryLog(log); + setDeliveryLogUnavailable(false); + } catch (error) { + logger.error('Failed to load notification delivery log', error); + setDeliveryLog(null); + setDeliveryLogUnavailable(true); + } finally { + setRefreshingDeliveryLog(false); + } + }; + + return { + deliveryLog, + deliveryLogUnavailable, + refreshingDeliveryLog, + loadDeliveryLog, + }; +} diff --git a/frontend-modern/src/utils/__tests__/alertDestinationsPresentation.test.ts b/frontend-modern/src/utils/__tests__/alertDestinationsPresentation.test.ts index 4515fbc2f..e24bd5db4 100644 --- a/frontend-modern/src/utils/__tests__/alertDestinationsPresentation.test.ts +++ b/frontend-modern/src/utils/__tests__/alertDestinationsPresentation.test.ts @@ -26,6 +26,13 @@ import { ALERT_DESTINATIONS_RETRYING_LABEL, ALERT_DESTINATIONS_RETRY_LABEL, ALERT_DESTINATIONS_WEBHOOK_LOAD_ERROR, + getAlertDeliveryLogFailureClassLabel, + getAlertDeliveryLogOutcomeLabel, + getAlertDestinationsDeliveryLogDescription, + getAlertDestinationsDeliveryLogEmpty, + getAlertDestinationsDeliveryLogTitle, + getAlertDestinationsDeliveryLogUnavailable, + getAlertDestinationsTestPausedWarning, getAlertDestinationsAppriseTargetsHelp, getAlertDestinationsAppriseTestLabel, getAlertDestinationsAppriseTestError, @@ -166,6 +173,42 @@ describe('alertDestinationsPresentation', () => { }); }); +describe('alert destinations delivery log copy', () => { + it('labels every delivery outcome in plain language, not queue jargon', () => { + expect(getAlertDeliveryLogOutcomeLabel('sent')).toBe('Delivered'); + expect(getAlertDeliveryLogOutcomeLabel('retry')).toBe('Retrying'); + expect(getAlertDeliveryLogOutcomeLabel('failed')).toBe('Failed'); + // "Dead letter" is queue vocabulary; a user needs to know retries stopped. + expect(getAlertDeliveryLogOutcomeLabel('dead_letter')).toBe('Failed, retries exhausted'); + expect(getAlertDeliveryLogOutcomeLabel('cancelled')).toBe('Cancelled'); + }); + + it('labels failure classes and falls back to unclassified for unknown values', () => { + expect(getAlertDeliveryLogFailureClassLabel('authentication')).toBe('Authentication failure'); + expect(getAlertDeliveryLogFailureClassLabel('rate_limited')).toBe('Rate limited'); + expect(getAlertDeliveryLogFailureClassLabel('made-up-class')).toBe('Unclassified failure'); + }); + + it('names the retention window and the test-send caveat so absence is not read as failure', () => { + expect(getAlertDestinationsDeliveryLogTitle()).toBe('Recent delivery activity'); + const description = getAlertDestinationsDeliveryLogDescription(7); + expect(description).toContain('last 7 days'); + expect(description).toContain('Test sends skip the queue'); + expect(getAlertDestinationsDeliveryLogEmpty()).toContain('No alert deliveries were attempted'); + // An unreadable log must never present itself as an empty one. + expect(getAlertDestinationsDeliveryLogUnavailable()).toContain( + 'could not read the delivery log', + ); + }); + + it('warns that a passing test does not mean live alerts flow while delivery is paused', () => { + const warning = getAlertDestinationsTestPausedWarning(); + expect(warning).toContain('Test sent'); + expect(warning).toContain('delivery is paused'); + expect(warning).toContain('real alerts are not being sent'); + }); +}); + describe('alert destinations delivery paused copy', () => { it('names the consequence and the test-send caveat for every paused reason', () => { expect(getAlertDestinationsDeliveryPausedTitle()).toBe('Notifications are paused'); diff --git a/frontend-modern/src/utils/alertDestinationsPresentation.ts b/frontend-modern/src/utils/alertDestinationsPresentation.ts index 51be5e31a..7b1b5efa3 100644 --- a/frontend-modern/src/utils/alertDestinationsPresentation.ts +++ b/frontend-modern/src/utils/alertDestinationsPresentation.ts @@ -199,6 +199,74 @@ export function getAlertDestinationsDeliveryRefreshLabel() { return ALERT_DESTINATIONS_DELIVERY_REFRESH_LABEL; } +// Shown instead of the plain test-success toast when the backend reports the +// test went out while real alert delivery is paused. Without this, a +// successful test is exactly how installs come to believe delivery works +// while every live alert is being suppressed. +export const ALERT_DESTINATIONS_TEST_PAUSED_WARNING = + 'Test sent, but notification delivery is paused: real alerts are not being sent. Turn on delivery at the top of this page.'; + +export function getAlertDestinationsTestPausedWarning() { + return ALERT_DESTINATIONS_TEST_PAUSED_WARNING; +} + +export const ALERT_DESTINATIONS_DELIVERY_LOG_TITLE = 'Recent delivery activity'; +export const ALERT_DESTINATIONS_DELIVERY_LOG_EMPTY = + 'No alert deliveries were attempted in this window.'; +export const ALERT_DESTINATIONS_DELIVERY_LOG_UNAVAILABLE = + 'Pulse could not read the delivery log, so recent delivery activity cannot be shown.'; + +export function getAlertDestinationsDeliveryLogTitle() { + return ALERT_DESTINATIONS_DELIVERY_LOG_TITLE; +} + +// The test-send caveat matters: test messages skip the queue entirely, so a +// user who sends a test and then checks this log would otherwise read its +// absence as a delivery failure. +export function getAlertDestinationsDeliveryLogDescription(windowDays: number) { + return `Delivery attempts for real alerts over the last ${windowDays} days. Test sends skip the queue and are not listed here.`; +} + +export function getAlertDestinationsDeliveryLogEmpty() { + return ALERT_DESTINATIONS_DELIVERY_LOG_EMPTY; +} + +export function getAlertDestinationsDeliveryLogUnavailable() { + return ALERT_DESTINATIONS_DELIVERY_LOG_UNAVAILABLE; +} + +export type AlertDeliveryLogOutcome = 'sent' | 'retry' | 'failed' | 'dead_letter' | 'cancelled'; + +// One plain-language label per outcome. "Dead letter" is queue jargon; what a +// user needs to know is that Pulse stopped retrying. +export function getAlertDeliveryLogOutcomeLabel(outcome: AlertDeliveryLogOutcome) { + switch (outcome) { + case 'sent': + return 'Delivered'; + case 'retry': + return 'Retrying'; + case 'dead_letter': + return 'Failed, retries exhausted'; + case 'cancelled': + return 'Cancelled'; + default: + return 'Failed'; + } +} + +export function getAlertDeliveryLogFailureClassLabel(failureClass: string) { + const labels: Record = { + authentication: 'Authentication failure', + rate_limited: 'Rate limited', + connectivity: 'Connectivity failure', + tls: 'TLS failure', + configuration: 'Configuration problem', + rejected: 'Rejected by destination', + unknown: 'Unclassified failure', + }; + return labels[failureClass] ?? labels.unknown; +} + const ALERT_DESTINATIONS_DELIVERY_PAUSED_TITLE = 'Notifications are paused'; const ALERT_DESTINATIONS_DELIVERY_PAUSED_ACTION = 'Turn on delivery'; diff --git a/internal/api/notifications.go b/internal/api/notifications.go index b77f75a00..8053b37fc 100644 --- a/internal/api/notifications.go +++ b/internal/api/notifications.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "sync" "time" @@ -45,6 +46,8 @@ type NotificationManager interface { TestEnhancedWebhook(notifications.EnhancedWebhookConfig) (int, string, error) GetQueueStats() (map[string]int, error) GetTelemetryStats(time.Time) (notifications.TelemetryStats, error) + GetDeliveryLog(time.Time, int) ([]notifications.DeliveryLogEntry, error) + IsEnabled() bool } // NotificationConfigPersistence defines the interface for saving notification configuration. @@ -710,8 +713,19 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R } } + // A test send bypasses the activation gate real alerts honor, so a bare + // success here is exactly how installs end up believing delivery works + // while every real alert is suppressed. Say so in the result itself. + response := map[string]interface{}{ + "status": "success", + "message": "Test notification sent", + } + if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() { + response["deliveryPaused"] = true + response["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent" + } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]string{"status": "success", "message": "Test notification sent"}) + json.NewEncoder(w).Encode(response) } // GetWebhookTemplates returns available webhook templates @@ -812,6 +826,11 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques w.WriteHeader(http.StatusBadRequest) } else { result["success"] = true + // Same honesty as TestNotification: a webhook test bypasses the + // activation gate, so its success must not imply live alerts flow. + if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() { + result["deliveryPaused"] = true + } } w.Header().Set("Content-Type", "application/json") @@ -903,6 +922,50 @@ func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *h json.NewEncoder(w).Encode(health) } +// GetDeliveryLog returns recent recorded delivery attempts, newest first. +// This is the per-attempt evidence behind the aggregate health verdict: what +// fired, which destination it went to, and what happened. Entries share the +// queue's retention windows, so the payload names the window rather than +// presenting itself as lifetime history. +func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) { + manager := h.getMonitor(r.Context()).GetNotificationManager() + + limit := 0 + if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed <= 0 { + http.Error(w, "limit must be a positive integer", http.StatusBadRequest) + return + } + limit = parsed + } + + entries, err := manager.GetDeliveryLog( + time.Now().Add(-completedQueueRetentionDays*24*time.Hour), limit, + ) + if err != nil { + log.Warn().Err(err).Msg("Failed to read notification delivery log") + http.Error(w, "Delivery log unavailable", http.StatusServiceUnavailable) + return + } + + // Webhook failure text can embed the destination URL, and destination URLs + // can embed credentials. + for i := range entries { + if entries[i].ErrorMessage != "" { + entries[i].ErrorMessage = notifications.RedactWebhookURLSecrets(entries[i].ErrorMessage) + } + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]interface{}{ + "entries": entries, + "window_days": completedQueueRetentionDays, + "dead_letter_retention_days": deadLetterQueueRetentionDays, + "entries_are_retention_bounded": true, + }) +} + // classifyNotificationQueueHealth delegates to the canonical rule in // internal/notifications so this endpoint and the monitoring loop that raises // the notification-delivery system alert cannot drift apart. @@ -1010,6 +1073,11 @@ func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *htt return } h.GetNotificationHealth(w, r) + case path == "/delivery-log" && r.Method == http.MethodGet: + if !requireAnyScope(config.ScopeSettingsRead, config.ScopeSettingsRead, config.ScopeSettingsWrite) { + return + } + h.GetDeliveryLog(w, r) default: http.Error(w, "Not found", http.StatusNotFound) } diff --git a/internal/api/notifications_health_test.go b/internal/api/notifications_health_test.go index 1380fee39..6dd736445 100644 --- a/internal/api/notifications_health_test.go +++ b/internal/api/notifications_health_test.go @@ -139,6 +139,182 @@ func TestGetNotificationHealthFailsClosedWhenQueueStatsAreUnavailable(t *testing } } +func TestGetDeliveryLogReturnsEntriesWithRedactedErrors(t *testing.T) { + mockMonitor := new(MockNotificationMonitor) + mockManager := new(MockNotificationManager) + mockMonitor.On("GetNotificationManager").Return(mockManager) + mockManager.On("GetDeliveryLog", mock.Anything, 25).Return([]notifications.DeliveryLogEntry{ + { + NotificationID: "webhook-1", + Type: "webhook", + DestinationID: "wh-ops", + Outcome: notifications.DeliveryOutcomeFailed, + AlertIDs: []string{"vm-offline-101"}, + AlertCount: 1, + Attempts: 3, + ErrorMessage: "post https://hooks.example.test/notify?token=supersecret returned 401", + FailureClass: "authentication", + }, + { + NotificationID: "email-1", + Type: "email", + Outcome: notifications.DeliveryOutcomeSent, + AlertIDs: []string{"disk-critical-1"}, + AlertCount: 1, + Attempts: 1, + Success: true, + }, + }, nil).Once() + + rec := httptest.NewRecorder() + NewNotificationHandlers(nil, mockMonitor).GetDeliveryLog( + rec, + httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log?limit=25", nil), + ) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + var response struct { + Entries []struct { + NotificationID string `json:"notificationId"` + Outcome string `json:"outcome"` + DestinationID string `json:"destinationId"` + AlertIDs []string `json:"alertIds"` + ErrorMessage string `json:"errorMessage"` + FailureClass string `json:"failureClass"` + } `json:"entries"` + WindowDays int `json:"window_days"` + DeadLetterRetentionDays int `json:"dead_letter_retention_days"` + EntriesAreRetentionBounded bool `json:"entries_are_retention_bounded"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("decode delivery log response: %v", err) + } + if len(response.Entries) != 2 { + t.Fatalf("entries = %#v, want 2", response.Entries) + } + if response.Entries[0].Outcome != "failed" || + response.Entries[0].DestinationID != "wh-ops" || + response.Entries[0].FailureClass != "authentication" || + len(response.Entries[0].AlertIDs) != 1 || + response.Entries[0].AlertIDs[0] != "vm-offline-101" { + t.Fatalf("failed entry = %#v", response.Entries[0]) + } + if strings.Contains(response.Entries[0].ErrorMessage, "supersecret") || + !strings.Contains(response.Entries[0].ErrorMessage, "token=REDACTED") { + t.Fatalf("error message not redacted: %q", response.Entries[0].ErrorMessage) + } + if response.WindowDays != 7 || + response.DeadLetterRetentionDays != 30 || + !response.EntriesAreRetentionBounded { + t.Fatalf("retention context = %#v", response) + } +} + +func TestGetDeliveryLogRejectsInvalidLimit(t *testing.T) { + mockMonitor := new(MockNotificationMonitor) + mockManager := new(MockNotificationManager) + mockMonitor.On("GetNotificationManager").Return(mockManager) + + for _, limit := range []string{"abc", "-1", "0"} { + rec := httptest.NewRecorder() + NewNotificationHandlers(nil, mockMonitor).GetDeliveryLog( + rec, + httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log?limit="+limit, nil), + ) + if rec.Code != http.StatusBadRequest { + t.Fatalf("limit=%q status = %d, want %d", limit, rec.Code, http.StatusBadRequest) + } + } + mockManager.AssertNotCalled(t, "GetDeliveryLog", mock.Anything, mock.Anything) +} + +func TestGetDeliveryLogFailsClosedWhenQueueUnavailable(t *testing.T) { + mockMonitor := new(MockNotificationMonitor) + mockManager := new(MockNotificationManager) + mockMonitor.On("GetNotificationManager").Return(mockManager) + mockManager.On("GetDeliveryLog", mock.Anything, 0).Return( + nil, errors.New("database path /secret unavailable"), + ).Once() + + rec := httptest.NewRecorder() + NewNotificationHandlers(nil, mockMonitor).GetDeliveryLog( + rec, + httptest.NewRequest(http.MethodGet, "/api/notifications/delivery-log", nil), + ) + + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable) + } + if body := rec.Body.String(); containsAny(body, "database path", "/secret") { + t.Fatalf("delivery log response exposed internal queue error: %s", body) + } +} + +// A test send bypasses the activation gate real alerts honor, so its result +// must say when real delivery is paused instead of reporting bare success. +func TestTestNotificationReportsPausedDeliveryInResult(t *testing.T) { + mockMonitor := new(MockNotificationMonitor) + mockManager := new(MockNotificationManager) + mockMonitor.On("GetNotificationManager").Return(mockManager) + mockManager.On("SendTestNotification", "email").Return(nil).Once() + mockManager.On("IsEnabled").Return(false).Once() + + rec := httptest.NewRecorder() + NewNotificationHandlers(nil, mockMonitor).TestNotification( + rec, + httptest.NewRequest( + http.MethodPost, + "/api/notifications/test", + strings.NewReader(`{"method":"email"}`), + ), + ) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + var response struct { + Status string `json:"status"` + Message string `json:"message"` + DeliveryPaused bool `json:"deliveryPaused"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("decode test notification response: %v", err) + } + if response.Status != "success" || !response.DeliveryPaused { + t.Fatalf("response = %#v, want success with deliveryPaused", response) + } + if !strings.Contains(response.Message, "paused") { + t.Fatalf("message %q does not warn about paused delivery", response.Message) + } +} + +func TestTestNotificationOmitsPausedFlagWhenDeliveryActive(t *testing.T) { + mockMonitor := new(MockNotificationMonitor) + mockManager := new(MockNotificationManager) + mockMonitor.On("GetNotificationManager").Return(mockManager) + mockManager.On("SendTestNotification", "email").Return(nil).Once() + mockManager.On("IsEnabled").Return(true).Once() + + rec := httptest.NewRecorder() + NewNotificationHandlers(nil, mockMonitor).TestNotification( + rec, + httptest.NewRequest( + http.MethodPost, + "/api/notifications/test", + strings.NewReader(`{"method":"email"}`), + ), + ) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK) + } + if strings.Contains(rec.Body.String(), "deliveryPaused") { + t.Fatalf("active delivery response carries paused flag: %s", rec.Body.String()) + } +} + func containsAny(value string, needles ...string) bool { for _, needle := range needles { if needle != "" && strings.Contains(value, needle) { diff --git a/internal/api/notifications_scope_test.go b/internal/api/notifications_scope_test.go index f16f44c6e..cbda77c32 100644 --- a/internal/api/notifications_scope_test.go +++ b/internal/api/notifications_scope_test.go @@ -41,6 +41,7 @@ func TestNotificationReadEndpointsRequireSettingsReadScope(t *testing.T) { "/api/notifications/webhook-history", "/api/notifications/email-providers", "/api/notifications/health", + "/api/notifications/delivery-log", } for _, path := range endpoints { diff --git a/internal/api/notifications_test.go b/internal/api/notifications_test.go index ba7886200..f097ce443 100644 --- a/internal/api/notifications_test.go +++ b/internal/api/notifications_test.go @@ -258,6 +258,19 @@ func (m *MockNotificationManager) GetTelemetryStats(since time.Time) (notificati return args.Get(0).(notifications.TelemetryStats), args.Error(1) } +func (m *MockNotificationManager) GetDeliveryLog(since time.Time, limit int) ([]notifications.DeliveryLogEntry, error) { + args := m.Called(since, limit) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]notifications.DeliveryLogEntry), args.Error(1) +} + +func (m *MockNotificationManager) IsEnabled() bool { + args := m.Called() + return args.Bool(0) +} + type MockNotificationConfigPersistence struct { mock.Mock } @@ -654,6 +667,7 @@ func TestNotificationHandlers(t *testing.T) { }}, {"POST", "/api/notifications/webhooks/test", func() { mockManager.On("TestEnhancedWebhook", mock.Anything).Return(200, "OK", nil).Once() + mockManager.On("IsEnabled").Return(true).Once() }}, {"PUT", "/api/notifications/webhooks/wh1", func() { mockManager.On("GetWebhooks").Return([]notifications.WebhookConfig{{ID: "wh1"}}).Once() @@ -705,6 +719,7 @@ func TestNotificationHandlers(t *testing.T) { t.Run("TestNotification", func(t *testing.T) { mockManager.On("SendTestNotification", "email").Return(nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]string{"method": "email"}) req := httptest.NewRequest("POST", "/api/notifications/test", bytes.NewReader(body)) w := httptest.NewRecorder() @@ -715,6 +730,7 @@ func TestNotificationHandlers(t *testing.T) { t.Run("TestNotification_Webhook", func(t *testing.T) { mockManager.On("GetWebhooks").Return([]notifications.WebhookConfig{{ID: "wh1"}}).Once() mockManager.On("SendTestWebhook", mock.Anything).Return(nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]string{"method": "webhook", "webhookId": "wh1"}) req := httptest.NewRequest("POST", "/api/notifications/test", bytes.NewReader(body)) w := httptest.NewRecorder() @@ -724,6 +740,7 @@ func TestNotificationHandlers(t *testing.T) { t.Run("TestWebhook", func(t *testing.T) { mockManager.On("TestEnhancedWebhook", mock.Anything).Return(200, "OK", nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]string{"url": "https://example.com/test", "service": "ntfy"}) req := httptest.NewRequest("POST", "/api/notifications/webhooks/test", bytes.NewReader(body)) w := httptest.NewRecorder() @@ -774,6 +791,7 @@ func TestNotificationHandlers(t *testing.T) { strings.Contains(webhook.PayloadTemplate, `"username": "Pulse Monitoring"`) && webhook.Headers["Content-Type"] == "application/json" })).Return(200, "OK", nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]string{"url": "https://example.com/test", "service": "discord"}) req := httptest.NewRequest("POST", "/api/notifications/webhooks/test", bytes.NewReader(body)) w := httptest.NewRecorder() @@ -796,6 +814,7 @@ func TestNotificationHandlers(t *testing.T) { !hasLegacyApp && !hasLegacyUser })).Return(200, "OK", nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]interface{}{ "url": "https://api.pushover.net/1/messages.json", @@ -814,6 +833,7 @@ func TestNotificationHandlers(t *testing.T) { t.Run("TestNotification_EmailWithConfig", func(t *testing.T) { mockManager.On("SendTestNotificationWithConfig", "email", mock.Anything, mock.Anything).Return(nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]interface{}{ "method": "email", "config": notifications.EmailConfig{Enabled: true, SMTPHost: "smtp.example.com", Password: "test"}, @@ -826,6 +846,7 @@ func TestNotificationHandlers(t *testing.T) { t.Run("TestNotification_AppriseWithConfig", func(t *testing.T) { mockManager.On("SendTestAppriseWithConfig", mock.Anything).Return(nil).Once() + mockManager.On("IsEnabled").Return(true).Once() body, _ := json.Marshal(map[string]interface{}{ "method": "apprise", "config": notifications.AppriseConfig{Enabled: true, APIKey: "test"}, diff --git a/internal/notifications/delivery_log.go b/internal/notifications/delivery_log.go new file mode 100644 index 000000000..2fc21c83a --- /dev/null +++ b/internal/notifications/delivery_log.go @@ -0,0 +1,215 @@ +package notifications + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" +) + +// The delivery log is the per-attempt record behind the delivery health +// verdict. Health says whether anything is wrong; the log is the evidence a +// user can read to see what fired, where it went, and what happened to it. +// It reads the same audit rows RecordAudit writes, so the log and the +// aggregate verdicts cannot disagree about what was attempted. + +// Delivery outcomes as surfaced to callers. These name what happened to one +// attempt, not the row's live queue status: a pending row with attempts is a +// retry in flight, not a success or a terminal failure. +const ( + DeliveryOutcomeSent = "sent" + DeliveryOutcomeRetry = "retry" + DeliveryOutcomeFailed = "failed" + DeliveryOutcomeDeadLetter = "dead_letter" + DeliveryOutcomeCancelled = "cancelled" +) + +// maxDeliveryLogEntries bounds one read so an install with a large retained +// audit trail cannot be asked for an unbounded payload. +const maxDeliveryLogEntries = 200 + +// defaultDeliveryLogEntries is the page size when the caller does not ask for +// a specific limit. +const defaultDeliveryLogEntries = 50 + +// DeliveryLogEntry is one recorded delivery attempt outcome. +type DeliveryLogEntry struct { + NotificationID string `json:"notificationId"` + Type string `json:"type"` + Method string `json:"method,omitempty"` + DestinationID string `json:"destinationId,omitempty"` + Outcome string `json:"outcome"` + AlertIDs []string `json:"alertIds"` + AlertCount int `json:"alertCount"` + Attempts int `json:"attempts"` + Success bool `json:"success"` + ErrorMessage string `json:"errorMessage,omitempty"` + FailureClass string `json:"failureClass,omitempty"` + Timestamp time.Time `json:"timestamp"` +} + +// deliveryOutcome is the single rule turning an audit row's status into the +// outcome callers act on. RecordAudit uses it for the operational-trust +// metric and GetDeliveryLog uses it for the user-facing log, so the two can +// never label the same attempt differently. +func deliveryOutcome(status NotificationQueueStatus, success bool, attempts int) string { + switch { + case success && status == QueueStatusSent: + return DeliveryOutcomeSent + case status == QueueStatusDLQ: + return DeliveryOutcomeDeadLetter + case status == QueueStatusCancelled: + return DeliveryOutcomeCancelled + case status == QueueStatusPending && attempts > 0: + return DeliveryOutcomeRetry + } + return DeliveryOutcomeFailed +} + +// GetDeliveryLog returns recorded delivery attempts on or after since, newest +// first. Rows share the queue's retention windows, so a window longer than +// the completed-row retention must not be presented as complete history. +func (nq *NotificationQueue) GetDeliveryLog(since time.Time, limit int) ([]DeliveryLogEntry, error) { + if nq == nil { + return nil, fmt.Errorf("notification queue not initialized") + } + if since.IsZero() { + since = time.Now().Add(-7 * 24 * time.Hour) + } + if limit <= 0 { + limit = defaultDeliveryLogEntries + } + if limit > maxDeliveryLogEntries { + limit = maxDeliveryLogEntries + } + + nq.mu.RLock() + defer nq.mu.RUnlock() + + rows, err := nq.db.Query(` + SELECT notification_id, type, method, status, alert_identifiers, + alert_count, operational_links, attempts, success, error_message, + failure_class, destination_id, timestamp + FROM notification_audit + WHERE timestamp >= ? + ORDER BY timestamp DESC, id DESC + LIMIT ? + `, since.UTC().Unix(), limit) + if err != nil { + return nil, fmt.Errorf("read notification delivery log: %w", err) + } + defer rows.Close() + + entries := make([]DeliveryLogEntry, 0, limit) + for rows.Next() { + var ( + entry DeliveryLogEntry + method sql.NullString + status sql.NullString + alertIdentifiers sql.NullString + alertCount sql.NullInt64 + operationalLinks sql.NullString + attempts sql.NullInt64 + success sql.NullBool + errorMessage sql.NullString + failureClass sql.NullString + destinationID sql.NullString + timestamp int64 + ) + if err := rows.Scan( + &entry.NotificationID, + &entry.Type, + &method, + &status, + &alertIdentifiers, + &alertCount, + &operationalLinks, + &attempts, + &success, + &errorMessage, + &failureClass, + &destinationID, + ×tamp, + ); err != nil { + return nil, fmt.Errorf("scan notification delivery log row: %w", err) + } + + entry.Method = method.String + entry.Attempts = int(attempts.Int64) + entry.Success = success.Bool + entry.ErrorMessage = errorMessage.String + entry.FailureClass = failureClass.String + entry.Timestamp = time.Unix(timestamp, 0).UTC() + entry.Outcome = deliveryOutcome( + NotificationQueueStatus(status.String), success.Bool, int(attempts.Int64), + ) + + entry.AlertIDs = decodeAuditAlertIdentifiers(alertIdentifiers.String) + entry.AlertCount = int(alertCount.Int64) + if entry.AlertCount == 0 { + entry.AlertCount = len(entry.AlertIDs) + } + // Rows written before the destination_id column existed still carry the + // destination inside their operational links. + entry.DestinationID = strings.TrimSpace(destinationID.String) + if entry.DestinationID == "" { + entry.DestinationID = decodeAuditDestinationID(operationalLinks.String) + } + + entries = append(entries, entry) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate notification delivery log: %w", err) + } + return entries, nil +} + +func decodeAuditAlertIdentifiers(raw string) []string { + if strings.TrimSpace(raw) == "" { + return []string{} + } + var identifiers []string + if err := json.Unmarshal([]byte(raw), &identifiers); err != nil { + return []string{} + } + if identifiers == nil { + return []string{} + } + return identifiers +} + +func decodeAuditDestinationID(raw string) string { + if strings.TrimSpace(raw) == "" { + return "" + } + var links []operationaltrust.NotificationLink + if err := json.Unmarshal([]byte(raw), &links); err != nil { + return "" + } + for _, link := range links { + if destination := strings.TrimSpace(link.DestinationID); destination != "" { + return destination + } + } + return "" +} + +// GetDeliveryLog exposes the queue's delivery log through the manager. A +// missing queue is an error, not an empty log: silence must not read as "no +// attempts were made". +func (n *NotificationManager) GetDeliveryLog(since time.Time, limit int) ([]DeliveryLogEntry, error) { + if n == nil { + return nil, fmt.Errorf("notification manager not initialized") + } + n.mu.RLock() + queue := n.queue + n.mu.RUnlock() + + if queue == nil { + return nil, fmt.Errorf("notification queue not initialized") + } + return queue.GetDeliveryLog(since, limit) +} diff --git a/internal/notifications/delivery_log_test.go b/internal/notifications/delivery_log_test.go new file mode 100644 index 000000000..8948b9df5 --- /dev/null +++ b/internal/notifications/delivery_log_test.go @@ -0,0 +1,213 @@ +package notifications + +import ( + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" +) + +func TestGetDeliveryLogReturnsWindowedAttemptsNewestFirst(t *testing.T) { + nq, err := NewNotificationQueue(t.TempDir()) + if err != nil { + t.Fatalf("NewNotificationQueue: %v", err) + } + defer func() { _ = nq.Stop() }() + + now := time.Now().UTC() + sent := &QueuedNotification{ + ID: "email-sent", + Type: "email", + DestinationID: "destination:aaaa", + Status: QueueStatusPending, + Alerts: []*alerts.Alert{{ID: "disk-critical-1", StartTime: now.Add(-time.Hour)}}, + Config: []byte(`{}`), + CreatedAt: now.Add(-time.Hour), + } + failed := &QueuedNotification{ + ID: "webhook-dlq", + Type: "webhook", + DestinationID: "webhook:wh-ops", + Status: QueueStatusPending, + Alerts: []*alerts.Alert{{ID: "vm-offline-101", StartTime: now.Add(-time.Hour)}}, + Config: []byte(`{}`), + CreatedAt: now.Add(-time.Hour), + } + old := &QueuedNotification{ + ID: "email-old", + Type: "email", + Status: QueueStatusPending, + Config: []byte(`{}`), + CreatedAt: now.Add(-8 * 24 * time.Hour), + } + for _, notif := range []*QueuedNotification{sent, failed, old} { + if err := nq.Enqueue(notif); err != nil { + t.Fatalf("enqueue %s: %v", notif.ID, err) + } + } + + old.Attempts = 1 + old.Status = QueueStatusSent + if err := nq.RecordAudit(old, true, ""); err != nil { + t.Fatalf("record old delivery: %v", err) + } + sent.Attempts = 1 + sent.Status = QueueStatusSent + if err := nq.RecordAudit(sent, true, ""); err != nil { + t.Fatalf("record sent delivery: %v", err) + } + failed.Attempts = 1 + failed.Status = QueueStatusPending + if err := nq.RecordAudit(failed, false, "HTTP 401 Unauthorized"); err != nil { + t.Fatalf("record retry attempt: %v", err) + } + failed.Attempts = 3 + failed.Status = QueueStatusDLQ + if err := nq.RecordAudit(failed, false, "HTTP 401 Unauthorized"); err != nil { + t.Fatalf("record dead-letter attempt: %v", err) + } + if _, err := nq.db.Exec( + `UPDATE notification_audit SET timestamp = ? WHERE notification_id = ?`, + now.Add(-8*24*time.Hour).Unix(), + "email-old", + ); err != nil { + t.Fatalf("age old audit row: %v", err) + } + + entries, err := nq.GetDeliveryLog(now.Add(-7*24*time.Hour), 0) + if err != nil { + t.Fatalf("GetDeliveryLog: %v", err) + } + if len(entries) != 3 { + t.Fatalf("entries = %d, want 3 (windowed rows only): %#v", len(entries), entries) + } + + deadLetter := entries[0] + if deadLetter.NotificationID != "webhook-dlq" || + deadLetter.Outcome != DeliveryOutcomeDeadLetter || + deadLetter.DestinationID != "webhook:wh-ops" || + deadLetter.Attempts != 3 || + deadLetter.Success || + deadLetter.FailureClass != "authentication" { + t.Fatalf("dead-letter entry = %#v", deadLetter) + } + if len(deadLetter.AlertIDs) != 1 || deadLetter.AlertIDs[0] != "vm-offline-101" || + deadLetter.AlertCount != 1 { + t.Fatalf("dead-letter alerts = %#v", deadLetter) + } + + retry := entries[1] + if retry.NotificationID != "webhook-dlq" || retry.Outcome != DeliveryOutcomeRetry { + t.Fatalf("retry entry = %#v", retry) + } + + delivered := entries[2] + if delivered.NotificationID != "email-sent" || + delivered.Outcome != DeliveryOutcomeSent || + !delivered.Success || + delivered.DestinationID != "destination:aaaa" || + delivered.FailureClass != "" || + delivered.ErrorMessage != "" { + t.Fatalf("delivered entry = %#v", delivered) + } + if delivered.Timestamp.IsZero() { + t.Fatalf("delivered entry has no timestamp: %#v", delivered) + } +} + +func TestGetDeliveryLogHonorsLimitAndCap(t *testing.T) { + nq, err := NewNotificationQueue(t.TempDir()) + if err != nil { + t.Fatalf("NewNotificationQueue: %v", err) + } + defer func() { _ = nq.Stop() }() + + now := time.Now().UTC() + notif := &QueuedNotification{ + ID: "email-1", + Type: "email", + Status: QueueStatusPending, + Config: []byte(`{}`), + CreatedAt: now, + } + if err := nq.Enqueue(notif); err != nil { + t.Fatalf("enqueue: %v", err) + } + notif.Status = QueueStatusSent + for i := 0; i < 3; i++ { + notif.Attempts = i + 1 + if err := nq.RecordAudit(notif, true, ""); err != nil { + t.Fatalf("record delivery %d: %v", i, err) + } + } + + entries, err := nq.GetDeliveryLog(now.Add(-time.Hour), 2) + if err != nil { + t.Fatalf("GetDeliveryLog: %v", err) + } + if len(entries) != 2 { + t.Fatalf("entries = %d, want limit of 2", len(entries)) + } + if entries[0].Attempts != 3 { + t.Fatalf("newest entry = %#v, want the latest attempt first", entries[0]) + } + + if capped, err := nq.GetDeliveryLog(now.Add(-time.Hour), maxDeliveryLogEntries*10); err != nil { + t.Fatalf("GetDeliveryLog capped: %v", err) + } else if len(capped) != 3 { + t.Fatalf("capped entries = %d, want all 3", len(capped)) + } +} + +func TestGetDeliveryLogFallsBackToOperationalLinksForLegacyRows(t *testing.T) { + nq, err := NewNotificationQueue(t.TempDir()) + if err != nil { + t.Fatalf("NewNotificationQueue: %v", err) + } + defer func() { _ = nq.Stop() }() + + now := time.Now().UTC() + notif := &QueuedNotification{ + ID: "webhook-legacy", + Type: "webhook", + Status: QueueStatusPending, + Config: []byte(`{}`), + CreatedAt: now, + } + if err := nq.Enqueue(notif); err != nil { + t.Fatalf("enqueue: %v", err) + } + notif.Status = QueueStatusSent + notif.Attempts = 1 + notif.Links = []operationaltrust.NotificationLink{{DestinationID: "webhook:legacy"}} + if err := nq.RecordAudit(notif, true, ""); err != nil { + t.Fatalf("record delivery: %v", err) + } + // Simulate a row written before the destination_id column existed. + if _, err := nq.db.Exec( + `UPDATE notification_audit SET destination_id = '' WHERE notification_id = ?`, + "webhook-legacy", + ); err != nil { + t.Fatalf("clear destination column: %v", err) + } + + entries, err := nq.GetDeliveryLog(now.Add(-time.Hour), 0) + if err != nil { + t.Fatalf("GetDeliveryLog: %v", err) + } + if len(entries) != 1 || entries[0].DestinationID != "webhook:legacy" { + t.Fatalf("entries = %#v, want destination decoded from links", entries) + } +} + +func TestGetDeliveryLogFailsClosedWithoutQueue(t *testing.T) { + var nq *NotificationQueue + if _, err := nq.GetDeliveryLog(time.Now().Add(-time.Hour), 0); err == nil { + t.Fatal("nil queue returned a delivery log instead of an error") + } + var manager *NotificationManager + if _, err := manager.GetDeliveryLog(time.Now().Add(-time.Hour), 0); err == nil { + t.Fatal("nil manager returned a delivery log instead of an error") + } +} diff --git a/internal/notifications/queue.go b/internal/notifications/queue.go index 220927d9d..6efa704a7 100644 --- a/internal/notifications/queue.go +++ b/internal/notifications/queue.go @@ -28,6 +28,7 @@ const ( legacyNotificationAuditAlertIdentifiersColumn = "alert_ids" notificationOperationalLinksColumn = "operational_links" notificationFailureClassColumn = "failure_class" + notificationAuditDestinationColumn = "destination_id" notificationQueueDirName = "notifications" notificationQueueFileName = "notification_queue.db" ) @@ -412,6 +413,7 @@ func (nq *NotificationQueue) initSchema() error { success BOOLEAN, error_message TEXT, failure_class TEXT NOT NULL DEFAULT '', + destination_id TEXT NOT NULL DEFAULT '', payload_size INTEGER, timestamp INTEGER NOT NULL, FOREIGN KEY (notification_id) REFERENCES notification_queue(id) @@ -452,9 +454,15 @@ func (nq *NotificationQueue) initSchema() error { ); err != nil { return err } - return nq.ensureTextColumn( + if err := nq.ensureTextColumn( "notification_audit", notificationFailureClassColumn, + ); err != nil { + return err + } + return nq.ensureTextColumn( + "notification_audit", + notificationAuditDestinationColumn, ) } @@ -1284,8 +1292,8 @@ func (nq *NotificationQueue) RecordAudit(notif *QueuedNotification, success bool query := ` INSERT INTO notification_audit - (notification_id, type, method, status, alert_identifiers, alert_count, operational_links, attempts, success, error_message, failure_class, payload_size, timestamp) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (notification_id, type, method, status, alert_identifiers, alert_count, operational_links, attempts, success, error_message, failure_class, destination_id, payload_size, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` failureClass := "" @@ -1304,22 +1312,14 @@ func (nq *NotificationQueue) RecordAudit(notif *QueuedNotification, success bool success, errorMsg, failureClass, + strings.TrimSpace(notif.DestinationID), notif.PayloadBytes, time.Now().Unix(), ) if err == nil { - outcome := "failed" - switch { - case success && notif.Status == QueueStatusSent: - outcome = "sent" - case notif.Status == QueueStatusDLQ: - outcome = "dead_letter" - case notif.Status == QueueStatusCancelled: - outcome = "cancelled" - case notif.Status == QueueStatusPending && notif.Attempts > 0: - outcome = "retry" - } - operationaltrust.GetMetrics().ObserveNotificationDelivery(outcome) + operationaltrust.GetMetrics().ObserveNotificationDelivery( + deliveryOutcome(notif.Status, success, notif.Attempts), + ) } return err }