From e0baecc418d15a6638ef80ed1e2b39d08aa9a5ad Mon Sep 17 00:00:00 2001 From: rcourtman Date: Thu, 27 Aug 2026 19:48:56 +0100 Subject: [PATCH] Make alert escalations destination-specific and repeatable --- .../v6/internal/subsystems/alerts.md | 20 +++ .../subsystems/frontend-primitives.md | 11 ++ .../v6/internal/subsystems/monitoring.md | 9 ++ .../v6/internal/subsystems/notifications.md | 14 ++ frontend-modern/browser-verification.json | 53 +++++-- .../alerts/AlertEscalationSection.tsx | 139 ++++++++++++++---- .../alerts/AlertScheduleSections.test.tsx | 20 ++- .../alerts/AlertsConfigurationSurface.tsx | 25 +++- .../alertsConfigurationModel.snapshot.test.ts | 33 +++++ .../alertsConfigurationModel.test.ts | 24 +++ .../useAlertDestinationsState.test.tsx | 17 +++ .../__tests__/useAlertScheduleState.test.tsx | 17 ++- .../alerts/alertsConfigurationModel.ts | 15 +- .../src/features/alerts/helpers.ts | 2 + .../src/features/alerts/tabs/ScheduleTab.tsx | 6 + frontend-modern/src/features/alerts/types.ts | 11 ++ .../alerts/useAlertDestinationsState.ts | 17 ++- .../alerts/useAlertDestinationsTabState.ts | 15 +- .../features/alerts/useAlertScheduleState.ts | 46 +++++- .../useAlertWebhookDestinationsState.ts | 22 ++- .../useAlertsConfigurationSnapshotState.ts | 14 +- .../alerts/useAlertsConfigurationState.ts | 2 + frontend-modern/src/types/alerts.ts | 4 +- .../__tests__/alertConfigPresentation.test.ts | 16 ++ .../src/utils/alertConfigPresentation.ts | 8 + internal/alerts/callbacks.go | 17 ++- .../notification_delivery_target_test.go | 36 ++++- internal/alerts/config/types.go | 75 +++++++++- internal/alerts/config_facade.go | 7 + internal/alerts/config_runtime.go | 6 +- internal/alerts/config_validation_test.go | 34 +++++ internal/alerts/escalation.go | 26 ++++ internal/alerts/escalation_repeat_test.go | 139 ++++++++++++++++++ .../monitor_alert_override_migration_test.go | 91 ++++++++++++ internal/monitoring/monitor_alerts.go | 7 +- internal/notifications/notifications.go | 67 +++++++-- internal/notifications/tag_routing_test.go | 35 +++++ 37 files changed, 1015 insertions(+), 85 deletions(-) create mode 100644 internal/alerts/escalation_repeat_test.go diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index fd7621860f..88d78dcddf 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -2184,3 +2184,23 @@ idempotent fingerprints. A restart interruption is raised into history even when the first successful external report immediately resolves it, so recovery does not erase the outage record. The watchdog must never invent resource identity or become a second lifecycle for infrastructure alerts. + +### Escalation targets are exact and critical repeats are bounded + +An escalation level may address credential-free logical destination IDs: +`email`, `apprise`, or `webhook:`. Once exact IDs are present they are the +authority for that level; a deleted or unknown ID fails closed and must never +widen to a channel or every webhook. The legacy `notify` channel remains a +mixed-version fallback for configurations that do not yet carry exact IDs. + +Optional critical-repeat policy applies only after the final escalation level. +It sends at most one repeat per checker pass, uses a normalized 5–180 minute +interval, and stops while the incident is acknowledged, resolved, snoozed, or +global detection or delivery controls are paused. A repeat is durable escalation +evidence with an explicit repeat marker rather than a new incident or a replay +of missed intervals. + +The Schedule surface loads the same email, Apprise, and webhook catalog as the +Destinations surface. It shows disabled and deleted selections explicitly, +prevents an escalation level from becoming destinationless, and explains the +critical-repeat stop conditions before save. diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index acb9ffcc37..6b8ab678e1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -6686,3 +6686,14 @@ its privacy and current-state boundary—but it must not redefine the policy. Alert feature state owns persistence and entitlement gating; the primitive owns presentation only. `alertDestinationsPresentation.test.ts` pins the shared vocabulary and the distinct mobile guidance. + +### Escalation configuration uses destination identity, not channel aliases + +The alerts-owned escalation section composes shared settings, toggle, and form +controls while presenting a feature-owned checkbox catalog keyed by logical +destination ID. It sources that catalog from the same loaded destination state +used by the Notifications tab, preserves selected-but-disabled and deleted +entries visibly, and prevents the final selection from being removed. The +critical-repeat control exposes a bounded numeric interval only when enabled +and states every lifecycle condition that stops paging. Desktop and mobile +layouts must retain associated labels and avoid horizontal overflow. diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 423c8c2010..4a644be5d0 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -3293,3 +3293,12 @@ in-flight request before durable stop or replacement state is published, so a revoked endpoint receives no trailing heartbeat and a removed destination becomes disabled immediately. State corruption and write failure are visible system conditions rather than silent loss of future outage evidence. + +### Escalation callbacks preserve exact routing intent + +The monitoring callback resolves the configured escalation level and forwards +exact logical destination IDs to notification delivery when present. Legacy +levels continue to route by channel, including Apprise; this compatibility path +must not silently skip a supported destination. Monitoring broadcasts the +escalated alert after dispatch but does not reinterpret destination identity, +retry semantics, acknowledgement, or the critical-repeat cadence. diff --git a/docs/release-control/v6/internal/subsystems/notifications.md b/docs/release-control/v6/internal/subsystems/notifications.md index 7b12484a4f..37bb0e5c29 100644 --- a/docs/release-control/v6/internal/subsystems/notifications.md +++ b/docs/release-control/v6/internal/subsystems/notifications.md @@ -472,3 +472,17 @@ have queued delivery cancelled by an uncommitted disable action. The forced failure paths are pinned in `internal/api/alerting/notifications_test.go`; the API ordering proof lives in `internal/api/contract_test.go`. + +### Escalation delivery can select exact logical destinations + +Escalation dispatch accepts optional logical destination IDs in addition to the +legacy channel target. Exact selection is applied before destination tag and +severity policy: `email` selects the enabled email job, `apprise` selects the +enabled Apprise job, and `webhook:` selects only that enabled webhook. +Unknown or unavailable IDs produce no job and never fall back to a broader +channel. A no-job exact selection also leaves notification cooldown state +untouched so configuration drift cannot masquerade as successful paging. + +Destination IDs never contain endpoint URLs, credentials, or Apprise targets. +All retry, queue, receipt, grouping, and recovery semantics remain owned by the +selected notification job after routing. diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index 5ea6f7f994..1eef1e114b 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,30 +1,55 @@ { "version": 1, - "base_sha": "8df9db7ab1159c9384b233d6f1e45a78720d231e", - "verified_at": "2026-08-27T18:21:17Z", + "base_sha": "5f6bfefa5603adc16c349115b8f453c8430ddce3", + "verified_at": "2026-08-27T18:39:55Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/components/Infrastructure/ResourceInvestigationContextTables.tsx", - "frontend-modern/src/components/shared/DetailSectionTable.tsx", - "frontend-modern/src/components/shared/detailSectionModel.ts" + "frontend-modern/src/features/alerts/AlertEscalationSection.tsx", + "frontend-modern/src/features/alerts/AlertsConfigurationSurface.tsx", + "frontend-modern/src/features/alerts/alertsConfigurationModel.ts", + "frontend-modern/src/features/alerts/helpers.ts", + "frontend-modern/src/features/alerts/tabs/ScheduleTab.tsx", + "frontend-modern/src/features/alerts/types.ts", + "frontend-modern/src/features/alerts/useAlertDestinationsState.ts", + "frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts", + "frontend-modern/src/features/alerts/useAlertScheduleState.ts", + "frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts", + "frontend-modern/src/features/alerts/useAlertsConfigurationSnapshotState.ts", + "frontend-modern/src/features/alerts/useAlertsConfigurationState.ts", + "frontend-modern/src/types/alerts.ts", + "frontend-modern/src/utils/alertConfigPresentation.ts" ], "content_sha256": { - "frontend-modern/src/components/Infrastructure/ResourceInvestigationContextTables.tsx": "86708854264fb59d14aa23766ccceb406301b9606e3af5ec5964ae524170f2d4", - "frontend-modern/src/components/shared/DetailSectionTable.tsx": "135153e1a29dc85afe8867643a31cbb66a6a13468f0f3af8efdd4be049f4d223", - "frontend-modern/src/components/shared/detailSectionModel.ts": "c98b6befd1348282abd990c47886fb5af60e362fdbb6c453d60f36e2fd3ebe1c" + "frontend-modern/src/features/alerts/AlertEscalationSection.tsx": "f4b12a0f7d5da49961ac952b94963d61a1b1f59bc3234ed7e961c9d264139021", + "frontend-modern/src/features/alerts/AlertsConfigurationSurface.tsx": "c514dd4b9f769c58af9af4fa5fcf647163224c85e6c4e640969a34c1d3b24708", + "frontend-modern/src/features/alerts/alertsConfigurationModel.ts": "855189aaf23c66b14ac43a1b7b0101e7fc0d59348a09415971a16a276f083f7f", + "frontend-modern/src/features/alerts/helpers.ts": "6ba556d57241fc7d7bfff7218d0dd924eaaeb9507f73ccda62204def14c3be61", + "frontend-modern/src/features/alerts/tabs/ScheduleTab.tsx": "7e5fa374edf72de0b9216f672cd179c183d348e22f087827c478bb29170ba9cf", + "frontend-modern/src/features/alerts/types.ts": "9b7c4331ac52f06d889fcdd02e75a95b247945cea6036a10e6fbea30e55d1eca", + "frontend-modern/src/features/alerts/useAlertDestinationsState.ts": "abf1a8c0acd32fbc4a1c7177dca213edee817b637af953c0f5a5ec275d41a600", + "frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts": "e9b2cabfaa1720ab430c410fdb269a164710478bcef775a5d163f69cc3965226", + "frontend-modern/src/features/alerts/useAlertScheduleState.ts": "fc439f16c0b79bfa6c01a74a3359ae0cb58e5765c074d75107c1ba8bd763cf31", + "frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts": "ee7a5d6c75902ab66a92ca0292b367bc02ba8dd268cab1e011fe719d09251fa9", + "frontend-modern/src/features/alerts/useAlertsConfigurationSnapshotState.ts": "af9014e6d93b0aa22399470efa00f8e048df7fbe54559092d8749b3fcf5bb43a", + "frontend-modern/src/features/alerts/useAlertsConfigurationState.ts": "20d61b9a97180d56aa6cef8445ba9d53e7043a22cc4020e6f72b505ed0c38fe7", + "frontend-modern/src/types/alerts.ts": "24999c6668c262214fcd3b8a92b09e7a68cab2163b8a4876780ce1148d9064cf", + "frontend-modern/src/utils/alertConfigPresentation.ts": "82d04fb6ce0c209fa968b1dabf19333dffae5a7317441a8a2cac5cfda86b831c" }, - "routes": ["/proxmox/overview"], + "routes": ["/alerts/schedule"], "viewports": [ { "width": 1920, "height": 800 }, { "width": 390, "height": 844 } ], "states": [ - "expanded backup-vault drawer after extending the shared responsive section primitive", - "Analysis and Governance responsive section state covered by focused rendered drawer tests" + "escalation enabled with one level and the unsaved-changes guard visible", + "exact Email selected while Apprise is unselected and both disabled states remain explicit", + "repeat critical escalation enabled with a 30 minute interval" ], "interactions": [ - "confirmed shared desktop sections remain aligned equal-width cards with adjacent labels and values", - "confirmed compact mobile table rows remain intact at 390x844 with zero horizontal overflow", - "reloaded the final implementation and confirmed no new browser console errors" + "enabled escalation and added a level without saving the live configuration", + "confirmed exact destination selection updates independently and never broadens to another destination", + "confirmed repeat-until-ack exposes its bounded interval and stop-condition guidance", + "confirmed the 390x844 schedule layout has zero horizontal overflow", + "confirmed no browser console errors on the exercised route" ] } diff --git a/frontend-modern/src/features/alerts/AlertEscalationSection.tsx b/frontend-modern/src/features/alerts/AlertEscalationSection.tsx index 0eae2c3d71..241f3a382b 100644 --- a/frontend-modern/src/features/alerts/AlertEscalationSection.tsx +++ b/frontend-modern/src/features/alerts/AlertEscalationSection.tsx @@ -1,29 +1,37 @@ import { For, Show, createUniqueId } from 'solid-js'; import { controlClass, formHelpText } from '@/components/shared/Form'; -import { FormSelect } from '@/components/shared/FormSelect'; import { SettingsPanel } from '@/components/shared/SettingsPanel'; import { Toggle } from '@/components/shared/Toggle'; import { ALERT_CONFIG_ESCALATION_ADD_LABEL, ALERT_CONFIG_ESCALATION_AFTER_LABEL, ALERT_CONFIG_ESCALATION_DESCRIPTION, + ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP, + ALERT_CONFIG_ESCALATION_DESTINATION_DISABLED, + ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE, ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX, ALERT_CONFIG_ESCALATION_NOTIFY_LABEL, ALERT_CONFIG_ESCALATION_REMOVE_TITLE, + ALERT_CONFIG_ESCALATION_REPEAT_EVERY_LABEL, + ALERT_CONFIG_ESCALATION_REPEAT_HELP, + ALERT_CONFIG_ESCALATION_REPEAT_LABEL, ALERT_CONFIG_ESCALATION_TITLE, getAlertConfigEscalationHelp, - getAlertConfigEscalationNotifyLabel, getAlertConfigToggleStatusLabel, } from '@/utils/alertConfigPresentation'; -import type { EscalationConfig, EscalationNotifyTarget } from './types'; +import type { EscalationConfig, EscalationDestination, EscalationNotifyTarget } from './types'; interface AlertEscalationSectionProps { escalation: EscalationConfig; setEscalationEnabled: (value: boolean) => void; setEscalationAfter: (index: number, value: string) => void; setEscalationNotify: (index: number, value: EscalationNotifyTarget) => void; + destinations: EscalationDestination[]; + setEscalationDestinationIds: (index: number, destinationIds: string[]) => void; + setEscalationRepeatCritical: (value: boolean) => void; + setEscalationRepeatEvery: (value: string) => void; removeEscalationLevel: (index: number) => void; addEscalationLevel: () => void; } @@ -59,11 +67,34 @@ export function AlertEscalationSection(props: AlertEscalationSectionProps) { {(level, index) => { const afterId = () => `${fieldIdPrefix}-after-${index()}`; - const notifyId = () => `${fieldIdPrefix}-notify-${index()}`; + const destinationsId = () => `${fieldIdPrefix}-destinations-${index()}`; + const selectedDestinationIds = () => + level.destinationIds && level.destinationIds.length > 0 + ? level.destinationIds + : props.destinations + .filter( + (destination) => + level.notify === 'all' || destination.kind === level.notify, + ) + .map((destination) => destination.id); + const destinationOptions = () => [ + ...props.destinations, + ...selectedDestinationIds() + .filter((id) => !props.destinations.some((destination) => destination.id === id)) + .map( + (id) => + ({ + id, + label: `${ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE} (${id})`, + kind: id === 'email' ? 'email' : id === 'apprise' ? 'apprise' : 'webhook', + enabled: false, + }) as EscalationDestination, + ), + ]; return ( -
-
+
+
- - props.setEscalationNotify( - index(), - event.currentTarget.value as EscalationNotifyTarget, - ) - } - fieldBaseClass="flex" - fieldClass="items-center gap-2" - labelClass="text-xs font-medium text-muted" - selectBaseClass={`${controlClass('px-2 py-1 text-sm')} flex-1`} - > - - - - - +
+ + {ALERT_CONFIG_ESCALATION_NOTIFY_LABEL} + +
+ + {(destination) => { + const checked = () => selectedDestinationIds().includes(destination.id); + return ( + + ); + }} + +
+

{ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP}

+