Make alert escalations destination-specific and repeatable

This commit is contained in:
rcourtman
2026-08-27 19:48:56 +01:00
parent 5f6bfefa56
commit e0baecc418
37 changed files with 1015 additions and 85 deletions
@@ -2184,3 +2184,23 @@ idempotent fingerprints. A restart interruption is raised into history even
when the first successful external report immediately resolves it, so recovery when the first successful external report immediately resolves it, so recovery
does not erase the outage record. The watchdog must never invent resource does not erase the outage record. The watchdog must never invent resource
identity or become a second lifecycle for infrastructure alerts. identity or become a second lifecycle for infrastructure alerts.
### Escalation targets are exact and critical repeats are bounded
An escalation level may address credential-free logical destination IDs:
`email`, `apprise`, or `webhook:<id>`. Once exact IDs are present they are the
authority for that level; a deleted or unknown ID fails closed and must never
widen to a channel or every webhook. The legacy `notify` channel remains a
mixed-version fallback for configurations that do not yet carry exact IDs.
Optional critical-repeat policy applies only after the final escalation level.
It sends at most one repeat per checker pass, uses a normalized 5–180 minute
interval, and stops while the incident is acknowledged, resolved, snoozed, or
global detection or delivery controls are paused. A repeat is durable escalation
evidence with an explicit repeat marker rather than a new incident or a replay
of missed intervals.
The Schedule surface loads the same email, Apprise, and webhook catalog as the
Destinations surface. It shows disabled and deleted selections explicitly,
prevents an escalation level from becoming destinationless, and explains the
critical-repeat stop conditions before save.
@@ -6686,3 +6686,14 @@ its privacy and current-state boundary—but it must not redefine the policy.
Alert feature state owns persistence and entitlement gating; the primitive Alert feature state owns persistence and entitlement gating; the primitive
owns presentation only. `alertDestinationsPresentation.test.ts` pins the owns presentation only. `alertDestinationsPresentation.test.ts` pins the
shared vocabulary and the distinct mobile guidance. shared vocabulary and the distinct mobile guidance.
### Escalation configuration uses destination identity, not channel aliases
The alerts-owned escalation section composes shared settings, toggle, and form
controls while presenting a feature-owned checkbox catalog keyed by logical
destination ID. It sources that catalog from the same loaded destination state
used by the Notifications tab, preserves selected-but-disabled and deleted
entries visibly, and prevents the final selection from being removed. The
critical-repeat control exposes a bounded numeric interval only when enabled
and states every lifecycle condition that stops paging. Desktop and mobile
layouts must retain associated labels and avoid horizontal overflow.
@@ -3293,3 +3293,12 @@ in-flight request before durable stop or replacement state is published, so a
revoked endpoint receives no trailing heartbeat and a removed destination revoked endpoint receives no trailing heartbeat and a removed destination
becomes disabled immediately. State corruption and write failure are visible becomes disabled immediately. State corruption and write failure are visible
system conditions rather than silent loss of future outage evidence. system conditions rather than silent loss of future outage evidence.
### Escalation callbacks preserve exact routing intent
The monitoring callback resolves the configured escalation level and forwards
exact logical destination IDs to notification delivery when present. Legacy
levels continue to route by channel, including Apprise; this compatibility path
must not silently skip a supported destination. Monitoring broadcasts the
escalated alert after dispatch but does not reinterpret destination identity,
retry semantics, acknowledgement, or the critical-repeat cadence.
@@ -472,3 +472,17 @@ have queued delivery cancelled by an uncommitted disable action. The forced
failure paths are pinned in failure paths are pinned in
`internal/api/alerting/notifications_test.go`; the API ordering proof lives in `internal/api/alerting/notifications_test.go`; the API ordering proof lives in
`internal/api/contract_test.go`. `internal/api/contract_test.go`.
### Escalation delivery can select exact logical destinations
Escalation dispatch accepts optional logical destination IDs in addition to the
legacy channel target. Exact selection is applied before destination tag and
severity policy: `email` selects the enabled email job, `apprise` selects the
enabled Apprise job, and `webhook:<id>` selects only that enabled webhook.
Unknown or unavailable IDs produce no job and never fall back to a broader
channel. A no-job exact selection also leaves notification cooldown state
untouched so configuration drift cannot masquerade as successful paging.
Destination IDs never contain endpoint URLs, credentials, or Apprise targets.
All retry, queue, receipt, grouping, and recovery semantics remain owned by the
selected notification job after routing.
+39 -14
View File
@@ -1,30 +1,55 @@
{ {
"version": 1, "version": 1,
"base_sha": "8df9db7ab1159c9384b233d6f1e45a78720d231e", "base_sha": "5f6bfefa5603adc16c349115b8f453c8430ddce3",
"verified_at": "2026-08-27T18:21:17Z", "verified_at": "2026-08-27T18:39:55Z",
"result": "passed", "result": "passed",
"changed_paths": [ "changed_paths": [
"frontend-modern/src/components/Infrastructure/ResourceInvestigationContextTables.tsx", "frontend-modern/src/features/alerts/AlertEscalationSection.tsx",
"frontend-modern/src/components/shared/DetailSectionTable.tsx", "frontend-modern/src/features/alerts/AlertsConfigurationSurface.tsx",
"frontend-modern/src/components/shared/detailSectionModel.ts" "frontend-modern/src/features/alerts/alertsConfigurationModel.ts",
"frontend-modern/src/features/alerts/helpers.ts",
"frontend-modern/src/features/alerts/tabs/ScheduleTab.tsx",
"frontend-modern/src/features/alerts/types.ts",
"frontend-modern/src/features/alerts/useAlertDestinationsState.ts",
"frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts",
"frontend-modern/src/features/alerts/useAlertScheduleState.ts",
"frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts",
"frontend-modern/src/features/alerts/useAlertsConfigurationSnapshotState.ts",
"frontend-modern/src/features/alerts/useAlertsConfigurationState.ts",
"frontend-modern/src/types/alerts.ts",
"frontend-modern/src/utils/alertConfigPresentation.ts"
], ],
"content_sha256": { "content_sha256": {
"frontend-modern/src/components/Infrastructure/ResourceInvestigationContextTables.tsx": "86708854264fb59d14aa23766ccceb406301b9606e3af5ec5964ae524170f2d4", "frontend-modern/src/features/alerts/AlertEscalationSection.tsx": "f4b12a0f7d5da49961ac952b94963d61a1b1f59bc3234ed7e961c9d264139021",
"frontend-modern/src/components/shared/DetailSectionTable.tsx": "135153e1a29dc85afe8867643a31cbb66a6a13468f0f3af8efdd4be049f4d223", "frontend-modern/src/features/alerts/AlertsConfigurationSurface.tsx": "c514dd4b9f769c58af9af4fa5fcf647163224c85e6c4e640969a34c1d3b24708",
"frontend-modern/src/components/shared/detailSectionModel.ts": "c98b6befd1348282abd990c47886fb5af60e362fdbb6c453d60f36e2fd3ebe1c" "frontend-modern/src/features/alerts/alertsConfigurationModel.ts": "855189aaf23c66b14ac43a1b7b0101e7fc0d59348a09415971a16a276f083f7f",
"frontend-modern/src/features/alerts/helpers.ts": "6ba556d57241fc7d7bfff7218d0dd924eaaeb9507f73ccda62204def14c3be61",
"frontend-modern/src/features/alerts/tabs/ScheduleTab.tsx": "7e5fa374edf72de0b9216f672cd179c183d348e22f087827c478bb29170ba9cf",
"frontend-modern/src/features/alerts/types.ts": "9b7c4331ac52f06d889fcdd02e75a95b247945cea6036a10e6fbea30e55d1eca",
"frontend-modern/src/features/alerts/useAlertDestinationsState.ts": "abf1a8c0acd32fbc4a1c7177dca213edee817b637af953c0f5a5ec275d41a600",
"frontend-modern/src/features/alerts/useAlertDestinationsTabState.ts": "e9b2cabfaa1720ab430c410fdb269a164710478bcef775a5d163f69cc3965226",
"frontend-modern/src/features/alerts/useAlertScheduleState.ts": "fc439f16c0b79bfa6c01a74a3359ae0cb58e5765c074d75107c1ba8bd763cf31",
"frontend-modern/src/features/alerts/useAlertWebhookDestinationsState.ts": "ee7a5d6c75902ab66a92ca0292b367bc02ba8dd268cab1e011fe719d09251fa9",
"frontend-modern/src/features/alerts/useAlertsConfigurationSnapshotState.ts": "af9014e6d93b0aa22399470efa00f8e048df7fbe54559092d8749b3fcf5bb43a",
"frontend-modern/src/features/alerts/useAlertsConfigurationState.ts": "20d61b9a97180d56aa6cef8445ba9d53e7043a22cc4020e6f72b505ed0c38fe7",
"frontend-modern/src/types/alerts.ts": "24999c6668c262214fcd3b8a92b09e7a68cab2163b8a4876780ce1148d9064cf",
"frontend-modern/src/utils/alertConfigPresentation.ts": "82d04fb6ce0c209fa968b1dabf19333dffae5a7317441a8a2cac5cfda86b831c"
}, },
"routes": ["/proxmox/overview"], "routes": ["/alerts/schedule"],
"viewports": [ "viewports": [
{ "width": 1920, "height": 800 }, { "width": 1920, "height": 800 },
{ "width": 390, "height": 844 } { "width": 390, "height": 844 }
], ],
"states": [ "states": [
"expanded backup-vault drawer after extending the shared responsive section primitive", "escalation enabled with one level and the unsaved-changes guard visible",
"Analysis and Governance responsive section state covered by focused rendered drawer tests" "exact Email selected while Apprise is unselected and both disabled states remain explicit",
"repeat critical escalation enabled with a 30 minute interval"
], ],
"interactions": [ "interactions": [
"confirmed shared desktop sections remain aligned equal-width cards with adjacent labels and values", "enabled escalation and added a level without saving the live configuration",
"confirmed compact mobile table rows remain intact at 390x844 with zero horizontal overflow", "confirmed exact destination selection updates independently and never broadens to another destination",
"reloaded the final implementation and confirmed no new browser console errors" "confirmed repeat-until-ack exposes its bounded interval and stop-condition guidance",
"confirmed the 390x844 schedule layout has zero horizontal overflow",
"confirmed no browser console errors on the exercised route"
] ]
} }
@@ -1,29 +1,37 @@
import { For, Show, createUniqueId } from 'solid-js'; import { For, Show, createUniqueId } from 'solid-js';
import { controlClass, formHelpText } from '@/components/shared/Form'; import { controlClass, formHelpText } from '@/components/shared/Form';
import { FormSelect } from '@/components/shared/FormSelect';
import { SettingsPanel } from '@/components/shared/SettingsPanel'; import { SettingsPanel } from '@/components/shared/SettingsPanel';
import { Toggle } from '@/components/shared/Toggle'; import { Toggle } from '@/components/shared/Toggle';
import { import {
ALERT_CONFIG_ESCALATION_ADD_LABEL, ALERT_CONFIG_ESCALATION_ADD_LABEL,
ALERT_CONFIG_ESCALATION_AFTER_LABEL, ALERT_CONFIG_ESCALATION_AFTER_LABEL,
ALERT_CONFIG_ESCALATION_DESCRIPTION, ALERT_CONFIG_ESCALATION_DESCRIPTION,
ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP,
ALERT_CONFIG_ESCALATION_DESTINATION_DISABLED,
ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE,
ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX, ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX,
ALERT_CONFIG_ESCALATION_NOTIFY_LABEL, ALERT_CONFIG_ESCALATION_NOTIFY_LABEL,
ALERT_CONFIG_ESCALATION_REMOVE_TITLE, ALERT_CONFIG_ESCALATION_REMOVE_TITLE,
ALERT_CONFIG_ESCALATION_REPEAT_EVERY_LABEL,
ALERT_CONFIG_ESCALATION_REPEAT_HELP,
ALERT_CONFIG_ESCALATION_REPEAT_LABEL,
ALERT_CONFIG_ESCALATION_TITLE, ALERT_CONFIG_ESCALATION_TITLE,
getAlertConfigEscalationHelp, getAlertConfigEscalationHelp,
getAlertConfigEscalationNotifyLabel,
getAlertConfigToggleStatusLabel, getAlertConfigToggleStatusLabel,
} from '@/utils/alertConfigPresentation'; } from '@/utils/alertConfigPresentation';
import type { EscalationConfig, EscalationNotifyTarget } from './types'; import type { EscalationConfig, EscalationDestination, EscalationNotifyTarget } from './types';
interface AlertEscalationSectionProps { interface AlertEscalationSectionProps {
escalation: EscalationConfig; escalation: EscalationConfig;
setEscalationEnabled: (value: boolean) => void; setEscalationEnabled: (value: boolean) => void;
setEscalationAfter: (index: number, value: string) => void; setEscalationAfter: (index: number, value: string) => void;
setEscalationNotify: (index: number, value: EscalationNotifyTarget) => void; setEscalationNotify: (index: number, value: EscalationNotifyTarget) => void;
destinations: EscalationDestination[];
setEscalationDestinationIds: (index: number, destinationIds: string[]) => void;
setEscalationRepeatCritical: (value: boolean) => void;
setEscalationRepeatEvery: (value: string) => void;
removeEscalationLevel: (index: number) => void; removeEscalationLevel: (index: number) => void;
addEscalationLevel: () => void; addEscalationLevel: () => void;
} }
@@ -59,11 +67,34 @@ export function AlertEscalationSection(props: AlertEscalationSectionProps) {
<For each={props.escalation.levels}> <For each={props.escalation.levels}>
{(level, index) => { {(level, index) => {
const afterId = () => `${fieldIdPrefix}-after-${index()}`; const afterId = () => `${fieldIdPrefix}-after-${index()}`;
const notifyId = () => `${fieldIdPrefix}-notify-${index()}`; const destinationsId = () => `${fieldIdPrefix}-destinations-${index()}`;
const selectedDestinationIds = () =>
level.destinationIds && level.destinationIds.length > 0
? level.destinationIds
: props.destinations
.filter(
(destination) =>
level.notify === 'all' || destination.kind === level.notify,
)
.map((destination) => destination.id);
const destinationOptions = () => [
...props.destinations,
...selectedDestinationIds()
.filter((id) => !props.destinations.some((destination) => destination.id === id))
.map(
(id) =>
({
id,
label: `${ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE} (${id})`,
kind: id === 'email' ? 'email' : id === 'apprise' ? 'apprise' : 'webhook',
enabled: false,
}) as EscalationDestination,
),
];
return ( return (
<div class="flex items-center gap-3 rounded-md border border-border bg-surface-hover p-3"> <div class="flex items-start gap-3 rounded-md border border-border bg-surface-hover p-3">
<div class="flex flex-1 flex-col gap-3 sm:grid sm:grid-cols-2 sm:items-center sm:gap-2"> <div class="flex flex-1 flex-col gap-3">
<div class="flex items-center gap-2"> <div class="flex items-center gap-2">
<label for={afterId()} class="text-xs font-medium text-muted"> <label for={afterId()} class="text-xs font-medium text-muted">
{ALERT_CONFIG_ESCALATION_AFTER_LABEL} {ALERT_CONFIG_ESCALATION_AFTER_LABEL}
@@ -83,30 +114,43 @@ export function AlertEscalationSection(props: AlertEscalationSectionProps) {
{ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX} {ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX}
</span> </span>
</div> </div>
<FormSelect <fieldset id={destinationsId()} class="space-y-2">
id={notifyId()} <legend class="text-xs font-medium text-muted">
label={ALERT_CONFIG_ESCALATION_NOTIFY_LABEL} {ALERT_CONFIG_ESCALATION_NOTIFY_LABEL}
value={level.notify} </legend>
onChange={(event) => <div class="grid gap-2 sm:grid-cols-2">
props.setEscalationNotify( <For each={destinationOptions()}>
index(), {(destination) => {
event.currentTarget.value as EscalationNotifyTarget, const checked = () => selectedDestinationIds().includes(destination.id);
) return (
} <label class="flex min-w-0 items-center gap-2 rounded-md border border-border bg-surface px-2.5 py-2 text-sm">
fieldBaseClass="flex" <input
fieldClass="items-center gap-2" type="checkbox"
labelClass="text-xs font-medium text-muted" checked={checked()}
selectBaseClass={`${controlClass('px-2 py-1 text-sm')} flex-1`} disabled={checked() && selectedDestinationIds().length === 1}
> onChange={(event) => {
<option value="email">{getAlertConfigEscalationNotifyLabel('email')}</option> const next = event.currentTarget.checked
<option value="webhook"> ? [...selectedDestinationIds(), destination.id]
{getAlertConfigEscalationNotifyLabel('webhook')} : selectedDestinationIds().filter(
</option> (id) => id !== destination.id,
<option value="apprise"> );
{getAlertConfigEscalationNotifyLabel('apprise')} props.setEscalationDestinationIds(index(), next);
</option> }}
<option value="all">{getAlertConfigEscalationNotifyLabel('all')}</option> class="h-4 w-4 rounded border-border"
</FormSelect> />
<span class="min-w-0 truncate">{destination.label}</span>
<Show when={!destination.enabled}>
<span class="ml-auto text-xs text-muted">
{ALERT_CONFIG_ESCALATION_DESTINATION_DISABLED}
</span>
</Show>
</label>
);
}}
</For>
</div>
<p class={formHelpText}>{ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP}</p>
</fieldset>
</div> </div>
<button <button
type="button" type="button"
@@ -129,6 +173,41 @@ export function AlertEscalationSection(props: AlertEscalationSectionProps) {
}} }}
</For> </For>
<Show when={props.escalation.levels.length > 0}>
<div class="space-y-3 rounded-md border border-border bg-surface p-3">
<Toggle
checked={props.escalation.repeatCritical}
onChange={(event) => props.setEscalationRepeatCritical(event.currentTarget.checked)}
label={
<span class="text-sm font-medium text-base-content">
{ALERT_CONFIG_ESCALATION_REPEAT_LABEL}
</span>
}
/>
<Show when={props.escalation.repeatCritical}>
<div class="flex items-center gap-2">
<label
for={`${fieldIdPrefix}-repeat-every`}
class="text-xs font-medium text-muted"
>
{ALERT_CONFIG_ESCALATION_REPEAT_EVERY_LABEL}
</label>
<input
id={`${fieldIdPrefix}-repeat-every`}
type="number"
min="5"
max="180"
value={props.escalation.repeatEvery}
onChange={(event) => props.setEscalationRepeatEvery(event.currentTarget.value)}
class={`${controlClass('px-2 py-1 text-sm')} w-20`}
/>
<span class="text-xs text-muted">{ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX}</span>
</div>
</Show>
<p class={formHelpText}>{ALERT_CONFIG_ESCALATION_REPEAT_HELP}</p>
</div>
</Show>
<button <button
type="button" type="button"
onClick={props.addEscalationLevel} onClick={props.addEscalationLevel}
@@ -119,7 +119,9 @@ describe('Alert schedule sections', () => {
it('associates visible labels with escalation controls', () => { it('associates visible labels with escalation controls', () => {
const escalation: EscalationConfig = { const escalation: EscalationConfig = {
enabled: true, enabled: true,
levels: [{ after: 15, notify: 'email' }], levels: [{ after: 15, notify: 'email', destinationIds: ['email'] }],
repeatCritical: true,
repeatEvery: 30,
}; };
render(() => ( render(() => (
@@ -128,6 +130,13 @@ describe('Alert schedule sections', () => {
setEscalationEnabled={vi.fn()} setEscalationEnabled={vi.fn()}
setEscalationAfter={vi.fn()} setEscalationAfter={vi.fn()}
setEscalationNotify={vi.fn()} setEscalationNotify={vi.fn()}
destinations={[
{ id: 'email', label: 'Email', kind: 'email', enabled: true },
{ id: 'apprise', label: 'Apprise', kind: 'apprise', enabled: false },
]}
setEscalationDestinationIds={vi.fn()}
setEscalationRepeatCritical={vi.fn()}
setEscalationRepeatEvery={vi.fn()}
removeEscalationLevel={vi.fn()} removeEscalationLevel={vi.fn()}
addEscalationLevel={vi.fn()} addEscalationLevel={vi.fn()}
/> />
@@ -138,8 +147,13 @@ describe('Alert schedule sections', () => {
'true', 'true',
); );
expect(screen.getByRole('spinbutton', { name: 'After' })).toBeInTheDocument(); expect(screen.getByRole('spinbutton', { name: 'After' })).toBeInTheDocument();
expect(screen.getByRole('combobox', { name: 'Notify' })).toBeInTheDocument(); expect(screen.getByRole('group', { name: 'Notify' })).toBeInTheDocument();
expect(screen.getByRole('option', { name: 'Apprise' })).toBeInTheDocument(); expect(screen.getByRole('checkbox', { name: 'Email' })).toBeChecked();
expect(screen.getByRole('checkbox', { name: /Apprise/ })).not.toBeChecked();
expect(
screen.getByRole('button', { name: 'Repeat critical escalation until acknowledged' }),
).toHaveAttribute('aria-pressed', 'true');
expect(screen.getByRole('spinbutton', { name: 'Repeat every' })).toHaveValue(30);
expect(screen.getByRole('button', { name: 'Remove escalation level' })).toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Remove escalation level' })).toBeInTheDocument();
}); });
@@ -1,4 +1,4 @@
import { Show } from 'solid-js'; import { createMemo, Show } from 'solid-js';
import { Card } from '@/components/shared/Card'; import { Card } from '@/components/shared/Card';
import { import {
@@ -20,6 +20,26 @@ import {
export function AlertsConfigurationSurface(props: AlertsConfigurationSurfaceProps) { export function AlertsConfigurationSurface(props: AlertsConfigurationSurfaceProps) {
const state = useAlertsConfigurationState(props); const state = useAlertsConfigurationState(props);
const escalationDestinations = createMemo(() => [
{
id: 'email',
label: 'Email',
kind: 'email' as const,
enabled: state.emailConfig().enabled,
},
{
id: 'apprise',
label: 'Apprise',
kind: 'apprise' as const,
enabled: state.appriseConfig().enabled,
},
...state.webhooks().map((webhook) => ({
id: `webhook:${webhook.id}`,
label: webhook.name || webhook.id,
kind: 'webhook' as const,
enabled: webhook.enabled,
})),
]);
return ( return (
<> <>
@@ -224,6 +244,8 @@ export function AlertsConfigurationSurface(props: AlertsConfigurationSurfaceProp
isRetrying={state.isReloadingConfig} isRetrying={state.isReloadingConfig}
isLoadingDestinations={state.isLoadingDestinations} isLoadingDestinations={state.isLoadingDestinations}
onRetryLoad={() => void state.loadAlertConfiguration()} onRetryLoad={() => void state.loadAlertConfiguration()}
webhooks={state.webhooks}
setWebhooks={state.setWebhooks}
/> />
</Show> </Show>
@@ -242,6 +264,7 @@ export function AlertsConfigurationSurface(props: AlertsConfigurationSurfaceProp
setNotifyOnResolve={state.setNotifyOnResolve} setNotifyOnResolve={state.setNotifyOnResolve}
escalation={state.scheduleEscalation} escalation={state.scheduleEscalation}
setEscalation={state.setScheduleEscalation} setEscalation={state.setScheduleEscalation}
escalationDestinations={escalationDestinations}
/> />
</Show> </Show>
</> </>
@@ -1180,6 +1180,39 @@ describe('readAlertsConfigurationSnapshot — schedule.escalation', () => {
]); ]);
}); });
it('preserves exact destination IDs and critical repeat policy', () => {
const snapshot = readAlertsConfigurationSnapshot(
cfg({
schedule: {
escalation: {
enabled: true,
repeatCritical: true,
repeatEvery: 15,
levels: [
{
after: 30,
notify: 'webhook',
destinationIds: ['webhook:pager', 'apprise'],
},
],
},
},
}),
);
expect(snapshot.scheduleEscalation).toEqual({
enabled: true,
repeatCritical: true,
repeatEvery: 15,
levels: [
{
after: 30,
notify: 'webhook',
destinationIds: ['webhook:pager', 'apprise'],
},
],
});
});
it('defaults level.after to 15 for non-number values', () => { it('defaults level.after to 15 for non-number values', () => {
const snapshot = readAlertsConfigurationSnapshot( const snapshot = readAlertsConfigurationSnapshot(
cfg({ cfg({
@@ -130,6 +130,18 @@ describe('alertsConfigurationModel', () => {
byGuest: false, byGuest: false,
}; };
snapshot.initialNotify = 'email'; snapshot.initialNotify = 'email';
snapshot.scheduleEscalation = {
enabled: true,
repeatCritical: true,
repeatEvery: 15,
levels: [
{
after: 30,
notify: 'webhook',
destinationIds: ['webhook:pager'],
},
],
};
snapshot.backupDefaults = { snapshot.backupDefaults = {
enabled: true, enabled: true,
warningDays: 30, warningDays: 30,
@@ -169,6 +181,18 @@ describe('alertsConfigurationModel', () => {
byNode: true, byNode: true,
byGuest: false, byGuest: false,
}); });
expect(result.alertConfig?.schedule?.escalation).toEqual({
enabled: true,
repeatCritical: true,
repeatEvery: 15,
levels: [
{
after: 30,
notify: 'webhook',
destinationIds: ['webhook:pager'],
},
],
});
expect(result.alertConfig?.backupDefaults).toMatchObject({ expect(result.alertConfig?.backupDefaults).toMatchObject({
warningDays: 10, warningDays: 10,
criticalDays: 10, criticalDays: 10,
@@ -13,6 +13,7 @@ vi.mock('@/api/notifications', () => ({
NotificationsAPI: { NotificationsAPI: {
getAppriseConfig: vi.fn(), getAppriseConfig: vi.fn(),
getEmailConfig: vi.fn(), getEmailConfig: vi.fn(),
getWebhooks: vi.fn(),
updateAppriseConfig: vi.fn(), updateAppriseConfig: vi.fn(),
updateEmailConfig: vi.fn(), updateEmailConfig: vi.fn(),
}, },
@@ -46,6 +47,7 @@ describe('useAlertDestinationsState', () => {
beforeEach(() => { beforeEach(() => {
vi.mocked(NotificationsAPI.getEmailConfig).mockReset(); vi.mocked(NotificationsAPI.getEmailConfig).mockReset();
vi.mocked(NotificationsAPI.getAppriseConfig).mockReset(); vi.mocked(NotificationsAPI.getAppriseConfig).mockReset();
vi.mocked(NotificationsAPI.getWebhooks).mockReset();
vi.mocked(NotificationsAPI.updateEmailConfig).mockReset(); vi.mocked(NotificationsAPI.updateEmailConfig).mockReset();
vi.mocked(NotificationsAPI.updateAppriseConfig).mockReset(); vi.mocked(NotificationsAPI.updateAppriseConfig).mockReset();
vi.mocked(AlertsAPI.getDeadManConfig).mockReset(); vi.mocked(AlertsAPI.getDeadManConfig).mockReset();
@@ -92,6 +94,16 @@ describe('useAlertDestinationsState', () => {
timeoutSeconds: 30, timeoutSeconds: 30,
skipTlsVerify: false, skipTlsVerify: false,
} as any); } as any);
vi.mocked(NotificationsAPI.getWebhooks).mockResolvedValue([
{
id: 'pager',
name: 'Pager',
url: 'https://pager.example.test',
method: 'POST',
headers: {},
enabled: true,
},
]);
vi.mocked(AlertsAPI.getDeadManConfig).mockResolvedValue({ vi.mocked(AlertsAPI.getDeadManConfig).mockResolvedValue({
pingUrl: '***REDACTED***', pingUrl: '***REDACTED***',
configured: true, configured: true,
@@ -112,6 +124,7 @@ describe('useAlertDestinationsState', () => {
await result.loadDestinations(); await result.loadDestinations();
expect(NotificationsAPI.getEmailConfig).toHaveBeenCalledTimes(1); expect(NotificationsAPI.getEmailConfig).toHaveBeenCalledTimes(1);
expect(NotificationsAPI.getAppriseConfig).toHaveBeenCalledTimes(1); expect(NotificationsAPI.getAppriseConfig).toHaveBeenCalledTimes(1);
expect(NotificationsAPI.getWebhooks).toHaveBeenCalledTimes(1);
expect(AlertsAPI.getDeadManConfig).toHaveBeenCalledTimes(1); expect(AlertsAPI.getDeadManConfig).toHaveBeenCalledTimes(1);
expect(RelayAPI.getConfig).toHaveBeenCalledTimes(1); expect(RelayAPI.getConfig).toHaveBeenCalledTimes(1);
expect(result.emailConfig().server).toBe('smtp.example.com'); expect(result.emailConfig().server).toBe('smtp.example.com');
@@ -120,12 +133,16 @@ describe('useAlertDestinationsState', () => {
expect(result.appriseConfig().minimumSeverity).toBe('critical'); expect(result.appriseConfig().minimumSeverity).toBe('critical');
expect(result.deadManPingUrl()).toBe('***REDACTED***'); expect(result.deadManPingUrl()).toBe('***REDACTED***');
expect(result.pushMinimumSeverity()).toBe('critical'); expect(result.pushMinimumSeverity()).toBe('critical');
expect(result.webhooks()).toEqual([
expect.objectContaining({ id: 'pager', service: 'generic' }),
]);
setActiveTab('destinations'); setActiveTab('destinations');
await Promise.resolve(); await Promise.resolve();
await Promise.resolve(); await Promise.resolve();
expect(NotificationsAPI.getEmailConfig).toHaveBeenCalledTimes(2); expect(NotificationsAPI.getEmailConfig).toHaveBeenCalledTimes(2);
expect(NotificationsAPI.getAppriseConfig).toHaveBeenCalledTimes(2); expect(NotificationsAPI.getAppriseConfig).toHaveBeenCalledTimes(2);
expect(NotificationsAPI.getWebhooks).toHaveBeenCalledTimes(2);
expect(AlertsAPI.getDeadManConfig).toHaveBeenCalledTimes(2); expect(AlertsAPI.getDeadManConfig).toHaveBeenCalledTimes(2);
expect(RelayAPI.getConfig).toHaveBeenCalledTimes(2); expect(RelayAPI.getConfig).toHaveBeenCalledTimes(2);
@@ -38,6 +38,10 @@ describe('useAlertScheduleState', () => {
setNotifyOnResolve, setNotifyOnResolve,
escalation, escalation,
setEscalation, setEscalation,
escalationDestinations: () => [
{ id: 'email', label: 'Email', kind: 'email', enabled: true },
{ id: 'webhook:pager', label: 'Pager', kind: 'webhook', enabled: true },
],
}), }),
); );
@@ -57,6 +61,9 @@ describe('useAlertScheduleState', () => {
result.addEscalationLevel(); result.addEscalationLevel();
result.setEscalationAfter(0, '30'); result.setEscalationAfter(0, '30');
result.setEscalationNotify(0, 'webhook'); result.setEscalationNotify(0, 'webhook');
result.setEscalationDestinationIds(0, ['webhook:pager']);
result.setEscalationRepeatCritical(true);
result.setEscalationRepeatEvery('15');
expect(hasUnsavedChanges()).toBe(true); expect(hasUnsavedChanges()).toBe(true);
expect(quietHours()).toMatchObject({ expect(quietHours()).toMatchObject({
@@ -72,7 +79,15 @@ describe('useAlertScheduleState', () => {
expect(notifyOnResolve()).toBe(true); expect(notifyOnResolve()).toBe(true);
expect(escalation()).toMatchObject({ expect(escalation()).toMatchObject({
enabled: true, enabled: true,
levels: [expect.objectContaining({ after: 30, notify: 'webhook' })], repeatCritical: true,
repeatEvery: 15,
levels: [
expect.objectContaining({
after: 30,
notify: 'webhook',
destinationIds: ['webhook:pager'],
}),
],
}); });
result.removeEscalationLevel(0); result.removeEscalationLevel(0);
@@ -690,9 +690,17 @@ export function readAlertsConfigurationSnapshot(config: AlertConfig): AlertsConf
if (config.schedule.escalation) { if (config.schedule.escalation) {
snapshot.scheduleEscalation = { snapshot.scheduleEscalation = {
enabled: Boolean(config.schedule.escalation.enabled), enabled: Boolean(config.schedule.escalation.enabled),
repeatCritical: Boolean(config.schedule.escalation.repeatCritical),
repeatEvery:
typeof config.schedule.escalation.repeatEvery === 'number'
? config.schedule.escalation.repeatEvery
: 30,
levels: (config.schedule.escalation.levels || []).map((level) => ({ levels: (config.schedule.escalation.levels || []).map((level) => ({
after: typeof level.after === 'number' ? level.after : 15, after: typeof level.after === 'number' ? level.after : 15,
notify: normalizeNotificationDeliveryTarget(level.notify) as EscalationNotifyTarget, notify: normalizeNotificationDeliveryTarget(level.notify) as EscalationNotifyTarget,
...(Array.isArray(level.destinationIds)
? { destinationIds: level.destinationIds.filter((id) => typeof id === 'string') }
: {}),
})), })),
}; };
} }
@@ -899,7 +907,12 @@ export function buildAlertsConfigurationPayload({
maxAlertsHour: normalizedMaxAlertsHour, maxAlertsHour: normalizedMaxAlertsHour,
escalation: { escalation: {
enabled: snapshot.scheduleEscalation.enabled, enabled: snapshot.scheduleEscalation.enabled,
levels: snapshot.scheduleEscalation.levels.map((level) => ({ ...level })), repeatCritical: snapshot.scheduleEscalation.repeatCritical,
repeatEvery: snapshot.scheduleEscalation.repeatEvery,
levels: snapshot.scheduleEscalation.levels.map((level) => ({
...level,
...(level.destinationIds ? { destinationIds: [...level.destinationIds] } : {}),
})),
}, },
grouping: { grouping: {
enabled: groupingEnabled, enabled: groupingEnabled,
@@ -184,6 +184,8 @@ export const normalizeMetricDelayMap = (
export const createDefaultEscalation = (): EscalationConfig => ({ export const createDefaultEscalation = (): EscalationConfig => ({
enabled: false, enabled: false,
levels: [], levels: [],
repeatCritical: false,
repeatEvery: 30,
}); });
export const getTriggerValue = ( export const getTriggerValue = (
@@ -16,6 +16,7 @@ import { useAlertScheduleState } from '../useAlertScheduleState';
import type { import type {
CooldownConfig, CooldownConfig,
EscalationConfig, EscalationConfig,
EscalationDestination,
GroupingConfig, GroupingConfig,
NotificationDeliveryTarget, NotificationDeliveryTarget,
QuietHoursConfig, QuietHoursConfig,
@@ -35,6 +36,7 @@ export interface ScheduleTabProps {
setNotifyOnResolve: (value: boolean) => void; setNotifyOnResolve: (value: boolean) => void;
escalation: () => EscalationConfig; escalation: () => EscalationConfig;
setEscalation: (value: EscalationConfig) => void; setEscalation: (value: EscalationConfig) => void;
escalationDestinations: () => EscalationDestination[];
} }
export function ScheduleTab(props: ScheduleTabProps) { export function ScheduleTab(props: ScheduleTabProps) {
@@ -119,6 +121,10 @@ export function ScheduleTab(props: ScheduleTabProps) {
setEscalationEnabled={scheduleState.setEscalationEnabled} setEscalationEnabled={scheduleState.setEscalationEnabled}
setEscalationAfter={scheduleState.setEscalationAfter} setEscalationAfter={scheduleState.setEscalationAfter}
setEscalationNotify={scheduleState.setEscalationNotify} setEscalationNotify={scheduleState.setEscalationNotify}
destinations={props.escalationDestinations()}
setEscalationDestinationIds={scheduleState.setEscalationDestinationIds}
setEscalationRepeatCritical={scheduleState.setEscalationRepeatCritical}
setEscalationRepeatEvery={scheduleState.setEscalationRepeatEvery}
removeEscalationLevel={scheduleState.removeEscalationLevel} removeEscalationLevel={scheduleState.removeEscalationLevel}
addEscalationLevel={scheduleState.addEscalationLevel} addEscalationLevel={scheduleState.addEscalationLevel}
/> />
@@ -202,15 +202,26 @@ export interface GroupingConfig {
export type NotificationDeliveryTarget = 'email' | 'webhook' | 'apprise' | 'all'; export type NotificationDeliveryTarget = 'email' | 'webhook' | 'apprise' | 'all';
export type EscalationNotifyTarget = NotificationDeliveryTarget; export type EscalationNotifyTarget = NotificationDeliveryTarget;
export type EscalationDestinationKind = Exclude<NotificationDeliveryTarget, 'all'>;
export interface EscalationDestination {
id: string;
label: string;
kind: EscalationDestinationKind;
enabled: boolean;
}
export interface EscalationLevel { export interface EscalationLevel {
after: number; after: number;
notify: EscalationNotifyTarget; notify: EscalationNotifyTarget;
destinationIds?: string[];
} }
export interface EscalationConfig { export interface EscalationConfig {
enabled: boolean; enabled: boolean;
levels: EscalationLevel[]; levels: EscalationLevel[];
repeatCritical: boolean;
repeatEvery: number;
} }
const COOLDOWN_MIN_MINUTES = 5; const COOLDOWN_MIN_MINUTES = 5;
@@ -3,6 +3,7 @@ import type { Accessor } from 'solid-js';
import { AlertsAPI } from '@/api/alerts'; import { AlertsAPI } from '@/api/alerts';
import { NotificationsAPI } from '@/api/notifications'; import { NotificationsAPI } from '@/api/notifications';
import type { Webhook } from '@/api/notifications';
import { RelayAPI } from '@/api/relay'; import { RelayAPI } from '@/api/relay';
import { hasFeature } from '@/stores/license'; import { hasFeature } from '@/stores/license';
import { getAlertDestinationsConfigLoadError } from '@/utils/alertDestinationsPresentation'; import { getAlertDestinationsConfigLoadError } from '@/utils/alertDestinationsPresentation';
@@ -30,6 +31,7 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
); );
const [deadManPingUrl, setDeadManPingUrl] = createSignal(''); const [deadManPingUrl, setDeadManPingUrl] = createSignal('');
const [pushMinimumSeverity, setPushMinimumSeverity] = createSignal<'all' | 'critical'>('all'); const [pushMinimumSeverity, setPushMinimumSeverity] = createSignal<'all' | 'critical'>('all');
const [webhooks, setWebhooks] = createSignal<Webhook[]>([]);
let reloadVersion = 0; let reloadVersion = 0;
let lastActiveTab: AlertTab | null = null; let lastActiveTab: AlertTab | null = null;
@@ -40,6 +42,7 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
setAppriseConfig(createDefaultAppriseConfig()); setAppriseConfig(createDefaultAppriseConfig());
setDeadManPingUrl(''); setDeadManPingUrl('');
setPushMinimumSeverity('all'); setPushMinimumSeverity('all');
setWebhooks([]);
}; };
const loadDestinations = async (options: { indicateLoading?: boolean } = {}) => { const loadDestinations = async (options: { indicateLoading?: boolean } = {}) => {
@@ -55,13 +58,14 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
NotificationsAPI.getAppriseConfig(), NotificationsAPI.getAppriseConfig(),
AlertsAPI.getDeadManConfig(), AlertsAPI.getDeadManConfig(),
hasFeature('relay') ? RelayAPI.getConfig() : Promise.resolve(null), hasFeature('relay') ? RelayAPI.getConfig() : Promise.resolve(null),
NotificationsAPI.getWebhooks(),
]); ]);
if (thisVersion !== reloadVersion) { if (thisVersion !== reloadVersion) {
return; return;
} }
const [emailResult, appriseResult, deadManResult, relayResult] = results; const [emailResult, appriseResult, deadManResult, relayResult, webhooksResult] = results;
if (emailResult.status === 'fulfilled') { if (emailResult.status === 'fulfilled') {
setEmailConfig(normalizeEmailConfigFromAPI(emailResult.value)); setEmailConfig(normalizeEmailConfigFromAPI(emailResult.value));
@@ -81,6 +85,15 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
); );
} }
if (webhooksResult.status === 'fulfilled') {
setWebhooks(
webhooksResult.value.map((webhook) => ({
...webhook,
service: webhook.service || 'generic',
})),
);
}
const failures = results.filter( const failures = results.filter(
(result): result is PromiseRejectedResult => result.status === 'rejected', (result): result is PromiseRejectedResult => result.status === 'rejected',
); );
@@ -138,6 +151,8 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
setDeadManPingUrl, setDeadManPingUrl,
pushMinimumSeverity, pushMinimumSeverity,
setPushMinimumSeverity, setPushMinimumSeverity,
webhooks,
setWebhooks,
resetDestinations, resetDestinations,
loadDestinations, loadDestinations,
saveDestinations, saveDestinations,
@@ -1,6 +1,7 @@
import { createMemo, createSignal, onMount, type Accessor } from 'solid-js'; import { createMemo, createSignal, onMount, type Accessor } from 'solid-js';
import { NotificationsAPI, type AppriseConfig } from '@/api/notifications'; import { NotificationsAPI, type AppriseConfig, type Webhook } from '@/api/notifications';
import type { Setter } from 'solid-js';
import { notificationStore } from '@/stores/notifications'; import { notificationStore } from '@/stores/notifications';
import { logger } from '@/utils/logger'; import { logger } from '@/utils/logger';
import { showErrorWithDetail } from '@/utils/toast'; import { showErrorWithDetail } from '@/utils/toast';
@@ -29,6 +30,8 @@ export interface AlertDestinationsTabStateProps {
isRetrying: Accessor<boolean>; isRetrying: Accessor<boolean>;
isLoadingDestinations: Accessor<boolean>; isLoadingDestinations: Accessor<boolean>;
onRetryLoad: () => void; onRetryLoad: () => void;
webhooks?: Accessor<Webhook[]>;
setWebhooks?: Setter<Webhook[]>;
} }
export function useAlertDestinationsTabState(props: AlertDestinationsTabStateProps) { export function useAlertDestinationsTabState(props: AlertDestinationsTabStateProps) {
@@ -50,7 +53,11 @@ export function useAlertDestinationsTabState(props: AlertDestinationsTabStatePro
heldEvents, heldEvents,
loadDeliveryLog, loadDeliveryLog,
} = useNotificationDeliveryLog(); } = useNotificationDeliveryLog();
const webhookState = useAlertWebhookDestinationsState(); const webhookState = useAlertWebhookDestinationsState({
webhooks: props.webhooks,
setWebhooks: props.setWebhooks,
autoLoad: props.webhooks === undefined,
});
const isLoading = createMemo( const isLoading = createMemo(
() => props.isLoadingDestinations() || webhookState.isLoadingWebhooks() || props.isRetrying(), () => props.isLoadingDestinations() || webhookState.isLoadingWebhooks() || props.isRetrying(),
@@ -142,7 +149,9 @@ export function useAlertDestinationsTabState(props: AlertDestinationsTabStatePro
const handleRetry = () => { const handleRetry = () => {
props.onRetryLoad(); props.onRetryLoad();
void webhookState.loadWebhooks(); if (props.webhooks === undefined) {
void webhookState.loadWebhooks();
}
void loadDeliveryHealth(); void loadDeliveryHealth();
void loadDeliveryLog(); void loadDeliveryLog();
}; };
@@ -11,6 +11,7 @@ import {
import type { import type {
CooldownConfig, CooldownConfig,
EscalationConfig, EscalationConfig,
EscalationDestination,
EscalationLevel, EscalationLevel,
EscalationNotifyTarget, EscalationNotifyTarget,
GroupingConfig, GroupingConfig,
@@ -33,6 +34,7 @@ export interface UseAlertScheduleStateProps {
setNotifyOnResolve: (value: boolean) => void; setNotifyOnResolve: (value: boolean) => void;
escalation: () => EscalationConfig; escalation: () => EscalationConfig;
setEscalation: (value: EscalationConfig) => void; setEscalation: (value: EscalationConfig) => void;
escalationDestinations: () => EscalationDestination[];
} }
export const ALERT_SCHEDULE_TIMEZONES = [ export const ALERT_SCHEDULE_TIMEZONES = [
@@ -328,6 +330,40 @@ export function useAlertScheduleState(props: UseAlertScheduleStateProps) {
markUnsaved(); markUnsaved();
}; };
const setEscalationDestinationIds = (index: number, destinationIds: string[]) => {
if (destinationIds.length === 0) return;
const destinations = props.escalationDestinations();
const selected = destinations.filter((destination) => destinationIds.includes(destination.id));
const allSelected = destinations.length > 0 && selected.length === destinations.length;
const fallbackNotify: EscalationNotifyTarget = allSelected
? 'all'
: (selected[0]?.kind ?? 'all');
const nextLevels = [...props.escalation().levels];
nextLevels[index] = {
...nextLevels[index],
notify: fallbackNotify,
destinationIds: [...new Set(destinationIds)],
};
props.setEscalation({ ...props.escalation(), levels: nextLevels });
markUnsaved();
};
const setEscalationRepeatCritical = (repeatCritical: boolean) => {
props.setEscalation({ ...props.escalation(), repeatCritical });
markUnsaved();
};
const setEscalationRepeatEvery = (rawValue: string) => {
const parsed = Number.parseInt(rawValue, 10);
props.setEscalation({
...props.escalation(),
repeatEvery: Number.isNaN(parsed)
? props.escalation().repeatEvery
: Math.min(180, Math.max(5, parsed)),
});
markUnsaved();
};
const removeEscalationLevel = (index: number) => { const removeEscalationLevel = (index: number) => {
const nextLevels = props const nextLevels = props
.escalation() .escalation()
@@ -342,11 +378,16 @@ export function useAlertScheduleState(props: UseAlertScheduleStateProps) {
const addEscalationLevel = () => { const addEscalationLevel = () => {
const lastLevel = props.escalation().levels[props.escalation().levels.length - 1]; const lastLevel = props.escalation().levels[props.escalation().levels.length - 1];
const nextAfter = typeof lastLevel?.after === 'number' ? lastLevel.after + 30 : 15; const nextAfter = typeof lastLevel?.after === 'number' ? lastLevel.after + 30 : 15;
const destinationIds = props.escalationDestinations().map((destination) => destination.id);
props.setEscalation({ props.setEscalation({
...props.escalation(), ...props.escalation(),
levels: [ levels: [
...props.escalation().levels, ...props.escalation().levels,
{ after: nextAfter, notify: 'all' as EscalationNotifyTarget }, {
after: nextAfter,
notify: 'all' as EscalationNotifyTarget,
...(destinationIds.length > 0 ? { destinationIds } : {}),
},
], ],
}); });
markUnsaved(); markUnsaved();
@@ -374,6 +415,9 @@ export function useAlertScheduleState(props: UseAlertScheduleStateProps) {
setEscalationEnabled, setEscalationEnabled,
setEscalationAfter, setEscalationAfter,
setEscalationNotify, setEscalationNotify,
setEscalationDestinationIds,
setEscalationRepeatCritical,
setEscalationRepeatEvery,
removeEscalationLevel, removeEscalationLevel,
addEscalationLevel, addEscalationLevel,
}; };
@@ -1,4 +1,4 @@
import { createSignal, onMount } from 'solid-js'; import { createSignal, onMount, type Accessor, type Setter } from 'solid-js';
import { NotificationsAPI, type Webhook } from '@/api/notifications'; import { NotificationsAPI, type Webhook } from '@/api/notifications';
import { notificationStore } from '@/stores/notifications'; import { notificationStore } from '@/stores/notifications';
@@ -20,10 +20,20 @@ const normalizeWebhook = (webhook: Webhook): Webhook => ({
service: webhook.service || 'generic', service: webhook.service || 'generic',
}); });
export function useAlertWebhookDestinationsState() { interface AlertWebhookDestinationsStateOptions {
const [webhooks, setWebhooks] = createSignal<Webhook[]>([]); webhooks?: Accessor<Webhook[]>;
setWebhooks?: Setter<Webhook[]>;
autoLoad?: boolean;
}
export function useAlertWebhookDestinationsState(
options: AlertWebhookDestinationsStateOptions = {},
) {
const [localWebhooks, setLocalWebhooks] = createSignal<Webhook[]>([]);
const webhooks = options.webhooks ?? localWebhooks;
const setWebhooks = options.setWebhooks ?? setLocalWebhooks;
const [webhookLoadError, setWebhookLoadError] = createSignal<string | null>(null); const [webhookLoadError, setWebhookLoadError] = createSignal<string | null>(null);
const [isLoadingWebhooks, setIsLoadingWebhooks] = createSignal(true); const [isLoadingWebhooks, setIsLoadingWebhooks] = createSignal(options.autoLoad !== false);
const [testingWebhook, setTestingWebhook] = createSignal<string | null>(null); const [testingWebhook, setTestingWebhook] = createSignal<string | null>(null);
const loadWebhooks = async () => { const loadWebhooks = async () => {
@@ -41,7 +51,9 @@ export function useAlertWebhookDestinationsState() {
}; };
onMount(() => { onMount(() => {
void loadWebhooks(); if (options.autoLoad !== false) {
void loadWebhooks();
}
}); });
const testWebhook = async (webhookId: string, webhookData?: Omit<Webhook, 'id'>) => { const testWebhook = async (webhookId: string, webhookData?: Omit<Webhook, 'id'>) => {
@@ -176,7 +176,12 @@ export function useAlertsConfigurationSnapshotState(
setScheduleGrouping({ ...snapshot.scheduleGrouping }); setScheduleGrouping({ ...snapshot.scheduleGrouping });
setScheduleEscalation({ setScheduleEscalation({
enabled: snapshot.scheduleEscalation.enabled, enabled: snapshot.scheduleEscalation.enabled,
levels: snapshot.scheduleEscalation.levels.map((level) => ({ ...level })), repeatCritical: snapshot.scheduleEscalation.repeatCritical,
repeatEvery: snapshot.scheduleEscalation.repeatEvery,
levels: snapshot.scheduleEscalation.levels.map((level) => ({
...level,
...(level.destinationIds ? { destinationIds: [...level.destinationIds] } : {}),
})),
}); });
setInitialNotify(snapshot.initialNotify); setInitialNotify(snapshot.initialNotify);
setNotifyOnResolve(snapshot.notifyOnResolve); setNotifyOnResolve(snapshot.notifyOnResolve);
@@ -239,7 +244,12 @@ export function useAlertsConfigurationSnapshotState(
scheduleGrouping: { ...scheduleGrouping() }, scheduleGrouping: { ...scheduleGrouping() },
scheduleEscalation: { scheduleEscalation: {
enabled: scheduleEscalation().enabled, enabled: scheduleEscalation().enabled,
levels: scheduleEscalation().levels.map((level) => ({ ...level })), repeatCritical: scheduleEscalation().repeatCritical,
repeatEvery: scheduleEscalation().repeatEvery,
levels: scheduleEscalation().levels.map((level) => ({
...level,
...(level.destinationIds ? { destinationIds: [...level.destinationIds] } : {}),
})),
}, },
initialNotify: initialNotify(), initialNotify: initialNotify(),
notifyOnResolve: notifyOnResolve(), notifyOnResolve: notifyOnResolve(),
@@ -141,6 +141,8 @@ export function useAlertsConfigurationState(props: AlertsConfigurationSurfacePro
setDeadManPingUrl: destinationsState.setDeadManPingUrl, setDeadManPingUrl: destinationsState.setDeadManPingUrl,
pushMinimumSeverity: destinationsState.pushMinimumSeverity, pushMinimumSeverity: destinationsState.pushMinimumSeverity,
setPushMinimumSeverity: destinationsState.setPushMinimumSeverity, setPushMinimumSeverity: destinationsState.setPushMinimumSeverity,
webhooks: destinationsState.webhooks,
setWebhooks: destinationsState.setWebhooks,
...configurationSnapshotState, ...configurationSnapshotState,
allGuests: overridesState.allGuests, allGuests: overridesState.allGuests,
agentResources: overridesState.agentResources, agentResources: overridesState.agentResources,
+3 -1
View File
@@ -238,7 +238,9 @@ export interface AlertConfig {
}; };
escalation?: { escalation?: {
enabled: boolean; enabled: boolean;
levels?: Array<{ after: number; notify: string }>; levels?: Array<{ after: number; notify: string; destinationIds?: string[] }>;
repeatCritical?: boolean;
repeatEvery?: number;
}; };
}; };
disableAllNodes?: boolean; disableAllNodes?: boolean;
@@ -13,10 +13,16 @@ import {
ALERT_CONFIG_DELIVERY_TARGET_LABEL, ALERT_CONFIG_DELIVERY_TARGET_LABEL,
ALERT_CONFIG_DELIVERY_TITLE, ALERT_CONFIG_DELIVERY_TITLE,
ALERT_CONFIG_ESCALATION_DESCRIPTION, ALERT_CONFIG_ESCALATION_DESCRIPTION,
ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP,
ALERT_CONFIG_ESCALATION_DESTINATION_DISABLED,
ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE,
ALERT_CONFIG_ESCALATION_NOTIFY_ALL, ALERT_CONFIG_ESCALATION_NOTIFY_ALL,
ALERT_CONFIG_ESCALATION_NOTIFY_APPRISE, ALERT_CONFIG_ESCALATION_NOTIFY_APPRISE,
ALERT_CONFIG_ESCALATION_NOTIFY_EMAIL, ALERT_CONFIG_ESCALATION_NOTIFY_EMAIL,
ALERT_CONFIG_ESCALATION_NOTIFY_WEBHOOKS, ALERT_CONFIG_ESCALATION_NOTIFY_WEBHOOKS,
ALERT_CONFIG_ESCALATION_REPEAT_EVERY_LABEL,
ALERT_CONFIG_ESCALATION_REPEAT_HELP,
ALERT_CONFIG_ESCALATION_REPEAT_LABEL,
ALERT_CONFIG_ESCALATION_TITLE, ALERT_CONFIG_ESCALATION_TITLE,
ALERT_CONFIG_GROUPING_BY_GUEST, ALERT_CONFIG_GROUPING_BY_GUEST,
ALERT_CONFIG_GROUPING_BY_NODE, ALERT_CONFIG_GROUPING_BY_NODE,
@@ -152,6 +158,16 @@ describe('alertConfigPresentation', () => {
expect(ALERT_CONFIG_ESCALATION_NOTIFY_WEBHOOKS).toBe('Webhooks'); expect(ALERT_CONFIG_ESCALATION_NOTIFY_WEBHOOKS).toBe('Webhooks');
expect(ALERT_CONFIG_ESCALATION_NOTIFY_APPRISE).toBe('Apprise'); expect(ALERT_CONFIG_ESCALATION_NOTIFY_APPRISE).toBe('Apprise');
expect(ALERT_CONFIG_ESCALATION_NOTIFY_ALL).toBe('All channels'); expect(ALERT_CONFIG_ESCALATION_NOTIFY_ALL).toBe('All channels');
expect(ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP).toContain('exact destinations');
expect(ALERT_CONFIG_ESCALATION_DESTINATION_DISABLED).toBe('disabled');
expect(ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE).toBe('Unavailable destination');
expect(ALERT_CONFIG_ESCALATION_REPEAT_LABEL).toBe(
'Repeat critical escalation until acknowledged',
);
expect(ALERT_CONFIG_ESCALATION_REPEAT_EVERY_LABEL).toBe('Repeat every');
expect(ALERT_CONFIG_ESCALATION_REPEAT_HELP).toContain(
'acknowledged, resolved, snoozed, or notifications are paused',
);
expect(ALERT_CONFIG_SUMMARY_TITLE).toBe('Configuration summary'); expect(ALERT_CONFIG_SUMMARY_TITLE).toBe('Configuration summary');
expect(ALERT_CONFIG_SUMMARY_DESCRIPTION).toBe('Preview of the active schedule settings.'); expect(ALERT_CONFIG_SUMMARY_DESCRIPTION).toBe('Preview of the active schedule settings.');
expect(ALERT_CONFIG_SUMMARY_ALL_DISABLED).toBe( expect(ALERT_CONFIG_SUMMARY_ALL_DISABLED).toBe(
@@ -72,6 +72,14 @@ export const ALERT_CONFIG_RECOVERY_HELP =
export const ALERT_CONFIG_ESCALATION_HELP = 'Define escalation levels for unresolved alerts:'; export const ALERT_CONFIG_ESCALATION_HELP = 'Define escalation levels for unresolved alerts:';
export const ALERT_CONFIG_ESCALATION_AFTER_LABEL = 'After'; export const ALERT_CONFIG_ESCALATION_AFTER_LABEL = 'After';
export const ALERT_CONFIG_ESCALATION_NOTIFY_LABEL = 'Notify'; export const ALERT_CONFIG_ESCALATION_NOTIFY_LABEL = 'Notify';
export const ALERT_CONFIG_ESCALATION_DESTINATIONS_HELP =
'Choose the exact destinations for this level. Disabled destinations remain selected but will not receive alerts until they are enabled.';
export const ALERT_CONFIG_ESCALATION_DESTINATION_DISABLED = 'disabled';
export const ALERT_CONFIG_ESCALATION_DESTINATION_UNAVAILABLE = 'Unavailable destination';
export const ALERT_CONFIG_ESCALATION_REPEAT_LABEL = 'Repeat critical escalation until acknowledged';
export const ALERT_CONFIG_ESCALATION_REPEAT_HELP =
'After the final level, repeat critical alerts at this interval. Repeats stop when the alert is acknowledged, resolved, snoozed, or notifications are paused.';
export const ALERT_CONFIG_ESCALATION_REPEAT_EVERY_LABEL = 'Repeat every';
export const ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX = 'min'; export const ALERT_CONFIG_ESCALATION_MINUTES_SUFFIX = 'min';
export const ALERT_CONFIG_ESCALATION_NOTIFY_EMAIL = 'Email'; export const ALERT_CONFIG_ESCALATION_NOTIFY_EMAIL = 'Email';
export const ALERT_CONFIG_ESCALATION_NOTIFY_WEBHOOKS = 'Webhooks'; export const ALERT_CONFIG_ESCALATION_NOTIFY_WEBHOOKS = 'Webhooks';
+15 -2
View File
@@ -487,8 +487,21 @@ func (m *Manager) safeCallUnacknowledgedCallback(alert *Alert, user string) {
// safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning. // safeCallEscalateCallback invokes onEscalate with panic recovery and alert cloning.
func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) { func (m *Manager) safeCallEscalateCallback(alert *Alert, level int) {
m.recordAlertEvent(eventlog.TypeEscalated, alert, "", "", "Alert escalated.", m.safeCallEscalateCallbackWithKind(alert, level, false)
map[string]string{"level": strconv.Itoa(level)}) }
func (m *Manager) safeCallEscalationRepeatCallback(alert *Alert, level int) {
m.safeCallEscalateCallbackWithKind(alert, level, true)
}
func (m *Manager) safeCallEscalateCallbackWithKind(alert *Alert, level int, repeat bool) {
details := map[string]string{"level": strconv.Itoa(level)}
message := "Alert escalated."
if repeat {
details["repeat"] = "true"
message = "Critical alert escalation repeated."
}
m.recordAlertEvent(eventlog.TypeEscalated, alert, "", "", message, details)
callback := m.getEscalateCallback() callback := m.getEscalateCallback()
if callback == nil { if callback == nil {
@@ -1,6 +1,9 @@
package config package config
import "testing" import (
"reflect"
"testing"
)
func TestNormalizeNotificationDeliveryTarget(t *testing.T) { func TestNormalizeNotificationDeliveryTarget(t *testing.T) {
tests := map[string]string{ tests := map[string]string{
@@ -21,3 +24,34 @@ func TestNormalizeNotificationDeliveryTarget(t *testing.T) {
}) })
} }
} }
func TestNormalizeEscalationConfigPreservesLegacyAndBoundsExactRouting(t *testing.T) {
config := EscalationConfig{
Enabled: true,
RepeatCritical: true,
RepeatEvery: 1,
Levels: []EscalationLevel{{
After: 15,
Notify: " WEBHOOKS ",
DestinationIDs: []string{" webhook:ops ", "email", "webhook:ops", "https://secret.example", "webhook:in\nvalid"},
}},
}
NormalizeEscalationConfig(&config)
if config.Levels[0].Notify != "webhook" {
t.Fatalf("legacy fallback = %q, want webhook", config.Levels[0].Notify)
}
if got, want := config.Levels[0].DestinationIDs, []string{"webhook:ops", "email"}; !reflect.DeepEqual(got, want) {
t.Fatalf("destination IDs = %v, want %v", got, want)
}
if config.RepeatEvery != DefaultEscalationRepeatMinutes {
t.Fatalf("repeat interval = %d, want %d", config.RepeatEvery, DefaultEscalationRepeatMinutes)
}
legacy := EscalationConfig{Enabled: true, Levels: []EscalationLevel{{After: 30, Notify: "apprise"}}}
NormalizeEscalationConfig(&legacy)
if legacy.RepeatCritical || legacy.RepeatEvery != 0 || legacy.Levels[0].DestinationIDs != nil {
t.Fatalf("legacy config gained new behavior: %+v", legacy)
}
}
+71 -4
View File
@@ -87,14 +87,17 @@ type QuietHoursSuppression struct {
// EscalationLevel represents an escalation rule // EscalationLevel represents an escalation rule
type EscalationLevel struct { type EscalationLevel struct {
After int `json:"after"` // minutes after initial alert After int `json:"after"` // minutes after initial alert
Notify string `json:"notify"` // "email", "webhook", "apprise", or "all" Notify string `json:"notify"` // backward-compatible channel fallback
DestinationIDs []string `json:"destinationIds,omitempty"` // exact logical destinations: email, apprise, webhook:<id>
} }
// EscalationConfig represents alert escalation configuration // EscalationConfig represents alert escalation configuration
type EscalationConfig struct { type EscalationConfig struct {
Enabled bool `json:"enabled"` Enabled bool `json:"enabled"`
Levels []EscalationLevel `json:"levels"` Levels []EscalationLevel `json:"levels"`
RepeatCritical bool `json:"repeatCritical,omitempty"`
RepeatEvery int `json:"repeatEvery,omitempty"` // minutes after the final level
} }
// GroupingConfig represents alert grouping configuration // GroupingConfig represents alert grouping configuration
@@ -131,6 +134,70 @@ func NormalizeNotificationDeliveryTarget(target string) string {
} }
} }
const (
DefaultEscalationRepeatMinutes = 30
MinEscalationRepeatMinutes = 5
MaxEscalationRepeatMinutes = 180
maxEscalationDestinations = 32
)
// NormalizeEscalationDestinationIDs keeps exact escalation routing bounded and
// credential-free. Unknown identifiers are dropped rather than widening to all
// destinations; an empty result preserves the legacy channel fallback.
func NormalizeEscalationDestinationIDs(destinationIDs []string) []string {
if len(destinationIDs) == 0 {
return nil
}
result := make([]string, 0, min(len(destinationIDs), maxEscalationDestinations))
seen := make(map[string]struct{}, len(destinationIDs))
for _, raw := range destinationIDs {
id := strings.TrimSpace(raw)
valid := id == "email" || id == "apprise"
if strings.HasPrefix(id, "webhook:") {
webhookID := strings.TrimSpace(strings.TrimPrefix(id, "webhook:"))
valid = webhookID != "" && len(webhookID) <= 128 && !strings.ContainsAny(webhookID, "\r\n\x00")
if valid {
id = "webhook:" + webhookID
}
}
if !valid {
continue
}
if _, exists := seen[id]; exists {
continue
}
seen[id] = struct{}{}
result = append(result, id)
if len(result) == maxEscalationDestinations {
break
}
}
if len(result) == 0 {
return nil
}
return result
}
// NormalizeEscalationConfig applies the schedule safety bounds without
// inventing repeat behavior for configurations written by older releases.
func NormalizeEscalationConfig(config *EscalationConfig) {
if config == nil {
return
}
for index := range config.Levels {
config.Levels[index].Notify = NormalizeNotificationDeliveryTarget(config.Levels[index].Notify)
config.Levels[index].DestinationIDs = NormalizeEscalationDestinationIDs(config.Levels[index].DestinationIDs)
}
if !config.RepeatCritical || len(config.Levels) == 0 {
config.RepeatCritical = false
config.RepeatEvery = 0
return
}
if config.RepeatEvery < MinEscalationRepeatMinutes || config.RepeatEvery > MaxEscalationRepeatMinutes {
config.RepeatEvery = DefaultEscalationRepeatMinutes
}
}
// FilterCondition represents a single filter condition // FilterCondition represents a single filter condition
type FilterCondition struct { type FilterCondition struct {
Type string `json:"type"` // "metric", "text", or "raw" Type string `json:"type"` // "metric", "text", or "raw"
+7
View File
@@ -42,6 +42,9 @@ const (
AlertIntentSignalDefault = alertconfig.AlertIntentSignalDefault AlertIntentSignalDefault = alertconfig.AlertIntentSignalDefault
AlertIntentSignalOffline = alertconfig.AlertIntentSignalOffline AlertIntentSignalOffline = alertconfig.AlertIntentSignalOffline
AlertIntentSignalAvailability = alertconfig.AlertIntentSignalAvailability AlertIntentSignalAvailability = alertconfig.AlertIntentSignalAvailability
DefaultEscalationRepeatMinutes = alertconfig.DefaultEscalationRepeatMinutes
MinEscalationRepeatMinutes = alertconfig.MinEscalationRepeatMinutes
MaxEscalationRepeatMinutes = alertconfig.MaxEscalationRepeatMinutes
) )
func NewAlertIntentPolicyDocument() AlertIntentPolicyDocument { func NewAlertIntentPolicyDocument() AlertIntentPolicyDocument {
@@ -78,6 +81,10 @@ func NormalizeDockerIgnoredPrefixes(prefixes []string) []string {
return alertconfig.NormalizeDockerIgnoredPrefixes(prefixes) return alertconfig.NormalizeDockerIgnoredPrefixes(prefixes)
} }
func NormalizeEscalationDestinationIDs(destinationIDs []string) []string {
return alertconfig.NormalizeEscalationDestinationIDs(destinationIDs)
}
func CanonicalResourceTypeKeys(resourceType string) []string { func CanonicalResourceTypeKeys(resourceType string) []string {
return alertconfig.CanonicalResourceTypeKeys(resourceType) return alertconfig.CanonicalResourceTypeKeys(resourceType)
} }
+1 -5
View File
@@ -52,11 +52,7 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
config.NodeDefaults.PoweredOffSeverity = alertconfig.NormalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity) config.NodeDefaults.PoweredOffSeverity = alertconfig.NormalizePoweredOffSeverity(config.NodeDefaults.PoweredOffSeverity)
config.DockerIgnoredContainerPrefixes = alertconfig.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes) config.DockerIgnoredContainerPrefixes = alertconfig.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
config.Schedule.InitialNotify = alertconfig.NormalizeNotificationDeliveryTarget(config.Schedule.InitialNotify) config.Schedule.InitialNotify = alertconfig.NormalizeNotificationDeliveryTarget(config.Schedule.InitialNotify)
for index := range config.Schedule.Escalation.Levels { alertconfig.NormalizeEscalationConfig(&config.Schedule.Escalation)
config.Schedule.Escalation.Levels[index].Notify = alertconfig.NormalizeNotificationDeliveryTarget(
config.Schedule.Escalation.Levels[index].Notify,
)
}
// Migration logic for activation state (backward compatibility) // Migration logic for activation state (backward compatibility)
m.migrateActivationState(&config) m.migrateActivationState(&config)
+34
View File
@@ -82,3 +82,37 @@ func TestUpdateConfigNormalizesFlappingSettings(t *testing.T) {
t.Fatalf("expected FlappingCooldownMinutes to default to 15, got %d", m.config.FlappingCooldownMinutes) t.Fatalf("expected FlappingCooldownMinutes to default to 15, got %d", m.config.FlappingCooldownMinutes)
} }
} }
func TestUpdateConfigNormalizesExactEscalationRoutingAndRepeatPolicy(t *testing.T) {
m := newTestManager(t)
cfg := m.GetConfig()
cfg.Schedule.Escalation = EscalationConfig{
Enabled: true,
RepeatCritical: true,
RepeatEvery: 1,
Levels: []EscalationLevel{{
After: 15,
Notify: "WEBHOOKS",
DestinationIDs: []string{" webhook:pager ", "email", "webhook:pager", "https://secret.example"},
}},
}
m.UpdateConfig(cfg)
got := m.GetConfig().Schedule.Escalation
if got.RepeatEvery != DefaultEscalationRepeatMinutes {
t.Fatalf("repeat interval = %d, want %d", got.RepeatEvery, DefaultEscalationRepeatMinutes)
}
if got.Levels[0].Notify != "webhook" {
t.Fatalf("legacy target = %q, want webhook", got.Levels[0].Notify)
}
wantIDs := []string{"webhook:pager", "email"}
if len(got.Levels[0].DestinationIDs) != len(wantIDs) {
t.Fatalf("destination IDs = %v, want %v", got.Levels[0].DestinationIDs, wantIDs)
}
for index, want := range wantIDs {
if got.Levels[0].DestinationIDs[index] != want {
t.Fatalf("destination IDs = %v, want %v", got.Levels[0].DestinationIDs, wantIDs)
}
}
}
+26
View File
@@ -86,5 +86,31 @@ func (m *Manager) checkEscalations() {
m.safeCallEscalateCallback(alert, i+1) m.safeCallEscalateCallback(alert, i+1)
} }
} }
escalation := m.config.Schedule.Escalation
if !escalation.RepeatCritical || alert.Level != AlertLevelCritical || len(escalation.Levels) == 0 || alert.LastEscalation < len(escalation.Levels) {
continue
}
repeatEvery := time.Duration(escalation.RepeatEvery) * time.Minute
if repeatEvery <= 0 {
continue
}
lastLevel := escalation.Levels[len(escalation.Levels)-1]
repeatBaseline := escalationBaseline.Add(time.Duration(lastLevel.After) * time.Minute)
if count := len(alert.EscalationTimes); count > 0 && alert.EscalationTimes[count-1].After(repeatBaseline) {
repeatBaseline = alert.EscalationTimes[count-1]
}
if !now.After(repeatBaseline.Add(repeatEvery)) {
continue
}
alert.EscalationTimes = append(alert.EscalationTimes, now)
log.Info().
Str("alertID", alert.ID).
Str("trackingKey", canonicalTrackingKeyForAlert(alert)).
Int("level", len(escalation.Levels)).
Int("repeatEveryMinutes", escalation.RepeatEvery).
Msg("Critical alert escalation repeated")
m.safeCallEscalationRepeatCallback(alert, len(escalation.Levels))
} }
} }
+139
View File
@@ -0,0 +1,139 @@
package alerts
import (
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
)
func TestCriticalEscalationRepeatsAtFinalLevelUntilAcknowledged(t *testing.T) {
m := newTestManager(t)
store, err := eventlog.OpenInMemory()
if err != nil {
t.Fatalf("open in-memory event log: %v", err)
}
m.SetEventLog(store)
now := time.Date(2026, time.August, 27, 18, 0, 0, 0, time.UTC)
m.now = func() time.Time { return now }
cfg := m.GetConfig()
cfg.Enabled = true
cfg.ActivationState = ActivationActive
cfg.Schedule.Escalation = EscalationConfig{
Enabled: true,
RepeatCritical: true,
RepeatEvery: 15,
Levels: []EscalationLevel{{
After: 5,
Notify: "webhook",
DestinationIDs: []string{"webhook:pager"},
}},
}
m.UpdateConfig(cfg)
var mu sync.Mutex
callbacks := 0
m.SetEscalateCallback(func(*Alert, int) {
mu.Lock()
callbacks++
mu.Unlock()
})
alert := &Alert{
ID: "critical-repeat",
Level: AlertLevelCritical,
ResourceID: "node-1",
CanonicalSpecID: "connectivity",
StartTime: now.Add(-6 * time.Minute),
LastSeen: now,
}
m.mu.Lock()
m.setActiveAlertNoLock(alert.ID, alert)
m.mu.Unlock()
m.checkEscalations()
waitForEscalationCallbacks(t, &mu, &callbacks, 1)
now = now.Add(14 * time.Minute)
m.checkEscalations()
waitForEscalationCallbacks(t, &mu, &callbacks, 1)
now = now.Add(2 * time.Minute)
m.checkEscalations()
waitForEscalationCallbacks(t, &mu, &callbacks, 2)
m.mu.RLock()
active := testRequireActiveAlert(t, m, "critical-repeat")
if active.LastEscalation != 1 || len(active.EscalationTimes) != 2 {
t.Fatalf("repeat lifecycle = %+v", active)
}
canonicalAlertID := exportedAlertID(active, active.ID)
m.mu.RUnlock()
if err := m.AcknowledgeAlert(canonicalAlertID, "operator"); err != nil {
t.Fatalf("AcknowledgeAlert: %v", err)
}
now = now.Add(20 * time.Minute)
m.checkEscalations()
waitForEscalationCallbacks(t, &mu, &callbacks, 2)
events, err := m.AlertEvents(eventlog.Filter{Types: []string{eventlog.TypeEscalated}, Limit: 10})
if err != nil {
t.Fatalf("list escalation events: %v", err)
}
foundRepeat := false
for _, event := range events {
if event.Details["repeat"] == "true" {
foundRepeat = true
}
}
if !foundRepeat {
t.Fatalf("repeat escalation was not distinguished in event log: %+v", events)
}
}
func TestEscalationRepeatSkipsWarningAlerts(t *testing.T) {
m := newTestManager(t)
now := time.Date(2026, time.August, 27, 18, 0, 0, 0, time.UTC)
m.now = func() time.Time { return now }
m.mu.Lock()
m.config.Enabled = true
m.config.ActivationState = ActivationActive
m.config.Schedule.Escalation = EscalationConfig{
Enabled: true, RepeatCritical: true, RepeatEvery: 5,
Levels: []EscalationLevel{{After: 5, Notify: "email"}},
}
m.activeAlerts["warning"] = &Alert{
ID: "warning", Level: AlertLevelWarning, StartTime: now.Add(-30 * time.Minute),
LastEscalation: 1, EscalationTimes: []time.Time{now.Add(-10 * time.Minute)},
}
m.mu.Unlock()
var callbacks atomic.Int32
m.SetEscalateCallback(func(*Alert, int) { callbacks.Add(1) })
m.checkEscalations()
time.Sleep(20 * time.Millisecond)
if got := callbacks.Load(); got != 0 {
t.Fatalf("warning alert repeated %d times", got)
}
}
func waitForEscalationCallbacks(t *testing.T, mu *sync.Mutex, callbacks *int, want int) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for {
mu.Lock()
got := *callbacks
mu.Unlock()
if got == want {
return
}
if got > want || time.Now().After(deadline) {
t.Fatalf("escalation callbacks = %d, want %d", got, want)
}
time.Sleep(5 * time.Millisecond)
}
}
@@ -1,6 +1,8 @@
package monitoring package monitoring
import ( import (
"net/http"
"net/http/httptest"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -8,6 +10,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
) )
@@ -33,6 +36,94 @@ func trueNASMemoryAlertResource(id, supersededID, hostname string, memory float6
return resource return resource
} }
func TestHandleAlertEscalatedPreservesLegacyAppriseAndExactWebhookRouting(t *testing.T) {
t.Run("legacy Apprise", func(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
requests := make(chan struct{}, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
requests <- struct{}{}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
notifMgr := notifications.NewNotificationManager("https://pulse.local")
defer notifMgr.Stop()
notifMgr.SetGroupingWindow(0)
if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
}
notifMgr.SetAppriseConfig(notifications.AppriseConfig{
Enabled: true, Mode: notifications.AppriseModeHTTP, ServerURL: server.URL,
})
manager := alerts.NewManager()
cfg := manager.GetConfig()
cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{After: 1, Notify: "apprise"}}
cfg.Schedule.QuietHours.Enabled = false
manager.UpdateConfig(cfg)
(&Monitor{notificationMgr: notifMgr, alertManager: manager}).handleAlertEscalated(nil, &alerts.Alert{
ID: "apprise-escalation", Type: "connectivity", Level: alerts.AlertLevelCritical,
ResourceID: "node/pve-1", ResourceName: "pve-1", StartTime: time.Now(),
}, 1)
select {
case <-requests:
case <-time.After(2 * time.Second):
t.Fatal("expected legacy Apprise escalation delivery")
}
})
t.Run("exact webhook", func(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
selectedRequests := make(chan struct{}, 1)
selectedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
selectedRequests <- struct{}{}
w.WriteHeader(http.StatusOK)
}))
defer selectedServer.Close()
unselectedRequests := make(chan struct{}, 1)
unselectedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
unselectedRequests <- struct{}{}
w.WriteHeader(http.StatusOK)
}))
defer unselectedServer.Close()
notifMgr := notifications.NewNotificationManager("https://pulse.local")
defer notifMgr.Stop()
notifMgr.SetGroupingWindow(0)
if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
}
notifMgr.AddWebhook(notifications.WebhookConfig{ID: "ops", Name: "ops", URL: unselectedServer.URL, Enabled: true})
notifMgr.AddWebhook(notifications.WebhookConfig{ID: "pager", Name: "pager", URL: selectedServer.URL, Enabled: true})
manager := alerts.NewManager()
cfg := manager.GetConfig()
cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{
After: 1, Notify: "webhook", DestinationIDs: []string{"webhook:pager"},
}}
cfg.Schedule.QuietHours.Enabled = false
manager.UpdateConfig(cfg)
(&Monitor{notificationMgr: notifMgr, alertManager: manager}).handleAlertEscalated(nil, &alerts.Alert{
ID: "exact-webhook-escalation", Type: "connectivity", Level: alerts.AlertLevelCritical,
ResourceID: "node/pve-1", ResourceName: "pve-1", StartTime: time.Now(),
}, 1)
select {
case <-selectedRequests:
case <-time.After(2 * time.Second):
t.Fatal("expected selected webhook escalation delivery")
}
select {
case <-unselectedRequests:
t.Fatal("exact escalation selection widened to an unselected webhook")
case <-time.After(300 * time.Millisecond):
}
})
}
func activeMemoryAlertForResource(manager *alerts.Manager, resourceID string) (alerts.Alert, bool) { func activeMemoryAlertForResource(manager *alerts.Manager, resourceID string) (alerts.Alert, bool) {
for _, alert := range manager.GetActiveAlerts() { for _, alert := range manager.GetActiveAlerts() {
if alert.ResourceID == resourceID && alert.Type == "memory" { if alert.ResourceID == resourceID && alert.Type == "memory" {
+6 -1
View File
@@ -295,8 +295,13 @@ func (m *Monitor) handleAlertEscalated(hub *websocket.Hub, alert *alerts.Alert,
if m.notificationMgr != nil { if m.notificationMgr != nil {
escalationLevel := config.Schedule.Escalation.Levels[level-1] escalationLevel := config.Schedule.Escalation.Levels[level-1]
if len(escalationLevel.DestinationIDs) > 0 {
m.notificationMgr.SendEscalatedAlertToDestinations(alert, escalationLevel.Notify, escalationLevel.DestinationIDs)
m.broadcastEscalatedAlert(hub, alert)
return
}
switch strings.ToLower(strings.TrimSpace(escalationLevel.Notify)) { switch strings.ToLower(strings.TrimSpace(escalationLevel.Notify)) {
case "", "all", "email", "webhook", "webhooks": case "", "all", "email", "webhook", "webhooks", "apprise":
m.notificationMgr.SendEscalatedAlert(alert, escalationLevel.Notify) m.notificationMgr.SendEscalatedAlert(alert, escalationLevel.Notify)
default: default:
log.Warn(). log.Warn().
+58 -9
View File
@@ -97,6 +97,7 @@ type alertSendOptions struct {
bypassCooldown bool bypassCooldown bool
immediate bool immediate bool
target notificationDeliveryTarget target notificationDeliveryTarget
destinationIDs []string
} }
// createSecureWebhookClient creates an HTTP client with security controls // createSecureWebhookClient creates an HTTP client with security controls
@@ -969,7 +970,7 @@ func (n *NotificationManager) SetGroupingConfig(enabled bool, seconds int, byNod
n.mu.Unlock() n.mu.Unlock()
for _, alert := range pending { for _, alert := range pending {
n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, []*alerts.Alert{alert}, initialTarget, queue) n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, []*alerts.Alert{alert}, initialTarget, nil, queue)
} }
log.Info(). log.Info().
@@ -1109,10 +1110,18 @@ func (n *NotificationManager) SendAlert(alert *alerts.Alert) {
// cadence is owned by the alert schedule, so delivery cooldown must not suppress // cadence is owned by the alert schedule, so delivery cooldown must not suppress
// a level that the alert manager has already decided is due. // a level that the alert manager has already decided is due.
func (n *NotificationManager) SendEscalatedAlert(alert *alerts.Alert, notifyTarget string) { func (n *NotificationManager) SendEscalatedAlert(alert *alerts.Alert, notifyTarget string) {
n.SendEscalatedAlertToDestinations(alert, notifyTarget, nil)
}
// SendEscalatedAlertToDestinations sends one due escalation to exact logical
// destinations when provided. Exact selection is intentionally fail-closed:
// stale or unknown IDs never widen into a channel-wide or all-destination send.
func (n *NotificationManager) SendEscalatedAlertToDestinations(alert *alerts.Alert, notifyTarget string, destinationIDs []string) {
n.sendAlert(alert, alertSendOptions{ n.sendAlert(alert, alertSendOptions{
bypassCooldown: true, bypassCooldown: true,
immediate: true, immediate: true,
target: normalizeNotificationDeliveryTarget(notifyTarget), target: normalizeNotificationDeliveryTarget(notifyTarget),
destinationIDs: normalizeEscalationDestinationIDs(destinationIDs),
}) })
} }
@@ -1172,7 +1181,7 @@ func (n *NotificationManager) sendAlert(alert *alerts.Alert, options alertSendOp
queue := n.queue queue := n.queue
n.mu.Unlock() n.mu.Unlock()
n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, []*alerts.Alert{alert}, options.target, queue) n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, []*alerts.Alert{alert}, options.target, options.destinationIDs, queue)
return return
} }
@@ -1224,9 +1233,10 @@ func (n *NotificationManager) dispatchFiringAlerts(
appriseConfig AppriseConfig, appriseConfig AppriseConfig,
alertsToSend []*alerts.Alert, alertsToSend []*alerts.Alert,
target notificationDeliveryTarget, target notificationDeliveryTarget,
destinationIDs []string,
queue *NotificationQueue, queue *NotificationQueue,
) { ) {
jobs := buildNotificationDeliveryJobsForTarget( jobs := buildNotificationDeliveryJobsForSelection(
emailConfig, emailConfig,
webhooks, webhooks,
appriseConfig, appriseConfig,
@@ -1234,9 +1244,12 @@ func (n *NotificationManager) dispatchFiringAlerts(
eventAlert, eventAlert,
time.Time{}, time.Time{},
target, target,
destinationIDs,
) )
if len(jobs) == 0 { if len(jobs) == 0 {
n.markAlertsNotified(alertsToSend, time.Now()) if len(destinationIDs) == 0 {
n.markAlertsNotified(alertsToSend, time.Now())
}
return return
} }
if queue != nil { if queue != nil {
@@ -1533,7 +1546,7 @@ func (n *NotificationManager) sendGroupedAlerts() {
queue := n.queue queue := n.queue
n.mu.Unlock() n.mu.Unlock()
n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, alertsToSend, initialTarget, queue) n.dispatchFiringAlerts(emailConfig, webhooks, appriseConfig, alertsToSend, initialTarget, nil, queue)
} }
func buildNotificationDeliveryJobs(emailConfig EmailConfig, webhooks []WebhookConfig, appriseConfig AppriseConfig, alertsToSend []*alerts.Alert, event notificationEvent, resolvedAt time.Time) []notificationDeliveryJob { func buildNotificationDeliveryJobs(emailConfig EmailConfig, webhooks []WebhookConfig, appriseConfig AppriseConfig, alertsToSend []*alerts.Alert, event notificationEvent, resolvedAt time.Time) []notificationDeliveryJob {
@@ -1556,13 +1569,45 @@ func buildNotificationDeliveryJobsForTarget(
event notificationEvent, event notificationEvent,
resolvedAt time.Time, resolvedAt time.Time,
target notificationDeliveryTarget, target notificationDeliveryTarget,
) []notificationDeliveryJob {
return buildNotificationDeliveryJobsForSelection(
emailConfig,
webhooks,
appriseConfig,
alertsToSend,
event,
resolvedAt,
target,
nil,
)
}
func buildNotificationDeliveryJobsForSelection(
emailConfig EmailConfig,
webhooks []WebhookConfig,
appriseConfig AppriseConfig,
alertsToSend []*alerts.Alert,
event notificationEvent,
resolvedAt time.Time,
target notificationDeliveryTarget,
destinationIDs []string,
) []notificationDeliveryJob { ) []notificationDeliveryJob {
if target == "" { if target == "" {
target = notificationDeliveryTargetAll target = notificationDeliveryTargetAll
} }
destinationIDs = normalizeEscalationDestinationIDs(destinationIDs)
exactSelection := len(destinationIDs) > 0
selected := make(map[string]struct{}, len(destinationIDs))
for _, id := range destinationIDs {
selected[id] = struct{}{}
}
isSelected := func(id string) bool {
_, ok := selected[id]
return ok
}
jobs := make([]notificationDeliveryJob, 0, 2+len(webhooks)) jobs := make([]notificationDeliveryJob, 0, 2+len(webhooks))
if emailConfig.Enabled && (target == notificationDeliveryTargetAll || target == notificationDeliveryTargetEmail) { if emailConfig.Enabled && ((exactSelection && isSelected("email")) || (!exactSelection && (target == notificationDeliveryTargetAll || target == notificationDeliveryTargetEmail))) {
routedAlerts := routeNotificationAlerts(alertsToSend, emailConfig.TagFilter, emailConfig.TagMode, emailConfig.MinimumSeverity, event) routedAlerts := routeNotificationAlerts(alertsToSend, emailConfig.TagFilter, emailConfig.TagMode, emailConfig.MinimumSeverity, event)
if len(routedAlerts) > 0 { if len(routedAlerts) > 0 {
emailCopy := emailConfig emailCopy := emailConfig
@@ -1575,9 +1620,9 @@ func buildNotificationDeliveryJobsForTarget(
}) })
} }
} }
if target == notificationDeliveryTargetAll || target == notificationDeliveryTargetWebhook { if exactSelection || target == notificationDeliveryTargetAll || target == notificationDeliveryTargetWebhook {
for _, webhook := range webhooks { for _, webhook := range webhooks {
if !webhook.Enabled { if !webhook.Enabled || (exactSelection && !isSelected("webhook:"+strings.TrimSpace(webhook.ID))) {
continue continue
} }
routedAlerts := routeNotificationAlerts(alertsToSend, webhook.TagFilter, webhook.TagMode, webhook.MinimumSeverity, event) routedAlerts := routeNotificationAlerts(alertsToSend, webhook.TagFilter, webhook.TagMode, webhook.MinimumSeverity, event)
@@ -1593,7 +1638,7 @@ func buildNotificationDeliveryJobsForTarget(
} }
} }
} }
if appriseConfig.Enabled && (target == notificationDeliveryTargetAll || target == notificationDeliveryTargetApprise) { if appriseConfig.Enabled && ((exactSelection && isSelected("apprise")) || (!exactSelection && (target == notificationDeliveryTargetAll || target == notificationDeliveryTargetApprise))) {
routedAlerts := routeNotificationAlerts(alertsToSend, nil, "", appriseConfig.MinimumSeverity, event) routedAlerts := routeNotificationAlerts(alertsToSend, nil, "", appriseConfig.MinimumSeverity, event)
if len(routedAlerts) > 0 { if len(routedAlerts) > 0 {
appriseCopy := appriseConfig appriseCopy := appriseConfig
@@ -1609,6 +1654,10 @@ func buildNotificationDeliveryJobsForTarget(
return jobs return jobs
} }
func normalizeEscalationDestinationIDs(destinationIDs []string) []string {
return alerts.NormalizeEscalationDestinationIDs(destinationIDs)
}
func queueTypeForNotificationDeliveryJob(job notificationDeliveryJob) string { func queueTypeForNotificationDeliveryJob(job notificationDeliveryJob) string {
queueType := job.Type queueType := job.Type
if job.Event == eventResolved { if job.Event == eventResolved {
@@ -121,6 +121,41 @@ func TestBuildNotificationDeliveryJobsRoutesGroupedAlertsByDestinationSeverity(t
} }
} }
func TestBuildNotificationDeliveryJobsTargetsExactEscalationDestinations(t *testing.T) {
alert := &alerts.Alert{ID: "critical", Level: alerts.AlertLevelCritical, StartTime: time.Unix(1_700_000_000, 0)}
jobs := buildNotificationDeliveryJobsForSelection(
EmailConfig{Enabled: true},
[]WebhookConfig{
{ID: "ops", Enabled: true},
{ID: "pager", Enabled: true},
},
AppriseConfig{Enabled: true, Mode: AppriseModeHTTP, ServerURL: "https://apprise.example.test"},
[]*alerts.Alert{alert},
eventAlert,
time.Time{},
notificationDeliveryTargetAll,
[]string{"webhook:pager", "apprise"},
)
if len(jobs) != 2 || jobs[0].WebhookConfig == nil || jobs[0].WebhookConfig.ID != "pager" || jobs[1].AppriseConfig == nil {
t.Fatalf("exact escalation jobs = %+v, want pager webhook plus Apprise", jobs)
}
jobs = buildNotificationDeliveryJobsForSelection(
EmailConfig{Enabled: true},
[]WebhookConfig{{ID: "ops", Enabled: true}},
AppriseConfig{Enabled: true},
[]*alerts.Alert{alert},
eventAlert,
time.Time{},
notificationDeliveryTargetAll,
[]string{"webhook:deleted"},
)
if len(jobs) != 0 {
t.Fatalf("unknown exact destination widened delivery: %+v", jobs)
}
}
func TestResolvedSeverityRoutingDefersToOccurrenceReceipts(t *testing.T) { func TestResolvedSeverityRoutingDefersToOccurrenceReceipts(t *testing.T) {
warning := &alerts.Alert{ID: "warning", Level: alerts.AlertLevelWarning} warning := &alerts.Alert{ID: "warning", Level: alerts.AlertLevelWarning}
routed := routeNotificationAlerts( routed := routeNotificationAlerts(