Add alert intent policies and delivery receipts

This commit is contained in:
rcourtman
2026-07-20 20:27:39 +01:00
parent e23f19459d
commit 057cf74629
66 changed files with 4124 additions and 268 deletions
@@ -4716,3 +4716,14 @@ mutations require `monitoring:write`, and slash-containing canonical record IDs
are parsed as opaque identities. Governed action planning still enters the
existing action lifecycle and entitlement boundary before any agent executor
is considered.
### Alert intent remains outside agent authority
Alert-intent policy reads and writes, UDP availability probes, operator-state
lookups, and backup-task context do not widen the agent lifecycle. The server
evaluates these read-state inputs without issuing an agent command, accepting a
new report shape, rotating credentials, or changing remote configuration.
Neither an expected-transient decision nor an indeterminate UDP outcome may be
translated into an agent action. Any later customer-infrastructure mutation
still requires the canonical Actions planner, approval, executor, receipt,
audit, and verification boundaries.
@@ -1186,3 +1186,34 @@ unambiguous/live/ambiguous decision matrix,
reload behavior, and
`frontend-modern/src/features/alerts/thresholds/hooks/__tests__/truenasThresholdPersistence.test.tsx`
proves the browser-side TrueNAS save/refetch contract.
### Versioned alert-intent policy
The alerts runtime owns one versioned alert-intent document and its durable
pending-condition state. Stable signal keys are `*`, `state.offline`,
`incident.availability`, and `metric.<name>`. Effective fields resolve
independently from legacy metric behavior, document defaults, resource-type
overrides, and canonical-resource overrides, in that order. Keys are
normalized before validation and collisions fail closed; updates use revision
compare-and-swap so a stale browser cannot overwrite a newer document.
Intent affects when detector truth becomes eligible for an active alert; it
does not mutate the underlying observation or create a second alert identity.
Without an explicit applicable rule, established alert behavior remains
compatible. With a rule, the first matched time is durable across restart and
becomes the canonical alert start time once eligible. Preview evaluates the
same resolver but restores pending state before returning, so it is read-only.
Invalid documents, persistence failures, or revision conflicts must leave the
prior in-memory and durable policy active.
Operator maintenance and intentionally-offline state are read only through the
canonical unified-resource identity. Backup-aware offline deferral consumes
fresh, matching, active task evidence, applies the configured post-backup grace,
and always terminates at its hard cap. Missing, stale, future-skewed,
finished, or mismatched backup evidence cannot suppress an outage. This policy
changes alert activation only: notification delivery, recovery assurance, and
customer-infrastructure mutation retain their existing owners.
`internal/alerts/intent_policy_test.go` proves precedence, normalization,
operator and backup contexts, preview immutability, restart continuity, and
first-match lifecycle identity.
@@ -7907,3 +7907,28 @@ authorize an alternate execution route.
acceptance and typed-conflict payloads, while
`frontend-modern/src/features/patrol/__tests__/patrolRunAcceptance.test.ts`
proves bounded status/history reconciliation against the returned run ID.
### Alert intent and UDP availability transport
`GET /api/alerts/intent-policies` and
`POST /api/alerts/intent-policies/preview` require `monitoring:read`;
`PUT /api/alerts/intent-policies` requires `monitoring:write`. The versioned
document carries schema version, revision, optional update time, and typed
default, resource-type, and canonical-resource rules. A stale revision returns
`409 Conflict`; invalid policy or preview input returns `400`; unavailable
runtime or persistence ownership returns `503`. A save failure restores the
previous in-memory document and returns `500`. Preview is bounded and read-only
and cannot advance pending grace state.
Availability targets add `udp` plus `response_required` and
`open_or_filtered` modes. Request and expected-response payloads remain
explicit configuration fields. Test responses add the probe `outcome` while
retaining the existing success, latency, and error fields for compatible
clients. A silent open-or-filtered result is `indeterminate`, not successful
reachability and not a transport failure that may be promoted to an outage.
`internal/api/alerts_endpoints_test.go` proves scopes, revision conflicts,
rollback, and preview behavior.
`frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts` and the
availability target API and settings tests prove the canonical browser routes
and additive UDP wire fields.
@@ -5165,3 +5165,28 @@ states show bounded guidance with no executable command. The focused proofs are
`frontend-modern/src/components/Settings/__tests__/DiagnosticsResultsPanel.test.tsx`,
`frontend-modern/src/components/Settings/__tests__/settingsHeaderMeta.branchcov0713.test.ts`,
and `frontend-modern/src/utils/__tests__/updatesPresentation.test.ts`.
### Alert intent and three-state availability presentation
The Alerts thresholds surface composes the versioned intent-policy editor from
the shared form, status, disclosure, and loading primitives. It must preserve
field-wise inheritance: omitted fields inherit, while explicit false and zero
values remain deliberate overrides. Save uses the displayed revision, reports
revision conflict without replacing local edits, and refreshes from the
server-owned document after success. Preview renders clear,
expected-transient, pending-grace, and would-activate as distinct states and
never presents preview as a write.
Availability controls expose UDP mode, request payload, and optional expected
response only where valid for the selected protocol. Unified-resource
presentation keeps `indeterminate` visibly distinct from reachable and
unreachable: open-or-filtered UDP uses warning treatment and bounded evidence
copy, never a green success tone or a fabricated latency. The primitive layer
does not infer detector, operator-intent, or recovery truth.
The focused proofs are
`frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx`,
`frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx`,
`frontend-modern/src/components/Settings/ConnectionEditor/__tests__/AvailabilityTargetSlot.test.tsx`,
and
`frontend-modern/src/utils/__tests__/availabilityProbePresentation.test.ts`.
@@ -1830,3 +1830,29 @@ before point reconciliation so completeness and permission failure cannot be
lost behind a successful cached-artifact path. Shared protection semantics stay
in `internal/recovery/`; PBS monitoring owns only this explicit evidence-quality
adapter.
### Alert-intent evidence adapters and UDP outcomes
Monitoring supplies read-only context to the alerts-owned intent resolver. The
operator-state adapter resolves source-native references to one canonical
unified-resource ID before reading durable operator intent. Lookup failure,
ambiguity, absence, or store error yields no suppression context; monitoring
does not synthesize maintenance state.
Backup-aware offline intent consumes a PVE task only when VMID, instance, and
node match and the task is active. `pollBackupTasks` stamps server observation
time. Evidence older than five minutes, more than one minute in the future,
finished, terminal, or missing an observation time fails closed. This
short-lived alert context is separate from PBS protection evidence and from
recovery assurance; it cannot claim that a backup is restorable or authorize a
restore.
Availability probing owns three outcomes: reachable, unreachable, and
indeterminate. UDP response-required mode needs a request and treats timeout or
mismatch as unreachable. Open-or-filtered mode may return indeterminate after
the full response deadline. Indeterminate clears accumulated failure count,
projects warning evidence, and emits no availability incident; it never claims
reachability. `internal/monitoring/availability_udp_test.go`,
`internal/monitoring/monitor_alert_intent_test.go`, and the backup polling
assertion in `internal/monitoring/monitor_full_coverage_test.go` are the focused
proofs.
@@ -256,3 +256,28 @@ retry/sent/failed/dead-letter/cancelled outcomes and open-to-enqueue latency
without destination, record, resource, or evidence labels. Delivery state
remains notification truth only and cannot resolve or reopen the alert
lifecycle.
### Occurrence-bound delivery receipts
The notification owner records successful firing delivery by exact alert ID,
nanosecond start time, and a normalized destination identity. Email recipients,
webhook ID plus URL, and Apprise mode/base/config/targets are part of that
identity, so a later occurrence or reconfigured destination cannot inherit an
older receipt. A resolved notification is eligible only for destinations that
received that exact firing occurrence. Successful recovery delivery deletes
the receipt; persistent receipts survive restart and are retention-bounded.
Receipt read failure suppresses recovery rather than inventing prior delivery.
Cooldown state is published only after firing receipts are recorded. Persistent
workers atomically claim a still-pending row after taking per-alert delivery
gates, and resolution cancellation takes the corresponding exclusive gates.
This prevents a stale pending snapshot from being sent after cancellation while
preserving grouped-row and operational-link lifecycle semantics. A destination
with delivery disabled or failed has no receipt and receives no misleading
recovery. These rules remain delivery truth only and cannot resolve the
alerts-owned lifecycle.
`internal/notifications/delivery_receipts_test.go`,
`internal/notifications/notifications_test.go`, and
`internal/notifications/queue_test.go` prove occurrence and destination
isolation, persistence, cleanup, and cancellation/claim ordering.
@@ -2064,6 +2064,7 @@
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"internal/alerts/intent_policy_test.go",
"internal/alerts/migration_characterization_test.go",
"internal/alerts/operational_contract_test.go",
"internal/alerts/unified_eval_parity_test.go",
@@ -2110,11 +2111,13 @@
"frontend-modern/src/components/Alerts/Thresholds/hooks/__tests__/useCollapsedSections.test.ts",
"frontend-modern/src/components/Alerts/Thresholds/sections/__tests__/CollapsibleSection.test.tsx",
"frontend-modern/src/components/Alerts/WebhookConfig.test.tsx",
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
"frontend-modern/src/features/alerts/__tests__/alertsConfigurationModel.test.ts",
"frontend-modern/src/features/alerts/__tests__/helpers.test.ts",
"frontend-modern/src/features/alerts/__tests__/OverviewTab.emptystate.test.tsx",
"frontend-modern/src/features/alerts/__tests__/OverviewTab.timelineerror.test.tsx",
"frontend-modern/src/features/alerts/__tests__/OverviewTab.total24h.test.tsx",
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
"frontend-modern/src/features/alerts/identity.test.ts",
"frontend-modern/src/features/alerts/thresholds/__tests__/helpers.test.ts",
"frontend-modern/src/features/alerts/thresholds/hooks/__tests__/truenasThresholdPersistence.test.tsx",
@@ -2270,6 +2273,7 @@
"internal/alerts/guest_snapshot_test.go",
"internal/alerts/history_concurrency_test.go",
"internal/alerts/history_test.go",
"internal/alerts/intent_policy_test.go",
"internal/alerts/operational_contract_test.go",
"internal/alerts/synology_test.go",
"internal/alerts/threshold_resolution_shared_test.go",
@@ -2639,6 +2643,21 @@
"internal/api/update_readiness_test.go"
]
},
{
"id": "alert-intent-policy-api",
"label": "alert intent policy transport proof",
"match_prefixes": [],
"match_files": [
"frontend-modern/src/api/alertIntentPolicies.ts",
"internal/api/alerts.go"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts",
"internal/api/alerts_endpoints_test.go"
]
},
{
"id": "backend-payload-contracts",
"label": "backend API payload proof",
@@ -5190,6 +5209,7 @@
"internal/models/deepcopy_test.go",
"internal/models/state_host_test.go",
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/monitoring/monitor_full_coverage_test.go",
"internal/monitoring/monitor_host_agents_test.go",
"internal/monitoring/monitor_package_updates_test.go",
"internal/unifiedresources/adapter_coverage_test.go",
@@ -5286,6 +5306,8 @@
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"internal/monitoring/monitor_alert_intent_test.go",
"internal/monitoring/monitor_full_coverage_test.go",
"internal/monitoring/pbs_protection_observation_test.go",
"internal/monitoring/recovery_ingest_test.go",
"internal/recovery/mapper/proxmox/mapper_test.go",
@@ -5338,8 +5360,10 @@
"test_prefixes": [],
"exact_files": [
"internal/monitoring/availability_poller_test.go",
"internal/monitoring/availability_udp_test.go",
"internal/monitoring/canonical_guardrails_test.go",
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/monitoring/monitor_alert_intent_test.go",
"internal/monitoring/monitor_alert_override_migration_test.go",
"internal/monitoring/monitor_backups_readstate_test.go",
"internal/monitoring/monitor_host_agents_test.go",
@@ -6782,6 +6806,7 @@
"internal/hostagent/issue1595_sas_collection_test.go",
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/truenas/contract_test.go",
"internal/unifiedresources/availability_projection_test.go",
"internal/unifiedresources/canonical_identity_test.go",
"internal/unifiedresources/canonical_ids_types_test.go",
"internal/unifiedresources/code_standards_test.go",
@@ -2187,6 +2187,21 @@ while storage detail drawers and filter controls must route summary series IDs,
source tones, and disk metrics through the shared storage helpers instead of
reconstructing them from local table state.
### Backup-aware alert intent is not recovery assurance
The alerts subsystem may defer an offline alert while monitoring supplies
fresh, matching evidence of an active PVE backup task. That evidence is a
short-lived operational context only. It does not create or update a recovery
point, prove backup completeness, validate restore permissions, establish
retention, or change protected/attention/unprotected/unknown posture.
Storage and recovery remain the sole owners of deterministic collection trust
and recovery assurance. Stale, future-skewed, terminal, or mismatched task
evidence fails closed for alert deferral, while PBS and provider recovery
evidence continues through its independent mapper and posture rules. An alert
grace period or hard-cap expiry cannot be interpreted as backup success or
failure and grants no restore or infrastructure mutation authority.
### Physical-disk collection truth
Storage surfaces consume the unified physical-disk collection contract without
@@ -1815,6 +1815,30 @@ through the canonical resource model, but unified-resource consumers must not
reintroduce removed workload aliases or feature-local resource-type shims just
to satisfy one table, drawer, or badge surface.
### Alert-intent identity and availability evidence projection
The monitor adapter may expose two optional read capabilities to the alert
intent resolver: source-reference resolution to the current canonical resource
ID and durable operator-state lookup for that ID. These are read-only views over
the existing registry and store. Display aliases do not become persistence
keys, an unresolved reference supplies no operator suppression, and the adapter
does not mutate operator state or alert policy.
The canonical availability facet additively carries `probeOutcome` and
`udpMode`. `indeterminate` is evidence that UDP reachability could not be
distinguished from filtering; it is neither available nor unavailable truth.
Backend and frontend resource types must retain these values through JSON and
read-state projection without changing canonical identity, correlation, or
incident ownership. Legacy clients may ignore the additive fields and continue
reading the established boolean and status fields, but new consumers must not
convert an indeterminate outcome into a successful check.
`internal/unifiedresources/availability_projection_test.go` proves the wire
shape, `internal/unifiedresources/monitor_adapter_read_state_test.go` proves
canonical identity and operator-state forwarding, and
`frontend-modern/src/types/__tests__/resource.test.ts` proves the browser
projection.
### Physical-disk identity and collection read state
`pkg/diskinventory` owns deterministic physical-disk fallback identity and
@@ -0,0 +1,75 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
AlertIntentPoliciesAPI,
type AlertIntentPolicyDocument,
type AlertIntentPolicyPreviewRequest,
} from '@/api/alertIntentPolicies';
import { apiFetchJSON } from '@/utils/apiClient';
vi.mock('@/utils/apiClient', () => ({
apiFetchJSON: vi.fn(),
}));
const mockedApiFetchJSON = vi.mocked(apiFetchJSON);
describe('AlertIntentPoliciesAPI', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('reads, updates, and previews policies through the canonical routes', async () => {
const document: AlertIntentPolicyDocument = {
schemaVersion: 1,
revision: 4,
defaults: {
'state.offline': {
graceSeconds: 90,
honorOperatorState: true,
},
},
};
const previewRequest: AlertIntentPolicyPreviewRequest = {
resourceId: 'vm:pve-a:101',
resourceType: 'vm',
signal: 'state.offline',
conditionActive: true,
firstMatchedAt: '2026-07-20T12:00:00Z',
};
const preview = {
resourceId: previewRequest.resourceId,
resourceType: previewRequest.resourceType,
signal: previewRequest.signal,
status: 'pending_grace' as const,
reason: 'grace period active',
effective: {
graceSeconds: 90,
honorOperatorState: true,
sources: { graceSeconds: 'defaults.state.offline' },
explicit: true,
},
contexts: [],
warnings: [],
};
mockedApiFetchJSON.mockResolvedValueOnce(document);
await expect(AlertIntentPoliciesAPI.get()).resolves.toEqual(document);
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/intent-policies');
mockedApiFetchJSON.mockResolvedValueOnce({ ...document, revision: 5 });
await expect(AlertIntentPoliciesAPI.update(document)).resolves.toEqual({
...document,
revision: 5,
});
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/intent-policies', {
method: 'PUT',
body: JSON.stringify(document),
});
mockedApiFetchJSON.mockResolvedValueOnce(preview);
await expect(AlertIntentPoliciesAPI.preview(previewRequest)).resolves.toEqual(preview);
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/intent-policies/preview', {
method: 'POST',
body: JSON.stringify(previewRequest),
});
});
});
@@ -0,0 +1,86 @@
import { apiFetchJSON } from '@/utils/apiClient';
const ALERT_INTENT_POLICIES_PATH = '/api/alerts/intent-policies';
export type AlertIntentSignal =
'*' | 'state.offline' | 'incident.availability' | `metric.${string}`;
export interface BackupOfflineIntentPolicy {
enabled: boolean;
postGraceSeconds?: number;
maxDeferralSeconds?: number;
}
export interface AlertIntentRule {
graceSeconds?: number;
honorOperatorState?: boolean;
backupOffline?: BackupOfflineIntentPolicy;
}
export interface AlertIntentPolicyDocument {
schemaVersion: number;
revision: number;
updatedAt?: string;
defaults?: Record<string, AlertIntentRule>;
resourceTypes?: Record<string, Record<string, AlertIntentRule>>;
resources?: Record<string, Record<string, AlertIntentRule>>;
}
export interface AlertIntentPolicyPreviewRequest {
resourceId: string;
resourceType: string;
signal: AlertIntentSignal;
conditionActive: boolean;
firstMatchedAt?: string;
backupActive?: boolean;
backupObservedAt?: string;
}
export interface AlertIntentPolicyPreview {
resourceId: string;
resourceType: string;
signal: string;
status: 'clear' | 'expected_transient' | 'pending_grace' | 'would_activate';
reason: string;
effective: {
graceSeconds: number;
honorOperatorState: boolean;
backupOffline?: BackupOfflineIntentPolicy;
sources: Record<string, string>;
explicit: boolean;
};
firstMatchedAt?: string;
eligibleAt?: string;
hardCapAt?: string;
remainingSeconds?: number;
contexts: Array<{
kind: string;
active: boolean;
evidence?: string;
observedAt?: string;
expiresAt?: string;
}>;
warnings: string[];
}
export class AlertIntentPoliciesAPI {
static async get(): Promise<AlertIntentPolicyDocument> {
return apiFetchJSON(ALERT_INTENT_POLICIES_PATH);
}
static async update(document: AlertIntentPolicyDocument): Promise<AlertIntentPolicyDocument> {
return apiFetchJSON(ALERT_INTENT_POLICIES_PATH, {
method: 'PUT',
body: JSON.stringify(document),
});
}
static async preview(
request: AlertIntentPolicyPreviewRequest,
): Promise<AlertIntentPolicyPreview> {
return apiFetchJSON(`${ALERT_INTENT_POLICIES_PATH}/preview`, {
method: 'POST',
body: JSON.stringify(request),
});
}
}
@@ -2,7 +2,8 @@ import { apiFetchJSON } from '@/utils/apiClient';
const AVAILABILITY_TARGETS_PATH = '/api/availability-targets';
export type AvailabilityProbeProtocol = 'icmp' | 'tcp' | 'http' | 'https';
export type AvailabilityProbeProtocol = 'icmp' | 'tcp' | 'udp' | 'http' | 'https';
export type AvailabilityUDPMode = 'response_required' | 'open_or_filtered';
export type AvailabilityTargetKind = 'machine' | 'service' | 'device';
export interface AvailabilityProbeStatus {
@@ -11,6 +12,7 @@ export interface AvailabilityProbeStatus {
targetKind?: AvailabilityTargetKind | string;
address: string;
protocol: AvailabilityProbeProtocol | string;
outcome?: 'reachable' | 'unreachable' | 'indeterminate' | string;
enabled: boolean;
available: boolean;
lastChecked?: string;
@@ -29,6 +31,9 @@ export interface AvailabilityTarget {
protocol: AvailabilityProbeProtocol;
port?: number;
path?: string;
udpMode?: AvailabilityUDPMode;
udpRequest?: string;
udpExpectedResponse?: string;
linkedResourceId?: string;
enabled: boolean;
pollIntervalSeconds?: number;
@@ -40,6 +45,7 @@ export interface AvailabilityTarget {
export interface AvailabilityTestResponse {
success: boolean;
latencyMillis: number;
outcome?: 'reachable' | 'unreachable' | 'indeterminate' | string;
error?: string;
}
@@ -15,6 +15,7 @@ import {
type AvailabilityTarget,
type AvailabilityTargetKind,
type AvailabilityTestResponse,
type AvailabilityUDPMode,
} from '@/api/availabilityTargets';
import {
AVAILABILITY_TARGET_PRESETS,
@@ -37,6 +38,9 @@ interface AvailabilityForm {
protocol: AvailabilityProbeProtocol;
port: string;
path: string;
udpMode: AvailabilityUDPMode;
udpRequest: string;
udpExpectedResponse: string;
linkedResourceId: string;
enabled: boolean;
pollIntervalSeconds: string;
@@ -68,6 +72,9 @@ const newAvailabilityForm = (
protocol: 'icmp',
port: '',
path: '',
udpMode: 'response_required',
udpRequest: '',
udpExpectedResponse: '',
linkedResourceId: '',
enabled: true,
pollIntervalSeconds: '60',
@@ -83,6 +90,9 @@ const formFromTarget = (target: AvailabilityTarget): AvailabilityForm => ({
protocol: target.protocol ?? 'icmp',
port: target.port ? String(target.port) : '',
path: target.path ?? '',
udpMode: target.udpMode ?? 'response_required',
udpRequest: target.udpRequest ?? '',
udpExpectedResponse: target.udpExpectedResponse ?? '',
linkedResourceId: target.linkedResourceId ?? '',
enabled: target.enabled ?? true,
pollIntervalSeconds: String(target.pollIntervalSeconds ?? 60),
@@ -104,7 +114,10 @@ const payloadFromForm = (form: AvailabilityForm): AvailabilityTarget => {
address: form.address.trim(),
protocol: form.protocol,
port: form.protocol === 'icmp' ? undefined : port,
path: form.protocol === 'http' ? form.path.trim() : undefined,
path: form.protocol === 'http' || form.protocol === 'https' ? form.path.trim() : undefined,
udpMode: form.protocol === 'udp' ? form.udpMode : undefined,
udpRequest: form.protocol === 'udp' ? form.udpRequest : undefined,
udpExpectedResponse: form.protocol === 'udp' ? form.udpExpectedResponse : undefined,
linkedResourceId: form.linkedResourceId.trim() || undefined,
enabled: form.enabled,
pollIntervalSeconds: parsePositiveInt(form.pollIntervalSeconds),
@@ -118,6 +131,9 @@ const presetSensitiveFormKeys: ReadonlySet<keyof AvailabilityForm> = new Set([
'port',
'protocol',
'targetKind',
'udpMode',
'udpRequest',
'udpExpectedResponse',
]);
const initialPresetForTargetKind = (
@@ -189,8 +205,8 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
const addressPlaceholder = () =>
selectedPresetConfig()?.addressPlaceholder ??
(form().protocol === 'http'
? 'http://service.local/status'
(form().protocol === 'http' || form().protocol === 'https'
? `${form().protocol}://service.local/status`
: form().targetKind === 'machine'
? 'server.local'
: form().targetKind === 'service'
@@ -198,7 +214,8 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
: 'device.local');
const portPlaceholder = () =>
selectedPresetConfig()?.portPlaceholder ?? (form().protocol === 'http' ? 'Optional' : '1883');
selectedPresetConfig()?.portPlaceholder ??
(form().protocol === 'http' || form().protocol === 'https' ? 'Optional' : '1883');
const namePlaceholder = () =>
form().targetKind === 'machine'
@@ -330,10 +347,14 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
>
<option value="icmp">ICMP ping</option>
<option value="tcp">TCP port</option>
<option value="udp">UDP datagram</option>
<option value="http">HTTP</option>
<option value="https">HTTPS</option>
</FormSelect>
<label class={`${formField} sm:col-span-2`}>
<span class={formLabel}>{form().protocol === 'http' ? 'URL or host' : 'Address'}</span>
<span class={formLabel}>
{form().protocol === 'http' || form().protocol === 'https' ? 'URL or host' : 'Address'}
</span>
<input
class={formControl}
value={form().address}
@@ -345,7 +366,9 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
? 'Use a hostname or IP address. Pulse will run one ping per poll.'
: form().protocol === 'tcp'
? 'Use a hostname or IP address and the port to open.'
: 'Use a full URL or a hostname. HTTP statuses below 500 count as reachable.'}
: form().protocol === 'udp'
? 'Use a hostname or unicast IP. UDP checks send one small datagram per poll.'
: 'Use a full URL or a hostname. HTTP statuses below 500 count as reachable.'}
</span>
</label>
<FormSelect
@@ -391,7 +414,7 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
/>
</label>
</Show>
<Show when={form().protocol === 'http'}>
<Show when={form().protocol === 'http' || form().protocol === 'https'}>
<label class={formField}>
<span class={formLabel}>Path override</span>
<input
@@ -402,6 +425,41 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
/>
</label>
</Show>
<Show when={form().protocol === 'udp'}>
<FormSelect
label="UDP result policy"
value={form().udpMode}
onChange={(event) =>
updateForm({ udpMode: event.currentTarget.value as AvailabilityUDPMode })
}
help="Response required is alert-safe. Open or filtered reports silence as indeterminate and only fails on an explicit port-unreachable response."
>
<option value="response_required">Require a response</option>
<option value="open_or_filtered">Accept open or filtered</option>
</FormSelect>
<label class={formField}>
<span class={formLabel}>Request payload</span>
<input
class={formControl}
value={form().udpRequest}
onInput={(event) => updateForm({ udpRequest: event.currentTarget.value })}
placeholder={form().udpMode === 'response_required' ? 'PING' : 'Optional'}
/>
<span class={formHelpText}>UTF-8 bytes, up to 512 bytes.</span>
</label>
<label class={`${formField} sm:col-span-2`}>
<span class={formLabel}>Expected response (optional)</span>
<input
class={formControl}
value={form().udpExpectedResponse}
onInput={(event) => updateForm({ udpExpectedResponse: event.currentTarget.value })}
placeholder="PONG"
/>
<span class={formHelpText}>
When set, the response must match these UTF-8 bytes exactly.
</span>
</label>
</Show>
<label class={formField}>
<span class={formLabel}>Poll interval (seconds)</span>
<input
@@ -452,13 +510,21 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
{(result) => (
<CalloutCard
role={result().success ? 'status' : 'alert'}
tone={result().success ? 'success' : 'danger'}
tone={
result().outcome === 'indeterminate'
? 'warning'
: result().success
? 'success'
: 'danger'
}
scale="compact"
padding="sm"
description={
result().success
? `Probe reached the target in ${result().latencyMillis} ms.`
: result().error || 'Probe failed.'
result().outcome === 'indeterminate'
? `No UDP rejection was received in ${result().latencyMillis} ms; the port is open or filtered, not proven reachable.`
: result().success
? `Probe reached the target in ${result().latencyMillis} ms.`
: result().error || 'Probe failed.'
}
/>
)}
@@ -146,4 +146,34 @@ describe('AvailabilityTargetSlot', () => {
),
);
});
it('creates response-validated UDP checks', async () => {
const onSaved = vi.fn();
render(() => <AvailabilityTargetSlot onCancel={vi.fn()} onSaved={onSaved} />);
fireEvent.change(screen.getByLabelText('Probe'), { target: { value: 'udp' } });
fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'DNS health' } });
fireEvent.input(screen.getByLabelText(/^Address/), { target: { value: 'dns.internal' } });
fireEvent.input(screen.getByLabelText('Port'), { target: { value: '53' } });
fireEvent.input(screen.getByLabelText(/^Request payload/), { target: { value: 'PING' } });
fireEvent.input(screen.getByLabelText(/^Expected response \(optional\)/), {
target: { value: 'PONG' },
});
fireEvent.click(screen.getByRole('button', { name: 'Add service/device check' }));
await waitFor(() =>
expect(mockedCreate).toHaveBeenCalledWith(
expect.objectContaining({
name: 'DNS health',
address: 'dns.internal',
protocol: 'udp',
port: 53,
udpMode: 'response_required',
udpRequest: 'PING',
udpExpectedResponse: 'PONG',
}),
),
);
expect(onSaved).toHaveBeenCalledTimes(1);
});
});
@@ -59,6 +59,8 @@ export function getAvailabilityTargetMethodLabel(target: AvailabilityTarget): st
return 'ICMP ping';
case 'tcp':
return target.port ? `TCP ${target.port}` : 'TCP port';
case 'udp':
return target.port ? `UDP ${target.port}` : 'UDP port';
case 'http':
return 'HTTP check';
default:
@@ -90,7 +92,7 @@ export function getAvailabilityTargetAddressLabel(target: AvailabilityTarget): s
}
return target.address;
}
if (target.protocol === 'tcp' && target.port) {
if ((target.protocol === 'tcp' || target.protocol === 'udp') && target.port) {
return `${target.address}:${target.port}`;
}
return target.address;
@@ -100,6 +102,7 @@ export function getAvailabilityTargetStatusLabel(target: AvailabilityTarget): st
if (!target.enabled) return 'Paused';
const status = target.status;
if (!status) return 'Not checked yet';
if (status.outcome === 'indeterminate') return 'Open or filtered';
if (status.available) {
return typeof status.latencyMillis === 'number'
? `Online · ${status.latencyMillis} ms`
@@ -111,6 +114,9 @@ export function getAvailabilityTargetStatusLabel(target: AvailabilityTarget): st
export function getAvailabilityTargetStatusClass(target: AvailabilityTarget): string {
if (!target.enabled) return 'bg-surface-alt text-muted';
if (!target.status) return 'bg-sky-100 text-sky-700 dark:bg-sky-900 dark:text-sky-300';
if (target.status.outcome === 'indeterminate') {
return 'bg-amber-100 text-amber-700 dark:bg-amber-900 dark:text-amber-300';
}
if (target.status.available) {
return 'bg-emerald-100 text-emerald-700 dark:bg-emerald-900 dark:text-emerald-300';
}
@@ -0,0 +1,513 @@
import { For, Show, createEffect, createMemo, createSignal, onMount } from 'solid-js';
import {
AlertIntentPoliciesAPI,
type AlertIntentPolicyDocument,
type AlertIntentPolicyPreview,
type AlertIntentRule,
type AlertIntentSignal,
} from '@/api/alertIntentPolicies';
import { Button } from '@/components/shared/Button';
import { CalloutCard } from '@/components/shared/CalloutCard';
import { Card } from '@/components/shared/Card';
import {
formCheckbox,
formControl,
formField,
formHelpText,
formLabel,
} from '@/components/shared/Form';
import { FormSelect } from '@/components/shared/FormSelect';
import type { Resource } from '@/types/resource';
import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity';
const SIGNALS: ReadonlyArray<{ value: AlertIntentSignal; label: string }> = [
{ value: 'state.offline', label: 'Offline / powered off' },
{ value: 'incident.availability', label: 'Availability incident' },
{ value: 'metric.cpu', label: 'CPU threshold' },
{ value: 'metric.memory', label: 'Memory threshold' },
{ value: 'metric.disk', label: 'Disk threshold' },
];
const nonNegativeInt = (value: string, fallback = 0): number => {
const parsed = Number.parseInt(value.trim(), 10);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
};
const detachedDocument = (document: AlertIntentPolicyDocument): AlertIntentPolicyDocument =>
structuredClone(document);
const ruleFor = (
document: AlertIntentPolicyDocument | null,
resourceId: string,
signal: AlertIntentSignal,
): AlertIntentRule => document?.resources?.[resourceId]?.[signal] ?? {};
export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] }) {
const [expanded, setExpanded] = createSignal(false);
const [document, setDocument] = createSignal<AlertIntentPolicyDocument | null>(null);
const [loading, setLoading] = createSignal(false);
const [saving, setSaving] = createSignal(false);
const [error, setError] = createSignal<string | null>(null);
const [message, setMessage] = createSignal<string | null>(null);
const [offlineGrace, setOfflineGrace] = createSignal('0');
const [availabilityGrace, setAvailabilityGrace] = createSignal('0');
const [honorOperatorState, setHonorOperatorState] = createSignal(false);
const [backupAware, setBackupAware] = createSignal(false);
const [backupPostGrace, setBackupPostGrace] = createSignal('60');
const [backupMaxDeferral, setBackupMaxDeferral] = createSignal('3600');
const sortedResources = createMemo(() =>
[...props.resources]
.filter((resource) => resource.id?.trim())
.sort((a, b) =>
getPreferredInfrastructureDisplayName(a).localeCompare(
getPreferredInfrastructureDisplayName(b),
),
),
);
const [resourceId, setResourceId] = createSignal('');
const [signal, setSignal] = createSignal<AlertIntentSignal>('state.offline');
const [overrideGrace, setOverrideGrace] = createSignal('');
const [overrideOperatorMode, setOverrideOperatorMode] = createSignal<
'inherit' | 'honor' | 'ignore'
>('inherit');
const [overrideBackupMode, setOverrideBackupMode] = createSignal<
'inherit' | 'enabled' | 'disabled'
>('inherit');
const [overrideBackupPostGrace, setOverrideBackupPostGrace] = createSignal('60');
const [overrideBackupMaxDeferral, setOverrideBackupMaxDeferral] = createSignal('3600');
const [previewBackupActive, setPreviewBackupActive] = createSignal(false);
const [previewing, setPreviewing] = createSignal(false);
const [preview, setPreview] = createSignal<AlertIntentPolicyPreview | null>(null);
const selectedResource = createMemo(() =>
sortedResources().find((resource) => resource.id === resourceId()),
);
const load = async () => {
setLoading(true);
setError(null);
try {
const loaded = await AlertIntentPoliciesAPI.get();
setDocument(loaded);
const offline = loaded.defaults?.['state.offline'] ?? {};
const availability = loaded.defaults?.['incident.availability'] ?? {};
setOfflineGrace(String(offline.graceSeconds ?? 0));
setAvailabilityGrace(String(availability.graceSeconds ?? 0));
setHonorOperatorState(offline.honorOperatorState ?? false);
setBackupAware(offline.backupOffline?.enabled ?? false);
setBackupPostGrace(String(offline.backupOffline?.postGraceSeconds ?? 60));
setBackupMaxDeferral(String(offline.backupOffline?.maxDeferralSeconds ?? 3600));
if (!resourceId() && sortedResources().length > 0) {
setResourceId(sortedResources()[0].id);
}
} catch (cause) {
setError(cause instanceof Error ? cause.message : 'Failed to load alert intent policies.');
} finally {
setLoading(false);
}
};
onMount(() => void load());
createEffect(() => {
const resources = sortedResources();
const selected = resourceId();
if (resources.length === 0) {
if (selected) setResourceId('');
return;
}
if (!resources.some((resource) => resource.id === selected)) {
setResourceId(resources[0].id);
}
});
createEffect(() => {
const current = ruleFor(document(), resourceId(), signal());
setOverrideGrace(current.graceSeconds === undefined ? '' : String(current.graceSeconds));
setOverrideOperatorMode(
current.honorOperatorState === undefined
? 'inherit'
: current.honorOperatorState
? 'honor'
: 'ignore',
);
setOverrideBackupMode(
current.backupOffline === undefined
? 'inherit'
: current.backupOffline.enabled
? 'enabled'
: 'disabled',
);
setOverrideBackupPostGrace(String(current.backupOffline?.postGraceSeconds ?? 60));
setOverrideBackupMaxDeferral(String(current.backupOffline?.maxDeferralSeconds ?? 3600));
setPreview(null);
});
const persist = async (next: AlertIntentPolicyDocument, successMessage: string) => {
setSaving(true);
setError(null);
setMessage(null);
try {
const updated = await AlertIntentPoliciesAPI.update(next);
setDocument(updated);
setMessage(successMessage);
} catch (cause) {
setError(cause instanceof Error ? cause.message : 'Failed to save alert intent policies.');
} finally {
setSaving(false);
}
};
const saveDefaults = async () => {
const current = document();
if (!current) return;
const next = detachedDocument(current);
next.defaults ??= {};
next.defaults['state.offline'] = {
...(next.defaults['state.offline'] ?? {}),
graceSeconds: nonNegativeInt(offlineGrace()),
honorOperatorState: honorOperatorState(),
backupOffline: {
enabled: backupAware(),
postGraceSeconds: nonNegativeInt(backupPostGrace(), 60),
maxDeferralSeconds: nonNegativeInt(backupMaxDeferral(), 3600),
},
};
next.defaults['incident.availability'] = {
...(next.defaults['incident.availability'] ?? {}),
graceSeconds: nonNegativeInt(availabilityGrace()),
honorOperatorState: honorOperatorState(),
};
await persist(next, 'Default intent policy saved.');
};
const saveOverride = async () => {
const current = document();
const id = resourceId();
if (!current || !id) return;
const next = detachedDocument(current);
next.resources ??= {};
next.resources[id] ??= {};
const rule: AlertIntentRule = {};
if (overrideGrace().trim() !== '') {
rule.graceSeconds = nonNegativeInt(overrideGrace());
}
if (overrideOperatorMode() !== 'inherit') {
rule.honorOperatorState = overrideOperatorMode() === 'honor';
}
if (signal() === 'state.offline' && overrideBackupMode() !== 'inherit') {
rule.backupOffline = {
enabled: overrideBackupMode() === 'enabled',
postGraceSeconds: nonNegativeInt(overrideBackupPostGrace(), 60),
maxDeferralSeconds: nonNegativeInt(overrideBackupMaxDeferral(), 3600),
};
}
if (Object.keys(rule).length === 0) {
delete next.resources[id][signal()];
if (Object.keys(next.resources[id]).length === 0) delete next.resources[id];
} else {
next.resources[id][signal()] = rule;
}
await persist(next, 'Resource intent override saved.');
};
const removeOverride = async () => {
const current = document();
const id = resourceId();
if (!current || !id || !current.resources?.[id]?.[signal()]) return;
const next = detachedDocument(current);
delete next.resources?.[id]?.[signal()];
if (next.resources?.[id] && Object.keys(next.resources[id]).length === 0) {
delete next.resources[id];
}
await persist(next, 'Resource intent override removed.');
};
const runPreview = async () => {
const resource = selectedResource();
if (!resource) return;
setPreviewing(true);
setError(null);
try {
setPreview(
await AlertIntentPoliciesAPI.preview({
resourceId: resource.id,
resourceType: resource.type,
signal: signal(),
conditionActive: true,
firstMatchedAt: new Date().toISOString(),
...(signal() === 'state.offline'
? {
backupActive: previewBackupActive(),
backupObservedAt: new Date().toISOString(),
}
: {}),
}),
);
} catch (cause) {
setError(cause instanceof Error ? cause.message : 'Failed to preview alert intent policy.');
} finally {
setPreviewing(false);
}
};
const hasSelectedOverride = createMemo(() =>
Boolean(document()?.resources?.[resourceId()]?.[signal()]),
);
return (
<Card padding="md" class="space-y-4">
<div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
<div>
<h2 class="text-base font-semibold text-base-content">Alert intent & grace</h2>
<p class="mt-1 text-sm text-muted">
Delay expected transients, honor operator maintenance state, and extend guest offline
grace while a backup is active.
</p>
</div>
<Button variant="secondary" size="sm" onClick={() => setExpanded((value) => !value)}>
{expanded() ? 'Hide policies' : 'Configure policies'}
</Button>
</div>
<Show when={error()}>
{(text) => <CalloutCard role="alert" tone="danger" scale="compact" description={text()} />}
</Show>
<Show when={message()}>
{(text) => (
<CalloutCard role="status" tone="success" scale="compact" description={text()} />
)}
</Show>
<Show when={expanded()}>
<Show
when={!loading() && document()}
fallback={<div class="text-sm text-muted">Loading intent policies</div>}
>
<div class="grid gap-4 border-t border-border pt-4 sm:grid-cols-2 lg:grid-cols-3">
<label class={formField}>
<span class={formLabel}>Default offline grace (seconds)</span>
<input
class={formControl}
inputMode="numeric"
value={offlineGrace()}
onInput={(event) => setOfflineGrace(event.currentTarget.value)}
/>
</label>
<label class={formField}>
<span class={formLabel}>Default availability grace (seconds)</span>
<input
class={formControl}
inputMode="numeric"
value={availabilityGrace()}
onInput={(event) => setAvailabilityGrace(event.currentTarget.value)}
/>
</label>
<label class="flex items-center gap-3 rounded-md border border-border px-3 py-2">
<input
type="checkbox"
class={formCheckbox}
checked={honorOperatorState()}
onChange={(event) => setHonorOperatorState(event.currentTarget.checked)}
/>
<span class="text-sm text-base-content">
Honor maintenance and intentionally-offline state
</span>
</label>
<label class="flex items-center gap-3 rounded-md border border-border px-3 py-2">
<input
type="checkbox"
class={formCheckbox}
checked={backupAware()}
onChange={(event) => setBackupAware(event.currentTarget.checked)}
/>
<span class="text-sm text-base-content">
Extend offline grace during Proxmox backups
</span>
</label>
<label class={formField}>
<span class={formLabel}>Post-backup grace (seconds)</span>
<input
class={formControl}
inputMode="numeric"
disabled={!backupAware()}
value={backupPostGrace()}
onInput={(event) => setBackupPostGrace(event.currentTarget.value)}
/>
</label>
<label class={formField}>
<span class={formLabel}>Maximum backup deferral (seconds)</span>
<input
class={formControl}
inputMode="numeric"
disabled={!backupAware()}
value={backupMaxDeferral()}
onInput={(event) => setBackupMaxDeferral(event.currentTarget.value)}
/>
<span class={formHelpText}>
Hard cap prevents a stale backup signal from hiding a real outage.
</span>
</label>
</div>
<div class="flex justify-end">
<Button
variant="primary"
isLoading={saving()}
disabled={saving()}
onClick={() => void saveDefaults()}
>
Save defaults
</Button>
</div>
<div class="space-y-4 border-t border-border pt-4">
<div>
<h3 class="text-sm font-semibold text-base-content">
Per-resource override and preview
</h3>
<p class="mt-1 text-xs text-muted">
Resource values override type and global policies field by field.
</p>
</div>
<div class="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
<FormSelect
label="Resource"
value={resourceId()}
onChange={(event) => setResourceId(event.currentTarget.value)}
fieldClass="lg:col-span-2"
>
<For each={sortedResources()}>
{(resource) => (
<option value={resource.id}>
{getPreferredInfrastructureDisplayName(resource)} · {resource.type}
</option>
)}
</For>
</FormSelect>
<FormSelect
label="Signal"
value={signal()}
onChange={(event) => setSignal(event.currentTarget.value as AlertIntentSignal)}
>
<For each={SIGNALS}>
{(item) => <option value={item.value}>{item.label}</option>}
</For>
</FormSelect>
<label class={formField}>
<span class={formLabel}>Grace override (seconds)</span>
<input
class={formControl}
inputMode="numeric"
value={overrideGrace()}
onInput={(event) => setOverrideGrace(event.currentTarget.value)}
placeholder="Inherit"
/>
<span class={formHelpText}>Leave blank to inherit.</span>
</label>
<FormSelect
label="Operator state override"
value={overrideOperatorMode()}
onChange={(event) =>
setOverrideOperatorMode(
event.currentTarget.value as 'inherit' | 'honor' | 'ignore',
)
}
>
<option value="inherit">Inherit</option>
<option value="honor">Honor operator state</option>
<option value="ignore">Ignore operator state</option>
</FormSelect>
<Show when={signal() === 'state.offline'}>
<FormSelect
label="Backup handling override"
value={overrideBackupMode()}
onChange={(event) =>
setOverrideBackupMode(
event.currentTarget.value as 'inherit' | 'enabled' | 'disabled',
)
}
>
<option value="inherit">Inherit</option>
<option value="enabled">Extend during backups</option>
<option value="disabled">Do not extend</option>
</FormSelect>
<label class={formField}>
<span class={formLabel}>Post-backup grace</span>
<input
class={formControl}
inputMode="numeric"
disabled={overrideBackupMode() !== 'enabled'}
value={overrideBackupPostGrace()}
onInput={(event) => setOverrideBackupPostGrace(event.currentTarget.value)}
/>
</label>
<label class={formField}>
<span class={formLabel}>Maximum deferral</span>
<input
class={formControl}
inputMode="numeric"
disabled={overrideBackupMode() !== 'enabled'}
value={overrideBackupMaxDeferral()}
onInput={(event) => setOverrideBackupMaxDeferral(event.currentTarget.value)}
/>
</label>
<label class="flex items-center gap-3 rounded-md border border-border px-3 py-2">
<input
type="checkbox"
class={formCheckbox}
checked={previewBackupActive()}
onChange={(event) => setPreviewBackupActive(event.currentTarget.checked)}
/>
<span class="text-sm text-base-content">Preview with backup active</span>
</label>
</Show>
</div>
<div class="flex flex-wrap justify-end gap-2">
<Show when={hasSelectedOverride()}>
<Button variant="danger" disabled={saving()} onClick={() => void removeOverride()}>
Remove override
</Button>
</Show>
<Button
variant="secondary"
isLoading={previewing()}
disabled={!resourceId() || previewing()}
onClick={() => void runPreview()}
>
Preview current policy
</Button>
<Button
variant="primary"
isLoading={saving()}
disabled={!resourceId() || saving()}
onClick={() => void saveOverride()}
>
Save override
</Button>
</div>
<Show when={preview()}>
{(result) => (
<CalloutCard
role="status"
tone={
result().status === 'would_activate'
? 'danger'
: result().status === 'expected_transient'
? 'info'
: 'warning'
}
scale="compact"
title={result().status.replace(/_/g, ' ')}
description={`${result().reason.replace(/_/g, ' ')} · ${result().effective.graceSeconds}s grace${result().remainingSeconds ? ` · ${result().remainingSeconds}s remaining` : ''}`}
/>
)}
</Show>
</div>
</Show>
</Show>
</Card>
);
}
@@ -0,0 +1,118 @@
import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library';
import { createSignal } from 'solid-js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { AlertIntentPoliciesAPI, type AlertIntentPolicyDocument } from '@/api/alertIntentPolicies';
import type { Resource } from '@/types/resource';
import { AlertIntentPolicyPanel } from '../AlertIntentPolicyPanel';
vi.mock('@/api/alertIntentPolicies', () => ({
AlertIntentPoliciesAPI: {
get: vi.fn(),
update: vi.fn(),
preview: vi.fn(),
},
}));
const baseDocument = (): AlertIntentPolicyDocument => ({
schemaVersion: 1,
revision: 0,
defaults: {
'state.offline': {
graceSeconds: 30,
honorOperatorState: true,
backupOffline: { enabled: true, postGraceSeconds: 60, maxDeferralSeconds: 3600 },
},
},
resourceTypes: {},
resources: {},
});
const resource: Resource = {
id: 'vm-canonical',
type: 'vm',
name: 'database',
displayName: 'database',
platformId: 'node-a',
platformType: 'proxmox-pve',
sourceType: 'api',
status: 'running',
lastSeen: Date.now(),
};
describe('AlertIntentPolicyPanel', () => {
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(AlertIntentPoliciesAPI.get).mockResolvedValue(baseDocument());
vi.mocked(AlertIntentPoliciesAPI.update).mockImplementation(async (document) => ({
...document,
revision: document.revision + 1,
}));
vi.mocked(AlertIntentPoliciesAPI.preview).mockResolvedValue({
resourceId: resource.id,
resourceType: resource.type,
signal: 'state.offline',
status: 'pending_grace',
reason: 'grace_period',
effective: {
graceSeconds: 30,
honorOperatorState: true,
sources: { graceSeconds: 'defaults.state.offline' },
explicit: true,
},
remainingSeconds: 30,
contexts: [],
warnings: [],
});
});
afterEach(() => cleanup());
it('selects the first canonical resource when inventory arrives after policy load', async () => {
const [resources, setResources] = createSignal<Resource[]>([]);
render(() => <AlertIntentPolicyPanel resources={resources()} />);
await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1));
setResources([resource]);
fireEvent.click(screen.getByRole('button', { name: 'Configure policies' }));
expect(await screen.findByLabelText('Resource')).toHaveValue(resource.id);
});
it('saves field-level resource overrides without disabling inherited policy', async () => {
render(() => <AlertIntentPolicyPanel resources={[resource]} />);
await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1));
fireEvent.click(screen.getByRole('button', { name: 'Configure policies' }));
const grace = await screen.findByLabelText(/^Grace override \(seconds\)/);
expect(grace).toHaveValue('');
expect(screen.getByLabelText('Operator state override')).toHaveValue('inherit');
expect(screen.getByLabelText('Backup handling override')).toHaveValue('inherit');
fireEvent.input(grace, { target: { value: '75' } });
fireEvent.click(screen.getByRole('button', { name: 'Save override' }));
await waitFor(() => expect(AlertIntentPoliciesAPI.update).toHaveBeenCalledTimes(1));
const saved = vi.mocked(AlertIntentPoliciesAPI.update).mock.calls[0][0];
expect(saved.resources?.[resource.id]?.['state.offline']).toEqual({ graceSeconds: 75 });
});
it('previews the selected canonical resource', async () => {
render(() => <AlertIntentPolicyPanel resources={[resource]} />);
await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1));
fireEvent.click(screen.getByRole('button', { name: 'Configure policies' }));
fireEvent.click(await screen.findByRole('button', { name: 'Preview current policy' }));
await waitFor(() =>
expect(AlertIntentPoliciesAPI.preview).toHaveBeenCalledWith(
expect.objectContaining({
resourceId: resource.id,
resourceType: resource.type,
signal: 'state.offline',
conditionActive: true,
}),
),
);
expect(await screen.findByText(/grace period · 30s grace/)).toBeInTheDocument();
});
});
@@ -5,6 +5,7 @@ import { ThresholdsTab } from '../tabs/ThresholdsTab';
import type { ThresholdsTabProps } from '../thresholds/thresholdsTabModel';
const captureThresholdsTableProps = vi.fn();
const captureIntentPolicyProps = vi.fn();
vi.mock('@/components/Alerts/ThresholdsTable', () => ({
ThresholdsTable: (props: unknown) => {
@@ -13,6 +14,13 @@ vi.mock('@/components/Alerts/ThresholdsTable', () => ({
},
}));
vi.mock('../AlertIntentPolicyPanel', () => ({
AlertIntentPolicyPanel: (props: unknown) => {
captureIntentPolicyProps(props);
return null;
},
}));
const buildProps = (): ThresholdsTabProps =>
({
overrides: () => [],
@@ -1,134 +1,138 @@
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
import { AlertIntentPolicyPanel } from '../AlertIntentPolicyPanel';
import type { ThresholdsTabProps } from '../thresholds/thresholdsTabModel';
export function ThresholdsTab(props: ThresholdsTabProps) {
return (
<ThresholdsTable
overrides={props.overrides}
setOverrides={props.setOverrides}
rawOverridesConfig={props.rawOverridesConfig}
setRawOverridesConfig={props.setRawOverridesConfig}
allGuests={props.allGuests}
nodes={props.nodes}
agents={props.agents}
storage={props.storage}
containerRuntimes={props.containerRuntimes}
dockerHosts={props.dockerHosts}
allResources={props.allResources}
pbsInstances={props.pbsInstances}
pmgInstances={props.pmgInstances}
pmgThresholds={props.pmgThresholds}
setPMGThresholds={props.setPMGThresholds}
guestDefaults={props.guestDefaults()}
setGuestDefaults={props.setGuestDefaults}
guestDisableConnectivity={props.guestDisableConnectivity}
setGuestDisableConnectivity={props.setGuestDisableConnectivity}
guestPoweredOffSeverity={props.guestPoweredOffSeverity}
setGuestPoweredOffSeverity={props.setGuestPoweredOffSeverity}
nodeDefaults={props.nodeDefaults()}
pbsDefaults={props.pbsDefaults()}
kubernetesDefaults={props.kubernetesDefaults()}
trueNASDefaults={props.trueNASDefaults()}
trueNASDiskDefaults={props.trueNASDiskDefaults()}
vmwareDefaults={props.vmwareDefaults()}
agentDefaults={props.agentDefaults()}
diskTempByType={props.diskTempByType()}
setDiskTempByType={props.setDiskTempByType}
setNodeDefaults={props.setNodeDefaults}
setPBSDefaults={props.setPBSDefaults}
setKubernetesDefaults={props.setKubernetesDefaults}
setTrueNASDefaults={props.setTrueNASDefaults}
setTrueNASDiskDefaults={props.setTrueNASDiskDefaults}
setVMwareDefaults={props.setVMwareDefaults}
setAgentDefaults={props.setAgentDefaults}
dockerDefaults={props.dockerDefaults()}
dockerDisableConnectivity={props.dockerDisableConnectivity}
setDockerDisableConnectivity={props.setDockerDisableConnectivity}
dockerPoweredOffSeverity={props.dockerPoweredOffSeverity}
setDockerPoweredOffSeverity={props.setDockerPoweredOffSeverity}
setDockerDefaults={props.setDockerDefaults}
dockerIgnoredPrefixes={props.dockerIgnoredPrefixes}
setDockerIgnoredPrefixes={props.setDockerIgnoredPrefixes}
ignoredGuestPrefixes={props.ignoredGuestPrefixes}
setIgnoredGuestPrefixes={props.setIgnoredGuestPrefixes}
guestTagWhitelist={props.guestTagWhitelist}
setGuestTagWhitelist={props.setGuestTagWhitelist}
guestTagBlacklist={props.guestTagBlacklist}
setGuestTagBlacklist={props.setGuestTagBlacklist}
storageDefault={props.storageDefault}
setStorageDefault={props.setStorageDefault}
resetGuestDefaults={props.resetGuestDefaults}
resetNodeDefaults={props.resetNodeDefaults}
resetPBSDefaults={props.resetPBSDefaults}
resetKubernetesDefaults={props.resetKubernetesDefaults}
resetTrueNASDefaults={props.resetTrueNASDefaults}
resetTrueNASDiskDefaults={props.resetTrueNASDiskDefaults}
resetVMwareDefaults={props.resetVMwareDefaults}
resetAgentDefaults={props.resetAgentDefaults}
resetDockerDefaults={props.resetDockerDefaults}
resetDockerIgnoredPrefixes={props.resetDockerIgnoredPrefixes}
resetStorageDefault={props.resetStorageDefault}
factoryGuestDefaults={props.factoryGuestDefaults}
factoryNodeDefaults={props.factoryNodeDefaults}
factoryPBSDefaults={props.factoryPBSDefaults}
factoryKubernetesDefaults={props.factoryKubernetesDefaults}
factoryTrueNASDefaults={props.factoryTrueNASDefaults}
factoryTrueNASDiskDefaults={props.factoryTrueNASDiskDefaults}
factoryVMwareDefaults={props.factoryVMwareDefaults}
factoryAgentDefaults={props.factoryAgentDefaults}
factoryDockerDefaults={props.factoryDockerDefaults}
factoryStorageDefault={props.factoryStorageDefault}
timeThresholds={props.timeThresholds}
metricTimeThresholds={props.metricTimeThresholds}
setMetricTimeThresholds={props.setMetricTimeThresholds}
snapshotDefaults={props.snapshotDefaults}
setSnapshotDefaults={props.setSnapshotDefaults}
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
resetSnapshotDefaults={props.resetSnapshotDefaults}
backupDefaults={props.backupDefaults}
setBackupDefaults={props.setBackupDefaults}
backupFactoryDefaults={props.backupFactoryDefaults}
resetBackupDefaults={props.resetBackupDefaults}
setHasUnsavedChanges={props.setHasUnsavedChanges}
activeAlerts={props.activeAlerts}
removeAlerts={props.removeAlerts}
disableAllNodes={props.disableAllNodes}
setDisableAllNodes={props.setDisableAllNodes}
disableAllGuests={props.disableAllGuests}
setDisableAllGuests={props.setDisableAllGuests}
disableAllAgents={props.disableAllAgents}
setDisableAllAgents={props.setDisableAllAgents}
disableAllStorage={props.disableAllStorage}
setDisableAllStorage={props.setDisableAllStorage}
disableAllPBS={props.disableAllPBS}
setDisableAllPBS={props.setDisableAllPBS}
disableAllPMG={props.disableAllPMG}
setDisableAllPMG={props.setDisableAllPMG}
disableAllDockerHosts={props.disableAllDockerHosts}
setDisableAllDockerHosts={props.setDisableAllDockerHosts}
disableAllDockerServices={props.disableAllDockerServices}
setDisableAllDockerServices={props.setDisableAllDockerServices}
disableAllDockerContainers={props.disableAllDockerContainers}
setDisableAllDockerContainers={props.setDisableAllDockerContainers}
disableAllKubernetes={props.disableAllKubernetes}
setDisableAllKubernetes={props.setDisableAllKubernetes}
disableAllTrueNAS={props.disableAllTrueNAS}
setDisableAllTrueNAS={props.setDisableAllTrueNAS}
disableAllVMware={props.disableAllVMware}
setDisableAllVMware={props.setDisableAllVMware}
disableAllNodesOffline={props.disableAllNodesOffline}
setDisableAllNodesOffline={props.setDisableAllNodesOffline}
disableAllGuestsOffline={props.disableAllGuestsOffline}
setDisableAllGuestsOffline={props.setDisableAllGuestsOffline}
disableAllAgentsOffline={props.disableAllAgentsOffline}
setDisableAllAgentsOffline={props.setDisableAllAgentsOffline}
disableAllPBSOffline={props.disableAllPBSOffline}
setDisableAllPBSOffline={props.setDisableAllPBSOffline}
disableAllPMGOffline={props.disableAllPMGOffline}
setDisableAllPMGOffline={props.setDisableAllPMGOffline}
disableAllDockerHostsOffline={props.disableAllDockerHostsOffline}
setDisableAllDockerHostsOffline={props.setDisableAllDockerHostsOffline}
/>
<div class="space-y-4">
<AlertIntentPolicyPanel resources={props.allResources} />
<ThresholdsTable
overrides={props.overrides}
setOverrides={props.setOverrides}
rawOverridesConfig={props.rawOverridesConfig}
setRawOverridesConfig={props.setRawOverridesConfig}
allGuests={props.allGuests}
nodes={props.nodes}
agents={props.agents}
storage={props.storage}
containerRuntimes={props.containerRuntimes}
dockerHosts={props.dockerHosts}
allResources={props.allResources}
pbsInstances={props.pbsInstances}
pmgInstances={props.pmgInstances}
pmgThresholds={props.pmgThresholds}
setPMGThresholds={props.setPMGThresholds}
guestDefaults={props.guestDefaults()}
setGuestDefaults={props.setGuestDefaults}
guestDisableConnectivity={props.guestDisableConnectivity}
setGuestDisableConnectivity={props.setGuestDisableConnectivity}
guestPoweredOffSeverity={props.guestPoweredOffSeverity}
setGuestPoweredOffSeverity={props.setGuestPoweredOffSeverity}
nodeDefaults={props.nodeDefaults()}
pbsDefaults={props.pbsDefaults()}
kubernetesDefaults={props.kubernetesDefaults()}
trueNASDefaults={props.trueNASDefaults()}
trueNASDiskDefaults={props.trueNASDiskDefaults()}
vmwareDefaults={props.vmwareDefaults()}
agentDefaults={props.agentDefaults()}
diskTempByType={props.diskTempByType()}
setDiskTempByType={props.setDiskTempByType}
setNodeDefaults={props.setNodeDefaults}
setPBSDefaults={props.setPBSDefaults}
setKubernetesDefaults={props.setKubernetesDefaults}
setTrueNASDefaults={props.setTrueNASDefaults}
setTrueNASDiskDefaults={props.setTrueNASDiskDefaults}
setVMwareDefaults={props.setVMwareDefaults}
setAgentDefaults={props.setAgentDefaults}
dockerDefaults={props.dockerDefaults()}
dockerDisableConnectivity={props.dockerDisableConnectivity}
setDockerDisableConnectivity={props.setDockerDisableConnectivity}
dockerPoweredOffSeverity={props.dockerPoweredOffSeverity}
setDockerPoweredOffSeverity={props.setDockerPoweredOffSeverity}
setDockerDefaults={props.setDockerDefaults}
dockerIgnoredPrefixes={props.dockerIgnoredPrefixes}
setDockerIgnoredPrefixes={props.setDockerIgnoredPrefixes}
ignoredGuestPrefixes={props.ignoredGuestPrefixes}
setIgnoredGuestPrefixes={props.setIgnoredGuestPrefixes}
guestTagWhitelist={props.guestTagWhitelist}
setGuestTagWhitelist={props.setGuestTagWhitelist}
guestTagBlacklist={props.guestTagBlacklist}
setGuestTagBlacklist={props.setGuestTagBlacklist}
storageDefault={props.storageDefault}
setStorageDefault={props.setStorageDefault}
resetGuestDefaults={props.resetGuestDefaults}
resetNodeDefaults={props.resetNodeDefaults}
resetPBSDefaults={props.resetPBSDefaults}
resetKubernetesDefaults={props.resetKubernetesDefaults}
resetTrueNASDefaults={props.resetTrueNASDefaults}
resetTrueNASDiskDefaults={props.resetTrueNASDiskDefaults}
resetVMwareDefaults={props.resetVMwareDefaults}
resetAgentDefaults={props.resetAgentDefaults}
resetDockerDefaults={props.resetDockerDefaults}
resetDockerIgnoredPrefixes={props.resetDockerIgnoredPrefixes}
resetStorageDefault={props.resetStorageDefault}
factoryGuestDefaults={props.factoryGuestDefaults}
factoryNodeDefaults={props.factoryNodeDefaults}
factoryPBSDefaults={props.factoryPBSDefaults}
factoryKubernetesDefaults={props.factoryKubernetesDefaults}
factoryTrueNASDefaults={props.factoryTrueNASDefaults}
factoryTrueNASDiskDefaults={props.factoryTrueNASDiskDefaults}
factoryVMwareDefaults={props.factoryVMwareDefaults}
factoryAgentDefaults={props.factoryAgentDefaults}
factoryDockerDefaults={props.factoryDockerDefaults}
factoryStorageDefault={props.factoryStorageDefault}
timeThresholds={props.timeThresholds}
metricTimeThresholds={props.metricTimeThresholds}
setMetricTimeThresholds={props.setMetricTimeThresholds}
snapshotDefaults={props.snapshotDefaults}
setSnapshotDefaults={props.setSnapshotDefaults}
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
resetSnapshotDefaults={props.resetSnapshotDefaults}
backupDefaults={props.backupDefaults}
setBackupDefaults={props.setBackupDefaults}
backupFactoryDefaults={props.backupFactoryDefaults}
resetBackupDefaults={props.resetBackupDefaults}
setHasUnsavedChanges={props.setHasUnsavedChanges}
activeAlerts={props.activeAlerts}
removeAlerts={props.removeAlerts}
disableAllNodes={props.disableAllNodes}
setDisableAllNodes={props.setDisableAllNodes}
disableAllGuests={props.disableAllGuests}
setDisableAllGuests={props.setDisableAllGuests}
disableAllAgents={props.disableAllAgents}
setDisableAllAgents={props.setDisableAllAgents}
disableAllStorage={props.disableAllStorage}
setDisableAllStorage={props.setDisableAllStorage}
disableAllPBS={props.disableAllPBS}
setDisableAllPBS={props.setDisableAllPBS}
disableAllPMG={props.disableAllPMG}
setDisableAllPMG={props.setDisableAllPMG}
disableAllDockerHosts={props.disableAllDockerHosts}
setDisableAllDockerHosts={props.setDisableAllDockerHosts}
disableAllDockerServices={props.disableAllDockerServices}
setDisableAllDockerServices={props.setDisableAllDockerServices}
disableAllDockerContainers={props.disableAllDockerContainers}
setDisableAllDockerContainers={props.setDisableAllDockerContainers}
disableAllKubernetes={props.disableAllKubernetes}
setDisableAllKubernetes={props.setDisableAllKubernetes}
disableAllTrueNAS={props.disableAllTrueNAS}
setDisableAllTrueNAS={props.setDisableAllTrueNAS}
disableAllVMware={props.disableAllVMware}
setDisableAllVMware={props.setDisableAllVMware}
disableAllNodesOffline={props.disableAllNodesOffline}
setDisableAllNodesOffline={props.setDisableAllNodesOffline}
disableAllGuestsOffline={props.disableAllGuestsOffline}
setDisableAllGuestsOffline={props.setDisableAllGuestsOffline}
disableAllAgentsOffline={props.disableAllAgentsOffline}
setDisableAllAgentsOffline={props.setDisableAllAgentsOffline}
disableAllPBSOffline={props.disableAllPBSOffline}
setDisableAllPBSOffline={props.setDisableAllPBSOffline}
disableAllPMGOffline={props.disableAllPMGOffline}
setDisableAllPMGOffline={props.setDisableAllPMGOffline}
disableAllDockerHostsOffline={props.disableAllDockerHostsOffline}
setDisableAllDockerHostsOffline={props.setDisableAllDockerHostsOffline}
/>
</div>
);
}
@@ -44,6 +44,25 @@ function createResource(overrides: Partial<Resource> = {}): Resource {
}
describe('Resource Type Guards', () => {
it('retains three-state UDP availability evidence on canonical resources', () => {
const resource = createResource({
type: 'network-endpoint',
availability: {
protocol: 'udp',
probeOutcome: 'indeterminate',
udpMode: 'open_or_filtered',
available: false,
},
});
expect(resource.availability).toMatchObject({
protocol: 'udp',
probeOutcome: 'indeterminate',
udpMode: 'open_or_filtered',
available: false,
});
});
it('keeps host maintenance posture scoped to agent metadata', () => {
const resource = createResource({
type: 'agent',
+2
View File
@@ -1390,6 +1390,8 @@ export interface ResourceAvailabilityMeta {
targetKind?: 'machine' | 'service' | 'device' | string;
address?: string;
protocol?: string;
probeOutcome?: string;
udpMode?: string;
port?: number;
path?: string;
enabled?: boolean;
@@ -47,6 +47,32 @@ describe('availabilityProbePresentation', () => {
expect(presentation?.toneClassName).toContain('emerald');
});
it('presents open-or-filtered UDP as indeterminate rather than reachable', () => {
vi.spyOn(Date, 'now').mockReturnValue(new Date('2026-05-06T13:00:20Z').getTime());
const presentation = getAvailabilityProbePresentation(
makeAvailabilityResource({
status: 'warning',
availability: {
protocol: 'udp',
port: 514,
available: false,
probeOutcome: 'indeterminate',
udpMode: 'open_or_filtered',
lastChecked: '2026-05-06T13:00:18Z',
},
}),
);
expect(presentation).toMatchObject({
methodLabel: 'UDP 514',
targetLabel: '514',
resultLabel: 'open or filtered',
netIoLabel: '514: open or filtered',
});
expect(presentation?.toneClassName).toContain('amber');
});
it('also reads availability evidence from platform data for live state rows', () => {
vi.spyOn(Date, 'now').mockReturnValue(new Date('2026-05-06T13:00:20Z').getTime());
@@ -31,8 +31,9 @@ export const getAvailabilityProbeMethodLabel = (
): string => {
const protocol = normalizeAvailabilityProtocol(availability?.protocol);
if (protocol === 'icmp') return 'ICMP';
if (protocol === 'tcp') {
return availability?.port ? `TCP ${availability.port}` : 'TCP';
if (protocol === 'tcp' || protocol === 'udp') {
const label = protocol.toUpperCase();
return availability?.port ? `${label} ${availability.port}` : label;
}
if (protocol === 'http' || protocol === 'https') {
const path = (availability?.path ?? '').trim();
@@ -45,7 +46,7 @@ export const getAvailabilityProbeTargetLabel = (
availability?: ResourceAvailabilityMeta | null,
): string | null => {
const protocol = normalizeAvailabilityProtocol(availability?.protocol);
if (protocol === 'tcp') {
if (protocol === 'tcp' || protocol === 'udp') {
const port = availability?.port;
return typeof port === 'number' && Number.isFinite(port) && port > 0 ? String(port) : null;
}
@@ -98,6 +99,7 @@ const getAvailabilityProbeResultLabel = (
): string => {
const latency = availability.latencyMillis;
const normalizedStatus = (resource.status ?? '').trim().toLowerCase();
if (availability.probeOutcome === 'indeterminate') return 'open or filtered';
if (availability.available === false || ['offline', 'degraded'].includes(normalizedStatus)) {
return getAvailabilityProbeFailureLabel(availability);
}
@@ -116,6 +118,9 @@ const getAvailabilityProbeToneClassName = (
freshnessLabel: AvailabilityProbePresentation['freshnessLabel'],
): string => {
const normalizedStatus = (resource.status ?? '').trim().toLowerCase();
if (availability.probeOutcome === 'indeterminate') {
return AVAILABILITY_PROBE_WARNING_CLASS;
}
if (availability.available === false || normalizedStatus === 'offline') {
return AVAILABILITY_PROBE_ERROR_CLASS;
}
+7 -1
View File
@@ -117,6 +117,9 @@ func (m *Manager) SaveActiveAlerts() error {
if err := os.Chmod(finalFile, alertsFilePerm); err != nil {
return fmt.Errorf("failed to set active alerts file permissions: %w", err)
}
if err := m.saveIntentPendingSnapshot(); err != nil {
return err
}
log.Debug().Int("count", len(alerts)).Msg("saved active alerts to disk")
return nil
@@ -132,7 +135,7 @@ func (m *Manager) LoadActiveAlerts() error {
if err != nil {
if errors.Is(err, os.ErrNotExist) {
log.Info().Msg("No active alerts file found, starting fresh")
return nil
return m.loadIntentPendingNoLock()
}
return fmt.Errorf("failed to read active alerts: %w", err)
}
@@ -260,6 +263,9 @@ func (m *Manager) LoadActiveAlerts() error {
}(alertCopy)
}
}
if err := m.loadIntentPendingNoLock(); err != nil {
return err
}
log.Info().
Int("restored", restoredCount).
+34
View File
@@ -26,6 +26,7 @@ type canonicalLifecycleAlertParams struct {
AddToHistory bool
RateLimit bool
DispatchAsync bool
IntentBackup BackupIntentContext
}
type canonicalStatefulAlertParams struct {
@@ -362,6 +363,9 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
return alertspecs.EvaluationResult{}, false
}
// Persist the canonical confirmation state before applying intent policy.
// Grace and operator context are an additional gate; they must not prevent
// the underlying evaluator from reaching its configured confirmation count.
if params.Tracking != nil {
if result.State.ConsecutiveMatches > 0 {
params.Tracking[params.TrackingKey] = result.State.ConsecutiveMatches
@@ -370,6 +374,33 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
}
}
intentSignal := ""
if params.Spec.Kind == alertspecs.AlertSpecKindConnectivity || params.Spec.Kind == alertspecs.AlertSpecKindPoweredState {
intentSignal = string(AlertIntentSignalOffline)
}
if intentSignal != "" && existing == nil {
conditionActive := result.State.State == alertspecs.AlertStatePending || result.State.State == alertspecs.AlertStateFiring
decision := m.evaluateIntentNoLock(params.Spec.ResourceID, string(params.Spec.ResourceType), intentSignal, storageKey, params.Evidence.ObservedAt, conditionActive, params.IntentBackup)
if decision.StateChanged {
m.saveActiveAlertsAsync("lifecycle intent state")
}
if decision.Effective.Explicit && conditionActive && !decision.ShouldActivate {
result.State.State = alertspecs.AlertStatePending
result.State.Reason = decision.Reason
if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() {
result.State.FirstMatchedAt = pending.FirstMatchedAt
}
return result, true
}
if decision.Effective.Explicit && result.State.State == alertspecs.AlertStateFiring {
if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() {
result.State.FirstMatchedAt = pending.FirstMatchedAt
}
delete(m.intentPending, storageKey)
m.saveActiveAlertsAsync("lifecycle intent activated")
}
}
switch result.State.State {
case alertspecs.AlertStatePending:
return result, true
@@ -400,6 +431,9 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
alert.Metadata["resourceType"] = string(params.Spec.ResourceType)
}
applyCanonicalIdentity(alert, params.Spec.ID, string(params.Spec.Kind))
if !result.State.FirstMatchedAt.IsZero() {
alert.StartTime = result.State.FirstMatchedAt
}
applyCanonicalOperationalEvidence(alert, params.Spec, params.Evidence, time.Now())
m.preserveAlertState(storageKey, alert)
m.setActiveAlertNoLock(storageKey, alert)
+19 -2
View File
@@ -144,8 +144,21 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
triggered := alertspecsMetricTriggered(spec.MetricThreshold, value)
alertStartTime := observedAt
if !exists && triggered {
timeThreshold := m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
if timeThreshold > 0 {
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
if effectiveIntent.Explicit {
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, true, BackupIntentContext{})
if pending, ok := m.intentPending[trackingKey]; ok && !pending.FirstMatchedAt.IsZero() {
alertStartTime = pending.FirstMatchedAt
}
if decision.StateChanged {
m.saveActiveAlertsAsync("canonical metric intent pending state")
}
if !decision.ShouldActivate {
return
}
delete(m.intentPending, trackingKey)
m.saveActiveAlertsAsync("canonical metric intent activated")
} else if timeThreshold := m.getTimeThreshold(spec.ResourceID, resourceType, metricType); timeThreshold > 0 {
if pendingTime, isPending := m.pendingAlerts[trackingKey]; isPending {
if time.Since(pendingTime) >= time.Duration(timeThreshold)*time.Second {
delete(m.pendingAlerts, trackingKey)
@@ -171,6 +184,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
}
if !triggered {
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, false, BackupIntentContext{})
if decision.StateChanged {
m.saveActiveAlertsAsync("canonical metric intent cleared")
}
delete(m.pendingAlerts, trackingKey)
}
+221
View File
@@ -0,0 +1,221 @@
package config
import (
"fmt"
"strings"
"time"
)
const (
CurrentAlertIntentPolicySchemaVersion = 1
maxAlertIntentGraceSeconds = 30 * 24 * 60 * 60
)
// AlertIntentSignal is a stable policy key shared by configuration, preview,
// and runtime evaluation. Metric signals use the metric.<name> namespace.
type AlertIntentSignal string
const (
AlertIntentSignalDefault AlertIntentSignal = "*"
AlertIntentSignalOffline AlertIntentSignal = "state.offline"
AlertIntentSignalAvailability AlertIntentSignal = "incident.availability"
)
// BackupOfflineIntentPolicy extends an offline candidate while Pulse has fresh
// evidence that a Proxmox backup is responsible for the transient state.
type BackupOfflineIntentPolicy struct {
Enabled bool `json:"enabled"`
PostGraceSeconds int `json:"postGraceSeconds,omitempty"`
MaxDeferralSeconds int `json:"maxDeferralSeconds,omitempty"`
}
// AlertIntentRule is deliberately small and typed. Pointer fields distinguish
// inheritance from an explicit zero/false override.
type AlertIntentRule struct {
GraceSeconds *int `json:"graceSeconds,omitempty"`
HonorOperatorState *bool `json:"honorOperatorState,omitempty"`
BackupOffline *BackupOfflineIntentPolicy `json:"backupOffline,omitempty"`
}
// AlertIntentPolicyDocument contains alert-intent defaults and overrides.
// Resources are keyed by canonical resource ID. ResourceTypes use canonical
// alert resource-type keys. Signal maps accept "*", state.offline,
// incident.availability, and metric.<name> keys.
type AlertIntentPolicyDocument struct {
SchemaVersion int `json:"schemaVersion"`
Revision int64 `json:"revision"`
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
Defaults map[string]AlertIntentRule `json:"defaults,omitempty"`
ResourceTypes map[string]map[string]AlertIntentRule `json:"resourceTypes,omitempty"`
Resources map[string]map[string]AlertIntentRule `json:"resources,omitempty"`
}
func NewAlertIntentPolicyDocument() AlertIntentPolicyDocument {
return AlertIntentPolicyDocument{
SchemaVersion: CurrentAlertIntentPolicySchemaVersion,
Defaults: make(map[string]AlertIntentRule),
ResourceTypes: make(map[string]map[string]AlertIntentRule),
Resources: make(map[string]map[string]AlertIntentRule),
}
}
func MetricAlertIntentSignal(metric string) string {
metric = strings.ToLower(strings.TrimSpace(metric))
if metric == "" {
return ""
}
return "metric." + metric
}
// NormalizeAlertIntentPolicyDocument returns a detached, canonical document.
func NormalizeAlertIntentPolicyDocument(input AlertIntentPolicyDocument) AlertIntentPolicyDocument {
out := NewAlertIntentPolicyDocument()
if input.SchemaVersion > 0 {
out.SchemaVersion = input.SchemaVersion
}
out.Revision = input.Revision
if input.UpdatedAt != nil {
updatedAt := input.UpdatedAt.UTC()
out.UpdatedAt = &updatedAt
}
out.Defaults = normalizeIntentSignalRules(input.Defaults)
out.ResourceTypes = normalizeIntentScopedRules(input.ResourceTypes, true)
out.Resources = normalizeIntentScopedRules(input.Resources, false)
return out
}
func normalizeIntentScopedRules(input map[string]map[string]AlertIntentRule, lowerScope bool) map[string]map[string]AlertIntentRule {
out := make(map[string]map[string]AlertIntentRule, len(input))
for rawScope, rules := range input {
scope := strings.TrimSpace(rawScope)
if lowerScope {
scope = CanonicalAlertResourceType(scope)
}
if scope == "" {
continue
}
normalized := normalizeIntentSignalRules(rules)
if len(normalized) > 0 {
out[scope] = normalized
}
}
return out
}
func normalizeIntentSignalRules(input map[string]AlertIntentRule) map[string]AlertIntentRule {
out := make(map[string]AlertIntentRule, len(input))
for rawSignal, rule := range input {
signal := strings.ToLower(strings.TrimSpace(rawSignal))
if signal == "default" || signal == "_default" {
signal = string(AlertIntentSignalDefault)
}
if signal == "" {
continue
}
out[signal] = cloneAlertIntentRule(rule)
}
return out
}
func cloneAlertIntentRule(rule AlertIntentRule) AlertIntentRule {
out := rule
if rule.GraceSeconds != nil {
value := *rule.GraceSeconds
out.GraceSeconds = &value
}
if rule.HonorOperatorState != nil {
value := *rule.HonorOperatorState
out.HonorOperatorState = &value
}
if rule.BackupOffline != nil {
value := *rule.BackupOffline
out.BackupOffline = &value
}
return out
}
func ValidateAlertIntentPolicyDocument(input AlertIntentPolicyDocument) error {
doc := NormalizeAlertIntentPolicyDocument(input)
if doc.SchemaVersion != CurrentAlertIntentPolicySchemaVersion {
return fmt.Errorf("unsupported alert intent policy schema version %d", doc.SchemaVersion)
}
if doc.Revision < 0 {
return fmt.Errorf("alert intent policy revision must be non-negative")
}
if err := validateIntentSignalRules("defaults", input.Defaults); err != nil {
return err
}
if err := validateIntentScopedRules("resource type", input.ResourceTypes, true); err != nil {
return err
}
if err := validateIntentScopedRules("resource", input.Resources, false); err != nil {
return err
}
return nil
}
func validateIntentScopedRules(label string, scopes map[string]map[string]AlertIntentRule, resourceType bool) error {
seen := make(map[string]string, len(scopes))
for rawScope, rules := range scopes {
scope := strings.TrimSpace(rawScope)
if resourceType {
scope = CanonicalAlertResourceType(scope)
}
if scope == "" {
return fmt.Errorf("alert intent %s is required", label)
}
if previous, exists := seen[scope]; exists {
return fmt.Errorf("alert intent %s keys %q and %q normalize to the same value", label, previous, rawScope)
}
seen[scope] = rawScope
if err := validateIntentSignalRules(label+" "+scope, rules); err != nil {
return err
}
}
return nil
}
func validateIntentSignalRules(scope string, rules map[string]AlertIntentRule) error {
seen := make(map[string]string, len(rules))
for rawSignal, rule := range rules {
signal := strings.ToLower(strings.TrimSpace(rawSignal))
if signal == "default" || signal == "_default" {
signal = string(AlertIntentSignalDefault)
}
if signal == "" {
return fmt.Errorf("%s has an empty alert intent signal", scope)
}
if previous, exists := seen[signal]; exists {
return fmt.Errorf("%s signal keys %q and %q normalize to the same value", scope, previous, rawSignal)
}
seen[signal] = rawSignal
if !validAlertIntentSignal(signal) {
return fmt.Errorf("%s has unsupported alert intent signal %q", scope, signal)
}
if rule.GraceSeconds != nil && (*rule.GraceSeconds < 0 || *rule.GraceSeconds > maxAlertIntentGraceSeconds) {
return fmt.Errorf("%s signal %q graceSeconds must be between 0 and %d", scope, signal, maxAlertIntentGraceSeconds)
}
if backup := rule.BackupOffline; backup != nil {
if signal != string(AlertIntentSignalOffline) && signal != string(AlertIntentSignalDefault) {
return fmt.Errorf("%s signal %q may not configure backupOffline", scope, signal)
}
if backup.PostGraceSeconds < 0 || backup.PostGraceSeconds > maxAlertIntentGraceSeconds {
return fmt.Errorf("%s signal %q postGraceSeconds must be between 0 and %d", scope, signal, maxAlertIntentGraceSeconds)
}
if backup.MaxDeferralSeconds < 0 || backup.MaxDeferralSeconds > maxAlertIntentGraceSeconds {
return fmt.Errorf("%s signal %q maxDeferralSeconds must be between 0 and %d", scope, signal, maxAlertIntentGraceSeconds)
}
if backup.Enabled && backup.MaxDeferralSeconds <= 0 {
return fmt.Errorf("%s signal %q backupOffline maxDeferralSeconds must be positive when enabled", scope, signal)
}
}
}
return nil
}
func validAlertIntentSignal(signal string) bool {
if signal == string(AlertIntentSignalDefault) || signal == string(AlertIntentSignalOffline) || signal == string(AlertIntentSignalAvailability) {
return true
}
return strings.HasPrefix(signal, "metric.") && strings.TrimPrefix(signal, "metric.") != ""
}
+29 -5
View File
@@ -27,15 +27,39 @@ type SnapshotAlertConfig = alertconfig.SnapshotAlertConfig
type BackupAlertConfig = alertconfig.BackupAlertConfig
type GuestLookup = alertconfig.GuestLookup
type AlertConfig = alertconfig.AlertConfig
type AlertIntentSignal = alertconfig.AlertIntentSignal
type BackupOfflineIntentPolicy = alertconfig.BackupOfflineIntentPolicy
type AlertIntentRule = alertconfig.AlertIntentRule
type AlertIntentPolicyDocument = alertconfig.AlertIntentPolicyDocument
const (
AlertLevelWarning = alertconfig.AlertLevelWarning
AlertLevelCritical = alertconfig.AlertLevelCritical
ActivationPending = alertconfig.ActivationPending
ActivationActive = alertconfig.ActivationActive
ActivationSnoozed = alertconfig.ActivationSnoozed
AlertLevelWarning = alertconfig.AlertLevelWarning
AlertLevelCritical = alertconfig.AlertLevelCritical
ActivationPending = alertconfig.ActivationPending
ActivationActive = alertconfig.ActivationActive
ActivationSnoozed = alertconfig.ActivationSnoozed
CurrentAlertIntentPolicySchemaVersion = alertconfig.CurrentAlertIntentPolicySchemaVersion
AlertIntentSignalDefault = alertconfig.AlertIntentSignalDefault
AlertIntentSignalOffline = alertconfig.AlertIntentSignalOffline
AlertIntentSignalAvailability = alertconfig.AlertIntentSignalAvailability
)
func NewAlertIntentPolicyDocument() AlertIntentPolicyDocument {
return alertconfig.NewAlertIntentPolicyDocument()
}
func MetricAlertIntentSignal(metric string) string {
return alertconfig.MetricAlertIntentSignal(metric)
}
func NormalizeAlertIntentPolicyDocument(document AlertIntentPolicyDocument) AlertIntentPolicyDocument {
return alertconfig.NormalizeAlertIntentPolicyDocument(document)
}
func ValidateAlertIntentPolicyDocument(document AlertIntentPolicyDocument) error {
return alertconfig.ValidateAlertIntentPolicyDocument(document)
}
var ErrAlertNotFound = errors.New("alert not found")
func NormalizeAlertConfigAliases(config *AlertConfig) {
+24 -1
View File
@@ -184,7 +184,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
m.mu.RLock()
thresholds := m.getGuestThresholds(guest, guestID)
m.mu.RUnlock()
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
backupContext := BackupIntentContext{
Active: strings.EqualFold(strings.TrimSpace(snapshot.Lock), "backup"),
ObservedAt: time.Now(),
Evidence: "guest_lock",
}
if !backupContext.Active {
m.mu.RLock()
resolver := m.backupIntentResolver
m.mu.RUnlock()
if resolver != nil {
if resolved, ok := resolver(guestID, instanceName, node, snapshot.VMID, backupContext.ObservedAt); ok {
backupContext = resolved
}
}
}
m.checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, backupContext)
}
} else {
// For paused/suspended, clear powered-off alert
@@ -359,6 +374,10 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT
}
func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool) {
m.checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, BackupIntentContext{})
}
func (m *Manager) checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool, backupContext BackupIntentContext) {
alertID := fmt.Sprintf("guest-powered-off-%s", guestID)
severity := NormalizePoweredOffSeverity(thresholds.PoweredOffSeverity)
resourceType := unifiedresources.ResourceTypeVM
@@ -400,6 +419,7 @@ func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instan
AddToRecent: true,
AddToHistory: true,
DispatchAsync: false,
IntentBackup: backupContext,
})
}
@@ -424,6 +444,9 @@ func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
Msg("Guest is running, resetting powered-off confirmation count")
delete(m.offlineConfirmations, guestID)
}
if decision := m.evaluateIntentNoLock(guestID, "", string(AlertIntentSignalOffline), alertID, time.Now(), false, BackupIntentContext{}); decision.StateChanged {
m.saveActiveAlertsAsync("guest offline intent cleared")
}
// Check if powered-off alert exists. A guest that moved nodes may still
// hold the alert under its old node-scoped identity; re-home it so it
+3
View File
@@ -23,6 +23,7 @@ type guestSnapshot struct {
Node string
Instance string
Status string
Lock string
CPUPercent float64
MemUsage float64
@@ -98,6 +99,7 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot {
Node: vm.Node,
Instance: vm.Instance,
Status: vm.Status,
Lock: vm.Lock,
CPUPercent: vm.CPU * 100,
MemUsage: vm.Memory.Usage,
DiskUsage: vm.Disk.Usage,
@@ -120,6 +122,7 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
Node: container.Node,
Instance: container.Instance,
Status: container.Status,
Lock: container.Lock,
CPUPercent: container.CPU * 100,
MemUsage: container.Memory.Usage,
DiskUsage: container.Disk.Usage,
+92
View File
@@ -0,0 +1,92 @@
package alerts
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"time"
"github.com/rs/zerolog/log"
)
const intentPendingFileName = "intent-pending.json"
func (m *Manager) saveIntentPendingSnapshot() error {
m.mu.RLock()
states := make([]IntentPendingState, 0, len(m.intentPending))
for _, state := range m.intentPending {
states = append(states, state)
}
m.mu.RUnlock()
sort.Slice(states, func(i, j int) bool { return states[i].TrackingKey < states[j].TrackingKey })
data, err := json.Marshal(states)
if err != nil {
return fmt.Errorf("marshal alert intent pending state: %w", err)
}
alertsDir := m.getAlertsDir()
if err := os.MkdirAll(alertsDir, alertsDirPerm); err != nil {
return fmt.Errorf("create alert intent state directory: %w", err)
}
tmp, err := os.CreateTemp(alertsDir, "intent-pending-*.json.tmp")
if err != nil {
return fmt.Errorf("create alert intent state temp file: %w", err)
}
tmpName := tmp.Name()
defer func() {
if err := os.Remove(tmpName); err != nil && !os.IsNotExist(err) {
log.Warn().Err(err).Str("file", tmpName).Msg("Failed to remove alert intent state temp file")
}
}()
if _, err := tmp.Write(data); err != nil {
tmp.Close()
return fmt.Errorf("write alert intent state temp file: %w", err)
}
if err := tmp.Chmod(alertsFilePerm); err != nil {
tmp.Close()
return fmt.Errorf("chmod alert intent state temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close alert intent state temp file: %w", err)
}
finalPath := filepath.Join(alertsDir, intentPendingFileName)
if err := os.Rename(tmpName, finalPath); err != nil {
return fmt.Errorf("persist alert intent pending state: %w", err)
}
if err := os.Chmod(finalPath, alertsFilePerm); err != nil {
return fmt.Errorf("chmod alert intent pending state: %w", err)
}
return nil
}
// loadIntentPendingNoLock restores pending policy candidates. Caller holds m.mu.
func (m *Manager) loadIntentPendingNoLock() error {
path := filepath.Join(m.getAlertsDir(), intentPendingFileName)
data, err := readLimitedRegularFile(path, maxActiveAlertsFileSizeBytes)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("read alert intent pending state: %w", err)
}
var states []IntentPendingState
if err := json.Unmarshal(data, &states); err != nil {
return fmt.Errorf("decode alert intent pending state: %w", err)
}
now := time.Now()
if m.intentPending == nil {
m.intentPending = make(map[string]IntentPendingState)
}
for _, state := range states {
if state.TrackingKey == "" || state.ResourceID == "" || state.Signal == "" || state.FirstMatchedAt.IsZero() {
continue
}
if now.Sub(state.LastObservedAt) > 24*time.Hour || state.LastObservedAt.After(now.Add(time.Minute)) || state.FirstMatchedAt.After(now.Add(time.Minute)) {
continue
}
m.intentPending[state.TrackingKey] = state
}
return nil
}
+453
View File
@@ -0,0 +1,453 @@
package alerts
import (
"errors"
"fmt"
"strings"
"time"
)
var ErrAlertIntentPolicyRevisionConflict = errors.New("alert_intent_policy_revision_conflict")
type OperatorIntentContext struct {
IntentionallyOffline bool `json:"intentionallyOffline"`
MaintenanceStartAt *time.Time `json:"maintenanceStartAt,omitempty"`
MaintenanceEndAt *time.Time `json:"maintenanceEndAt,omitempty"`
MaintenanceReason string `json:"maintenanceReason,omitempty"`
}
func (c OperatorIntentContext) MaintenanceActiveAt(now time.Time) bool {
return c.MaintenanceStartAt != nil && c.MaintenanceEndAt != nil &&
!now.Before(*c.MaintenanceStartAt) && now.Before(*c.MaintenanceEndAt)
}
type BackupIntentContext struct {
Active bool `json:"active"`
ObservedAt time.Time `json:"observedAt,omitempty"`
Evidence string `json:"evidence,omitempty"`
}
type OperatorIntentContextResolver func(resourceID string, now time.Time) (OperatorIntentContext, bool)
type BackupIntentContextResolver func(resourceID, instance, node string, vmid int, now time.Time) (BackupIntentContext, bool)
type ResourceIntentIdentityResolver func(resourceID string) (canonicalID string, found bool)
type IntentPendingState struct {
TrackingKey string `json:"trackingKey"`
ResourceID string `json:"resourceId"`
ResourceType string `json:"resourceType"`
Signal string `json:"signal"`
FirstMatchedAt time.Time `json:"firstMatchedAt"`
LastObservedAt time.Time `json:"lastObservedAt"`
BackupActive bool `json:"backupActive,omitempty"`
BackupEndedAt *time.Time `json:"backupEndedAt,omitempty"`
BackupEvidence string `json:"backupEvidence,omitempty"`
}
type EffectiveAlertIntentPolicy struct {
GraceSeconds int `json:"graceSeconds"`
HonorOperatorState bool `json:"honorOperatorState"`
BackupOffline *BackupOfflineIntentPolicy `json:"backupOffline,omitempty"`
Sources map[string]string `json:"sources"`
Explicit bool `json:"explicit"`
}
type AlertIntentPolicyPreviewRequest struct {
ResourceID string `json:"resourceId"`
ResourceType string `json:"resourceType"`
Signal string `json:"signal"`
ConditionActive bool `json:"conditionActive"`
FirstMatchedAt *time.Time `json:"firstMatchedAt,omitempty"`
BackupActive *bool `json:"backupActive,omitempty"`
BackupObservedAt *time.Time `json:"backupObservedAt,omitempty"`
}
type AlertIntentPolicyPreviewContext struct {
Kind string `json:"kind"`
Active bool `json:"active"`
Evidence string `json:"evidence,omitempty"`
ObservedAt *time.Time `json:"observedAt,omitempty"`
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
}
type AlertIntentPolicyPreview struct {
ResourceID string `json:"resourceId"`
ResourceType string `json:"resourceType"`
Signal string `json:"signal"`
Status string `json:"status"`
Reason string `json:"reason"`
Effective EffectiveAlertIntentPolicy `json:"effective"`
FirstMatchedAt *time.Time `json:"firstMatchedAt,omitempty"`
EligibleAt *time.Time `json:"eligibleAt,omitempty"`
HardCapAt *time.Time `json:"hardCapAt,omitempty"`
RemainingSecs int64 `json:"remainingSeconds,omitempty"`
Contexts []AlertIntentPolicyPreviewContext `json:"contexts"`
Warnings []string `json:"warnings"`
}
type intentDecision struct {
ShouldActivate bool
Pending bool
Suppressed bool
Reason string
EligibleAt time.Time
HardCapAt time.Time
StateChanged bool
Effective EffectiveAlertIntentPolicy
}
func (m *Manager) SetOperatorIntentContextResolver(resolver OperatorIntentContextResolver) {
if m == nil {
return
}
m.mu.Lock()
m.operatorIntentResolver = resolver
m.mu.Unlock()
}
func (m *Manager) SetBackupIntentContextResolver(resolver BackupIntentContextResolver) {
if m == nil {
return
}
m.mu.Lock()
m.backupIntentResolver = resolver
m.mu.Unlock()
}
func (m *Manager) SetResourceIntentIdentityResolver(resolver ResourceIntentIdentityResolver) {
if m == nil {
return
}
m.mu.Lock()
m.resourceIntentResolver = resolver
m.mu.Unlock()
}
// LoadIntentPolicies installs a persisted policy document without changing its
// revision. API writes should use UpdateIntentPolicies.
func (m *Manager) LoadIntentPolicies(document AlertIntentPolicyDocument) error {
if err := ValidateAlertIntentPolicyDocument(document); err != nil {
return err
}
if m == nil {
return errors.New("alert manager is nil")
}
m.mu.Lock()
m.intentPolicies = NormalizeAlertIntentPolicyDocument(document)
m.mu.Unlock()
return nil
}
func (m *Manager) GetIntentPolicies() AlertIntentPolicyDocument {
if m == nil {
return NewAlertIntentPolicyDocument()
}
m.mu.RLock()
document := NormalizeAlertIntentPolicyDocument(m.intentPolicies)
m.mu.RUnlock()
return document
}
func (m *Manager) UpdateIntentPolicies(document AlertIntentPolicyDocument) (AlertIntentPolicyDocument, error) {
if err := ValidateAlertIntentPolicyDocument(document); err != nil {
return AlertIntentPolicyDocument{}, err
}
if m == nil {
return AlertIntentPolicyDocument{}, errors.New("alert manager is nil")
}
now := m.policyNow().UTC()
m.mu.Lock()
defer m.mu.Unlock()
if document.Revision != m.intentPolicies.Revision {
return AlertIntentPolicyDocument{}, fmt.Errorf("%w: got %d want %d", ErrAlertIntentPolicyRevisionConflict, document.Revision, m.intentPolicies.Revision)
}
normalized := NormalizeAlertIntentPolicyDocument(document)
normalized.Revision = m.intentPolicies.Revision + 1
normalized.UpdatedAt = &now
m.intentPolicies = normalized
return NormalizeAlertIntentPolicyDocument(normalized), nil
}
func (m *Manager) resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal string) EffectiveAlertIntentPolicy {
resourceID = strings.TrimSpace(resourceID)
resourceType = strings.ToLower(strings.TrimSpace(resourceType))
signal = strings.ToLower(strings.TrimSpace(signal))
effective := EffectiveAlertIntentPolicy{Sources: make(map[string]string)}
if strings.HasPrefix(signal, "metric.") {
metric := strings.TrimPrefix(signal, "metric.")
if delay, source, ok := m.getLegacyTimeThresholdWithSource(resourceType, metric); ok {
effective.GraceSeconds = delay
effective.Sources["graceSeconds"] = source
}
}
apply := func(rule AlertIntentRule, source string) {
if rule.GraceSeconds != nil {
effective.GraceSeconds = *rule.GraceSeconds
effective.Sources["graceSeconds"] = source
effective.Explicit = true
}
if rule.HonorOperatorState != nil {
effective.HonorOperatorState = *rule.HonorOperatorState
effective.Sources["honorOperatorState"] = source
effective.Explicit = true
}
if rule.BackupOffline != nil {
backup := *rule.BackupOffline
effective.BackupOffline = &backup
effective.Sources["backupOffline"] = source
effective.Explicit = true
}
}
applyRules := func(rules map[string]AlertIntentRule, source string) {
if rule, ok := rules[string(AlertIntentSignalDefault)]; ok {
apply(rule, source+".*")
}
if rule, ok := rules[signal]; ok {
apply(rule, source+"."+signal)
}
}
applyRules(m.intentPolicies.Defaults, "defaults")
for _, typeKey := range CanonicalResourceTypeKeys(resourceType) {
if rules, ok := m.intentPolicies.ResourceTypes[typeKey]; ok {
applyRules(rules, "resourceTypes."+typeKey)
break
}
}
resourceIDs := make([]string, 0, 2)
if m.resourceIntentResolver != nil {
if canonicalID, ok := m.resourceIntentResolver(resourceID); ok {
canonicalID = strings.TrimSpace(canonicalID)
if canonicalID != "" && canonicalID != resourceID {
resourceIDs = append(resourceIDs, canonicalID)
}
}
}
resourceIDs = append(resourceIDs, resourceID)
for _, candidateID := range resourceIDs {
if rules, ok := m.intentPolicies.Resources[candidateID]; ok {
applyRules(rules, "resources."+candidateID)
}
}
if _, ok := effective.Sources["graceSeconds"]; !ok {
effective.Sources["graceSeconds"] = "factory"
}
if _, ok := effective.Sources["honorOperatorState"]; !ok {
effective.Sources["honorOperatorState"] = "factory"
}
return effective
}
func (m *Manager) ResolveEffectiveIntentPolicy(resourceID, resourceType, signal string) EffectiveAlertIntentPolicy {
if m == nil {
return EffectiveAlertIntentPolicy{Sources: map[string]string{"graceSeconds": "factory", "honorOperatorState": "factory"}}
}
m.mu.RLock()
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal)
m.mu.RUnlock()
return effective
}
func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackingKey string, observedAt time.Time, conditionActive bool, backup BackupIntentContext) intentDecision {
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal)
decision := intentDecision{Effective: effective}
if !conditionActive {
if _, exists := m.intentPending[trackingKey]; exists {
delete(m.intentPending, trackingKey)
decision.StateChanged = true
}
decision.Reason = "condition_clear"
return decision
}
if !effective.Explicit {
if _, exists := m.intentPending[trackingKey]; exists {
delete(m.intentPending, trackingKey)
decision.StateChanged = true
}
decision.ShouldActivate = true
return decision
}
if observedAt.IsZero() {
observedAt = m.policyNow()
}
state, exists := m.intentPending[trackingKey]
if !exists || state.FirstMatchedAt.IsZero() {
state = IntentPendingState{
TrackingKey: trackingKey, ResourceID: resourceID, ResourceType: resourceType,
Signal: signal, FirstMatchedAt: observedAt,
}
decision.StateChanged = true
}
state.LastObservedAt = observedAt
if effective.HonorOperatorState && m.operatorIntentResolver != nil {
if operator, ok := m.operatorIntentResolver(resourceID, observedAt); ok {
if operator.MaintenanceActiveAt(observedAt) {
m.intentPending[trackingKey] = state
decision.Pending = true
decision.Suppressed = true
decision.Reason = "operator_maintenance"
if operator.MaintenanceEndAt != nil {
decision.EligibleAt = *operator.MaintenanceEndAt
}
return decision
}
if signal == string(AlertIntentSignalOffline) && operator.IntentionallyOffline {
m.intentPending[trackingKey] = state
decision.Pending = true
decision.Suppressed = true
decision.Reason = "operator_intentionally_offline"
return decision
}
}
}
eligibleAt := state.FirstMatchedAt.Add(time.Duration(effective.GraceSeconds) * time.Second)
if effective.BackupOffline != nil && effective.BackupOffline.Enabled && signal == string(AlertIntentSignalOffline) {
if backup.Active {
if !state.BackupActive || state.BackupEvidence != backup.Evidence {
decision.StateChanged = true
}
state.BackupActive = true
state.BackupEndedAt = nil
state.BackupEvidence = backup.Evidence
} else if state.BackupActive {
endedAt := observedAt
state.BackupActive = false
state.BackupEndedAt = &endedAt
decision.StateChanged = true
}
capAt := state.FirstMatchedAt.Add(time.Duration(effective.BackupOffline.MaxDeferralSeconds) * time.Second)
decision.HardCapAt = capAt
if state.BackupActive && observedAt.Before(capAt) {
m.intentPending[trackingKey] = state
decision.Pending = true
decision.Suppressed = true
decision.Reason = "backup_active"
decision.EligibleAt = capAt
return decision
}
if state.BackupEndedAt != nil {
postEligible := state.BackupEndedAt.Add(time.Duration(effective.BackupOffline.PostGraceSeconds) * time.Second)
if postEligible.After(eligibleAt) {
eligibleAt = postEligible
}
}
if eligibleAt.After(capAt) {
eligibleAt = capAt
}
if !observedAt.Before(capAt) {
decision.Reason = "backup_grace_cap_exceeded"
}
}
m.intentPending[trackingKey] = state
decision.EligibleAt = eligibleAt
if observedAt.Before(eligibleAt) {
decision.Pending = true
if decision.Reason == "" {
decision.Reason = "grace_period"
}
return decision
}
decision.ShouldActivate = true
if decision.Reason == "" {
decision.Reason = "eligible"
}
return decision
}
func (m *Manager) PreviewIntentPolicy(request AlertIntentPolicyPreviewRequest) (AlertIntentPolicyPreview, error) {
request.ResourceID = strings.TrimSpace(request.ResourceID)
request.ResourceType = strings.ToLower(strings.TrimSpace(request.ResourceType))
request.Signal = strings.ToLower(strings.TrimSpace(request.Signal))
if request.ResourceID == "" || request.ResourceType == "" || request.Signal == "" {
return AlertIntentPolicyPreview{}, errors.New("resourceId, resourceType, and signal are required")
}
now := m.policyNow().UTC()
trackingKey := "preview:" + request.ResourceID + ":" + request.Signal
backup := BackupIntentContext{}
if request.BackupActive != nil {
backup.Active = *request.BackupActive
backup.Evidence = "preview"
if request.BackupObservedAt != nil {
backup.ObservedAt = request.BackupObservedAt.UTC()
}
}
m.mu.Lock()
previous, hadPrevious := m.intentPending[trackingKey]
if request.FirstMatchedAt != nil {
m.intentPending[trackingKey] = IntentPendingState{
TrackingKey: trackingKey, ResourceID: request.ResourceID, ResourceType: request.ResourceType,
Signal: request.Signal, FirstMatchedAt: request.FirstMatchedAt.UTC(),
}
}
decision := m.evaluateIntentNoLock(request.ResourceID, request.ResourceType, request.Signal, trackingKey, now, request.ConditionActive, backup)
state, stateExists := m.intentPending[trackingKey]
operator := OperatorIntentContext{}
operatorFound := false
if decision.Effective.HonorOperatorState && m.operatorIntentResolver != nil {
operator, operatorFound = m.operatorIntentResolver(request.ResourceID, now)
}
if hadPrevious {
m.intentPending[trackingKey] = previous
} else {
delete(m.intentPending, trackingKey)
}
m.mu.Unlock()
preview := AlertIntentPolicyPreview{
ResourceID: request.ResourceID, ResourceType: request.ResourceType, Signal: request.Signal,
Effective: decision.Effective, Contexts: []AlertIntentPolicyPreviewContext{}, Warnings: []string{},
}
if stateExists && !state.FirstMatchedAt.IsZero() {
first := state.FirstMatchedAt
preview.FirstMatchedAt = &first
}
if !decision.EligibleAt.IsZero() {
eligible := decision.EligibleAt
preview.EligibleAt = &eligible
if eligible.After(now) {
preview.RemainingSecs = int64(eligible.Sub(now).Seconds())
}
}
if !decision.HardCapAt.IsZero() {
capAt := decision.HardCapAt
preview.HardCapAt = &capAt
}
switch {
case !request.ConditionActive:
preview.Status, preview.Reason = "clear", "condition_clear"
case decision.Suppressed:
preview.Status, preview.Reason = "expected_transient", decision.Reason
case decision.Pending:
preview.Status, preview.Reason = "pending_grace", decision.Reason
default:
preview.Status, preview.Reason = "would_activate", decision.Reason
}
if request.BackupActive != nil {
ctx := AlertIntentPolicyPreviewContext{Kind: "backup", Active: *request.BackupActive, Evidence: backup.Evidence}
if !backup.ObservedAt.IsZero() {
observedAt := backup.ObservedAt
ctx.ObservedAt = &observedAt
}
preview.Contexts = append(preview.Contexts, ctx)
}
if operatorFound {
active := operator.IntentionallyOffline || operator.MaintenanceActiveAt(now)
ctx := AlertIntentPolicyPreviewContext{Kind: "operator_state", Active: active, Evidence: operator.MaintenanceReason}
if operator.MaintenanceEndAt != nil {
expiresAt := operator.MaintenanceEndAt.UTC()
ctx.ExpiresAt = &expiresAt
}
preview.Contexts = append(preview.Contexts, ctx)
}
if request.Signal == string(AlertIntentSignalAvailability) {
preview.Warnings = append(preview.Warnings, "Availability probe failure thresholds are evaluated before alert grace.")
}
return preview, nil
}
+231
View File
@@ -0,0 +1,231 @@
package alerts
import (
"testing"
"time"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func intPointer(value int) *int { return &value }
func boolPointer(value bool) *bool { return &value }
func TestAlertIntentPolicyResolutionPrecedenceIsFieldByField(t *testing.T) {
m := NewManagerWithDataDir(t.TempDir())
t.Cleanup(m.Stop)
document := NewAlertIntentPolicyDocument()
document.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{
GraceSeconds: intPointer(30),
HonorOperatorState: boolPointer(true),
BackupOffline: &BackupOfflineIntentPolicy{
Enabled: true, PostGraceSeconds: 45, MaxDeferralSeconds: 600,
},
}
document.ResourceTypes["vm"] = map[string]AlertIntentRule{
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(60)},
}
document.Resources["vm:101"] = map[string]AlertIntentRule{
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(0)},
}
if err := m.LoadIntentPolicies(document); err != nil {
t.Fatalf("LoadIntentPolicies() error = %v", err)
}
effective := m.ResolveEffectiveIntentPolicy("vm:101", "vm", string(AlertIntentSignalOffline))
if effective.GraceSeconds != 0 || !effective.HonorOperatorState || effective.BackupOffline == nil || !effective.BackupOffline.Enabled {
t.Fatalf("effective policy = %+v", effective)
}
if got := effective.Sources["graceSeconds"]; got != "resources.vm:101.state.offline" {
t.Fatalf("grace source = %q", got)
}
if got := effective.Sources["honorOperatorState"]; got != "defaults.state.offline" {
t.Fatalf("operator source = %q", got)
}
}
func TestAlertIntentPolicyResolvesCanonicalResourceFromSourceID(t *testing.T) {
m := NewManagerWithDataDir(t.TempDir())
t.Cleanup(m.Stop)
document := NewAlertIntentPolicyDocument()
document.Resources["vm-canonical"] = map[string]AlertIntentRule{
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(90)},
}
if err := m.LoadIntentPolicies(document); err != nil {
t.Fatal(err)
}
m.SetResourceIntentIdentityResolver(func(resourceID string) (string, bool) {
if resourceID == "cluster-a:node-a:101" {
return "vm-canonical", true
}
return "", false
})
effective := m.ResolveEffectiveIntentPolicy("cluster-a:node-a:101", "vm", string(AlertIntentSignalOffline))
if effective.GraceSeconds != 90 || effective.Sources["graceSeconds"] != "resources.vm-canonical.state.offline" {
t.Fatalf("effective policy = %+v", effective)
}
}
func TestAlertIntentPolicyValidationRejectsAmbiguousNormalizedKeys(t *testing.T) {
document := NewAlertIntentPolicyDocument()
document.ResourceTypes["VM"] = map[string]AlertIntentRule{"default": {GraceSeconds: intPointer(10)}}
document.ResourceTypes[" vm "] = map[string]AlertIntentRule{"*": {GraceSeconds: intPointer(20)}}
if err := ValidateAlertIntentPolicyDocument(document); err == nil {
t.Fatal("expected normalized resource type collision to fail validation")
}
document = NewAlertIntentPolicyDocument()
document.Defaults["default"] = AlertIntentRule{GraceSeconds: intPointer(10)}
document.Defaults["*"] = AlertIntentRule{GraceSeconds: intPointer(20)}
if err := ValidateAlertIntentPolicyDocument(document); err == nil {
t.Fatal("expected normalized signal collision to fail validation")
}
}
func TestAlertIntentBackupDeferralEndsWithPostGraceAndHardCap(t *testing.T) {
m := NewManagerWithDataDir(t.TempDir())
t.Cleanup(m.Stop)
document := NewAlertIntentPolicyDocument()
document.Resources["vm:101"] = map[string]AlertIntentRule{
string(AlertIntentSignalOffline): {
GraceSeconds: intPointer(30),
BackupOffline: &BackupOfflineIntentPolicy{
Enabled: true, PostGraceSeconds: 60, MaxDeferralSeconds: 300,
},
},
}
if err := m.LoadIntentPolicies(document); err != nil {
t.Fatal(err)
}
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
m.mu.Lock()
active := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start, true, BackupIntentContext{Active: true, Evidence: "guest_lock"})
ended := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(100*time.Second), true, BackupIntentContext{})
pending := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(159*time.Second), true, BackupIntentContext{})
eligible := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(160*time.Second), true, BackupIntentContext{})
m.mu.Unlock()
if !active.Pending || active.Reason != "backup_active" {
t.Fatalf("active backup decision = %+v", active)
}
if !ended.Pending || ended.EligibleAt != start.Add(160*time.Second) {
t.Fatalf("ended backup decision = %+v", ended)
}
if !pending.Pending || pending.ShouldActivate {
t.Fatalf("post-backup decision = %+v", pending)
}
if !eligible.ShouldActivate || eligible.HardCapAt != start.Add(300*time.Second) {
t.Fatalf("eligible decision = %+v", eligible)
}
}
func TestAlertIntentPreviewHonorsOperatorStateWithoutMutatingRuntime(t *testing.T) {
m := NewManagerWithDataDir(t.TempDir())
t.Cleanup(m.Stop)
document := NewAlertIntentPolicyDocument()
document.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{
GraceSeconds: intPointer(10), HonorOperatorState: boolPointer(true),
}
if err := m.LoadIntentPolicies(document); err != nil {
t.Fatal(err)
}
now := time.Date(2026, 7, 13, 10, 0, 0, 0, time.UTC)
m.now = func() time.Time { return now }
end := now.Add(time.Hour)
m.SetOperatorIntentContextResolver(func(resourceID string, observedAt time.Time) (OperatorIntentContext, bool) {
return OperatorIntentContext{MaintenanceStartAt: &now, MaintenanceEndAt: &end, MaintenanceReason: "upgrade"}, true
})
preview, err := m.PreviewIntentPolicy(AlertIntentPolicyPreviewRequest{
ResourceID: "vm:101", ResourceType: "vm", Signal: string(AlertIntentSignalOffline), ConditionActive: true,
})
if err != nil {
t.Fatalf("PreviewIntentPolicy() error = %v", err)
}
if preview.Status != "expected_transient" || preview.Reason != "operator_maintenance" {
t.Fatalf("preview = %+v", preview)
}
if len(preview.Contexts) != 1 || preview.Contexts[0].Kind != "operator_state" || !preview.Contexts[0].Active {
t.Fatalf("preview contexts = %+v", preview.Contexts)
}
if len(m.intentPending) != 0 {
t.Fatalf("preview leaked runtime state: %+v", m.intentPending)
}
}
func TestLifecycleAlertStartsAtFirstIntentMatch(t *testing.T) {
m := NewManagerWithDataDir(t.TempDir())
t.Cleanup(m.Stop)
document := NewAlertIntentPolicyDocument()
document.Resources["vm:101"] = map[string]AlertIntentRule{
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(60)},
}
if err := m.LoadIntentPolicies(document); err != nil {
t.Fatal(err)
}
spec, err := buildCanonicalPoweredStateSpec("vm:101", "database", unifiedresources.ResourceTypeVM, AlertLevelWarning, 1, false)
if err != nil {
t.Fatal(err)
}
tracking := make(map[string]int)
start := time.Date(2026, 7, 13, 11, 0, 0, 0, time.UTC)
params := canonicalLifecycleAlertParams{
Spec: spec,
Evidence: alertspecs.AlertEvidence{
ObservedAt: start,
PoweredState: &alertspecs.PoweredStateEvidence{
Expected: alertspecs.PowerStateOn,
Observed: alertspecs.PowerStateOff,
},
},
Tracking: tracking, TrackingKey: "vm:101", AlertID: "guest-powered-off-vm:101",
AlertType: "powered-off", ResourceID: "vm:101", ResourceName: "database",
}
if result, _ := m.evaluateCanonicalLifecycleAlert(params); result.State.State != alertspecs.AlertStatePending {
t.Fatalf("initial state = %s, want pending", result.State.State)
}
params.Evidence.ObservedAt = start.Add(60 * time.Second)
if result, _ := m.evaluateCanonicalLifecycleAlert(params); result.State.State != alertspecs.AlertStateFiring {
t.Fatalf("eligible state = %s, want firing", result.State.State)
}
m.mu.RLock()
alert, ok := m.getActiveAlertNoLock(canonicalPoweredStateStateID("vm:101"))
m.mu.RUnlock()
if !ok || alert == nil {
t.Fatal("expected powered-state alert")
}
if !alert.StartTime.Equal(start) {
t.Fatalf("alert start = %v, want first match %v", alert.StartTime, start)
}
}
func TestIntentPendingStatePersistsAcrossRestart(t *testing.T) {
dataDir := t.TempDir()
first := NewManagerWithDataDir(dataDir)
t.Cleanup(first.Stop)
now := time.Now().UTC().Truncate(time.Second)
first.mu.Lock()
first.intentPending["state:vm:101"] = IntentPendingState{
TrackingKey: "state:vm:101", ResourceID: "vm:101", ResourceType: "vm",
Signal: string(AlertIntentSignalOffline), FirstMatchedAt: now.Add(-time.Minute), LastObservedAt: now,
}
first.mu.Unlock()
if err := first.SaveActiveAlerts(); err != nil {
t.Fatalf("SaveActiveAlerts: %v", err)
}
second := NewManagerWithDataDir(dataDir)
t.Cleanup(second.Stop)
if err := second.LoadActiveAlerts(); err != nil {
t.Fatalf("LoadActiveAlerts: %v", err)
}
second.mu.RLock()
restored, ok := second.intentPending["state:vm:101"]
second.mu.RUnlock()
if !ok || !restored.FirstMatchedAt.Equal(now.Add(-time.Minute)) {
t.Fatalf("restored intent state = %+v, found %v", restored, ok)
}
}
+10
View File
@@ -48,6 +48,14 @@ type Manager struct {
resolvedMutex sync.RWMutex // Secondary lock - see Lock Ordering Documentation above
// Time threshold tracking
pendingAlerts map[string]time.Time // Track when thresholds were first exceeded
// Intent-policy pending state retains wall-clock and transient-context
// evidence for policy-enabled candidates. It is keyed by canonical alert
// tracking key and persisted with the alert manager's transition state.
intentPending map[string]IntentPendingState
intentPolicies AlertIntentPolicyDocument
operatorIntentResolver OperatorIntentContextResolver
backupIntentResolver BackupIntentContextResolver
resourceIntentResolver ResourceIntentIdentityResolver
// Offline confirmation tracking
nodeOfflineCount map[string]int // Track consecutive offline counts for nodes (legacy)
connectionDegradedCount map[string]int // Track consecutive degraded counts for platform connections (pve/pbs/pmg/vmware/truenas)
@@ -128,6 +136,8 @@ func NewManagerWithDataDir(dataDir string) *Manager {
recentlyResolved: make(map[string]*ResolvedAlert),
resolvedAlias: make(map[string]string),
pendingAlerts: make(map[string]time.Time),
intentPending: make(map[string]IntentPendingState),
intentPolicies: NewAlertIntentPolicyDocument(),
nodeOfflineCount: make(map[string]int),
connectionDegradedCount: make(map[string]int),
offlineConfirmations: make(map[string]int),
+34 -10
View File
@@ -46,20 +46,30 @@ func getThresholdForMetricFromConfig(config ThresholdConfig, metricType string)
}
// getTimeThreshold determines the delay to apply for a metric/resource combination.
func (m *Manager) getTimeThreshold(_ string, resourceType, metricType string) int {
func (m *Manager) getTimeThreshold(resourceID string, resourceType, metricType string) int {
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
return effective.GraceSeconds
}
func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType string) (int, string, bool) {
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
return delay
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
}
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
if !hasTypeSpecific {
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
return delay
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
}
}
return base
if hasTypeSpecific {
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
}
if base != 0 {
return base, "legacy.timeThresholds.all", true
}
return 0, "", false
}
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
@@ -215,11 +225,21 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
if !exists {
alertStartTime := time.Now()
// Determine the appropriate time threshold based on resource/metric type
timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType)
// Check if we have a time threshold configured
if timeThreshold > 0 {
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
if effectiveIntent.Explicit {
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, alertStartTime, true, BackupIntentContext{})
if pending, ok := m.intentPending[trackingKey]; ok && !pending.FirstMatchedAt.IsZero() {
alertStartTime = pending.FirstMatchedAt
}
if decision.StateChanged {
m.saveActiveAlertsAsync("metric intent pending state")
}
if !decision.ShouldActivate {
return
}
delete(m.intentPending, trackingKey)
m.saveActiveAlertsAsync("metric intent activated")
} else if timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType); timeThreshold > 0 {
// Check if this threshold was already pending
if pendingTime, isPending := m.pendingAlerts[trackingKey]; isPending {
// Check if enough time has passed
@@ -472,6 +492,10 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
}
} else {
// Value is below trigger threshold
intentDecision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), false, BackupIntentContext{})
if intentDecision.StateChanged {
m.saveActiveAlertsAsync("metric intent cleared")
}
// Clear any pending alert for this metric
if _, isPending := m.pendingAlerts[trackingKey]; isPending {
delete(m.pendingAlerts, trackingKey)
+59
View File
@@ -113,6 +113,53 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
m.mu.Lock()
defer m.mu.Unlock()
// Availability failure thresholds establish that an incident exists;
// intent grace is a second, operator-controlled gate before activation.
// Track observed keys separately so a pending incident is not mistaken for
// a cleared condition when it is intentionally removed from desired.
observedAvailability := make(map[string]struct{})
intentStateChanged := false
for storageKey, alert := range desired {
if !isAvailabilityIncidentAlert(alert) {
continue
}
observedAvailability[storageKey] = struct{}{}
if existing, exists := m.getActiveAlertNoLock(storageKey); exists && existing != nil {
if _, pending := m.intentPending[storageKey]; pending {
delete(m.intentPending, storageKey)
intentStateChanged = true
}
continue
}
decision := m.evaluateIntentNoLock(alert.ResourceID, alertMetadataString(alert, "resourceType"), string(AlertIntentSignalAvailability), storageKey, alert.LastSeen, true, BackupIntentContext{})
intentStateChanged = intentStateChanged || decision.StateChanged
if decision.Effective.Explicit && !decision.ShouldActivate {
delete(desired, storageKey)
continue
}
if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() {
alert.StartTime = pending.FirstMatchedAt
}
if _, ok := m.intentPending[storageKey]; ok {
delete(m.intentPending, storageKey)
intentStateChanged = true
}
}
for storageKey, pending := range m.intentPending {
if pending.Signal != string(AlertIntentSignalAvailability) {
continue
}
if _, observed := observedAvailability[storageKey]; observed {
continue
}
delete(m.intentPending, storageKey)
intentStateChanged = true
}
if intentStateChanged {
m.saveActiveAlertsAsync("availability intent state")
}
for storageKey, alert := range m.activeAlerts {
if !strings.HasPrefix(storageKey, unifiedresources.CanonicalResourceID(alert.ResourceID)+canonicalStateSeparator+"alertspec:provider-incident:") {
continue
@@ -147,6 +194,18 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
}
}
func isAvailabilityIncidentAlert(alert *Alert) bool {
return alertMetadataString(alert, "incidentCode") == "availability_unreachable"
}
func alertMetadataString(alert *Alert, key string) string {
if alert == nil || alert.Metadata == nil {
return ""
}
value, _ := alert.Metadata[key].(string)
return strings.TrimSpace(value)
}
func shouldSuppressCanonicalUnifiedIncidentSpec(spec alertspecs.ResourceAlertSpec, providerSpecsByResource map[string][]alertspecs.ResourceAlertSpec) bool {
if len(spec.SuppressionKeys) == 0 {
return false
+110
View File
@@ -3,6 +3,7 @@ package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
@@ -42,6 +43,17 @@ type ConfigPersistence interface {
SaveAlertConfig(alerts.AlertConfig) error
}
type alertIntentPolicyManager interface {
GetIntentPolicies() alerts.AlertIntentPolicyDocument
LoadIntentPolicies(alerts.AlertIntentPolicyDocument) error
UpdateIntentPolicies(alerts.AlertIntentPolicyDocument) (alerts.AlertIntentPolicyDocument, error)
PreviewIntentPolicy(alerts.AlertIntentPolicyPreviewRequest) (alerts.AlertIntentPolicyPreview, error)
}
type alertIntentPolicyPersistence interface {
SaveAlertIntentPolicies(alerts.AlertIntentPolicyDocument) error
}
// AlertMonitor defines the interface for monitoring operations used by alert handlers.
type AlertMonitor interface {
GetAlertManager() AlertManager
@@ -55,6 +67,7 @@ type AlertMonitor interface {
// AlertHandlers handles alert-related HTTP endpoints
type AlertHandlers struct {
stateMu sync.RWMutex
intentPolicyMu sync.Mutex
mtMonitor *monitoring.MultiTenantMonitor
defaultMonitor AlertMonitor
wsHub *websocket.Hub
@@ -297,6 +310,88 @@ func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request)
}
}
func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
h.intentPolicyMu.Lock()
defer h.intentPolicyMu.Unlock()
manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
if !ok {
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
return
}
if err := utils.WriteJSONResponse(w, manager.GetIntentPolicies()); err != nil {
log.Error().Err(err).Msg("Failed to write alert intent policies response")
}
}
func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, 256*1024)
var document alerts.AlertIntentPolicyDocument
if err := json.NewDecoder(r.Body).Decode(&document); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Treat revision check, in-memory install, and durable save as one API
// transaction. This also prevents a failed earlier writer from rolling
// back a later successful update.
h.intentPolicyMu.Lock()
defer h.intentPolicyMu.Unlock()
monitor := h.getMonitor(r.Context())
manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
if !ok {
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
return
}
persistence, ok := monitor.GetConfigPersistence().(alertIntentPolicyPersistence)
if !ok {
http.Error(w, "alert intent policy persistence is unavailable", http.StatusServiceUnavailable)
return
}
previous := manager.GetIntentPolicies()
updated, err := manager.UpdateIntentPolicies(document)
if err != nil {
if errors.Is(err, alerts.ErrAlertIntentPolicyRevisionConflict) {
http.Error(w, err.Error(), http.StatusConflict)
return
}
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := persistence.SaveAlertIntentPolicies(updated); err != nil {
if rollbackErr := manager.LoadIntentPolicies(previous); rollbackErr != nil {
log.Error().Err(rollbackErr).Msg("Failed to roll back in-memory alert intent policies after persistence failure")
}
http.Error(w, fmt.Sprintf("Failed to save alert intent policies: %v", err), http.StatusInternalServerError)
return
}
if err := utils.WriteJSONResponse(w, updated); err != nil {
log.Error().Err(err).Msg("Failed to write updated alert intent policies response")
}
}
func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
var request alerts.AlertIntentPolicyPreviewRequest
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
h.intentPolicyMu.Lock()
defer h.intentPolicyMu.Unlock()
manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
if !ok {
http.Error(w, "alert intent policy preview is unavailable", http.StatusServiceUnavailable)
return
}
preview, err := manager.PreviewIntentPolicy(request)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := utils.WriteJSONResponse(w, preview); err != nil {
log.Error().Err(err).Msg("Failed to write alert intent policy preview response")
}
}
// GetAlertDeliveryDiagnosis explains current notification delivery policy for
// one active alert without sending a notification or mutating delivery state.
func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http.Request) {
@@ -1049,6 +1144,21 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
return
}
h.UpdateAlertConfig(w, r)
case path == "intent-policies" && r.Method == http.MethodGet:
if !ensureScope(w, r, config.ScopeMonitoringRead) {
return
}
h.GetAlertIntentPolicies(w, r)
case path == "intent-policies" && r.Method == http.MethodPut:
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
return
}
h.UpdateAlertIntentPolicies(w, r)
case path == "intent-policies/preview" && r.Method == http.MethodPost:
if !ensureScope(w, r, config.ScopeMonitoringRead) {
return
}
h.PreviewAlertIntentPolicy(w, r)
case path == "activate" && r.Method == http.MethodPost:
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
return
+76
View File
@@ -72,6 +72,82 @@ func TestAlertsEndpoints(t *testing.T) {
}
})
t.Run("AlertIntentPolicies", func(t *testing.T) {
res, err := http.Get(srv.server.URL + "/api/alerts/intent-policies")
if err != nil {
t.Fatalf("get policies: %v", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
t.Fatalf("get policies status = %d, want %d", res.StatusCode, http.StatusOK)
}
var document alerts.AlertIntentPolicyDocument
if err := json.NewDecoder(res.Body).Decode(&document); err != nil {
t.Fatalf("decode policies: %v", err)
}
grace := 45
if document.Defaults == nil {
document.Defaults = make(map[string]alerts.AlertIntentRule)
}
document.Defaults[string(alerts.AlertIntentSignalOffline)] = alerts.AlertIntentRule{GraceSeconds: &grace}
body, err := json.Marshal(document)
if err != nil {
t.Fatalf("marshal policies: %v", err)
}
req, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/intent-policies", bytes.NewReader(body))
if err != nil {
t.Fatalf("create policy update: %v", err)
}
req.Header.Set("Content-Type", "application/json")
updatedResponse, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("update policies: %v", err)
}
defer updatedResponse.Body.Close()
if updatedResponse.StatusCode != http.StatusOK {
t.Fatalf("update policies status = %d, want %d", updatedResponse.StatusCode, http.StatusOK)
}
var updated alerts.AlertIntentPolicyDocument
if err := json.NewDecoder(updatedResponse.Body).Decode(&updated); err != nil {
t.Fatalf("decode updated policies: %v", err)
}
if updated.Revision != document.Revision+1 {
t.Fatalf("updated revision = %d, want %d", updated.Revision, document.Revision+1)
}
staleReq, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/intent-policies", bytes.NewReader(body))
if err != nil {
t.Fatalf("create stale update: %v", err)
}
staleReq.Header.Set("Content-Type", "application/json")
staleResponse, err := http.DefaultClient.Do(staleReq)
if err != nil {
t.Fatalf("stale update: %v", err)
}
defer staleResponse.Body.Close()
if staleResponse.StatusCode != http.StatusConflict {
t.Fatalf("stale update status = %d, want %d", staleResponse.StatusCode, http.StatusConflict)
}
previewBody := []byte(`{"resourceId":"vm:101","resourceType":"vm","signal":"state.offline","conditionActive":true}`)
previewResponse, err := http.Post(srv.server.URL+"/api/alerts/intent-policies/preview", "application/json", bytes.NewReader(previewBody))
if err != nil {
t.Fatalf("preview policy: %v", err)
}
defer previewResponse.Body.Close()
if previewResponse.StatusCode != http.StatusOK {
t.Fatalf("preview status = %d, want %d", previewResponse.StatusCode, http.StatusOK)
}
var preview alerts.AlertIntentPolicyPreview
if err := json.NewDecoder(previewResponse.Body).Decode(&preview); err != nil {
t.Fatalf("decode preview: %v", err)
}
if preview.Status != "pending_grace" || preview.Effective.GraceSeconds != grace {
t.Fatalf("preview = %+v", preview)
}
})
// 3. Activate alerts
t.Run("ActivateAlerts", func(t *testing.T) {
req, err := http.NewRequest(http.MethodPost, srv.server.URL+"/api/alerts/activate", nil)
+3 -1
View File
@@ -27,6 +27,7 @@ type availabilityTargetResponse struct {
type availabilityTestResponse struct {
Success bool `json:"success"`
LatencyMillis int64 `json:"latencyMillis"`
Outcome string `json:"outcome,omitempty"`
Error string `json:"error,omitempty"`
}
@@ -250,7 +251,7 @@ func (h *AvailabilityHandlers) testTarget(w http.ResponseWriter, r *http.Request
return
}
start := time.Now()
err := monitoring.ProbeAvailabilityTarget(r.Context(), target)
outcome, err := monitoring.ProbeAvailabilityTargetResult(r.Context(), target)
latencyMs := time.Since(start).Milliseconds()
if err == nil && latencyMs == 0 {
latencyMs = 1
@@ -258,6 +259,7 @@ func (h *AvailabilityHandlers) testTarget(w http.ResponseWriter, r *http.Request
response := availabilityTestResponse{
Success: err == nil,
LatencyMillis: latencyMs,
Outcome: string(outcome),
}
if err != nil {
response.Error = err.Error()
+52 -2
View File
@@ -22,10 +22,22 @@ const (
AvailabilityProbeTCP AvailabilityProbeProtocol = "tcp"
AvailabilityProbeHTTP AvailabilityProbeProtocol = "http"
AvailabilityProbeHTTPS AvailabilityProbeProtocol = "https"
AvailabilityProbeUDP AvailabilityProbeProtocol = "udp"
availabilityProbePingAlias AvailabilityProbeProtocol = "ping"
)
type AvailabilityUDPMode string
const (
// AvailabilityUDPResponseRequired requires a datagram response and treats
// silence as failure. This is the safe default for alerting.
AvailabilityUDPResponseRequired AvailabilityUDPMode = "response_required"
// AvailabilityUDPOpenOrFiltered treats silence as an indeterminate,
// non-failing result; an ICMP port-unreachable response still fails.
AvailabilityUDPOpenOrFiltered AvailabilityUDPMode = "open_or_filtered"
)
type AvailabilityTargetKind string
const (
@@ -44,6 +56,9 @@ type AvailabilityTarget struct {
Protocol AvailabilityProbeProtocol `json:"protocol"`
Port int `json:"port,omitempty"`
Path string `json:"path,omitempty"`
UDPMode AvailabilityUDPMode `json:"udpMode,omitempty"`
UDPRequest string `json:"udpRequest,omitempty"`
UDPExpected string `json:"udpExpectedResponse,omitempty"`
Enabled bool `json:"enabled"`
PollIntervalSecs int `json:"pollIntervalSeconds,omitempty"`
TimeoutMillis int `json:"timeoutMillis,omitempty"`
@@ -88,6 +103,12 @@ func (t *AvailabilityTarget) ApplyDefaults() {
if t.FailureThreshold <= 0 {
t.FailureThreshold = DefaultAvailabilityFailureThreshold
}
if t.Protocol == AvailabilityProbeUDP {
t.UDPMode = normalizeAvailabilityUDPMode(t.UDPMode)
if t.UDPMode == "" {
t.UDPMode = AvailabilityUDPResponseRequired
}
}
}
func (t AvailabilityTarget) EffectivePollIntervalSecs() int {
@@ -137,9 +158,9 @@ func (t AvailabilityTarget) Validate() error {
if t.Port != 0 {
return fmt.Errorf("icmp availability targets must not set a port")
}
case AvailabilityProbeTCP:
case AvailabilityProbeTCP, AvailabilityProbeUDP:
if t.Port <= 0 || t.Port > 65535 {
return fmt.Errorf("tcp availability targets require a valid port")
return fmt.Errorf("%s availability targets require a valid port", protocol)
}
case AvailabilityProbeHTTP, AvailabilityProbeHTTPS:
if t.Port < 0 || t.Port > 65535 {
@@ -148,6 +169,25 @@ func (t AvailabilityTarget) Validate() error {
default:
return fmt.Errorf("unsupported availability protocol %q", t.Protocol)
}
if protocol == AvailabilityProbeUDP {
mode := normalizeAvailabilityUDPMode(t.UDPMode)
switch mode {
case AvailabilityUDPResponseRequired, AvailabilityUDPOpenOrFiltered:
default:
return fmt.Errorf("unsupported UDP availability mode %q", t.UDPMode)
}
if mode == AvailabilityUDPResponseRequired && len(t.UDPRequest) == 0 {
return fmt.Errorf("UDP response-required checks require a request payload")
}
if len(t.UDPRequest) > 512 {
return fmt.Errorf("UDP request payload must be 512 bytes or less")
}
if len(t.UDPExpected) > 4096 {
return fmt.Errorf("UDP expected response must be 4096 bytes or less")
}
} else if t.UDPMode != "" || t.UDPRequest != "" || t.UDPExpected != "" {
return fmt.Errorf("UDP settings may only be used with UDP availability targets")
}
if t.PollIntervalSecs > 0 && t.PollIntervalSecs < 10 {
return fmt.Errorf("availability poll interval must be at least 10 seconds")
}
@@ -218,11 +258,21 @@ func NormalizeAvailabilityTarget(target AvailabilityTarget) AvailabilityTarget {
target.Address = normalizeAvailabilityAddress(target.Address)
}
target.Path = strings.TrimSpace(target.Path)
target.UDPMode = normalizeAvailabilityUDPMode(target.UDPMode)
if target.Protocol != AvailabilityProbeUDP {
target.UDPMode = ""
target.UDPRequest = ""
target.UDPExpected = ""
}
target.LinkedResourceID = strings.TrimSpace(target.LinkedResourceID)
target.ApplyDefaults()
return target
}
func normalizeAvailabilityUDPMode(mode AvailabilityUDPMode) AvailabilityUDPMode {
return AvailabilityUDPMode(strings.ToLower(strings.TrimSpace(string(mode))))
}
func normalizeAvailabilityProbeProtocol(protocol AvailabilityProbeProtocol) AvailabilityProbeProtocol {
normalized := AvailabilityProbeProtocol(strings.ToLower(strings.TrimSpace(string(protocol))))
if normalized == availabilityProbePingAlias {
@@ -225,7 +225,7 @@ func TestBranchCovValidate(t *testing.T) {
// --- Protocol switch: default arm. ---
{
name: "unsupported protocol rejected",
target: AvailabilityTarget{Address: "device.local", TargetKind: AvailabilityTargetService, Protocol: AvailabilityProbeProtocol("udp")},
target: AvailabilityTarget{Address: "device.local", TargetKind: AvailabilityTargetService, Protocol: AvailabilityProbeProtocol("smtp")},
wantErr: true,
wantErrContains: "unsupported availability protocol",
},
+29 -13
View File
@@ -24,17 +24,18 @@ const (
// ExportData contains all configuration data for export
type ExportData struct {
Version string `json:"version"`
ExportedAt time.Time `json:"exportedAt"`
Nodes NodesConfig `json:"nodes"`
Alerts alerts.AlertConfig `json:"alerts"`
Email notifications.EmailConfig `json:"email"`
Webhooks []notifications.WebhookConfig `json:"webhooks"`
Apprise notifications.AppriseConfig `json:"apprise"`
System SystemSettings `json:"system"`
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
SSO *SSOConfig `json:"sso,omitempty"`
APITokens []APITokenRecord `json:"apiTokens,omitempty"`
Version string `json:"version"`
ExportedAt time.Time `json:"exportedAt"`
Nodes NodesConfig `json:"nodes"`
Alerts alerts.AlertConfig `json:"alerts"`
AlertIntent *alerts.AlertIntentPolicyDocument `json:"alertIntentPolicies,omitempty"`
Email notifications.EmailConfig `json:"email"`
Webhooks []notifications.WebhookConfig `json:"webhooks"`
Apprise notifications.AppriseConfig `json:"apprise"`
System SystemSettings `json:"system"`
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
SSO *SSOConfig `json:"sso,omitempty"`
APITokens []APITokenRecord `json:"apiTokens,omitempty"`
}
// ExportConfig exports all configuration with passphrase-based encryption
@@ -55,6 +56,10 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
if err != nil {
return "", fmt.Errorf("failed to load alert config: %w", err)
}
alertIntent, err := c.LoadAlertIntentPolicies()
if err != nil {
return "", fmt.Errorf("failed to load alert intent policies: %w", err)
}
emailConfig, err := c.LoadEmailConfig()
if err != nil {
@@ -97,10 +102,11 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
// Create export data
exportData := ExportData{
Version: "4.2",
Version: "4.3",
ExportedAt: time.Now(),
Nodes: *nodes,
Alerts: *alertConfig,
AlertIntent: alertIntent,
Email: *emailConfig,
Webhooks: webhooks,
Apprise: *appriseConfig,
@@ -152,8 +158,10 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
// Check version compatibility (warn but don't fail)
switch exportData.Version {
case "4.2", "":
case "4.3", "":
// current version, nothing to do
case "4.2":
log.Info().Msg("Config was exported from version 4.2. Alert intent policies were not included in that format.")
case "4.1":
log.Info().Msg("Config was exported from version 4.1. SSO settings were not included in that format.")
case "4.0":
@@ -186,6 +194,14 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
if err := c.SaveAlertConfig(exportData.Alerts); err != nil {
return fmt.Errorf("failed to import alert config: %w", err)
}
intentPolicies := exportData.AlertIntent
if intentPolicies == nil {
defaults := alerts.NewAlertIntentPolicyDocument()
intentPolicies = &defaults
}
if err := c.SaveAlertIntentPolicies(*intentPolicies); err != nil {
return fmt.Errorf("failed to import alert intent policies: %w", err)
}
if err := c.SaveEmailConfig(exportData.Email); err != nil {
return fmt.Errorf("failed to import email config: %w", err)
+8
View File
@@ -29,6 +29,7 @@ type ConfigPersistence struct {
tx *importTransaction
configDir string
alertFile string
alertIntentFile string
emailFile string
webhookFile string
appriseFile string
@@ -107,6 +108,7 @@ type alertScheduleGroupingPresence struct {
type resolvedConfigPersistencePaths struct {
alertFile string
alertIntentFile string
emailFile string
webhookFile string
appriseFile string
@@ -146,6 +148,10 @@ func resolveConfigPersistencePaths(configDir string) (string, resolvedConfigPers
if err != nil {
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve alerts.json: %w", err)
}
alertIntentFile, err := resolveLeaf("alert_intent_policies.json")
if err != nil {
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve alert_intent_policies.json: %w", err)
}
emailFile, err := resolveLeaf("email.enc")
if err != nil {
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve email.enc: %w", err)
@@ -241,6 +247,7 @@ func resolveConfigPersistencePaths(configDir string) (string, resolvedConfigPers
return normalizedConfigDir, resolvedConfigPersistencePaths{
alertFile: alertFile,
alertIntentFile: alertIntentFile,
emailFile: emailFile,
webhookFile: webhookFile,
appriseFile: appriseFile,
@@ -296,6 +303,7 @@ func newConfigPersistence(configDir string) (*ConfigPersistence, error) {
cp := &ConfigPersistence{
configDir: resolvedConfigDir,
alertFile: resolvedPaths.alertFile,
alertIntentFile: resolvedPaths.alertIntentFile,
emailFile: resolvedPaths.emailFile,
webhookFile: resolvedPaths.webhookFile,
appriseFile: resolvedPaths.appriseFile,
@@ -0,0 +1,31 @@
package config
import (
"errors"
"io/fs"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
)
func (c *ConfigPersistence) SaveAlertIntentPolicies(document alerts.AlertIntentPolicyDocument) error {
if err := alerts.ValidateAlertIntentPolicyDocument(document); err != nil {
return err
}
document = alerts.NormalizeAlertIntentPolicyDocument(document)
return saveJSON(c, c.alertIntentFile, document, false)
}
func (c *ConfigPersistence) LoadAlertIntentPolicies() (*alerts.AlertIntentPolicyDocument, error) {
document := alerts.NewAlertIntentPolicyDocument()
if err := loadJSON(c, c.alertIntentFile, false, &document); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return &document, nil
}
return nil, err
}
document = alerts.NormalizeAlertIntentPolicyDocument(document)
if err := alerts.ValidateAlertIntentPolicyDocument(document); err != nil {
return nil, err
}
return &document, nil
}
@@ -0,0 +1,90 @@
package config_test
import (
"reflect"
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
)
func TestAlertIntentPolicyPersistenceMissingAndRoundTrip(t *testing.T) {
cp := config.NewConfigPersistence(t.TempDir())
if err := cp.EnsureConfigDir(); err != nil {
t.Fatalf("EnsureConfigDir: %v", err)
}
missing, err := cp.LoadAlertIntentPolicies()
if err != nil {
t.Fatalf("LoadAlertIntentPolicies missing: %v", err)
}
if missing.SchemaVersion != alerts.CurrentAlertIntentPolicySchemaVersion || missing.Revision != 0 {
t.Fatalf("missing policy document = %+v", missing)
}
grace := 75
document := alerts.NewAlertIntentPolicyDocument()
document.Revision = 4
document.Resources["vm:101"] = map[string]alerts.AlertIntentRule{
string(alerts.AlertIntentSignalOffline): {
GraceSeconds: &grace,
BackupOffline: &alerts.BackupOfflineIntentPolicy{
Enabled: true, PostGraceSeconds: 30, MaxDeferralSeconds: 900,
},
},
}
if err := cp.SaveAlertIntentPolicies(document); err != nil {
t.Fatalf("SaveAlertIntentPolicies: %v", err)
}
loaded, err := cp.LoadAlertIntentPolicies()
if err != nil {
t.Fatalf("LoadAlertIntentPolicies: %v", err)
}
want := alerts.NormalizeAlertIntentPolicyDocument(document)
if !reflect.DeepEqual(*loaded, want) {
t.Fatalf("loaded policy = %#v, want %#v", *loaded, want)
}
}
func TestExportImportIncludesAlertIntentPolicies(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
source := config.NewConfigPersistence(t.TempDir())
if err := source.EnsureConfigDir(); err != nil {
t.Fatalf("EnsureConfigDir source: %v", err)
}
grace := 120
document := alerts.NewAlertIntentPolicyDocument()
document.Resources["service:dns"] = map[string]alerts.AlertIntentRule{
string(alerts.AlertIntentSignalAvailability): {GraceSeconds: &grace},
}
if err := source.SaveAlertIntentPolicies(document); err != nil {
t.Fatalf("SaveAlertIntentPolicies: %v", err)
}
const passphrase = "intent-policy-round-trip"
bundle, err := source.ExportConfig(passphrase)
if err != nil {
t.Fatalf("ExportConfig: %v", err)
}
decoded := mustDecodeExport(t, bundle, passphrase)
if decoded.Version != "4.3" || decoded.AlertIntent == nil {
t.Fatalf("export metadata = version %q intent %#v", decoded.Version, decoded.AlertIntent)
}
destination := config.NewConfigPersistence(t.TempDir())
if err := destination.EnsureConfigDir(); err != nil {
t.Fatalf("EnsureConfigDir destination: %v", err)
}
if err := destination.ImportConfig(bundle, passphrase); err != nil {
t.Fatalf("ImportConfig: %v", err)
}
loaded, err := destination.LoadAlertIntentPolicies()
if err != nil {
t.Fatalf("LoadAlertIntentPolicies: %v", err)
}
want := alerts.NormalizeAlertIntentPolicyDocument(document)
if !reflect.DeepEqual(*loaded, want) {
t.Fatalf("imported policy = %#v, want %#v", *loaded, want)
}
}
+2 -2
View File
@@ -589,8 +589,8 @@ func TestExportConfigIncludesAPITokens(t *testing.T) {
decoded := mustDecodeExport(t, exported, passphrase)
if decoded.Version != "4.2" {
t.Fatalf("expected export version 4.2, got %q", decoded.Version)
if decoded.Version != "4.3" {
t.Fatalf("expected export version 4.3, got %q", decoded.Version)
}
assertJSONEqual(t, decoded.APITokens, tokens, "api tokens")
+11 -10
View File
@@ -2944,16 +2944,17 @@ func (b PVEBackups) NormalizeCollections() PVEBackups {
// BackupTask represents a PVE backup task
type BackupTask struct {
ID string `json:"id"`
Node string `json:"node"`
Instance string `json:"instance"` // Unique instance identifier
Type string `json:"type"`
VMID int `json:"vmid"`
Status string `json:"status"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime,omitempty"`
Size int64 `json:"size,omitempty"`
Error string `json:"error,omitempty"`
ID string `json:"id"`
Node string `json:"node"`
Instance string `json:"instance"` // Unique instance identifier
Type string `json:"type"`
VMID int `json:"vmid"`
Status string `json:"status"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime,omitempty"`
ObservedAt time.Time `json:"observedAt,omitempty"`
Size int64 `json:"size,omitempty"`
Error string `json:"error,omitempty"`
}
// StorageBackup represents a backup file in storage
+108 -11
View File
@@ -29,6 +29,7 @@ type AvailabilityProbeStatus struct {
TargetKind string `json:"targetKind,omitempty"`
Address string `json:"address"`
Protocol string `json:"protocol"`
Outcome string `json:"outcome,omitempty"`
Enabled bool `json:"enabled"`
Available bool `json:"available"`
LastChecked time.Time `json:"lastChecked,omitempty"`
@@ -39,6 +40,14 @@ type AvailabilityProbeStatus struct {
FailureThreshold int `json:"failureThreshold,omitempty"`
}
type AvailabilityProbeOutcome string
const (
AvailabilityProbeReachable AvailabilityProbeOutcome = "reachable"
AvailabilityProbeUnreachable AvailabilityProbeOutcome = "unreachable"
AvailabilityProbeIndeterminate AvailabilityProbeOutcome = "indeterminate"
)
type availabilityPollProvider struct{}
func newAvailabilityPollProvider() PollProvider {
@@ -255,10 +264,10 @@ func (m *Monitor) RefreshAvailabilityTargets() {
func (m *Monitor) pollAvailabilityTarget(ctx context.Context, target config.AvailabilityTarget) {
target = config.NormalizeAvailabilityTarget(target)
start := time.Now()
err := ProbeAvailabilityTarget(ctx, target)
outcome, err := ProbeAvailabilityTargetResult(ctx, target)
latency := time.Since(start)
checkedAt := time.Now().UTC()
m.setAvailabilityStatus(target, checkedAt, latency, err)
m.setAvailabilityStatus(target, checkedAt, latency, outcome, err)
if err == nil {
if m.stalenessTracker != nil {
@@ -275,11 +284,12 @@ func (m *Monitor) pollAvailabilityTarget(ctx context.Context, target config.Avai
m.updateResourceStore(m.GetState())
}
func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checkedAt time.Time, latency time.Duration, probeErr error) {
func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checkedAt time.Time, latency time.Duration, outcome AvailabilityProbeOutcome, probeErr error) {
if m == nil {
return
}
status := availabilityStatusFromTarget(target)
status.Outcome = string(outcome)
status.LastChecked = checkedAt
latencyMs := latency.Milliseconds()
if probeErr == nil && latencyMs == 0 {
@@ -287,8 +297,13 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke
}
status.LatencyMillis = latencyMs
if probeErr == nil {
status.Available = true
status.LastSuccess = checkedAt
// An open-or-filtered UDP timeout is healthy probe execution but does
// not prove endpoint reachability. Keep it non-failing without claiming
// the endpoint is available.
status.Available = outcome == AvailabilityProbeReachable
if outcome == AvailabilityProbeReachable {
status.LastSuccess = checkedAt
}
} else {
status.Available = false
status.LastError = probeErr.Error()
@@ -303,7 +318,9 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke
if probeErr == nil {
status.ConsecutiveFailures = 0
status.LastError = ""
status.LastSuccess = checkedAt
if outcome == AvailabilityProbeReachable {
status.LastSuccess = checkedAt
}
} else {
status.ConsecutiveFailures = previous.ConsecutiveFailures + 1
}
@@ -316,9 +333,16 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke
// ProbeAvailabilityTarget executes one agentless availability check.
func ProbeAvailabilityTarget(ctx context.Context, target config.AvailabilityTarget) error {
_, err := ProbeAvailabilityTargetResult(ctx, target)
return err
}
// ProbeAvailabilityTargetResult preserves UDP's open-or-filtered state rather
// than incorrectly claiming that a silent UDP endpoint was proven reachable.
func ProbeAvailabilityTargetResult(ctx context.Context, target config.AvailabilityTarget) (AvailabilityProbeOutcome, error) {
target = config.NormalizeAvailabilityTarget(target)
if err := target.Validate(); err != nil {
return err
return AvailabilityProbeUnreachable, err
}
timeout := time.Duration(target.EffectiveTimeoutMillis()) * time.Millisecond
@@ -330,16 +354,87 @@ func ProbeAvailabilityTarget(ctx context.Context, target config.AvailabilityTarg
switch target.Protocol {
case config.AvailabilityProbeICMP:
return probeICMP(probeCtx, target)
return outcomeFromProbeError(probeICMP(probeCtx, target))
case config.AvailabilityProbeTCP:
return probeTCP(probeCtx, target)
return outcomeFromProbeError(probeTCP(probeCtx, target))
case config.AvailabilityProbeUDP:
return probeUDP(probeCtx, target)
case config.AvailabilityProbeHTTP, config.AvailabilityProbeHTTPS:
return probeHTTP(probeCtx, target, timeout)
return outcomeFromProbeError(probeHTTP(probeCtx, target, timeout))
default:
return fmt.Errorf("unsupported availability protocol %q", target.Protocol)
return AvailabilityProbeUnreachable, fmt.Errorf("unsupported availability protocol %q", target.Protocol)
}
}
func outcomeFromProbeError(err error) (AvailabilityProbeOutcome, error) {
if err != nil {
return AvailabilityProbeUnreachable, err
}
return AvailabilityProbeReachable, nil
}
func probeUDP(ctx context.Context, target config.AvailabilityTarget) (AvailabilityProbeOutcome, error) {
host := target.ProbeAddress()
addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return AvailabilityProbeUnreachable, fmt.Errorf("resolve UDP availability target: %w", err)
}
var selected net.IP
for _, address := range addresses {
if address.IP == nil || address.IP.IsUnspecified() || address.IP.IsMulticast() || address.IP.Equal(net.IPv4bcast) {
continue
}
selected = address.IP
break
}
if selected == nil {
return AvailabilityProbeUnreachable, fmt.Errorf("UDP availability target did not resolve to an allowed unicast address")
}
dialer := net.Dialer{}
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(selected.String(), strconv.Itoa(target.Port)))
if err != nil {
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe dial failed: %w", err)
}
defer conn.Close()
if deadline, ok := ctx.Deadline(); ok {
if err := conn.SetDeadline(deadline); err != nil {
return AvailabilityProbeUnreachable, fmt.Errorf("set UDP probe deadline: %w", err)
}
}
payload := []byte(target.UDPRequest)
if len(payload) == 0 {
// A one-byte datagram gives the kernel an opportunity to surface an
// ICMP port-unreachable result in open-or-filtered mode.
payload = []byte{0}
}
if _, err := conn.Write(payload); err != nil {
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe write failed: %w", err)
}
response := make([]byte, 4096)
n, err := conn.Read(response)
if err == nil {
if target.UDPExpected != "" && string(response[:n]) != target.UDPExpected {
return AvailabilityProbeUnreachable, fmt.Errorf("UDP response did not match the expected payload")
}
return AvailabilityProbeReachable, nil
}
if ctxErr := ctx.Err(); ctxErr != nil {
if target.UDPMode == config.AvailabilityUDPOpenOrFiltered && ctxErr == context.DeadlineExceeded {
return AvailabilityProbeIndeterminate, nil
}
return AvailabilityProbeUnreachable, ctxErr
}
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
if target.UDPMode == config.AvailabilityUDPOpenOrFiltered {
return AvailabilityProbeIndeterminate, nil
}
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe timed out waiting for a response")
}
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe failed: %w", err)
}
func probeICMP(ctx context.Context, target config.AvailabilityTarget) error {
host := target.ProbeAddress()
if host == "" {
@@ -522,6 +617,8 @@ func availabilityResourceFromTarget(target config.AvailabilityTarget, status Ava
TargetKind: string(target.TargetKind),
Address: target.Address,
Protocol: string(target.Protocol),
ProbeOutcome: status.Outcome,
UDPMode: string(target.UDPMode),
Port: target.Port,
Path: target.Path,
Enabled: target.Enabled,
@@ -0,0 +1,106 @@
package monitoring
import (
"context"
"net"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
)
func udpTestTarget(address string, port int) config.AvailabilityTarget {
return config.NormalizeAvailabilityTarget(config.AvailabilityTarget{
ID: "udp-test", Name: "UDP test", TargetKind: config.AvailabilityTargetService,
Address: address, Protocol: config.AvailabilityProbeUDP, Port: port, Enabled: true,
TimeoutMillis: 250, UDPMode: config.AvailabilityUDPResponseRequired, UDPRequest: "PING",
})
}
func TestProbeAvailabilityTargetUDPResponseRequired(t *testing.T) {
listener, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0})
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go func() {
buffer := make([]byte, 32)
_, remote, readErr := listener.ReadFromUDP(buffer)
if readErr == nil {
_, _ = listener.WriteToUDP([]byte("PONG"), remote)
}
}()
port := listener.LocalAddr().(*net.UDPAddr).Port
target := udpTestTarget("127.0.0.1", port)
target.UDPExpected = "PONG"
outcome, err := ProbeAvailabilityTargetResult(context.Background(), target)
if err != nil || outcome != AvailabilityProbeReachable {
t.Fatalf("UDP outcome = %q, error = %v", outcome, err)
}
}
func TestProbeAvailabilityTargetUDPOpenOrFilteredIsIndeterminate(t *testing.T) {
listener, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0})
if err != nil {
t.Fatal(err)
}
defer listener.Close()
go func() {
buffer := make([]byte, 32)
_, _, _ = listener.ReadFromUDP(buffer)
}()
port := listener.LocalAddr().(*net.UDPAddr).Port
target := udpTestTarget("127.0.0.1", port)
target.UDPMode = config.AvailabilityUDPOpenOrFiltered
target.UDPRequest = ""
started := time.Now()
outcome, err := ProbeAvailabilityTargetResult(context.Background(), target)
if err != nil || outcome != AvailabilityProbeIndeterminate {
t.Fatalf("UDP outcome = %q, error = %v", outcome, err)
}
if time.Since(started) < 200*time.Millisecond {
t.Fatalf("indeterminate result returned before the response deadline")
}
}
func TestAvailabilityUDPValidationRequiresPayloadInResponseMode(t *testing.T) {
target := udpTestTarget("127.0.0.1", 53)
target.UDPRequest = ""
if err := target.Validate(); err == nil {
t.Fatal("Validate() error = nil, want response-required payload error")
}
target.UDPMode = config.AvailabilityUDPOpenOrFiltered
if err := target.Validate(); err != nil {
t.Fatalf("open-or-filtered Validate() error = %v", err)
}
}
func TestAvailabilityUDPIndeterminateDoesNotClaimReachabilityOrAccumulateFailures(t *testing.T) {
target := udpTestTarget("127.0.0.1", 53)
target.UDPMode = config.AvailabilityUDPOpenOrFiltered
target.UDPRequest = ""
monitor := &Monitor{availabilityStatuses: map[string]AvailabilityProbeStatus{
target.ID: {TargetID: target.ID, ConsecutiveFailures: 2},
}}
checkedAt := time.Now().UTC()
monitor.setAvailabilityStatus(target, checkedAt, 250*time.Millisecond, AvailabilityProbeIndeterminate, nil)
status := monitor.AvailabilityStatusSnapshot()[target.ID]
if status.Available {
t.Fatal("indeterminate UDP result must not be reported as reachable")
}
if status.Outcome != string(AvailabilityProbeIndeterminate) {
t.Fatalf("outcome = %q, want indeterminate", status.Outcome)
}
if status.ConsecutiveFailures != 0 {
t.Fatalf("consecutive failures = %d, want 0", status.ConsecutiveFailures)
}
if got := availabilityResourceStatus(target, status); got != "warning" {
t.Fatalf("resource status = %q, want warning", got)
}
if incident := availabilityIncident(target, status, checkedAt); incident != nil {
t.Fatalf("indeterminate UDP result created an incident: %#v", incident)
}
}
+9
View File
@@ -1716,6 +1716,7 @@ func New(cfg *config.Config) (*Monitor, error) {
}
m.executor = newRealExecutor(m)
m.alertManager.SetBackupIntentContextResolver(m.resolveBackupIntentContext)
m.registerBuiltInPollProviders()
m.buildInstanceInfoCache(cfg)
@@ -1741,6 +1742,13 @@ func New(cfg *config.Config) (*Monitor, error) {
} else {
log.Warn().Err(err).Msg("failed to load alert configuration")
}
if intentPolicies, err := m.configPersist.LoadAlertIntentPolicies(); err == nil {
if err := m.alertManager.LoadIntentPolicies(*intentPolicies); err != nil {
log.Warn().Err(err).Msg("failed to install alert intent policies")
}
} else {
log.Warn().Err(err).Msg("failed to load alert intent policies")
}
if emailConfig, err := m.configPersist.LoadEmailConfig(); err == nil {
m.notificationMgr.SetEmailConfig(*emailConfig)
@@ -4281,6 +4289,7 @@ func (m *Monitor) SetResourceStore(store ResourceStoreInterface) {
m.resourceStore = store
incidentStore := m.incidentStore
m.mu.Unlock()
m.installOperatorIntentResolver(store)
log.Info().Msg("resource store set for polling optimization")
if incidentStore != nil {
@@ -295,6 +295,7 @@ func TestMonitor_HandleAlertResolved_SendsRecoveryForGuestPoweredOffState(t *tes
t.Cleanup(srv.Close)
notifMgr := notifications.NewNotificationManagerWithDataDir("http://pulse.example", t.TempDir())
t.Cleanup(notifMgr.Stop)
if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
}
@@ -308,7 +309,8 @@ func TestMonitor_HandleAlertResolved_SendsRecoveryForGuestPoweredOffState(t *tes
notifMgr.SetNotifyOnResolve(true)
notifMgr.SetGroupingWindow(0)
alertMgr := alerts.NewManager()
alertMgr := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(alertMgr.Stop)
cfg := alertMgr.GetConfig()
cfg.Enabled = true
cfg.ActivationState = alerts.ActivationActive
@@ -0,0 +1,88 @@
package monitoring
import (
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
const backupIntentEvidenceMaxAge = 5 * time.Minute
type resourceOperatorIntentReader interface {
GetResourceOperatorState(canonicalID string) (unifiedresources.ResourceOperatorState, bool, error)
}
type resourceIntentIdentityReader interface {
ResolveCanonicalResourceID(ref string) (string, bool)
}
func (m *Monitor) installOperatorIntentResolver(store ResourceStoreInterface) {
if m == nil || m.alertManager == nil {
return
}
identityReader, hasIdentityReader := store.(resourceIntentIdentityReader)
if hasIdentityReader && identityReader != nil {
m.alertManager.SetResourceIntentIdentityResolver(identityReader.ResolveCanonicalResourceID)
} else {
m.alertManager.SetResourceIntentIdentityResolver(nil)
}
reader, ok := store.(resourceOperatorIntentReader)
if !ok || reader == nil {
m.alertManager.SetOperatorIntentContextResolver(nil)
return
}
m.alertManager.SetOperatorIntentContextResolver(func(resourceID string, _ time.Time) (alerts.OperatorIntentContext, bool) {
if hasIdentityReader {
if canonicalID, found := identityReader.ResolveCanonicalResourceID(resourceID); found {
resourceID = canonicalID
}
}
state, found, err := reader.GetResourceOperatorState(resourceID)
if err != nil {
log.Warn().Err(err).Str("resourceID", resourceID).Msg("Failed to read operator state for alert intent")
return alerts.OperatorIntentContext{}, false
}
if !found {
return alerts.OperatorIntentContext{}, false
}
return alerts.OperatorIntentContext{
IntentionallyOffline: state.IntentionallyOffline,
MaintenanceStartAt: state.MaintenanceStartAt,
MaintenanceEndAt: state.MaintenanceEndAt,
MaintenanceReason: state.MaintenanceReason,
}, true
})
}
func (m *Monitor) resolveBackupIntentContext(_ string, instance, node string, vmid int, now time.Time) (alerts.BackupIntentContext, bool) {
if m == nil || m.state == nil || vmid <= 0 {
return alerts.BackupIntentContext{}, false
}
instance = strings.TrimSpace(instance)
node = strings.TrimSpace(node)
if now.IsZero() {
now = time.Now().UTC()
}
for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {
if task.VMID != vmid || (instance != "" && task.Instance != instance) || (node != "" && task.Node != "" && task.Node != node) {
continue
}
if task.ObservedAt.IsZero() || now.Sub(task.ObservedAt) > backupIntentEvidenceMaxAge || task.ObservedAt.After(now.Add(time.Minute)) {
continue
}
status := strings.ToLower(strings.TrimSpace(task.Status))
if !task.EndTime.IsZero() || status == "ok" || status == "stopped" || status == "error" || status == "warning" {
continue
}
return alerts.BackupIntentContext{
Active: true,
ObservedAt: task.ObservedAt,
Evidence: "pve_vzdump_task:" + task.ID,
}, true
}
return alerts.BackupIntentContext{}, false
}
@@ -0,0 +1,70 @@
package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestResolveBackupIntentContextRequiresFreshActiveMatchingEvidence(t *testing.T) {
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
state := models.NewState()
state.UpdateBackupTasksForInstance("pve-a", []models.BackupTask{
{
ID: "active-101",
Instance: "pve-a",
Node: "node-a",
VMID: 101,
Status: "running",
ObservedAt: now.Add(-time.Minute),
},
{
ID: "stale-102",
Instance: "pve-a",
Node: "node-a",
VMID: 102,
Status: "running",
ObservedAt: now.Add(-backupIntentEvidenceMaxAge - time.Second),
},
{
ID: "finished-103",
Instance: "pve-a",
Node: "node-a",
VMID: 103,
Status: "OK",
ObservedAt: now.Add(-time.Minute),
EndTime: now.Add(-30 * time.Second),
},
})
monitor := &Monitor{state: state}
context, found := monitor.resolveBackupIntentContext("", "pve-a", "node-a", 101, now)
if !found || !context.Active {
t.Fatalf("fresh active task did not resolve: found=%v context=%+v", found, context)
}
if context.ObservedAt != now.Add(-time.Minute) {
t.Fatalf("observedAt = %v, want %v", context.ObservedAt, now.Add(-time.Minute))
}
if context.Evidence != "pve_vzdump_task:active-101" {
t.Fatalf("evidence = %q, want active task identity", context.Evidence)
}
for _, tc := range []struct {
name string
instance string
node string
vmid int
}{
{name: "wrong instance", instance: "pve-b", node: "node-a", vmid: 101},
{name: "wrong node", instance: "pve-a", node: "node-b", vmid: 101},
{name: "stale", instance: "pve-a", node: "node-a", vmid: 102},
{name: "finished", instance: "pve-a", node: "node-a", vmid: 103},
} {
t.Run(tc.name, func(t *testing.T) {
if got, ok := monitor.resolveBackupIntentContext("", tc.instance, tc.node, tc.vmid, now); ok {
t.Fatalf("unexpected backup intent context: %+v", got)
}
})
}
}
+8 -7
View File
@@ -1993,13 +1993,14 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
taskID := fmt.Sprintf("%s-%s", instanceName, task.UPID)
backupTask := models.BackupTask{
ID: taskID,
Node: task.Node,
Instance: instanceName,
Type: task.Type,
VMID: vmid,
Status: task.Status,
StartTime: time.Unix(task.StartTime, 0),
ID: taskID,
Node: task.Node,
Instance: instanceName,
Type: task.Type,
VMID: vmid,
Status: task.Status,
StartTime: time.Unix(task.StartTime, 0),
ObservedAt: time.Now().UTC(),
}
if task.EndTime > 0 {
@@ -763,6 +763,9 @@ func TestMonitor_PollBackupAndReplication(t *testing.T) {
if len(state.PVEBackups.BackupTasks) != 1 {
t.Errorf("Expected 1 backup task, got %d", len(state.PVEBackups.BackupTasks))
}
if len(state.PVEBackups.BackupTasks) == 1 && state.PVEBackups.BackupTasks[0].ObservedAt.IsZero() {
t.Error("Expected polled backup task to carry producer observation time")
}
m.pollReplicationStatus(context.Background(), "pve-test", client, []models.VM{{VMID: 101, Name: "vm1"}})
state = m.state.GetSnapshot()
@@ -0,0 +1,73 @@
package notifications
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
)
func TestResolvedJobsRequireReceiptForSameOccurrenceAndDestination(t *testing.T) {
manager := NewNotificationManagerWithDataDir("", t.TempDir())
defer manager.Stop()
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
alert := &alerts.Alert{ID: "vm-offline-101", StartTime: start}
first := WebhookConfig{ID: "first", URL: "https://first.example.test", Enabled: true}
second := WebhookConfig{ID: "second", URL: "https://second.example.test", Enabled: true}
firingJob := notificationDeliveryJob{Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &first}
manager.recordSuccessfulDelivery(firingJob, start.Add(time.Second))
resolvedJobs := buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{first, second}, AppriseConfig{}, []*alerts.Alert{alert}, eventResolved, start.Add(time.Minute))
filtered := manager.filterResolvedJobsByDeliveryReceipt(resolvedJobs)
if len(filtered) != 1 || filtered[0].WebhookConfig == nil || filtered[0].WebhookConfig.ID != "first" {
t.Fatalf("filtered jobs = %+v", filtered)
}
newOccurrence := alert.Clone()
newOccurrence.StartTime = start.Add(2 * time.Hour)
resolvedJobs = buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{first}, AppriseConfig{}, []*alerts.Alert{newOccurrence}, eventResolved, start.Add(3*time.Hour))
if got := manager.filterResolvedJobsByDeliveryReceipt(resolvedJobs); len(got) != 0 {
t.Fatalf("new occurrence inherited old receipt: %+v", got)
}
}
func TestResolvedJobsDoNotReuseReceiptAfterWebhookURLChanges(t *testing.T) {
manager := NewNotificationManagerWithDataDir("", t.TempDir())
defer manager.Stop()
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
alert := &alerts.Alert{ID: "vm-offline-102", StartTime: start}
original := WebhookConfig{ID: "ops", URL: "https://first.example.test", Enabled: true}
reconfigured := WebhookConfig{ID: "ops", URL: "https://second.example.test", Enabled: true}
manager.recordSuccessfulDelivery(notificationDeliveryJob{
Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &original,
}, start.Add(time.Second))
resolvedJobs := buildNotificationDeliveryJobs(
EmailConfig{}, []WebhookConfig{reconfigured}, AppriseConfig{}, []*alerts.Alert{alert}, eventResolved, start.Add(time.Minute),
)
if got := manager.filterResolvedJobsByDeliveryReceipt(resolvedJobs); len(got) != 0 {
t.Fatalf("reconfigured webhook inherited receipt for old URL: %+v", got)
}
}
func TestDeliveryReceiptPersistsAndIsClearedAfterRecovery(t *testing.T) {
dataDir := t.TempDir()
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
alert := &alerts.Alert{ID: "disk-critical-1", StartTime: start}
webhook := WebhookConfig{ID: "ops", URL: "https://ops.example.test", Enabled: true}
first := NewNotificationManagerWithDataDir("", dataDir)
first.recordSuccessfulDelivery(notificationDeliveryJob{Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook}, start.Add(time.Second))
first.Stop()
second := NewNotificationManagerWithDataDir("", dataDir)
defer second.Stop()
resolved := notificationDeliveryJob{Type: "webhook", Event: eventResolved, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook}
if got := second.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 1 {
t.Fatalf("persisted receipt not found: %+v", got)
}
second.recordSuccessfulDelivery(resolved, start.Add(time.Minute))
if got := second.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 0 {
t.Fatalf("receipt remained after recovery delivery: %+v", got)
}
}
+187 -24
View File
@@ -238,6 +238,7 @@ type NotificationManager struct {
cooldown time.Duration
notifyOnResolve bool
lastNotified map[string]notificationRecord
deliveryReceipts map[string]struct{}
groupWindow time.Duration
pendingAlerts []*alerts.Alert
groupTimer *time.Timer
@@ -686,11 +687,12 @@ func NewNotificationManagerWithDataDir(publicURL string, dataDir string) *Notifi
}
nm := &NotificationManager{
enabled: true,
cooldown: 5 * time.Minute,
notifyOnResolve: true,
lastNotified: make(map[string]notificationRecord),
webhooks: []WebhookConfig{},
enabled: true,
cooldown: 5 * time.Minute,
notifyOnResolve: true,
lastNotified: make(map[string]notificationRecord),
deliveryReceipts: make(map[string]struct{}),
webhooks: []WebhookConfig{},
appriseConfig: AppriseConfig{
Enabled: false,
Mode: AppriseModeCLI,
@@ -1075,13 +1077,14 @@ func (n *NotificationManager) sendAlert(alert *alerts.Alert, options alertSendOp
time.Time{},
options.target,
)
if len(jobs) == 0 {
n.markAlertsNotified(alertsToSend, time.Now())
return
}
if queue != nil {
if anyFailed := n.enqueueNotificationJobs(queue, jobs); anyFailed {
n.markAlertsNotified(alertsToSend, time.Now())
}
n.enqueueNotificationJobs(queue, jobs)
} else {
n.dispatchNotificationJobsAsync(jobs)
n.markAlertsNotified(alertsToSend, time.Now())
}
return
}
@@ -1128,6 +1131,155 @@ func (n *NotificationManager) markAlertsNotified(alertsToSend []*alerts.Alert, s
n.mu.Unlock()
}
func notificationDeliveryDestinationKey(job notificationDeliveryJob) string {
var identity string
switch job.Type {
case "email":
if job.EmailConfig == nil {
return ""
}
recipients := append([]string(nil), job.EmailConfig.To...)
if len(recipients) == 0 && strings.TrimSpace(job.EmailConfig.From) != "" {
recipients = append(recipients, job.EmailConfig.From)
}
for i := range recipients {
recipients[i] = strings.ToLower(strings.TrimSpace(recipients[i]))
}
sort.Strings(recipients)
identity = strings.Join(recipients, "\x1f")
case "webhook":
if job.WebhookConfig == nil {
return ""
}
// Include both the logical configuration ID and the actual endpoint.
// Reusing an ID after changing its URL must not send a recovery to an
// endpoint that never received the firing occurrence.
webhookID := strings.TrimSpace(job.WebhookConfig.ID)
webhookURL := strings.TrimSpace(job.WebhookConfig.URL)
if webhookID == "" && webhookURL == "" {
return ""
}
identity = strings.Join([]string{webhookID, webhookURL}, "\x1f")
case "apprise":
if job.AppriseConfig == nil {
return ""
}
targets := append([]string(nil), job.AppriseConfig.Targets...)
for i := range targets {
targets[i] = strings.TrimSpace(targets[i])
}
sort.Strings(targets)
identity = strings.Join([]string{
string(job.AppriseConfig.Mode),
strings.TrimSpace(job.AppriseConfig.ServerURL),
strings.TrimSpace(job.AppriseConfig.ConfigKey),
strings.Join(targets, "\x1f"),
}, "\x1e")
default:
return ""
}
if identity == "" {
return ""
}
digest := sha256.Sum256([]byte(job.Type + "\x00" + identity))
return job.Type + ":" + hex.EncodeToString(digest[:16])
}
func notificationDeliveryReceiptKey(alert *alerts.Alert, destinationKey string) string {
if alert == nil || alert.ID == "" || alert.StartTime.IsZero() || destinationKey == "" {
return ""
}
return alert.ID + "\x00" + strconv.FormatInt(alert.StartTime.UnixNano(), 10) + "\x00" + destinationKey
}
func (n *NotificationManager) recordSuccessfulDelivery(job notificationDeliveryJob, deliveredAt time.Time) {
destinationKey := notificationDeliveryDestinationKey(job)
if destinationKey == "" {
return
}
queue := n.GetQueue()
for _, alert := range job.Alerts {
if alert == nil || alert.StartTime.IsZero() {
continue
}
if queue != nil {
var err error
if job.Event == eventResolved {
err = queue.DeleteDeliveryReceipt(alert.ID, alert.StartTime, destinationKey)
} else {
err = queue.RecordDeliveryReceipt(alert.ID, alert.StartTime, destinationKey, deliveredAt)
}
if err != nil {
log.Error().Err(err).Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Failed to update notification delivery receipt")
}
continue
}
key := notificationDeliveryReceiptKey(alert, destinationKey)
if key == "" {
continue
}
n.mu.Lock()
if n.deliveryReceipts == nil {
n.deliveryReceipts = make(map[string]struct{})
}
if job.Event == eventResolved {
delete(n.deliveryReceipts, key)
} else {
n.deliveryReceipts[key] = struct{}{}
}
n.mu.Unlock()
}
if job.Event == eventAlert {
// Publish the cooldown/delivered marker only after the per-destination
// receipts are durable. Resolution can race the delivery goroutine and
// uses this marker as proof that receipt recording has completed.
n.markAlertsNotified(job.Alerts, deliveredAt)
}
}
func (n *NotificationManager) filterResolvedJobsByDeliveryReceipt(jobs []notificationDeliveryJob) []notificationDeliveryJob {
queue := n.GetQueue()
filtered := make([]notificationDeliveryJob, 0, len(jobs))
for _, job := range jobs {
destinationKey := notificationDeliveryDestinationKey(job)
if destinationKey == "" {
continue
}
eligibleAlerts := make([]*alerts.Alert, 0, len(job.Alerts))
for _, alert := range job.Alerts {
if alert == nil || alert.StartTime.IsZero() {
continue
}
eligible := false
if queue != nil {
var err error
eligible, err = queue.HasDeliveryReceipt(alert.ID, alert.StartTime, destinationKey)
if err != nil {
log.Error().Err(err).Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Failed to read notification delivery receipt; suppressing recovery")
continue
}
} else {
key := notificationDeliveryReceiptKey(alert, destinationKey)
n.mu.RLock()
_, eligible = n.deliveryReceipts[key]
n.mu.RUnlock()
}
if eligible {
eligibleAlerts = append(eligibleAlerts, alert)
continue
}
log.Debug().Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Resolved notification suppressed because this destination did not receive the firing occurrence")
}
if len(eligibleAlerts) == 0 {
continue
}
job.Alerts = eligibleAlerts
filtered = append(filtered, job)
}
return filtered
}
// SendResolvedAlert delivers notifications for a resolved alert immediately.
func (n *NotificationManager) SendResolvedAlert(resolved *alerts.ResolvedAlert) {
if resolved == nil || resolved.Alert == nil {
@@ -1163,6 +1315,7 @@ func (n *NotificationManager) SendResolvedAlert(resolved *alerts.ResolvedAlert)
alertsToSend := []*alerts.Alert{alertCopy}
jobs := buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertsToSend, eventResolved, resolvedAt)
jobs = n.filterResolvedJobsByDeliveryReceipt(jobs)
if queue != nil {
n.enqueueNotificationJobs(queue, jobs)
@@ -1265,17 +1418,20 @@ func (n *NotificationManager) sendGroupedAlerts() {
n.mu.Unlock()
jobs := buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertsToSend, eventAlert, time.Time{})
if len(jobs) == 0 {
// Preserve cooldown semantics when notifications are globally enabled
// but no destination is configured. Delivery receipts remain empty, so
// a later recovery is still correctly suppressed.
n.markAlertsNotified(alertsToSend, time.Now())
return
}
// Use persistent queue if available, otherwise send directly
if queue != nil {
if anyFailed := n.enqueueNotificationJobs(queue, jobs); anyFailed {
n.markAlertsNotified(alertsToSend, time.Now())
}
n.enqueueNotificationJobs(queue, jobs)
// Note: Cooldown will be marked after successful dequeue and send
} else {
n.dispatchNotificationJobsAsync(jobs)
// For direct sends, mark cooldown immediately (fire-and-forget)
n.markAlertsNotified(alertsToSend, time.Now())
}
}
@@ -1512,7 +1668,9 @@ func (n *NotificationManager) dispatchNotificationJobAsync(job notificationDeliv
spawnAsync(func() {
if err := n.deliverNotificationJob(job); err != nil {
n.logNotificationJobError(job, err, failureMessage)
return
}
n.recordSuccessfulDelivery(job, time.Now())
})
}
@@ -3469,7 +3627,10 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
Int("alertCount", len(notif.Alerts)).
Msg("processing queued notification")
var err error
var (
err error
deliveredJob notificationDeliveryJob
)
switch baseType {
case "email":
var emailConfig EmailConfig
@@ -3485,13 +3646,14 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
Msg("skipping queued email notification because email delivery is disabled")
return nil
}
err = n.deliverNotificationJob(notificationDeliveryJob{
deliveredJob = notificationDeliveryJob{
Type: "email",
Event: event,
Alerts: notif.Alerts,
ResolvedAt: resolvedTimeFromAlerts(notif.Alerts),
EmailConfig: &emailConfig,
})
}
err = n.deliverNotificationJob(deliveredJob)
case "webhook":
var webhookConfig WebhookConfig
@@ -3507,13 +3669,14 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
Msg("skipping queued webhook notification because delivery is disabled")
return nil
}
err = n.deliverNotificationJob(notificationDeliveryJob{
deliveredJob = notificationDeliveryJob{
Type: "webhook",
Event: event,
Alerts: notif.Alerts,
ResolvedAt: resolvedTimeFromAlerts(notif.Alerts),
WebhookConfig: &webhookConfig,
})
}
err = n.deliverNotificationJob(deliveredJob)
case "apprise":
var appriseConfig AppriseConfig
@@ -3529,21 +3692,21 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
Msg("skipping queued Apprise notification because delivery is disabled")
return nil
}
err = n.deliverNotificationJob(notificationDeliveryJob{
deliveredJob = notificationDeliveryJob{
Type: "apprise",
Event: event,
Alerts: notif.Alerts,
ResolvedAt: resolvedTimeFromAlerts(notif.Alerts),
AppriseConfig: &appriseConfig,
})
}
err = n.deliverNotificationJob(deliveredJob)
default:
return fmt.Errorf("unknown notification type: %s", baseType)
}
// Mark cooldown after successful send for active alerts only
if err == nil && event == eventAlert {
n.markAlertsNotified(notif.Alerts, time.Now())
if err == nil {
n.recordSuccessfulDelivery(deliveredJob, time.Now())
}
if err != nil {
+21 -8
View File
@@ -3927,12 +3927,13 @@ func TestSendResolvedAlert(t *testing.T) {
defer server.Close()
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
nm.AddWebhook(WebhookConfig{
webhook := WebhookConfig{
ID: "test-webhook",
Name: "Test",
URL: server.URL,
Enabled: true,
})
}
nm.AddWebhook(webhook)
nm.SetNotifyOnResolve(true)
@@ -3948,6 +3949,9 @@ func TestSendResolvedAlert(t *testing.T) {
Alert: alert,
ResolvedTime: time.Now(),
}
nm.recordSuccessfulDelivery(notificationDeliveryJob{
Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook,
}, time.Now())
// This should send immediately (no grouping)
nm.SendResolvedAlert(resolved)
@@ -4770,8 +4774,8 @@ func TestSendGroupedAlertsQueueEnqueueFailureDoesNotDeadlock(t *testing.T) {
nm.mu.RLock()
_, notified := nm.lastNotified[alert.ID]
nm.mu.RUnlock()
if !notified {
t.Fatal("expected cooldown record to be marked for fallback direct send")
if notified {
t.Fatal("did not expect a cooldown record when fallback delivery never ran")
}
}
@@ -4815,21 +4819,21 @@ func TestSendResolvedAlertQueueEnqueueFailureFallsBackToSharedDeliveryExecutor(t
}))
defer server.Close()
nm := NewNotificationManager("https://pulse.local")
nm := NewNotificationManagerWithDataDir("https://pulse.local", t.TempDir())
defer nm.Stop()
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
nm.AddWebhook(WebhookConfig{
webhook := WebhookConfig{
ID: "resolved-fallback-hook",
Name: "resolved-fallback",
URL: server.URL + "/hook",
Enabled: true,
})
}
nm.AddWebhook(webhook)
queue := nm.GetQueue()
if queue == nil {
t.Skip("queue unavailable in this environment")
}
_ = queue.Stop()
alert := &alerts.Alert{
ID: "resolved-fallback-1",
@@ -4843,6 +4847,15 @@ func TestSendResolvedAlertQueueEnqueueFailureFallsBackToSharedDeliveryExecutor(t
StartTime: time.Now().Add(-5 * time.Minute),
}
resolvedAt := time.Now()
nm.recordSuccessfulDelivery(notificationDeliveryJob{
Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook,
}, resolvedAt.Add(-time.Minute))
// Keep the receipt store available while forcing only queue insertion to
// fail. Recovery delivery must remain fail-closed if receipt lookup itself
// is unavailable.
if _, err := queue.db.Exec(`DROP TABLE notification_queue`); err != nil {
t.Fatalf("force queue enqueue failure: %v", err)
}
nm.SendResolvedAlert(&alerts.ResolvedAlert{
Alert: alert,
+269 -27
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"net/url"
"os"
"sort"
"strings"
"sync"
"time"
@@ -65,6 +66,8 @@ type QueuedNotification struct {
// NotificationQueue manages persistent notification delivery with retries and DLQ
type NotificationQueue struct {
mu sync.RWMutex
deliveryGateMu sync.Mutex
deliveryGates map[string]*notificationDeliveryGate
stopOnce sync.Once
stopErr error
db *sql.DB
@@ -78,6 +81,14 @@ type NotificationQueue struct {
workerSem chan struct{} // Semaphore for limiting concurrent workers
}
// notificationDeliveryGate orders delivery and cancellation for a single
// alert identifier. References are counted so high-cardinality alert IDs do
// not accumulate in the queue for the lifetime of the process.
type notificationDeliveryGate struct {
mu sync.RWMutex
refs int
}
func queueSQLiteDSN(path string) string {
separator := "?"
if strings.Contains(path, "?") {
@@ -131,6 +142,7 @@ func newNotificationQueueFromDSN(dbPath, dsn string) (*NotificationQueue, error)
nq := &NotificationQueue{
db: db,
dbPath: dbPath,
deliveryGates: make(map[string]*notificationDeliveryGate),
stopChan: make(chan struct{}),
processorTicker: time.NewTicker(5 * time.Second),
cleanupTicker: time.NewTicker(1 * time.Hour),
@@ -189,6 +201,86 @@ func NewInMemoryNotificationQueue() (*NotificationQueue, error) {
return newNotificationQueueFromDSN(":memory:", queueSQLiteDSN("file::memory:?mode=memory"))
}
func normalizeAlertIdentifiers(alertIdentifiers []string) []string {
seen := make(map[string]struct{}, len(alertIdentifiers))
normalized := make([]string, 0, len(alertIdentifiers))
for _, identifier := range alertIdentifiers {
identifier = strings.TrimSpace(identifier)
if identifier == "" {
continue
}
if _, exists := seen[identifier]; exists {
continue
}
seen[identifier] = struct{}{}
normalized = append(normalized, identifier)
}
sort.Strings(normalized)
return normalized
}
func alertIdentifiersFromAlerts(alertList []*alerts.Alert) []string {
identifiers := make([]string, 0, len(alertList))
for _, alert := range alertList {
if alert != nil {
identifiers = append(identifiers, alert.ID)
}
}
return normalizeAlertIdentifiers(identifiers)
}
// acquireAlertDeliveryGates coordinates a firing delivery with resolution
// cancellation for only the alert IDs involved in that operation. Acquiring
// IDs in sorted order prevents grouped notifications from deadlocking when
// different alerts resolve concurrently.
func (nq *NotificationQueue) acquireAlertDeliveryGates(alertIdentifiers []string, write bool) func() {
identifiers := normalizeAlertIdentifiers(alertIdentifiers)
if len(identifiers) == 0 {
return func() {}
}
nq.deliveryGateMu.Lock()
gates := make([]*notificationDeliveryGate, 0, len(identifiers))
for _, identifier := range identifiers {
gate := nq.deliveryGates[identifier]
if gate == nil {
gate = &notificationDeliveryGate{}
nq.deliveryGates[identifier] = gate
}
gate.refs++
gates = append(gates, gate)
}
nq.deliveryGateMu.Unlock()
for _, gate := range gates {
if write {
gate.mu.Lock()
} else {
gate.mu.RLock()
}
}
return func() {
for i := len(gates) - 1; i >= 0; i-- {
if write {
gates[i].mu.Unlock()
} else {
gates[i].mu.RUnlock()
}
}
nq.deliveryGateMu.Lock()
for i, identifier := range identifiers {
gate := gates[i]
gate.refs--
if gate.refs == 0 && nq.deliveryGates[identifier] == gate {
delete(nq.deliveryGates, identifier)
}
}
nq.deliveryGateMu.Unlock()
}
}
// initSchema creates the database tables
func (nq *NotificationQueue) initSchema() error {
schema := `
@@ -235,6 +327,17 @@ func (nq *NotificationQueue) initSchema() error {
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON notification_audit(timestamp);
CREATE INDEX IF NOT EXISTS idx_audit_notification_id ON notification_audit(notification_id);
CREATE INDEX IF NOT EXISTS idx_audit_status ON notification_audit(status);
CREATE TABLE IF NOT EXISTS notification_delivery_receipts (
alert_id TEXT NOT NULL,
alert_start_nanos INTEGER NOT NULL,
destination_key TEXT NOT NULL,
delivered_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, alert_start_nanos, destination_key)
);
CREATE INDEX IF NOT EXISTS idx_delivery_receipts_delivered_at
ON notification_delivery_receipts(delivered_at);
`
if _, err := nq.db.Exec(schema); err != nil {
@@ -256,6 +359,63 @@ func (nq *NotificationQueue) initSchema() error {
)
}
func (nq *NotificationQueue) RecordDeliveryReceipt(alertID string, alertStart time.Time, destinationKey string, deliveredAt time.Time) error {
if nq == nil || strings.TrimSpace(alertID) == "" || strings.TrimSpace(destinationKey) == "" || alertStart.IsZero() {
return fmt.Errorf("alert occurrence and destination are required for a delivery receipt")
}
if deliveredAt.IsZero() {
deliveredAt = time.Now()
}
nq.mu.Lock()
defer nq.mu.Unlock()
_, err := nq.db.Exec(`
INSERT INTO notification_delivery_receipts
(alert_id, alert_start_nanos, destination_key, delivered_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(alert_id, alert_start_nanos, destination_key)
DO UPDATE SET delivered_at = excluded.delivered_at
`, alertID, alertStart.UnixNano(), destinationKey, deliveredAt.Unix())
if err != nil {
return fmt.Errorf("record notification delivery receipt: %w", err)
}
return nil
}
func (nq *NotificationQueue) HasDeliveryReceipt(alertID string, alertStart time.Time, destinationKey string) (bool, error) {
if nq == nil || strings.TrimSpace(alertID) == "" || strings.TrimSpace(destinationKey) == "" || alertStart.IsZero() {
return false, nil
}
nq.mu.RLock()
defer nq.mu.RUnlock()
var marker int
err := nq.db.QueryRow(`
SELECT 1 FROM notification_delivery_receipts
WHERE alert_id = ? AND alert_start_nanos = ? AND destination_key = ?
`, alertID, alertStart.UnixNano(), destinationKey).Scan(&marker)
if err == sql.ErrNoRows {
return false, nil
}
if err != nil {
return false, fmt.Errorf("read notification delivery receipt: %w", err)
}
return true, nil
}
func (nq *NotificationQueue) DeleteDeliveryReceipt(alertID string, alertStart time.Time, destinationKey string) error {
if nq == nil || strings.TrimSpace(alertID) == "" || strings.TrimSpace(destinationKey) == "" || alertStart.IsZero() {
return nil
}
nq.mu.Lock()
defer nq.mu.Unlock()
if _, err := nq.db.Exec(`
DELETE FROM notification_delivery_receipts
WHERE alert_id = ? AND alert_start_nanos = ? AND destination_key = ?
`, alertID, alertStart.UnixNano(), destinationKey); err != nil {
return fmt.Errorf("delete notification delivery receipt: %w", err)
}
return nil
}
func (nq *NotificationQueue) migrateAlertIdentifierColumns() error {
columns, err := nq.tableColumns("notification_audit")
if err != nil {
@@ -698,7 +858,10 @@ func (nq *NotificationQueue) IncrementAttempt(id string) error {
return nil
}
// IncrementAttemptAndSetStatus atomically increments attempt counter and sets status in a single operation
// IncrementAttemptAndSetStatus updates a queue row and its operational links
// atomically. Delivery workers use claimPendingForDelivery so a concurrent
// resolution can cancel a pending row before it is claimed; this method
// remains available for explicit lifecycle transitions and compatibility.
func (nq *NotificationQueue) IncrementAttemptAndSetStatus(id string, status NotificationQueueStatus) error {
nq.mu.Lock()
defer nq.mu.Unlock()
@@ -725,13 +888,75 @@ func (nq *NotificationQueue) IncrementAttemptAndSetStatus(id string, status Noti
operational_links = ?
WHERE id = ?
`
_, err = nq.db.Exec(query, status, now.Unix(), string(linksJSON), id)
if err != nil {
if _, err := nq.db.Exec(query, status, now.Unix(), string(linksJSON), id); err != nil {
return fmt.Errorf("failed to increment attempt and set status: %w", err)
}
return nil
}
// claimPendingForDelivery atomically claims a pending queue row and refreshes
// its alert list. The refresh is required because resolution cancellation may
// have rewritten a grouped row after GetPending returned its snapshot.
func (nq *NotificationQueue) claimPendingForDelivery(notif *QueuedNotification) (bool, error) {
nq.mu.Lock()
defer nq.mu.Unlock()
now := time.Now()
links, err := nq.readNotificationLinksNoLock(notif.ID)
if err != nil {
return false, err
}
links = transitionNotificationLinks(
links,
notificationDeliveryStateForQueueStatus(QueueStatusSending),
now,
)
linksJSON, err := json.Marshal(links)
if err != nil {
return false, fmt.Errorf("marshal notification links for attempt: %w", err)
}
query := `
UPDATE notification_queue
SET attempts = attempts + 1,
status = ?,
last_attempt = ?,
operational_links = ?
WHERE id = ? AND status = 'pending'
`
result, err := nq.db.Exec(
query,
QueueStatusSending,
now.Unix(),
string(linksJSON),
notif.ID,
)
if err != nil {
return false, fmt.Errorf("failed to claim pending notification: %w", err)
}
rows, err := result.RowsAffected()
if err != nil {
return false, fmt.Errorf("read claimed notification row count: %w", err)
}
if rows == 0 {
return false, nil
}
var alertsJSON []byte
if err := nq.db.QueryRow(`SELECT alerts FROM notification_queue WHERE id = ?`, notif.ID).Scan(&alertsJSON); err != nil {
return false, fmt.Errorf("reload claimed notification alerts: %w", err)
}
var refreshedAlerts []*alerts.Alert
if err := json.Unmarshal(alertsJSON, &refreshedAlerts); err != nil {
return false, fmt.Errorf("decode claimed notification alerts: %w", err)
}
notif.Alerts = refreshedAlerts
notif.Attempts++
notif.Status = QueueStatusSending
notif.LastAttempt = &now
notif.Links = links
return true, nil
}
// GetPending returns notifications ready for processing
func (nq *NotificationQueue) GetPending(limit int) ([]*QueuedNotification, error) {
nq.mu.RLock()
@@ -1106,9 +1331,13 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
Msg("Skipping cancelled notification")
return
}
releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiersFromAlerts(notif.Alerts), false)
defer releaseDeliveryGates()
// Atomically increment attempt counter and set status to sending
if err := nq.IncrementAttemptAndSetStatus(notif.ID, QueueStatusSending); err != nil {
// Atomically claim the pending row. A concurrent resolution may have
// cancelled it while it was waiting for its per-alert delivery gate.
claimed, err := nq.claimPendingForDelivery(notif)
if err != nil {
log.Error().
Err(err).
Str("component", "notification_queue").
@@ -1117,19 +1346,18 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
Str("type", notif.Type).
Int("attempt", notif.Attempts+1).
Int("maxAttempts", notif.MaxAttempts).
Msg("Failed to increment attempt and set status")
Msg("Failed to claim pending notification")
return
}
if !claimed {
log.Debug().
Str("component", "notification_queue").
Str("action", "skip_unclaimed").
Str("id", notif.ID).
Str("type", notif.Type).
Msg("Skipping notification because it is no longer pending")
return
}
attemptedAt := time.Now()
notif.Attempts++
notif.Status = QueueStatusSending
notif.LastAttempt = &attemptedAt
notif.Links = transitionNotificationLinks(
notif.Links,
operationaltrust.NotificationDelivering,
attemptedAt,
)
// Call processor if set
nq.mu.RLock()
processor := nq.processor
@@ -1138,7 +1366,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
return
}
err := processor(notif)
err = processor(notif)
success := err == nil
errorMsg := ""
@@ -1285,6 +1513,26 @@ func (nq *NotificationQueue) performCleanup() {
// Keep completed/failed for 7 days, DLQ for 30 days
completedCutoff := time.Now().Add(-7 * 24 * time.Hour).Unix()
dlqCutoff := time.Now().Add(-30 * 24 * time.Hour).Unix()
receiptCutoff := dlqCutoff
// Receipts are only needed to correlate a later recovery with a firing
// occurrence. Bound their lifetime so installations with recovery delivery
// disabled do not grow this side table indefinitely.
if result, err := nq.db.Exec(`DELETE FROM notification_delivery_receipts WHERE delivered_at < ?`, receiptCutoff); err != nil {
log.Error().
Err(err).
Str("component", "notification_queue").
Str("action", "cleanup_delivery_receipts").
Int64("receiptCutoff", receiptCutoff).
Msg("Failed to cleanup old notification delivery receipts")
} else if rows, _ := result.RowsAffected(); rows > 0 {
log.Debug().
Str("component", "notification_queue").
Str("action", "cleanup_delivery_receipts").
Int64("count", rows).
Int64("receiptCutoff", receiptCutoff).
Msg("Cleaned up old notification delivery receipts")
}
// Delete audit records for notifications about to be cleaned up (FK constraint)
auditCleanup := `DELETE FROM notification_audit WHERE notification_id IN (
@@ -1419,9 +1667,12 @@ func calculateBackoff(attempt int) time.Duration {
// mid-send ('sending') are cancelled best-effort but not counted, because
// their delivery may still complete.
func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string) (int, error) {
alertIdentifiers = normalizeAlertIdentifiers(alertIdentifiers)
if len(alertIdentifiers) == 0 {
return 0, nil
}
releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiers, true)
defer releaseDeliveryGates()
nq.mu.Lock()
defer nq.mu.Unlock()
@@ -1439,16 +1690,7 @@ func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string)
alertIdentifierSet := make(map[string]struct{})
for _, id := range alertIdentifiers {
id = strings.TrimSpace(id)
if id != "" {
alertIdentifierSet[id] = struct{}{}
}
}
if len(alertIdentifierSet) == 0 {
if err := rows.Close(); err != nil {
return 0, fmt.Errorf("failed to close cancellation query: %w", err)
}
return 0, nil
alertIdentifierSet[id] = struct{}{}
}
type queuedAlertCancellation struct {
+120
View File
@@ -750,6 +750,126 @@ func TestCancelByAlertIdentifiers_MultipleAlertsPartialMatch(t *testing.T) {
}
}
func TestProcessNotificationReloadsAlertsAfterCancellationRewrite(t *testing.T) {
nq, err := NewNotificationQueue(t.TempDir())
if err != nil {
t.Fatalf("NewNotificationQueue: %v", err)
}
defer func() { _ = nq.Stop() }()
futureRetry := time.Now().Add(time.Hour)
queued := &QueuedNotification{
ID: "notif-rewritten-before-claim", Type: "webhook", Status: QueueStatusPending,
MaxAttempts: 3, Config: []byte(`{}`), NextRetryAt: &futureRetry,
Alerts: []*alerts.Alert{{ID: "alert-1"}, {ID: "alert-2"}},
}
if err := nq.Enqueue(queued); err != nil {
t.Fatalf("Enqueue: %v", err)
}
// Model a worker snapshot returned by GetPending before resolution rewrites
// the durable grouped row.
workerSnapshot := *queued
workerSnapshot.Alerts = append([]*alerts.Alert(nil), queued.Alerts...)
if count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"}); err != nil {
t.Fatalf("CancelByAlertIdentifiers: %v", err)
} else if count != 1 {
t.Fatalf("cancelled alert count = %d, want 1", count)
}
var delivered []string
nq.SetProcessor(func(notif *QueuedNotification) error {
for _, alert := range notif.Alerts {
if alert != nil {
delivered = append(delivered, alert.ID)
}
}
return nil
})
nq.processNotification(&workerSnapshot)
if len(delivered) != 1 || delivered[0] != "alert-2" {
t.Fatalf("delivered alerts = %v, want only alert-2", delivered)
}
}
func TestCancelByAlertIdentifiersWaitsForInFlightDelivery(t *testing.T) {
nq, err := NewNotificationQueue(t.TempDir())
if err != nil {
t.Fatalf("NewNotificationQueue: %v", err)
}
defer func() { _ = nq.Stop() }()
futureRetry := time.Now().Add(time.Hour)
queued := &QueuedNotification{
ID: "notif-in-flight", Type: "webhook", Status: QueueStatusPending,
MaxAttempts: 3, Config: []byte(`{}`), NextRetryAt: &futureRetry,
Alerts: []*alerts.Alert{{ID: "alert-1"}},
}
if err := nq.Enqueue(queued); err != nil {
t.Fatalf("Enqueue: %v", err)
}
started := make(chan struct{})
release := make(chan struct{})
nq.SetProcessor(func(*QueuedNotification) error {
close(started)
<-release
return nil
})
processed := make(chan struct{})
go func() {
nq.processNotification(queued)
close(processed)
}()
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("delivery did not start")
}
cancelled := make(chan int, 1)
cancelErr := make(chan error, 1)
go func() {
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"})
cancelled <- count
cancelErr <- err
}()
select {
case count := <-cancelled:
close(release)
t.Fatalf("cancellation returned before in-flight delivery completed: %d", count)
case <-time.After(50 * time.Millisecond):
}
close(release)
select {
case <-processed:
case <-time.After(2 * time.Second):
t.Fatal("delivery did not finish")
}
select {
case err := <-cancelErr:
if err != nil {
t.Fatalf("CancelByAlertIdentifiers: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("cancellation did not finish")
}
if count := <-cancelled; count != 0 {
t.Fatalf("cancelled alert count = %d, want 0 after delivery", count)
}
var status string
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, queued.ID).Scan(&status); err != nil {
t.Fatalf("read notification status: %v", err)
}
if status != string(QueueStatusSent) {
t.Fatalf("status = %q, want %q", status, QueueStatusSent)
}
}
func TestCancelByAlertIdentifiers_SendingRowsCancelledButNotCounted(t *testing.T) {
tempDir := t.TempDir()
nq, err := NewNotificationQueue(tempDir)
@@ -0,0 +1,38 @@
package unifiedresources
import (
"encoding/json"
"strings"
"testing"
)
func TestAvailabilityDataPreservesThreeStateUDPWireEvidence(t *testing.T) {
resource := Resource{
ID: "availability:syslog",
Type: ResourceTypeNetworkEndpoint,
Availability: &AvailabilityData{
Protocol: "udp",
ProbeOutcome: "indeterminate",
UDPMode: "open_or_filtered",
Available: false,
},
}
payload, err := json.Marshal(resource)
if err != nil {
t.Fatalf("Marshal() error = %v", err)
}
wire := string(payload)
for _, expected := range []string{
`"protocol":"udp"`,
`"probeOutcome":"indeterminate"`,
`"udpMode":"open_or_filtered"`,
} {
if !strings.Contains(wire, expected) {
t.Fatalf("wire payload %s does not contain %s", wire, expected)
}
}
if strings.Contains(wire, `"available":true`) {
t.Fatalf("indeterminate UDP evidence must not claim availability: %s", wire)
}
}
@@ -118,6 +118,28 @@ func (a *MonitorAdapter) GetRecentChanges(canonicalID string, since time.Time, l
return registry.store.GetRecentChanges(canonicalID, since, limit)
}
// GetResourceOperatorState exposes the durable operator-intent record through
// the monitor adapter without widening monitoring's core resource-store
// interface. Alert policy resolution discovers this capability optionally.
func (a *MonitorAdapter) GetResourceOperatorState(canonicalID string) (ResourceOperatorState, bool, error) {
registry := a.currentRegistry()
if registry == nil || registry.store == nil {
return ResourceOperatorState{}, false, nil
}
return registry.store.GetResourceOperatorState(canonicalID)
}
// ResolveCanonicalResourceID bridges source-native alert identifiers to the
// canonical resource identity used by durable operator state and policy UI.
func (a *MonitorAdapter) ResolveCanonicalResourceID(ref string) (string, bool) {
registry := a.currentRegistry()
if registry == nil {
return "", false
}
_, canonicalID, ok := registry.GetByReference(ref)
return canonicalID, ok
}
func (a *MonitorAdapter) replaceRegistry(snapshot models.StateSnapshot, recordsBySource map[DataSource][]IngestRecord) {
registry := a.currentRegistry()
if registry == nil {
@@ -99,6 +99,39 @@ func TestMonitorAdapterReadStateForwardsToRegistry(t *testing.T) {
}
}
func TestMonitorAdapterResolvesCanonicalOperatorIntentCapabilities(t *testing.T) {
registry := NewRegistry(NewMemoryStore())
adapter := NewMonitorAdapter(registry)
adapter.PopulateSupplementalRecords(SourceProxmox, []IngestRecord{{
SourceID: "pve-a:vm:101",
Resource: Resource{
ID: "vm:pve-a:101",
Type: ResourceTypeVM,
Name: "vm-101",
},
}})
canonicalID, found := adapter.ResolveCanonicalResourceID("pve-a:vm:101")
if !found || canonicalID == "" {
t.Fatalf("ResolveCanonicalResourceID() = %q, %v", canonicalID, found)
}
operatorState := ResourceOperatorState{
CanonicalID: canonicalID,
IntentionallyOffline: true,
MaintenanceReason: "planned hardware work",
}
if err := registry.store.SetResourceOperatorState(operatorState); err != nil {
t.Fatalf("SetResourceOperatorState() error = %v", err)
}
got, found, err := adapter.GetResourceOperatorState(canonicalID)
if err != nil || !found {
t.Fatalf("GetResourceOperatorState() found=%v error=%v", found, err)
}
if !got.IntentionallyOffline || got.MaintenanceReason != operatorState.MaintenanceReason {
t.Fatalf("operator state = %+v, want persisted intent", got)
}
}
func TestMonitorAdapterPhysicalDiskReadStateRetainsProxmoxIdentityAfterSMARTMerge(t *testing.T) {
adapter := NewMonitorAdapter(NewRegistry(nil))
now := time.Date(2026, 7, 7, 10, 0, 0, 0, time.UTC)
+2
View File
@@ -1639,6 +1639,8 @@ type AvailabilityData struct {
TargetKind string `json:"targetKind,omitempty"`
Address string `json:"address,omitempty"`
Protocol string `json:"protocol,omitempty"`
ProbeOutcome string `json:"probeOutcome,omitempty"`
UDPMode string `json:"udpMode,omitempty"`
Port int `json:"port,omitempty"`
Path string `json:"path,omitempty"`
Enabled bool `json:"enabled"`
@@ -247,8 +247,10 @@ class CanonicalCompletionGuardTest(unittest.TestCase):
"test_prefixes": [],
"exact_files": [
"internal/monitoring/availability_poller_test.go",
"internal/monitoring/availability_udp_test.go",
"internal/monitoring/canonical_guardrails_test.go",
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/monitoring/monitor_alert_intent_test.go",
"internal/monitoring/monitor_alert_override_migration_test.go",
"internal/monitoring/monitor_backups_readstate_test.go",
"internal/monitoring/monitor_host_agents_test.go",
@@ -2152,9 +2154,11 @@ None yet.
"frontend-modern/src/components/Alerts/__tests__/BulkEditDialog.test.tsx",
"frontend-modern/src/components/Alerts/__tests__/InvestigateAlertButton.test.tsx",
"frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx",
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
"frontend-modern/src/features/alerts/__tests__/OverviewTab.emptystate.test.tsx",
"frontend-modern/src/features/alerts/__tests__/OverviewTab.timelineerror.test.tsx",
"frontend-modern/src/features/alerts/__tests__/OverviewTab.total24h.test.tsx",
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
"frontend-modern/src/features/alerts/__tests__/alertsConfigurationModel.test.ts",
"frontend-modern/src/features/alerts/__tests__/helpers.test.ts",
"frontend-modern/src/features/alerts/identity.test.ts",
@@ -81,12 +81,22 @@ class RegistryAuditTest(unittest.TestCase):
"internal/api/ai_handlers_patrol_actions_additional_test.go",
},
("alerts", "alerts-frontend-surface"): {
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
"frontend-modern/src/features/alerts/thresholds/hooks/__tests__/truenasThresholdPersistence.test.tsx",
},
("alerts", "canonical-alert-runtime"): {
"internal/alerts/intent_policy_test.go",
},
("alerts", "alerts-runtime-support"): {
"internal/alerts/canonical_override_migration_test.go",
"internal/alerts/intent_policy_test.go",
"internal/monitoring/monitor_alert_override_migration_test.go",
},
("api-contracts", "alert-intent-policy-api"): {
"frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts",
"internal/api/alerts_endpoints_test.go",
},
("api-contracts", "backend-payload-contracts"): {
"internal/api/ai_handlers_patrol_actions_additional_test.go",
},
@@ -107,9 +117,16 @@ class RegistryAuditTest(unittest.TestCase):
},
("monitoring", "runtime-report-model"): {
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/monitoring/monitor_full_coverage_test.go",
},
("monitoring", "pbs-protection-evidence-runtime"): {
"internal/monitoring/monitor_alert_intent_test.go",
"internal/monitoring/monitor_full_coverage_test.go",
},
("monitoring", "monitoring-runtime"): {
"internal/monitoring/availability_udp_test.go",
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/monitoring/monitor_alert_intent_test.go",
"internal/monitoring/monitor_alert_override_migration_test.go",
},
("monitoring", "agent-fleet-diagnostics-runtime"): {
@@ -138,6 +155,7 @@ class RegistryAuditTest(unittest.TestCase):
"internal/hostagent/issue1595_sas_collection_test.go",
"internal/monitoring/issue1595_collection_trust_test.go",
"internal/truenas/contract_test.go",
"internal/unifiedresources/availability_projection_test.go",
"pkg/diskinventory/identity_test.go",
"pkg/diskinventory/status_test.go",
},