mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Add alert intent policies and delivery receipts
This commit is contained in:
@@ -4716,3 +4716,14 @@ mutations require `monitoring:write`, and slash-containing canonical record IDs
|
||||
are parsed as opaque identities. Governed action planning still enters the
|
||||
existing action lifecycle and entitlement boundary before any agent executor
|
||||
is considered.
|
||||
|
||||
### Alert intent remains outside agent authority
|
||||
|
||||
Alert-intent policy reads and writes, UDP availability probes, operator-state
|
||||
lookups, and backup-task context do not widen the agent lifecycle. The server
|
||||
evaluates these read-state inputs without issuing an agent command, accepting a
|
||||
new report shape, rotating credentials, or changing remote configuration.
|
||||
Neither an expected-transient decision nor an indeterminate UDP outcome may be
|
||||
translated into an agent action. Any later customer-infrastructure mutation
|
||||
still requires the canonical Actions planner, approval, executor, receipt,
|
||||
audit, and verification boundaries.
|
||||
|
||||
@@ -1186,3 +1186,34 @@ unambiguous/live/ambiguous decision matrix,
|
||||
reload behavior, and
|
||||
`frontend-modern/src/features/alerts/thresholds/hooks/__tests__/truenasThresholdPersistence.test.tsx`
|
||||
proves the browser-side TrueNAS save/refetch contract.
|
||||
|
||||
### Versioned alert-intent policy
|
||||
|
||||
The alerts runtime owns one versioned alert-intent document and its durable
|
||||
pending-condition state. Stable signal keys are `*`, `state.offline`,
|
||||
`incident.availability`, and `metric.<name>`. Effective fields resolve
|
||||
independently from legacy metric behavior, document defaults, resource-type
|
||||
overrides, and canonical-resource overrides, in that order. Keys are
|
||||
normalized before validation and collisions fail closed; updates use revision
|
||||
compare-and-swap so a stale browser cannot overwrite a newer document.
|
||||
|
||||
Intent affects when detector truth becomes eligible for an active alert; it
|
||||
does not mutate the underlying observation or create a second alert identity.
|
||||
Without an explicit applicable rule, established alert behavior remains
|
||||
compatible. With a rule, the first matched time is durable across restart and
|
||||
becomes the canonical alert start time once eligible. Preview evaluates the
|
||||
same resolver but restores pending state before returning, so it is read-only.
|
||||
Invalid documents, persistence failures, or revision conflicts must leave the
|
||||
prior in-memory and durable policy active.
|
||||
|
||||
Operator maintenance and intentionally-offline state are read only through the
|
||||
canonical unified-resource identity. Backup-aware offline deferral consumes
|
||||
fresh, matching, active task evidence, applies the configured post-backup grace,
|
||||
and always terminates at its hard cap. Missing, stale, future-skewed,
|
||||
finished, or mismatched backup evidence cannot suppress an outage. This policy
|
||||
changes alert activation only: notification delivery, recovery assurance, and
|
||||
customer-infrastructure mutation retain their existing owners.
|
||||
|
||||
`internal/alerts/intent_policy_test.go` proves precedence, normalization,
|
||||
operator and backup contexts, preview immutability, restart continuity, and
|
||||
first-match lifecycle identity.
|
||||
|
||||
@@ -7907,3 +7907,28 @@ authorize an alternate execution route.
|
||||
acceptance and typed-conflict payloads, while
|
||||
`frontend-modern/src/features/patrol/__tests__/patrolRunAcceptance.test.ts`
|
||||
proves bounded status/history reconciliation against the returned run ID.
|
||||
|
||||
### Alert intent and UDP availability transport
|
||||
|
||||
`GET /api/alerts/intent-policies` and
|
||||
`POST /api/alerts/intent-policies/preview` require `monitoring:read`;
|
||||
`PUT /api/alerts/intent-policies` requires `monitoring:write`. The versioned
|
||||
document carries schema version, revision, optional update time, and typed
|
||||
default, resource-type, and canonical-resource rules. A stale revision returns
|
||||
`409 Conflict`; invalid policy or preview input returns `400`; unavailable
|
||||
runtime or persistence ownership returns `503`. A save failure restores the
|
||||
previous in-memory document and returns `500`. Preview is bounded and read-only
|
||||
and cannot advance pending grace state.
|
||||
|
||||
Availability targets add `udp` plus `response_required` and
|
||||
`open_or_filtered` modes. Request and expected-response payloads remain
|
||||
explicit configuration fields. Test responses add the probe `outcome` while
|
||||
retaining the existing success, latency, and error fields for compatible
|
||||
clients. A silent open-or-filtered result is `indeterminate`, not successful
|
||||
reachability and not a transport failure that may be promoted to an outage.
|
||||
|
||||
`internal/api/alerts_endpoints_test.go` proves scopes, revision conflicts,
|
||||
rollback, and preview behavior.
|
||||
`frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts` and the
|
||||
availability target API and settings tests prove the canonical browser routes
|
||||
and additive UDP wire fields.
|
||||
|
||||
@@ -5165,3 +5165,28 @@ states show bounded guidance with no executable command. The focused proofs are
|
||||
`frontend-modern/src/components/Settings/__tests__/DiagnosticsResultsPanel.test.tsx`,
|
||||
`frontend-modern/src/components/Settings/__tests__/settingsHeaderMeta.branchcov0713.test.ts`,
|
||||
and `frontend-modern/src/utils/__tests__/updatesPresentation.test.ts`.
|
||||
|
||||
### Alert intent and three-state availability presentation
|
||||
|
||||
The Alerts thresholds surface composes the versioned intent-policy editor from
|
||||
the shared form, status, disclosure, and loading primitives. It must preserve
|
||||
field-wise inheritance: omitted fields inherit, while explicit false and zero
|
||||
values remain deliberate overrides. Save uses the displayed revision, reports
|
||||
revision conflict without replacing local edits, and refreshes from the
|
||||
server-owned document after success. Preview renders clear,
|
||||
expected-transient, pending-grace, and would-activate as distinct states and
|
||||
never presents preview as a write.
|
||||
|
||||
Availability controls expose UDP mode, request payload, and optional expected
|
||||
response only where valid for the selected protocol. Unified-resource
|
||||
presentation keeps `indeterminate` visibly distinct from reachable and
|
||||
unreachable: open-or-filtered UDP uses warning treatment and bounded evidence
|
||||
copy, never a green success tone or a fabricated latency. The primitive layer
|
||||
does not infer detector, operator-intent, or recovery truth.
|
||||
|
||||
The focused proofs are
|
||||
`frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx`,
|
||||
`frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx`,
|
||||
`frontend-modern/src/components/Settings/ConnectionEditor/__tests__/AvailabilityTargetSlot.test.tsx`,
|
||||
and
|
||||
`frontend-modern/src/utils/__tests__/availabilityProbePresentation.test.ts`.
|
||||
|
||||
@@ -1830,3 +1830,29 @@ before point reconciliation so completeness and permission failure cannot be
|
||||
lost behind a successful cached-artifact path. Shared protection semantics stay
|
||||
in `internal/recovery/`; PBS monitoring owns only this explicit evidence-quality
|
||||
adapter.
|
||||
|
||||
### Alert-intent evidence adapters and UDP outcomes
|
||||
|
||||
Monitoring supplies read-only context to the alerts-owned intent resolver. The
|
||||
operator-state adapter resolves source-native references to one canonical
|
||||
unified-resource ID before reading durable operator intent. Lookup failure,
|
||||
ambiguity, absence, or store error yields no suppression context; monitoring
|
||||
does not synthesize maintenance state.
|
||||
|
||||
Backup-aware offline intent consumes a PVE task only when VMID, instance, and
|
||||
node match and the task is active. `pollBackupTasks` stamps server observation
|
||||
time. Evidence older than five minutes, more than one minute in the future,
|
||||
finished, terminal, or missing an observation time fails closed. This
|
||||
short-lived alert context is separate from PBS protection evidence and from
|
||||
recovery assurance; it cannot claim that a backup is restorable or authorize a
|
||||
restore.
|
||||
|
||||
Availability probing owns three outcomes: reachable, unreachable, and
|
||||
indeterminate. UDP response-required mode needs a request and treats timeout or
|
||||
mismatch as unreachable. Open-or-filtered mode may return indeterminate after
|
||||
the full response deadline. Indeterminate clears accumulated failure count,
|
||||
projects warning evidence, and emits no availability incident; it never claims
|
||||
reachability. `internal/monitoring/availability_udp_test.go`,
|
||||
`internal/monitoring/monitor_alert_intent_test.go`, and the backup polling
|
||||
assertion in `internal/monitoring/monitor_full_coverage_test.go` are the focused
|
||||
proofs.
|
||||
|
||||
@@ -256,3 +256,28 @@ retry/sent/failed/dead-letter/cancelled outcomes and open-to-enqueue latency
|
||||
without destination, record, resource, or evidence labels. Delivery state
|
||||
remains notification truth only and cannot resolve or reopen the alert
|
||||
lifecycle.
|
||||
|
||||
### Occurrence-bound delivery receipts
|
||||
|
||||
The notification owner records successful firing delivery by exact alert ID,
|
||||
nanosecond start time, and a normalized destination identity. Email recipients,
|
||||
webhook ID plus URL, and Apprise mode/base/config/targets are part of that
|
||||
identity, so a later occurrence or reconfigured destination cannot inherit an
|
||||
older receipt. A resolved notification is eligible only for destinations that
|
||||
received that exact firing occurrence. Successful recovery delivery deletes
|
||||
the receipt; persistent receipts survive restart and are retention-bounded.
|
||||
Receipt read failure suppresses recovery rather than inventing prior delivery.
|
||||
|
||||
Cooldown state is published only after firing receipts are recorded. Persistent
|
||||
workers atomically claim a still-pending row after taking per-alert delivery
|
||||
gates, and resolution cancellation takes the corresponding exclusive gates.
|
||||
This prevents a stale pending snapshot from being sent after cancellation while
|
||||
preserving grouped-row and operational-link lifecycle semantics. A destination
|
||||
with delivery disabled or failed has no receipt and receives no misleading
|
||||
recovery. These rules remain delivery truth only and cannot resolve the
|
||||
alerts-owned lifecycle.
|
||||
|
||||
`internal/notifications/delivery_receipts_test.go`,
|
||||
`internal/notifications/notifications_test.go`, and
|
||||
`internal/notifications/queue_test.go` prove occurrence and destination
|
||||
isolation, persistence, cleanup, and cancellation/claim ordering.
|
||||
|
||||
@@ -2064,6 +2064,7 @@
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"internal/alerts/intent_policy_test.go",
|
||||
"internal/alerts/migration_characterization_test.go",
|
||||
"internal/alerts/operational_contract_test.go",
|
||||
"internal/alerts/unified_eval_parity_test.go",
|
||||
@@ -2110,11 +2111,13 @@
|
||||
"frontend-modern/src/components/Alerts/Thresholds/hooks/__tests__/useCollapsedSections.test.ts",
|
||||
"frontend-modern/src/components/Alerts/Thresholds/sections/__tests__/CollapsibleSection.test.tsx",
|
||||
"frontend-modern/src/components/Alerts/WebhookConfig.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/alertsConfigurationModel.test.ts",
|
||||
"frontend-modern/src/features/alerts/__tests__/helpers.test.ts",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.emptystate.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.timelineerror.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.total24h.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
|
||||
"frontend-modern/src/features/alerts/identity.test.ts",
|
||||
"frontend-modern/src/features/alerts/thresholds/__tests__/helpers.test.ts",
|
||||
"frontend-modern/src/features/alerts/thresholds/hooks/__tests__/truenasThresholdPersistence.test.tsx",
|
||||
@@ -2270,6 +2273,7 @@
|
||||
"internal/alerts/guest_snapshot_test.go",
|
||||
"internal/alerts/history_concurrency_test.go",
|
||||
"internal/alerts/history_test.go",
|
||||
"internal/alerts/intent_policy_test.go",
|
||||
"internal/alerts/operational_contract_test.go",
|
||||
"internal/alerts/synology_test.go",
|
||||
"internal/alerts/threshold_resolution_shared_test.go",
|
||||
@@ -2639,6 +2643,21 @@
|
||||
"internal/api/update_readiness_test.go"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "alert-intent-policy-api",
|
||||
"label": "alert intent policy transport proof",
|
||||
"match_prefixes": [],
|
||||
"match_files": [
|
||||
"frontend-modern/src/api/alertIntentPolicies.ts",
|
||||
"internal/api/alerts.go"
|
||||
],
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts",
|
||||
"internal/api/alerts_endpoints_test.go"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "backend-payload-contracts",
|
||||
"label": "backend API payload proof",
|
||||
@@ -5190,6 +5209,7 @@
|
||||
"internal/models/deepcopy_test.go",
|
||||
"internal/models/state_host_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/monitor_full_coverage_test.go",
|
||||
"internal/monitoring/monitor_host_agents_test.go",
|
||||
"internal/monitoring/monitor_package_updates_test.go",
|
||||
"internal/unifiedresources/adapter_coverage_test.go",
|
||||
@@ -5286,6 +5306,8 @@
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"internal/monitoring/monitor_alert_intent_test.go",
|
||||
"internal/monitoring/monitor_full_coverage_test.go",
|
||||
"internal/monitoring/pbs_protection_observation_test.go",
|
||||
"internal/monitoring/recovery_ingest_test.go",
|
||||
"internal/recovery/mapper/proxmox/mapper_test.go",
|
||||
@@ -5338,8 +5360,10 @@
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"internal/monitoring/availability_poller_test.go",
|
||||
"internal/monitoring/availability_udp_test.go",
|
||||
"internal/monitoring/canonical_guardrails_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/monitor_alert_intent_test.go",
|
||||
"internal/monitoring/monitor_alert_override_migration_test.go",
|
||||
"internal/monitoring/monitor_backups_readstate_test.go",
|
||||
"internal/monitoring/monitor_host_agents_test.go",
|
||||
@@ -6782,6 +6806,7 @@
|
||||
"internal/hostagent/issue1595_sas_collection_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/truenas/contract_test.go",
|
||||
"internal/unifiedresources/availability_projection_test.go",
|
||||
"internal/unifiedresources/canonical_identity_test.go",
|
||||
"internal/unifiedresources/canonical_ids_types_test.go",
|
||||
"internal/unifiedresources/code_standards_test.go",
|
||||
|
||||
@@ -2187,6 +2187,21 @@ while storage detail drawers and filter controls must route summary series IDs,
|
||||
source tones, and disk metrics through the shared storage helpers instead of
|
||||
reconstructing them from local table state.
|
||||
|
||||
### Backup-aware alert intent is not recovery assurance
|
||||
|
||||
The alerts subsystem may defer an offline alert while monitoring supplies
|
||||
fresh, matching evidence of an active PVE backup task. That evidence is a
|
||||
short-lived operational context only. It does not create or update a recovery
|
||||
point, prove backup completeness, validate restore permissions, establish
|
||||
retention, or change protected/attention/unprotected/unknown posture.
|
||||
|
||||
Storage and recovery remain the sole owners of deterministic collection trust
|
||||
and recovery assurance. Stale, future-skewed, terminal, or mismatched task
|
||||
evidence fails closed for alert deferral, while PBS and provider recovery
|
||||
evidence continues through its independent mapper and posture rules. An alert
|
||||
grace period or hard-cap expiry cannot be interpreted as backup success or
|
||||
failure and grants no restore or infrastructure mutation authority.
|
||||
|
||||
### Physical-disk collection truth
|
||||
|
||||
Storage surfaces consume the unified physical-disk collection contract without
|
||||
|
||||
@@ -1815,6 +1815,30 @@ through the canonical resource model, but unified-resource consumers must not
|
||||
reintroduce removed workload aliases or feature-local resource-type shims just
|
||||
to satisfy one table, drawer, or badge surface.
|
||||
|
||||
### Alert-intent identity and availability evidence projection
|
||||
|
||||
The monitor adapter may expose two optional read capabilities to the alert
|
||||
intent resolver: source-reference resolution to the current canonical resource
|
||||
ID and durable operator-state lookup for that ID. These are read-only views over
|
||||
the existing registry and store. Display aliases do not become persistence
|
||||
keys, an unresolved reference supplies no operator suppression, and the adapter
|
||||
does not mutate operator state or alert policy.
|
||||
|
||||
The canonical availability facet additively carries `probeOutcome` and
|
||||
`udpMode`. `indeterminate` is evidence that UDP reachability could not be
|
||||
distinguished from filtering; it is neither available nor unavailable truth.
|
||||
Backend and frontend resource types must retain these values through JSON and
|
||||
read-state projection without changing canonical identity, correlation, or
|
||||
incident ownership. Legacy clients may ignore the additive fields and continue
|
||||
reading the established boolean and status fields, but new consumers must not
|
||||
convert an indeterminate outcome into a successful check.
|
||||
|
||||
`internal/unifiedresources/availability_projection_test.go` proves the wire
|
||||
shape, `internal/unifiedresources/monitor_adapter_read_state_test.go` proves
|
||||
canonical identity and operator-state forwarding, and
|
||||
`frontend-modern/src/types/__tests__/resource.test.ts` proves the browser
|
||||
projection.
|
||||
|
||||
### Physical-disk identity and collection read state
|
||||
|
||||
`pkg/diskinventory` owns deterministic physical-disk fallback identity and
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
AlertIntentPoliciesAPI,
|
||||
type AlertIntentPolicyDocument,
|
||||
type AlertIntentPolicyPreviewRequest,
|
||||
} from '@/api/alertIntentPolicies';
|
||||
import { apiFetchJSON } from '@/utils/apiClient';
|
||||
|
||||
vi.mock('@/utils/apiClient', () => ({
|
||||
apiFetchJSON: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockedApiFetchJSON = vi.mocked(apiFetchJSON);
|
||||
|
||||
describe('AlertIntentPoliciesAPI', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('reads, updates, and previews policies through the canonical routes', async () => {
|
||||
const document: AlertIntentPolicyDocument = {
|
||||
schemaVersion: 1,
|
||||
revision: 4,
|
||||
defaults: {
|
||||
'state.offline': {
|
||||
graceSeconds: 90,
|
||||
honorOperatorState: true,
|
||||
},
|
||||
},
|
||||
};
|
||||
const previewRequest: AlertIntentPolicyPreviewRequest = {
|
||||
resourceId: 'vm:pve-a:101',
|
||||
resourceType: 'vm',
|
||||
signal: 'state.offline',
|
||||
conditionActive: true,
|
||||
firstMatchedAt: '2026-07-20T12:00:00Z',
|
||||
};
|
||||
const preview = {
|
||||
resourceId: previewRequest.resourceId,
|
||||
resourceType: previewRequest.resourceType,
|
||||
signal: previewRequest.signal,
|
||||
status: 'pending_grace' as const,
|
||||
reason: 'grace period active',
|
||||
effective: {
|
||||
graceSeconds: 90,
|
||||
honorOperatorState: true,
|
||||
sources: { graceSeconds: 'defaults.state.offline' },
|
||||
explicit: true,
|
||||
},
|
||||
contexts: [],
|
||||
warnings: [],
|
||||
};
|
||||
|
||||
mockedApiFetchJSON.mockResolvedValueOnce(document);
|
||||
await expect(AlertIntentPoliciesAPI.get()).resolves.toEqual(document);
|
||||
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/intent-policies');
|
||||
|
||||
mockedApiFetchJSON.mockResolvedValueOnce({ ...document, revision: 5 });
|
||||
await expect(AlertIntentPoliciesAPI.update(document)).resolves.toEqual({
|
||||
...document,
|
||||
revision: 5,
|
||||
});
|
||||
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/intent-policies', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(document),
|
||||
});
|
||||
|
||||
mockedApiFetchJSON.mockResolvedValueOnce(preview);
|
||||
await expect(AlertIntentPoliciesAPI.preview(previewRequest)).resolves.toEqual(preview);
|
||||
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/intent-policies/preview', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(previewRequest),
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import { apiFetchJSON } from '@/utils/apiClient';
|
||||
|
||||
const ALERT_INTENT_POLICIES_PATH = '/api/alerts/intent-policies';
|
||||
|
||||
export type AlertIntentSignal =
|
||||
'*' | 'state.offline' | 'incident.availability' | `metric.${string}`;
|
||||
|
||||
export interface BackupOfflineIntentPolicy {
|
||||
enabled: boolean;
|
||||
postGraceSeconds?: number;
|
||||
maxDeferralSeconds?: number;
|
||||
}
|
||||
|
||||
export interface AlertIntentRule {
|
||||
graceSeconds?: number;
|
||||
honorOperatorState?: boolean;
|
||||
backupOffline?: BackupOfflineIntentPolicy;
|
||||
}
|
||||
|
||||
export interface AlertIntentPolicyDocument {
|
||||
schemaVersion: number;
|
||||
revision: number;
|
||||
updatedAt?: string;
|
||||
defaults?: Record<string, AlertIntentRule>;
|
||||
resourceTypes?: Record<string, Record<string, AlertIntentRule>>;
|
||||
resources?: Record<string, Record<string, AlertIntentRule>>;
|
||||
}
|
||||
|
||||
export interface AlertIntentPolicyPreviewRequest {
|
||||
resourceId: string;
|
||||
resourceType: string;
|
||||
signal: AlertIntentSignal;
|
||||
conditionActive: boolean;
|
||||
firstMatchedAt?: string;
|
||||
backupActive?: boolean;
|
||||
backupObservedAt?: string;
|
||||
}
|
||||
|
||||
export interface AlertIntentPolicyPreview {
|
||||
resourceId: string;
|
||||
resourceType: string;
|
||||
signal: string;
|
||||
status: 'clear' | 'expected_transient' | 'pending_grace' | 'would_activate';
|
||||
reason: string;
|
||||
effective: {
|
||||
graceSeconds: number;
|
||||
honorOperatorState: boolean;
|
||||
backupOffline?: BackupOfflineIntentPolicy;
|
||||
sources: Record<string, string>;
|
||||
explicit: boolean;
|
||||
};
|
||||
firstMatchedAt?: string;
|
||||
eligibleAt?: string;
|
||||
hardCapAt?: string;
|
||||
remainingSeconds?: number;
|
||||
contexts: Array<{
|
||||
kind: string;
|
||||
active: boolean;
|
||||
evidence?: string;
|
||||
observedAt?: string;
|
||||
expiresAt?: string;
|
||||
}>;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export class AlertIntentPoliciesAPI {
|
||||
static async get(): Promise<AlertIntentPolicyDocument> {
|
||||
return apiFetchJSON(ALERT_INTENT_POLICIES_PATH);
|
||||
}
|
||||
|
||||
static async update(document: AlertIntentPolicyDocument): Promise<AlertIntentPolicyDocument> {
|
||||
return apiFetchJSON(ALERT_INTENT_POLICIES_PATH, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(document),
|
||||
});
|
||||
}
|
||||
|
||||
static async preview(
|
||||
request: AlertIntentPolicyPreviewRequest,
|
||||
): Promise<AlertIntentPolicyPreview> {
|
||||
return apiFetchJSON(`${ALERT_INTENT_POLICIES_PATH}/preview`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,8 @@ import { apiFetchJSON } from '@/utils/apiClient';
|
||||
|
||||
const AVAILABILITY_TARGETS_PATH = '/api/availability-targets';
|
||||
|
||||
export type AvailabilityProbeProtocol = 'icmp' | 'tcp' | 'http' | 'https';
|
||||
export type AvailabilityProbeProtocol = 'icmp' | 'tcp' | 'udp' | 'http' | 'https';
|
||||
export type AvailabilityUDPMode = 'response_required' | 'open_or_filtered';
|
||||
export type AvailabilityTargetKind = 'machine' | 'service' | 'device';
|
||||
|
||||
export interface AvailabilityProbeStatus {
|
||||
@@ -11,6 +12,7 @@ export interface AvailabilityProbeStatus {
|
||||
targetKind?: AvailabilityTargetKind | string;
|
||||
address: string;
|
||||
protocol: AvailabilityProbeProtocol | string;
|
||||
outcome?: 'reachable' | 'unreachable' | 'indeterminate' | string;
|
||||
enabled: boolean;
|
||||
available: boolean;
|
||||
lastChecked?: string;
|
||||
@@ -29,6 +31,9 @@ export interface AvailabilityTarget {
|
||||
protocol: AvailabilityProbeProtocol;
|
||||
port?: number;
|
||||
path?: string;
|
||||
udpMode?: AvailabilityUDPMode;
|
||||
udpRequest?: string;
|
||||
udpExpectedResponse?: string;
|
||||
linkedResourceId?: string;
|
||||
enabled: boolean;
|
||||
pollIntervalSeconds?: number;
|
||||
@@ -40,6 +45,7 @@ export interface AvailabilityTarget {
|
||||
export interface AvailabilityTestResponse {
|
||||
success: boolean;
|
||||
latencyMillis: number;
|
||||
outcome?: 'reachable' | 'unreachable' | 'indeterminate' | string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
|
||||
+77
-11
@@ -15,6 +15,7 @@ import {
|
||||
type AvailabilityTarget,
|
||||
type AvailabilityTargetKind,
|
||||
type AvailabilityTestResponse,
|
||||
type AvailabilityUDPMode,
|
||||
} from '@/api/availabilityTargets';
|
||||
import {
|
||||
AVAILABILITY_TARGET_PRESETS,
|
||||
@@ -37,6 +38,9 @@ interface AvailabilityForm {
|
||||
protocol: AvailabilityProbeProtocol;
|
||||
port: string;
|
||||
path: string;
|
||||
udpMode: AvailabilityUDPMode;
|
||||
udpRequest: string;
|
||||
udpExpectedResponse: string;
|
||||
linkedResourceId: string;
|
||||
enabled: boolean;
|
||||
pollIntervalSeconds: string;
|
||||
@@ -68,6 +72,9 @@ const newAvailabilityForm = (
|
||||
protocol: 'icmp',
|
||||
port: '',
|
||||
path: '',
|
||||
udpMode: 'response_required',
|
||||
udpRequest: '',
|
||||
udpExpectedResponse: '',
|
||||
linkedResourceId: '',
|
||||
enabled: true,
|
||||
pollIntervalSeconds: '60',
|
||||
@@ -83,6 +90,9 @@ const formFromTarget = (target: AvailabilityTarget): AvailabilityForm => ({
|
||||
protocol: target.protocol ?? 'icmp',
|
||||
port: target.port ? String(target.port) : '',
|
||||
path: target.path ?? '',
|
||||
udpMode: target.udpMode ?? 'response_required',
|
||||
udpRequest: target.udpRequest ?? '',
|
||||
udpExpectedResponse: target.udpExpectedResponse ?? '',
|
||||
linkedResourceId: target.linkedResourceId ?? '',
|
||||
enabled: target.enabled ?? true,
|
||||
pollIntervalSeconds: String(target.pollIntervalSeconds ?? 60),
|
||||
@@ -104,7 +114,10 @@ const payloadFromForm = (form: AvailabilityForm): AvailabilityTarget => {
|
||||
address: form.address.trim(),
|
||||
protocol: form.protocol,
|
||||
port: form.protocol === 'icmp' ? undefined : port,
|
||||
path: form.protocol === 'http' ? form.path.trim() : undefined,
|
||||
path: form.protocol === 'http' || form.protocol === 'https' ? form.path.trim() : undefined,
|
||||
udpMode: form.protocol === 'udp' ? form.udpMode : undefined,
|
||||
udpRequest: form.protocol === 'udp' ? form.udpRequest : undefined,
|
||||
udpExpectedResponse: form.protocol === 'udp' ? form.udpExpectedResponse : undefined,
|
||||
linkedResourceId: form.linkedResourceId.trim() || undefined,
|
||||
enabled: form.enabled,
|
||||
pollIntervalSeconds: parsePositiveInt(form.pollIntervalSeconds),
|
||||
@@ -118,6 +131,9 @@ const presetSensitiveFormKeys: ReadonlySet<keyof AvailabilityForm> = new Set([
|
||||
'port',
|
||||
'protocol',
|
||||
'targetKind',
|
||||
'udpMode',
|
||||
'udpRequest',
|
||||
'udpExpectedResponse',
|
||||
]);
|
||||
|
||||
const initialPresetForTargetKind = (
|
||||
@@ -189,8 +205,8 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
|
||||
const addressPlaceholder = () =>
|
||||
selectedPresetConfig()?.addressPlaceholder ??
|
||||
(form().protocol === 'http'
|
||||
? 'http://service.local/status'
|
||||
(form().protocol === 'http' || form().protocol === 'https'
|
||||
? `${form().protocol}://service.local/status`
|
||||
: form().targetKind === 'machine'
|
||||
? 'server.local'
|
||||
: form().targetKind === 'service'
|
||||
@@ -198,7 +214,8 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
: 'device.local');
|
||||
|
||||
const portPlaceholder = () =>
|
||||
selectedPresetConfig()?.portPlaceholder ?? (form().protocol === 'http' ? 'Optional' : '1883');
|
||||
selectedPresetConfig()?.portPlaceholder ??
|
||||
(form().protocol === 'http' || form().protocol === 'https' ? 'Optional' : '1883');
|
||||
|
||||
const namePlaceholder = () =>
|
||||
form().targetKind === 'machine'
|
||||
@@ -330,10 +347,14 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
>
|
||||
<option value="icmp">ICMP ping</option>
|
||||
<option value="tcp">TCP port</option>
|
||||
<option value="udp">UDP datagram</option>
|
||||
<option value="http">HTTP</option>
|
||||
<option value="https">HTTPS</option>
|
||||
</FormSelect>
|
||||
<label class={`${formField} sm:col-span-2`}>
|
||||
<span class={formLabel}>{form().protocol === 'http' ? 'URL or host' : 'Address'}</span>
|
||||
<span class={formLabel}>
|
||||
{form().protocol === 'http' || form().protocol === 'https' ? 'URL or host' : 'Address'}
|
||||
</span>
|
||||
<input
|
||||
class={formControl}
|
||||
value={form().address}
|
||||
@@ -345,7 +366,9 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
? 'Use a hostname or IP address. Pulse will run one ping per poll.'
|
||||
: form().protocol === 'tcp'
|
||||
? 'Use a hostname or IP address and the port to open.'
|
||||
: 'Use a full URL or a hostname. HTTP statuses below 500 count as reachable.'}
|
||||
: form().protocol === 'udp'
|
||||
? 'Use a hostname or unicast IP. UDP checks send one small datagram per poll.'
|
||||
: 'Use a full URL or a hostname. HTTP statuses below 500 count as reachable.'}
|
||||
</span>
|
||||
</label>
|
||||
<FormSelect
|
||||
@@ -391,7 +414,7 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
/>
|
||||
</label>
|
||||
</Show>
|
||||
<Show when={form().protocol === 'http'}>
|
||||
<Show when={form().protocol === 'http' || form().protocol === 'https'}>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Path override</span>
|
||||
<input
|
||||
@@ -402,6 +425,41 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
/>
|
||||
</label>
|
||||
</Show>
|
||||
<Show when={form().protocol === 'udp'}>
|
||||
<FormSelect
|
||||
label="UDP result policy"
|
||||
value={form().udpMode}
|
||||
onChange={(event) =>
|
||||
updateForm({ udpMode: event.currentTarget.value as AvailabilityUDPMode })
|
||||
}
|
||||
help="Response required is alert-safe. Open or filtered reports silence as indeterminate and only fails on an explicit port-unreachable response."
|
||||
>
|
||||
<option value="response_required">Require a response</option>
|
||||
<option value="open_or_filtered">Accept open or filtered</option>
|
||||
</FormSelect>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Request payload</span>
|
||||
<input
|
||||
class={formControl}
|
||||
value={form().udpRequest}
|
||||
onInput={(event) => updateForm({ udpRequest: event.currentTarget.value })}
|
||||
placeholder={form().udpMode === 'response_required' ? 'PING' : 'Optional'}
|
||||
/>
|
||||
<span class={formHelpText}>UTF-8 bytes, up to 512 bytes.</span>
|
||||
</label>
|
||||
<label class={`${formField} sm:col-span-2`}>
|
||||
<span class={formLabel}>Expected response (optional)</span>
|
||||
<input
|
||||
class={formControl}
|
||||
value={form().udpExpectedResponse}
|
||||
onInput={(event) => updateForm({ udpExpectedResponse: event.currentTarget.value })}
|
||||
placeholder="PONG"
|
||||
/>
|
||||
<span class={formHelpText}>
|
||||
When set, the response must match these UTF-8 bytes exactly.
|
||||
</span>
|
||||
</label>
|
||||
</Show>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Poll interval (seconds)</span>
|
||||
<input
|
||||
@@ -452,13 +510,21 @@ export const AvailabilityTargetSlot: Component<AvailabilityTargetSlotProps> = (p
|
||||
{(result) => (
|
||||
<CalloutCard
|
||||
role={result().success ? 'status' : 'alert'}
|
||||
tone={result().success ? 'success' : 'danger'}
|
||||
tone={
|
||||
result().outcome === 'indeterminate'
|
||||
? 'warning'
|
||||
: result().success
|
||||
? 'success'
|
||||
: 'danger'
|
||||
}
|
||||
scale="compact"
|
||||
padding="sm"
|
||||
description={
|
||||
result().success
|
||||
? `Probe reached the target in ${result().latencyMillis} ms.`
|
||||
: result().error || 'Probe failed.'
|
||||
result().outcome === 'indeterminate'
|
||||
? `No UDP rejection was received in ${result().latencyMillis} ms; the port is open or filtered, not proven reachable.`
|
||||
: result().success
|
||||
? `Probe reached the target in ${result().latencyMillis} ms.`
|
||||
: result().error || 'Probe failed.'
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
+30
@@ -146,4 +146,34 @@ describe('AvailabilityTargetSlot', () => {
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('creates response-validated UDP checks', async () => {
|
||||
const onSaved = vi.fn();
|
||||
render(() => <AvailabilityTargetSlot onCancel={vi.fn()} onSaved={onSaved} />);
|
||||
|
||||
fireEvent.change(screen.getByLabelText('Probe'), { target: { value: 'udp' } });
|
||||
fireEvent.input(screen.getByLabelText('Name'), { target: { value: 'DNS health' } });
|
||||
fireEvent.input(screen.getByLabelText(/^Address/), { target: { value: 'dns.internal' } });
|
||||
fireEvent.input(screen.getByLabelText('Port'), { target: { value: '53' } });
|
||||
fireEvent.input(screen.getByLabelText(/^Request payload/), { target: { value: 'PING' } });
|
||||
fireEvent.input(screen.getByLabelText(/^Expected response \(optional\)/), {
|
||||
target: { value: 'PONG' },
|
||||
});
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Add service/device check' }));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mockedCreate).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
name: 'DNS health',
|
||||
address: 'dns.internal',
|
||||
protocol: 'udp',
|
||||
port: 53,
|
||||
udpMode: 'response_required',
|
||||
udpRequest: 'PING',
|
||||
udpExpectedResponse: 'PONG',
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(onSaved).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -59,6 +59,8 @@ export function getAvailabilityTargetMethodLabel(target: AvailabilityTarget): st
|
||||
return 'ICMP ping';
|
||||
case 'tcp':
|
||||
return target.port ? `TCP ${target.port}` : 'TCP port';
|
||||
case 'udp':
|
||||
return target.port ? `UDP ${target.port}` : 'UDP port';
|
||||
case 'http':
|
||||
return 'HTTP check';
|
||||
default:
|
||||
@@ -90,7 +92,7 @@ export function getAvailabilityTargetAddressLabel(target: AvailabilityTarget): s
|
||||
}
|
||||
return target.address;
|
||||
}
|
||||
if (target.protocol === 'tcp' && target.port) {
|
||||
if ((target.protocol === 'tcp' || target.protocol === 'udp') && target.port) {
|
||||
return `${target.address}:${target.port}`;
|
||||
}
|
||||
return target.address;
|
||||
@@ -100,6 +102,7 @@ export function getAvailabilityTargetStatusLabel(target: AvailabilityTarget): st
|
||||
if (!target.enabled) return 'Paused';
|
||||
const status = target.status;
|
||||
if (!status) return 'Not checked yet';
|
||||
if (status.outcome === 'indeterminate') return 'Open or filtered';
|
||||
if (status.available) {
|
||||
return typeof status.latencyMillis === 'number'
|
||||
? `Online · ${status.latencyMillis} ms`
|
||||
@@ -111,6 +114,9 @@ export function getAvailabilityTargetStatusLabel(target: AvailabilityTarget): st
|
||||
export function getAvailabilityTargetStatusClass(target: AvailabilityTarget): string {
|
||||
if (!target.enabled) return 'bg-surface-alt text-muted';
|
||||
if (!target.status) return 'bg-sky-100 text-sky-700 dark:bg-sky-900 dark:text-sky-300';
|
||||
if (target.status.outcome === 'indeterminate') {
|
||||
return 'bg-amber-100 text-amber-700 dark:bg-amber-900 dark:text-amber-300';
|
||||
}
|
||||
if (target.status.available) {
|
||||
return 'bg-emerald-100 text-emerald-700 dark:bg-emerald-900 dark:text-emerald-300';
|
||||
}
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
import { For, Show, createEffect, createMemo, createSignal, onMount } from 'solid-js';
|
||||
|
||||
import {
|
||||
AlertIntentPoliciesAPI,
|
||||
type AlertIntentPolicyDocument,
|
||||
type AlertIntentPolicyPreview,
|
||||
type AlertIntentRule,
|
||||
type AlertIntentSignal,
|
||||
} from '@/api/alertIntentPolicies';
|
||||
import { Button } from '@/components/shared/Button';
|
||||
import { CalloutCard } from '@/components/shared/CalloutCard';
|
||||
import { Card } from '@/components/shared/Card';
|
||||
import {
|
||||
formCheckbox,
|
||||
formControl,
|
||||
formField,
|
||||
formHelpText,
|
||||
formLabel,
|
||||
} from '@/components/shared/Form';
|
||||
import { FormSelect } from '@/components/shared/FormSelect';
|
||||
import type { Resource } from '@/types/resource';
|
||||
import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity';
|
||||
|
||||
const SIGNALS: ReadonlyArray<{ value: AlertIntentSignal; label: string }> = [
|
||||
{ value: 'state.offline', label: 'Offline / powered off' },
|
||||
{ value: 'incident.availability', label: 'Availability incident' },
|
||||
{ value: 'metric.cpu', label: 'CPU threshold' },
|
||||
{ value: 'metric.memory', label: 'Memory threshold' },
|
||||
{ value: 'metric.disk', label: 'Disk threshold' },
|
||||
];
|
||||
|
||||
const nonNegativeInt = (value: string, fallback = 0): number => {
|
||||
const parsed = Number.parseInt(value.trim(), 10);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
};
|
||||
|
||||
const detachedDocument = (document: AlertIntentPolicyDocument): AlertIntentPolicyDocument =>
|
||||
structuredClone(document);
|
||||
|
||||
const ruleFor = (
|
||||
document: AlertIntentPolicyDocument | null,
|
||||
resourceId: string,
|
||||
signal: AlertIntentSignal,
|
||||
): AlertIntentRule => document?.resources?.[resourceId]?.[signal] ?? {};
|
||||
|
||||
export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] }) {
|
||||
const [expanded, setExpanded] = createSignal(false);
|
||||
const [document, setDocument] = createSignal<AlertIntentPolicyDocument | null>(null);
|
||||
const [loading, setLoading] = createSignal(false);
|
||||
const [saving, setSaving] = createSignal(false);
|
||||
const [error, setError] = createSignal<string | null>(null);
|
||||
const [message, setMessage] = createSignal<string | null>(null);
|
||||
|
||||
const [offlineGrace, setOfflineGrace] = createSignal('0');
|
||||
const [availabilityGrace, setAvailabilityGrace] = createSignal('0');
|
||||
const [honorOperatorState, setHonorOperatorState] = createSignal(false);
|
||||
const [backupAware, setBackupAware] = createSignal(false);
|
||||
const [backupPostGrace, setBackupPostGrace] = createSignal('60');
|
||||
const [backupMaxDeferral, setBackupMaxDeferral] = createSignal('3600');
|
||||
|
||||
const sortedResources = createMemo(() =>
|
||||
[...props.resources]
|
||||
.filter((resource) => resource.id?.trim())
|
||||
.sort((a, b) =>
|
||||
getPreferredInfrastructureDisplayName(a).localeCompare(
|
||||
getPreferredInfrastructureDisplayName(b),
|
||||
),
|
||||
),
|
||||
);
|
||||
const [resourceId, setResourceId] = createSignal('');
|
||||
const [signal, setSignal] = createSignal<AlertIntentSignal>('state.offline');
|
||||
const [overrideGrace, setOverrideGrace] = createSignal('');
|
||||
const [overrideOperatorMode, setOverrideOperatorMode] = createSignal<
|
||||
'inherit' | 'honor' | 'ignore'
|
||||
>('inherit');
|
||||
const [overrideBackupMode, setOverrideBackupMode] = createSignal<
|
||||
'inherit' | 'enabled' | 'disabled'
|
||||
>('inherit');
|
||||
const [overrideBackupPostGrace, setOverrideBackupPostGrace] = createSignal('60');
|
||||
const [overrideBackupMaxDeferral, setOverrideBackupMaxDeferral] = createSignal('3600');
|
||||
const [previewBackupActive, setPreviewBackupActive] = createSignal(false);
|
||||
const [previewing, setPreviewing] = createSignal(false);
|
||||
const [preview, setPreview] = createSignal<AlertIntentPolicyPreview | null>(null);
|
||||
|
||||
const selectedResource = createMemo(() =>
|
||||
sortedResources().find((resource) => resource.id === resourceId()),
|
||||
);
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const loaded = await AlertIntentPoliciesAPI.get();
|
||||
setDocument(loaded);
|
||||
const offline = loaded.defaults?.['state.offline'] ?? {};
|
||||
const availability = loaded.defaults?.['incident.availability'] ?? {};
|
||||
setOfflineGrace(String(offline.graceSeconds ?? 0));
|
||||
setAvailabilityGrace(String(availability.graceSeconds ?? 0));
|
||||
setHonorOperatorState(offline.honorOperatorState ?? false);
|
||||
setBackupAware(offline.backupOffline?.enabled ?? false);
|
||||
setBackupPostGrace(String(offline.backupOffline?.postGraceSeconds ?? 60));
|
||||
setBackupMaxDeferral(String(offline.backupOffline?.maxDeferralSeconds ?? 3600));
|
||||
if (!resourceId() && sortedResources().length > 0) {
|
||||
setResourceId(sortedResources()[0].id);
|
||||
}
|
||||
} catch (cause) {
|
||||
setError(cause instanceof Error ? cause.message : 'Failed to load alert intent policies.');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
onMount(() => void load());
|
||||
|
||||
createEffect(() => {
|
||||
const resources = sortedResources();
|
||||
const selected = resourceId();
|
||||
if (resources.length === 0) {
|
||||
if (selected) setResourceId('');
|
||||
return;
|
||||
}
|
||||
if (!resources.some((resource) => resource.id === selected)) {
|
||||
setResourceId(resources[0].id);
|
||||
}
|
||||
});
|
||||
|
||||
createEffect(() => {
|
||||
const current = ruleFor(document(), resourceId(), signal());
|
||||
setOverrideGrace(current.graceSeconds === undefined ? '' : String(current.graceSeconds));
|
||||
setOverrideOperatorMode(
|
||||
current.honorOperatorState === undefined
|
||||
? 'inherit'
|
||||
: current.honorOperatorState
|
||||
? 'honor'
|
||||
: 'ignore',
|
||||
);
|
||||
setOverrideBackupMode(
|
||||
current.backupOffline === undefined
|
||||
? 'inherit'
|
||||
: current.backupOffline.enabled
|
||||
? 'enabled'
|
||||
: 'disabled',
|
||||
);
|
||||
setOverrideBackupPostGrace(String(current.backupOffline?.postGraceSeconds ?? 60));
|
||||
setOverrideBackupMaxDeferral(String(current.backupOffline?.maxDeferralSeconds ?? 3600));
|
||||
setPreview(null);
|
||||
});
|
||||
|
||||
const persist = async (next: AlertIntentPolicyDocument, successMessage: string) => {
|
||||
setSaving(true);
|
||||
setError(null);
|
||||
setMessage(null);
|
||||
try {
|
||||
const updated = await AlertIntentPoliciesAPI.update(next);
|
||||
setDocument(updated);
|
||||
setMessage(successMessage);
|
||||
} catch (cause) {
|
||||
setError(cause instanceof Error ? cause.message : 'Failed to save alert intent policies.');
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const saveDefaults = async () => {
|
||||
const current = document();
|
||||
if (!current) return;
|
||||
const next = detachedDocument(current);
|
||||
next.defaults ??= {};
|
||||
next.defaults['state.offline'] = {
|
||||
...(next.defaults['state.offline'] ?? {}),
|
||||
graceSeconds: nonNegativeInt(offlineGrace()),
|
||||
honorOperatorState: honorOperatorState(),
|
||||
backupOffline: {
|
||||
enabled: backupAware(),
|
||||
postGraceSeconds: nonNegativeInt(backupPostGrace(), 60),
|
||||
maxDeferralSeconds: nonNegativeInt(backupMaxDeferral(), 3600),
|
||||
},
|
||||
};
|
||||
next.defaults['incident.availability'] = {
|
||||
...(next.defaults['incident.availability'] ?? {}),
|
||||
graceSeconds: nonNegativeInt(availabilityGrace()),
|
||||
honorOperatorState: honorOperatorState(),
|
||||
};
|
||||
await persist(next, 'Default intent policy saved.');
|
||||
};
|
||||
|
||||
const saveOverride = async () => {
|
||||
const current = document();
|
||||
const id = resourceId();
|
||||
if (!current || !id) return;
|
||||
const next = detachedDocument(current);
|
||||
next.resources ??= {};
|
||||
next.resources[id] ??= {};
|
||||
const rule: AlertIntentRule = {};
|
||||
if (overrideGrace().trim() !== '') {
|
||||
rule.graceSeconds = nonNegativeInt(overrideGrace());
|
||||
}
|
||||
if (overrideOperatorMode() !== 'inherit') {
|
||||
rule.honorOperatorState = overrideOperatorMode() === 'honor';
|
||||
}
|
||||
if (signal() === 'state.offline' && overrideBackupMode() !== 'inherit') {
|
||||
rule.backupOffline = {
|
||||
enabled: overrideBackupMode() === 'enabled',
|
||||
postGraceSeconds: nonNegativeInt(overrideBackupPostGrace(), 60),
|
||||
maxDeferralSeconds: nonNegativeInt(overrideBackupMaxDeferral(), 3600),
|
||||
};
|
||||
}
|
||||
if (Object.keys(rule).length === 0) {
|
||||
delete next.resources[id][signal()];
|
||||
if (Object.keys(next.resources[id]).length === 0) delete next.resources[id];
|
||||
} else {
|
||||
next.resources[id][signal()] = rule;
|
||||
}
|
||||
await persist(next, 'Resource intent override saved.');
|
||||
};
|
||||
|
||||
const removeOverride = async () => {
|
||||
const current = document();
|
||||
const id = resourceId();
|
||||
if (!current || !id || !current.resources?.[id]?.[signal()]) return;
|
||||
const next = detachedDocument(current);
|
||||
delete next.resources?.[id]?.[signal()];
|
||||
if (next.resources?.[id] && Object.keys(next.resources[id]).length === 0) {
|
||||
delete next.resources[id];
|
||||
}
|
||||
await persist(next, 'Resource intent override removed.');
|
||||
};
|
||||
|
||||
const runPreview = async () => {
|
||||
const resource = selectedResource();
|
||||
if (!resource) return;
|
||||
setPreviewing(true);
|
||||
setError(null);
|
||||
try {
|
||||
setPreview(
|
||||
await AlertIntentPoliciesAPI.preview({
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
signal: signal(),
|
||||
conditionActive: true,
|
||||
firstMatchedAt: new Date().toISOString(),
|
||||
...(signal() === 'state.offline'
|
||||
? {
|
||||
backupActive: previewBackupActive(),
|
||||
backupObservedAt: new Date().toISOString(),
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
} catch (cause) {
|
||||
setError(cause instanceof Error ? cause.message : 'Failed to preview alert intent policy.');
|
||||
} finally {
|
||||
setPreviewing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const hasSelectedOverride = createMemo(() =>
|
||||
Boolean(document()?.resources?.[resourceId()]?.[signal()]),
|
||||
);
|
||||
|
||||
return (
|
||||
<Card padding="md" class="space-y-4">
|
||||
<div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
|
||||
<div>
|
||||
<h2 class="text-base font-semibold text-base-content">Alert intent & grace</h2>
|
||||
<p class="mt-1 text-sm text-muted">
|
||||
Delay expected transients, honor operator maintenance state, and extend guest offline
|
||||
grace while a backup is active.
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="secondary" size="sm" onClick={() => setExpanded((value) => !value)}>
|
||||
{expanded() ? 'Hide policies' : 'Configure policies'}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<Show when={error()}>
|
||||
{(text) => <CalloutCard role="alert" tone="danger" scale="compact" description={text()} />}
|
||||
</Show>
|
||||
<Show when={message()}>
|
||||
{(text) => (
|
||||
<CalloutCard role="status" tone="success" scale="compact" description={text()} />
|
||||
)}
|
||||
</Show>
|
||||
|
||||
<Show when={expanded()}>
|
||||
<Show
|
||||
when={!loading() && document()}
|
||||
fallback={<div class="text-sm text-muted">Loading intent policies…</div>}
|
||||
>
|
||||
<div class="grid gap-4 border-t border-border pt-4 sm:grid-cols-2 lg:grid-cols-3">
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Default offline grace (seconds)</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
value={offlineGrace()}
|
||||
onInput={(event) => setOfflineGrace(event.currentTarget.value)}
|
||||
/>
|
||||
</label>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Default availability grace (seconds)</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
value={availabilityGrace()}
|
||||
onInput={(event) => setAvailabilityGrace(event.currentTarget.value)}
|
||||
/>
|
||||
</label>
|
||||
<label class="flex items-center gap-3 rounded-md border border-border px-3 py-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
class={formCheckbox}
|
||||
checked={honorOperatorState()}
|
||||
onChange={(event) => setHonorOperatorState(event.currentTarget.checked)}
|
||||
/>
|
||||
<span class="text-sm text-base-content">
|
||||
Honor maintenance and intentionally-offline state
|
||||
</span>
|
||||
</label>
|
||||
<label class="flex items-center gap-3 rounded-md border border-border px-3 py-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
class={formCheckbox}
|
||||
checked={backupAware()}
|
||||
onChange={(event) => setBackupAware(event.currentTarget.checked)}
|
||||
/>
|
||||
<span class="text-sm text-base-content">
|
||||
Extend offline grace during Proxmox backups
|
||||
</span>
|
||||
</label>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Post-backup grace (seconds)</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
disabled={!backupAware()}
|
||||
value={backupPostGrace()}
|
||||
onInput={(event) => setBackupPostGrace(event.currentTarget.value)}
|
||||
/>
|
||||
</label>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Maximum backup deferral (seconds)</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
disabled={!backupAware()}
|
||||
value={backupMaxDeferral()}
|
||||
onInput={(event) => setBackupMaxDeferral(event.currentTarget.value)}
|
||||
/>
|
||||
<span class={formHelpText}>
|
||||
Hard cap prevents a stale backup signal from hiding a real outage.
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="flex justify-end">
|
||||
<Button
|
||||
variant="primary"
|
||||
isLoading={saving()}
|
||||
disabled={saving()}
|
||||
onClick={() => void saveDefaults()}
|
||||
>
|
||||
Save defaults
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div class="space-y-4 border-t border-border pt-4">
|
||||
<div>
|
||||
<h3 class="text-sm font-semibold text-base-content">
|
||||
Per-resource override and preview
|
||||
</h3>
|
||||
<p class="mt-1 text-xs text-muted">
|
||||
Resource values override type and global policies field by field.
|
||||
</p>
|
||||
</div>
|
||||
<div class="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
|
||||
<FormSelect
|
||||
label="Resource"
|
||||
value={resourceId()}
|
||||
onChange={(event) => setResourceId(event.currentTarget.value)}
|
||||
fieldClass="lg:col-span-2"
|
||||
>
|
||||
<For each={sortedResources()}>
|
||||
{(resource) => (
|
||||
<option value={resource.id}>
|
||||
{getPreferredInfrastructureDisplayName(resource)} · {resource.type}
|
||||
</option>
|
||||
)}
|
||||
</For>
|
||||
</FormSelect>
|
||||
<FormSelect
|
||||
label="Signal"
|
||||
value={signal()}
|
||||
onChange={(event) => setSignal(event.currentTarget.value as AlertIntentSignal)}
|
||||
>
|
||||
<For each={SIGNALS}>
|
||||
{(item) => <option value={item.value}>{item.label}</option>}
|
||||
</For>
|
||||
</FormSelect>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Grace override (seconds)</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
value={overrideGrace()}
|
||||
onInput={(event) => setOverrideGrace(event.currentTarget.value)}
|
||||
placeholder="Inherit"
|
||||
/>
|
||||
<span class={formHelpText}>Leave blank to inherit.</span>
|
||||
</label>
|
||||
<FormSelect
|
||||
label="Operator state override"
|
||||
value={overrideOperatorMode()}
|
||||
onChange={(event) =>
|
||||
setOverrideOperatorMode(
|
||||
event.currentTarget.value as 'inherit' | 'honor' | 'ignore',
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="inherit">Inherit</option>
|
||||
<option value="honor">Honor operator state</option>
|
||||
<option value="ignore">Ignore operator state</option>
|
||||
</FormSelect>
|
||||
<Show when={signal() === 'state.offline'}>
|
||||
<FormSelect
|
||||
label="Backup handling override"
|
||||
value={overrideBackupMode()}
|
||||
onChange={(event) =>
|
||||
setOverrideBackupMode(
|
||||
event.currentTarget.value as 'inherit' | 'enabled' | 'disabled',
|
||||
)
|
||||
}
|
||||
>
|
||||
<option value="inherit">Inherit</option>
|
||||
<option value="enabled">Extend during backups</option>
|
||||
<option value="disabled">Do not extend</option>
|
||||
</FormSelect>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Post-backup grace</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
disabled={overrideBackupMode() !== 'enabled'}
|
||||
value={overrideBackupPostGrace()}
|
||||
onInput={(event) => setOverrideBackupPostGrace(event.currentTarget.value)}
|
||||
/>
|
||||
</label>
|
||||
<label class={formField}>
|
||||
<span class={formLabel}>Maximum deferral</span>
|
||||
<input
|
||||
class={formControl}
|
||||
inputMode="numeric"
|
||||
disabled={overrideBackupMode() !== 'enabled'}
|
||||
value={overrideBackupMaxDeferral()}
|
||||
onInput={(event) => setOverrideBackupMaxDeferral(event.currentTarget.value)}
|
||||
/>
|
||||
</label>
|
||||
<label class="flex items-center gap-3 rounded-md border border-border px-3 py-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
class={formCheckbox}
|
||||
checked={previewBackupActive()}
|
||||
onChange={(event) => setPreviewBackupActive(event.currentTarget.checked)}
|
||||
/>
|
||||
<span class="text-sm text-base-content">Preview with backup active</span>
|
||||
</label>
|
||||
</Show>
|
||||
</div>
|
||||
<div class="flex flex-wrap justify-end gap-2">
|
||||
<Show when={hasSelectedOverride()}>
|
||||
<Button variant="danger" disabled={saving()} onClick={() => void removeOverride()}>
|
||||
Remove override
|
||||
</Button>
|
||||
</Show>
|
||||
<Button
|
||||
variant="secondary"
|
||||
isLoading={previewing()}
|
||||
disabled={!resourceId() || previewing()}
|
||||
onClick={() => void runPreview()}
|
||||
>
|
||||
Preview current policy
|
||||
</Button>
|
||||
<Button
|
||||
variant="primary"
|
||||
isLoading={saving()}
|
||||
disabled={!resourceId() || saving()}
|
||||
onClick={() => void saveOverride()}
|
||||
>
|
||||
Save override
|
||||
</Button>
|
||||
</div>
|
||||
<Show when={preview()}>
|
||||
{(result) => (
|
||||
<CalloutCard
|
||||
role="status"
|
||||
tone={
|
||||
result().status === 'would_activate'
|
||||
? 'danger'
|
||||
: result().status === 'expected_transient'
|
||||
? 'info'
|
||||
: 'warning'
|
||||
}
|
||||
scale="compact"
|
||||
title={result().status.replace(/_/g, ' ')}
|
||||
description={`${result().reason.replace(/_/g, ' ')} · ${result().effective.graceSeconds}s grace${result().remainingSeconds ? ` · ${result().remainingSeconds}s remaining` : ''}`}
|
||||
/>
|
||||
)}
|
||||
</Show>
|
||||
</div>
|
||||
</Show>
|
||||
</Show>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library';
|
||||
import { createSignal } from 'solid-js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { AlertIntentPoliciesAPI, type AlertIntentPolicyDocument } from '@/api/alertIntentPolicies';
|
||||
import type { Resource } from '@/types/resource';
|
||||
import { AlertIntentPolicyPanel } from '../AlertIntentPolicyPanel';
|
||||
|
||||
vi.mock('@/api/alertIntentPolicies', () => ({
|
||||
AlertIntentPoliciesAPI: {
|
||||
get: vi.fn(),
|
||||
update: vi.fn(),
|
||||
preview: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const baseDocument = (): AlertIntentPolicyDocument => ({
|
||||
schemaVersion: 1,
|
||||
revision: 0,
|
||||
defaults: {
|
||||
'state.offline': {
|
||||
graceSeconds: 30,
|
||||
honorOperatorState: true,
|
||||
backupOffline: { enabled: true, postGraceSeconds: 60, maxDeferralSeconds: 3600 },
|
||||
},
|
||||
},
|
||||
resourceTypes: {},
|
||||
resources: {},
|
||||
});
|
||||
|
||||
const resource: Resource = {
|
||||
id: 'vm-canonical',
|
||||
type: 'vm',
|
||||
name: 'database',
|
||||
displayName: 'database',
|
||||
platformId: 'node-a',
|
||||
platformType: 'proxmox-pve',
|
||||
sourceType: 'api',
|
||||
status: 'running',
|
||||
lastSeen: Date.now(),
|
||||
};
|
||||
|
||||
describe('AlertIntentPolicyPanel', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(AlertIntentPoliciesAPI.get).mockResolvedValue(baseDocument());
|
||||
vi.mocked(AlertIntentPoliciesAPI.update).mockImplementation(async (document) => ({
|
||||
...document,
|
||||
revision: document.revision + 1,
|
||||
}));
|
||||
vi.mocked(AlertIntentPoliciesAPI.preview).mockResolvedValue({
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
signal: 'state.offline',
|
||||
status: 'pending_grace',
|
||||
reason: 'grace_period',
|
||||
effective: {
|
||||
graceSeconds: 30,
|
||||
honorOperatorState: true,
|
||||
sources: { graceSeconds: 'defaults.state.offline' },
|
||||
explicit: true,
|
||||
},
|
||||
remainingSeconds: 30,
|
||||
contexts: [],
|
||||
warnings: [],
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => cleanup());
|
||||
|
||||
it('selects the first canonical resource when inventory arrives after policy load', async () => {
|
||||
const [resources, setResources] = createSignal<Resource[]>([]);
|
||||
render(() => <AlertIntentPolicyPanel resources={resources()} />);
|
||||
|
||||
await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1));
|
||||
setResources([resource]);
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Configure policies' }));
|
||||
|
||||
expect(await screen.findByLabelText('Resource')).toHaveValue(resource.id);
|
||||
});
|
||||
|
||||
it('saves field-level resource overrides without disabling inherited policy', async () => {
|
||||
render(() => <AlertIntentPolicyPanel resources={[resource]} />);
|
||||
|
||||
await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1));
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Configure policies' }));
|
||||
const grace = await screen.findByLabelText(/^Grace override \(seconds\)/);
|
||||
expect(grace).toHaveValue('');
|
||||
expect(screen.getByLabelText('Operator state override')).toHaveValue('inherit');
|
||||
expect(screen.getByLabelText('Backup handling override')).toHaveValue('inherit');
|
||||
|
||||
fireEvent.input(grace, { target: { value: '75' } });
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Save override' }));
|
||||
|
||||
await waitFor(() => expect(AlertIntentPoliciesAPI.update).toHaveBeenCalledTimes(1));
|
||||
const saved = vi.mocked(AlertIntentPoliciesAPI.update).mock.calls[0][0];
|
||||
expect(saved.resources?.[resource.id]?.['state.offline']).toEqual({ graceSeconds: 75 });
|
||||
});
|
||||
|
||||
it('previews the selected canonical resource', async () => {
|
||||
render(() => <AlertIntentPolicyPanel resources={[resource]} />);
|
||||
await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1));
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Configure policies' }));
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Preview current policy' }));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(AlertIntentPoliciesAPI.preview).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
resourceId: resource.id,
|
||||
resourceType: resource.type,
|
||||
signal: 'state.offline',
|
||||
conditionActive: true,
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(await screen.findByText(/grace period · 30s grace/)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,7 @@ import { ThresholdsTab } from '../tabs/ThresholdsTab';
|
||||
import type { ThresholdsTabProps } from '../thresholds/thresholdsTabModel';
|
||||
|
||||
const captureThresholdsTableProps = vi.fn();
|
||||
const captureIntentPolicyProps = vi.fn();
|
||||
|
||||
vi.mock('@/components/Alerts/ThresholdsTable', () => ({
|
||||
ThresholdsTable: (props: unknown) => {
|
||||
@@ -13,6 +14,13 @@ vi.mock('@/components/Alerts/ThresholdsTable', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../AlertIntentPolicyPanel', () => ({
|
||||
AlertIntentPolicyPanel: (props: unknown) => {
|
||||
captureIntentPolicyProps(props);
|
||||
return null;
|
||||
},
|
||||
}));
|
||||
|
||||
const buildProps = (): ThresholdsTabProps =>
|
||||
({
|
||||
overrides: () => [],
|
||||
|
||||
@@ -1,134 +1,138 @@
|
||||
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
|
||||
import { AlertIntentPolicyPanel } from '../AlertIntentPolicyPanel';
|
||||
|
||||
import type { ThresholdsTabProps } from '../thresholds/thresholdsTabModel';
|
||||
|
||||
export function ThresholdsTab(props: ThresholdsTabProps) {
|
||||
return (
|
||||
<ThresholdsTable
|
||||
overrides={props.overrides}
|
||||
setOverrides={props.setOverrides}
|
||||
rawOverridesConfig={props.rawOverridesConfig}
|
||||
setRawOverridesConfig={props.setRawOverridesConfig}
|
||||
allGuests={props.allGuests}
|
||||
nodes={props.nodes}
|
||||
agents={props.agents}
|
||||
storage={props.storage}
|
||||
containerRuntimes={props.containerRuntimes}
|
||||
dockerHosts={props.dockerHosts}
|
||||
allResources={props.allResources}
|
||||
pbsInstances={props.pbsInstances}
|
||||
pmgInstances={props.pmgInstances}
|
||||
pmgThresholds={props.pmgThresholds}
|
||||
setPMGThresholds={props.setPMGThresholds}
|
||||
guestDefaults={props.guestDefaults()}
|
||||
setGuestDefaults={props.setGuestDefaults}
|
||||
guestDisableConnectivity={props.guestDisableConnectivity}
|
||||
setGuestDisableConnectivity={props.setGuestDisableConnectivity}
|
||||
guestPoweredOffSeverity={props.guestPoweredOffSeverity}
|
||||
setGuestPoweredOffSeverity={props.setGuestPoweredOffSeverity}
|
||||
nodeDefaults={props.nodeDefaults()}
|
||||
pbsDefaults={props.pbsDefaults()}
|
||||
kubernetesDefaults={props.kubernetesDefaults()}
|
||||
trueNASDefaults={props.trueNASDefaults()}
|
||||
trueNASDiskDefaults={props.trueNASDiskDefaults()}
|
||||
vmwareDefaults={props.vmwareDefaults()}
|
||||
agentDefaults={props.agentDefaults()}
|
||||
diskTempByType={props.diskTempByType()}
|
||||
setDiskTempByType={props.setDiskTempByType}
|
||||
setNodeDefaults={props.setNodeDefaults}
|
||||
setPBSDefaults={props.setPBSDefaults}
|
||||
setKubernetesDefaults={props.setKubernetesDefaults}
|
||||
setTrueNASDefaults={props.setTrueNASDefaults}
|
||||
setTrueNASDiskDefaults={props.setTrueNASDiskDefaults}
|
||||
setVMwareDefaults={props.setVMwareDefaults}
|
||||
setAgentDefaults={props.setAgentDefaults}
|
||||
dockerDefaults={props.dockerDefaults()}
|
||||
dockerDisableConnectivity={props.dockerDisableConnectivity}
|
||||
setDockerDisableConnectivity={props.setDockerDisableConnectivity}
|
||||
dockerPoweredOffSeverity={props.dockerPoweredOffSeverity}
|
||||
setDockerPoweredOffSeverity={props.setDockerPoweredOffSeverity}
|
||||
setDockerDefaults={props.setDockerDefaults}
|
||||
dockerIgnoredPrefixes={props.dockerIgnoredPrefixes}
|
||||
setDockerIgnoredPrefixes={props.setDockerIgnoredPrefixes}
|
||||
ignoredGuestPrefixes={props.ignoredGuestPrefixes}
|
||||
setIgnoredGuestPrefixes={props.setIgnoredGuestPrefixes}
|
||||
guestTagWhitelist={props.guestTagWhitelist}
|
||||
setGuestTagWhitelist={props.setGuestTagWhitelist}
|
||||
guestTagBlacklist={props.guestTagBlacklist}
|
||||
setGuestTagBlacklist={props.setGuestTagBlacklist}
|
||||
storageDefault={props.storageDefault}
|
||||
setStorageDefault={props.setStorageDefault}
|
||||
resetGuestDefaults={props.resetGuestDefaults}
|
||||
resetNodeDefaults={props.resetNodeDefaults}
|
||||
resetPBSDefaults={props.resetPBSDefaults}
|
||||
resetKubernetesDefaults={props.resetKubernetesDefaults}
|
||||
resetTrueNASDefaults={props.resetTrueNASDefaults}
|
||||
resetTrueNASDiskDefaults={props.resetTrueNASDiskDefaults}
|
||||
resetVMwareDefaults={props.resetVMwareDefaults}
|
||||
resetAgentDefaults={props.resetAgentDefaults}
|
||||
resetDockerDefaults={props.resetDockerDefaults}
|
||||
resetDockerIgnoredPrefixes={props.resetDockerIgnoredPrefixes}
|
||||
resetStorageDefault={props.resetStorageDefault}
|
||||
factoryGuestDefaults={props.factoryGuestDefaults}
|
||||
factoryNodeDefaults={props.factoryNodeDefaults}
|
||||
factoryPBSDefaults={props.factoryPBSDefaults}
|
||||
factoryKubernetesDefaults={props.factoryKubernetesDefaults}
|
||||
factoryTrueNASDefaults={props.factoryTrueNASDefaults}
|
||||
factoryTrueNASDiskDefaults={props.factoryTrueNASDiskDefaults}
|
||||
factoryVMwareDefaults={props.factoryVMwareDefaults}
|
||||
factoryAgentDefaults={props.factoryAgentDefaults}
|
||||
factoryDockerDefaults={props.factoryDockerDefaults}
|
||||
factoryStorageDefault={props.factoryStorageDefault}
|
||||
timeThresholds={props.timeThresholds}
|
||||
metricTimeThresholds={props.metricTimeThresholds}
|
||||
setMetricTimeThresholds={props.setMetricTimeThresholds}
|
||||
snapshotDefaults={props.snapshotDefaults}
|
||||
setSnapshotDefaults={props.setSnapshotDefaults}
|
||||
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
|
||||
resetSnapshotDefaults={props.resetSnapshotDefaults}
|
||||
backupDefaults={props.backupDefaults}
|
||||
setBackupDefaults={props.setBackupDefaults}
|
||||
backupFactoryDefaults={props.backupFactoryDefaults}
|
||||
resetBackupDefaults={props.resetBackupDefaults}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
activeAlerts={props.activeAlerts}
|
||||
removeAlerts={props.removeAlerts}
|
||||
disableAllNodes={props.disableAllNodes}
|
||||
setDisableAllNodes={props.setDisableAllNodes}
|
||||
disableAllGuests={props.disableAllGuests}
|
||||
setDisableAllGuests={props.setDisableAllGuests}
|
||||
disableAllAgents={props.disableAllAgents}
|
||||
setDisableAllAgents={props.setDisableAllAgents}
|
||||
disableAllStorage={props.disableAllStorage}
|
||||
setDisableAllStorage={props.setDisableAllStorage}
|
||||
disableAllPBS={props.disableAllPBS}
|
||||
setDisableAllPBS={props.setDisableAllPBS}
|
||||
disableAllPMG={props.disableAllPMG}
|
||||
setDisableAllPMG={props.setDisableAllPMG}
|
||||
disableAllDockerHosts={props.disableAllDockerHosts}
|
||||
setDisableAllDockerHosts={props.setDisableAllDockerHosts}
|
||||
disableAllDockerServices={props.disableAllDockerServices}
|
||||
setDisableAllDockerServices={props.setDisableAllDockerServices}
|
||||
disableAllDockerContainers={props.disableAllDockerContainers}
|
||||
setDisableAllDockerContainers={props.setDisableAllDockerContainers}
|
||||
disableAllKubernetes={props.disableAllKubernetes}
|
||||
setDisableAllKubernetes={props.setDisableAllKubernetes}
|
||||
disableAllTrueNAS={props.disableAllTrueNAS}
|
||||
setDisableAllTrueNAS={props.setDisableAllTrueNAS}
|
||||
disableAllVMware={props.disableAllVMware}
|
||||
setDisableAllVMware={props.setDisableAllVMware}
|
||||
disableAllNodesOffline={props.disableAllNodesOffline}
|
||||
setDisableAllNodesOffline={props.setDisableAllNodesOffline}
|
||||
disableAllGuestsOffline={props.disableAllGuestsOffline}
|
||||
setDisableAllGuestsOffline={props.setDisableAllGuestsOffline}
|
||||
disableAllAgentsOffline={props.disableAllAgentsOffline}
|
||||
setDisableAllAgentsOffline={props.setDisableAllAgentsOffline}
|
||||
disableAllPBSOffline={props.disableAllPBSOffline}
|
||||
setDisableAllPBSOffline={props.setDisableAllPBSOffline}
|
||||
disableAllPMGOffline={props.disableAllPMGOffline}
|
||||
setDisableAllPMGOffline={props.setDisableAllPMGOffline}
|
||||
disableAllDockerHostsOffline={props.disableAllDockerHostsOffline}
|
||||
setDisableAllDockerHostsOffline={props.setDisableAllDockerHostsOffline}
|
||||
/>
|
||||
<div class="space-y-4">
|
||||
<AlertIntentPolicyPanel resources={props.allResources} />
|
||||
<ThresholdsTable
|
||||
overrides={props.overrides}
|
||||
setOverrides={props.setOverrides}
|
||||
rawOverridesConfig={props.rawOverridesConfig}
|
||||
setRawOverridesConfig={props.setRawOverridesConfig}
|
||||
allGuests={props.allGuests}
|
||||
nodes={props.nodes}
|
||||
agents={props.agents}
|
||||
storage={props.storage}
|
||||
containerRuntimes={props.containerRuntimes}
|
||||
dockerHosts={props.dockerHosts}
|
||||
allResources={props.allResources}
|
||||
pbsInstances={props.pbsInstances}
|
||||
pmgInstances={props.pmgInstances}
|
||||
pmgThresholds={props.pmgThresholds}
|
||||
setPMGThresholds={props.setPMGThresholds}
|
||||
guestDefaults={props.guestDefaults()}
|
||||
setGuestDefaults={props.setGuestDefaults}
|
||||
guestDisableConnectivity={props.guestDisableConnectivity}
|
||||
setGuestDisableConnectivity={props.setGuestDisableConnectivity}
|
||||
guestPoweredOffSeverity={props.guestPoweredOffSeverity}
|
||||
setGuestPoweredOffSeverity={props.setGuestPoweredOffSeverity}
|
||||
nodeDefaults={props.nodeDefaults()}
|
||||
pbsDefaults={props.pbsDefaults()}
|
||||
kubernetesDefaults={props.kubernetesDefaults()}
|
||||
trueNASDefaults={props.trueNASDefaults()}
|
||||
trueNASDiskDefaults={props.trueNASDiskDefaults()}
|
||||
vmwareDefaults={props.vmwareDefaults()}
|
||||
agentDefaults={props.agentDefaults()}
|
||||
diskTempByType={props.diskTempByType()}
|
||||
setDiskTempByType={props.setDiskTempByType}
|
||||
setNodeDefaults={props.setNodeDefaults}
|
||||
setPBSDefaults={props.setPBSDefaults}
|
||||
setKubernetesDefaults={props.setKubernetesDefaults}
|
||||
setTrueNASDefaults={props.setTrueNASDefaults}
|
||||
setTrueNASDiskDefaults={props.setTrueNASDiskDefaults}
|
||||
setVMwareDefaults={props.setVMwareDefaults}
|
||||
setAgentDefaults={props.setAgentDefaults}
|
||||
dockerDefaults={props.dockerDefaults()}
|
||||
dockerDisableConnectivity={props.dockerDisableConnectivity}
|
||||
setDockerDisableConnectivity={props.setDockerDisableConnectivity}
|
||||
dockerPoweredOffSeverity={props.dockerPoweredOffSeverity}
|
||||
setDockerPoweredOffSeverity={props.setDockerPoweredOffSeverity}
|
||||
setDockerDefaults={props.setDockerDefaults}
|
||||
dockerIgnoredPrefixes={props.dockerIgnoredPrefixes}
|
||||
setDockerIgnoredPrefixes={props.setDockerIgnoredPrefixes}
|
||||
ignoredGuestPrefixes={props.ignoredGuestPrefixes}
|
||||
setIgnoredGuestPrefixes={props.setIgnoredGuestPrefixes}
|
||||
guestTagWhitelist={props.guestTagWhitelist}
|
||||
setGuestTagWhitelist={props.setGuestTagWhitelist}
|
||||
guestTagBlacklist={props.guestTagBlacklist}
|
||||
setGuestTagBlacklist={props.setGuestTagBlacklist}
|
||||
storageDefault={props.storageDefault}
|
||||
setStorageDefault={props.setStorageDefault}
|
||||
resetGuestDefaults={props.resetGuestDefaults}
|
||||
resetNodeDefaults={props.resetNodeDefaults}
|
||||
resetPBSDefaults={props.resetPBSDefaults}
|
||||
resetKubernetesDefaults={props.resetKubernetesDefaults}
|
||||
resetTrueNASDefaults={props.resetTrueNASDefaults}
|
||||
resetTrueNASDiskDefaults={props.resetTrueNASDiskDefaults}
|
||||
resetVMwareDefaults={props.resetVMwareDefaults}
|
||||
resetAgentDefaults={props.resetAgentDefaults}
|
||||
resetDockerDefaults={props.resetDockerDefaults}
|
||||
resetDockerIgnoredPrefixes={props.resetDockerIgnoredPrefixes}
|
||||
resetStorageDefault={props.resetStorageDefault}
|
||||
factoryGuestDefaults={props.factoryGuestDefaults}
|
||||
factoryNodeDefaults={props.factoryNodeDefaults}
|
||||
factoryPBSDefaults={props.factoryPBSDefaults}
|
||||
factoryKubernetesDefaults={props.factoryKubernetesDefaults}
|
||||
factoryTrueNASDefaults={props.factoryTrueNASDefaults}
|
||||
factoryTrueNASDiskDefaults={props.factoryTrueNASDiskDefaults}
|
||||
factoryVMwareDefaults={props.factoryVMwareDefaults}
|
||||
factoryAgentDefaults={props.factoryAgentDefaults}
|
||||
factoryDockerDefaults={props.factoryDockerDefaults}
|
||||
factoryStorageDefault={props.factoryStorageDefault}
|
||||
timeThresholds={props.timeThresholds}
|
||||
metricTimeThresholds={props.metricTimeThresholds}
|
||||
setMetricTimeThresholds={props.setMetricTimeThresholds}
|
||||
snapshotDefaults={props.snapshotDefaults}
|
||||
setSnapshotDefaults={props.setSnapshotDefaults}
|
||||
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
|
||||
resetSnapshotDefaults={props.resetSnapshotDefaults}
|
||||
backupDefaults={props.backupDefaults}
|
||||
setBackupDefaults={props.setBackupDefaults}
|
||||
backupFactoryDefaults={props.backupFactoryDefaults}
|
||||
resetBackupDefaults={props.resetBackupDefaults}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
activeAlerts={props.activeAlerts}
|
||||
removeAlerts={props.removeAlerts}
|
||||
disableAllNodes={props.disableAllNodes}
|
||||
setDisableAllNodes={props.setDisableAllNodes}
|
||||
disableAllGuests={props.disableAllGuests}
|
||||
setDisableAllGuests={props.setDisableAllGuests}
|
||||
disableAllAgents={props.disableAllAgents}
|
||||
setDisableAllAgents={props.setDisableAllAgents}
|
||||
disableAllStorage={props.disableAllStorage}
|
||||
setDisableAllStorage={props.setDisableAllStorage}
|
||||
disableAllPBS={props.disableAllPBS}
|
||||
setDisableAllPBS={props.setDisableAllPBS}
|
||||
disableAllPMG={props.disableAllPMG}
|
||||
setDisableAllPMG={props.setDisableAllPMG}
|
||||
disableAllDockerHosts={props.disableAllDockerHosts}
|
||||
setDisableAllDockerHosts={props.setDisableAllDockerHosts}
|
||||
disableAllDockerServices={props.disableAllDockerServices}
|
||||
setDisableAllDockerServices={props.setDisableAllDockerServices}
|
||||
disableAllDockerContainers={props.disableAllDockerContainers}
|
||||
setDisableAllDockerContainers={props.setDisableAllDockerContainers}
|
||||
disableAllKubernetes={props.disableAllKubernetes}
|
||||
setDisableAllKubernetes={props.setDisableAllKubernetes}
|
||||
disableAllTrueNAS={props.disableAllTrueNAS}
|
||||
setDisableAllTrueNAS={props.setDisableAllTrueNAS}
|
||||
disableAllVMware={props.disableAllVMware}
|
||||
setDisableAllVMware={props.setDisableAllVMware}
|
||||
disableAllNodesOffline={props.disableAllNodesOffline}
|
||||
setDisableAllNodesOffline={props.setDisableAllNodesOffline}
|
||||
disableAllGuestsOffline={props.disableAllGuestsOffline}
|
||||
setDisableAllGuestsOffline={props.setDisableAllGuestsOffline}
|
||||
disableAllAgentsOffline={props.disableAllAgentsOffline}
|
||||
setDisableAllAgentsOffline={props.setDisableAllAgentsOffline}
|
||||
disableAllPBSOffline={props.disableAllPBSOffline}
|
||||
setDisableAllPBSOffline={props.setDisableAllPBSOffline}
|
||||
disableAllPMGOffline={props.disableAllPMGOffline}
|
||||
setDisableAllPMGOffline={props.setDisableAllPMGOffline}
|
||||
disableAllDockerHostsOffline={props.disableAllDockerHostsOffline}
|
||||
setDisableAllDockerHostsOffline={props.setDisableAllDockerHostsOffline}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -44,6 +44,25 @@ function createResource(overrides: Partial<Resource> = {}): Resource {
|
||||
}
|
||||
|
||||
describe('Resource Type Guards', () => {
|
||||
it('retains three-state UDP availability evidence on canonical resources', () => {
|
||||
const resource = createResource({
|
||||
type: 'network-endpoint',
|
||||
availability: {
|
||||
protocol: 'udp',
|
||||
probeOutcome: 'indeterminate',
|
||||
udpMode: 'open_or_filtered',
|
||||
available: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(resource.availability).toMatchObject({
|
||||
protocol: 'udp',
|
||||
probeOutcome: 'indeterminate',
|
||||
udpMode: 'open_or_filtered',
|
||||
available: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps host maintenance posture scoped to agent metadata', () => {
|
||||
const resource = createResource({
|
||||
type: 'agent',
|
||||
|
||||
@@ -1390,6 +1390,8 @@ export interface ResourceAvailabilityMeta {
|
||||
targetKind?: 'machine' | 'service' | 'device' | string;
|
||||
address?: string;
|
||||
protocol?: string;
|
||||
probeOutcome?: string;
|
||||
udpMode?: string;
|
||||
port?: number;
|
||||
path?: string;
|
||||
enabled?: boolean;
|
||||
|
||||
@@ -47,6 +47,32 @@ describe('availabilityProbePresentation', () => {
|
||||
expect(presentation?.toneClassName).toContain('emerald');
|
||||
});
|
||||
|
||||
it('presents open-or-filtered UDP as indeterminate rather than reachable', () => {
|
||||
vi.spyOn(Date, 'now').mockReturnValue(new Date('2026-05-06T13:00:20Z').getTime());
|
||||
|
||||
const presentation = getAvailabilityProbePresentation(
|
||||
makeAvailabilityResource({
|
||||
status: 'warning',
|
||||
availability: {
|
||||
protocol: 'udp',
|
||||
port: 514,
|
||||
available: false,
|
||||
probeOutcome: 'indeterminate',
|
||||
udpMode: 'open_or_filtered',
|
||||
lastChecked: '2026-05-06T13:00:18Z',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(presentation).toMatchObject({
|
||||
methodLabel: 'UDP 514',
|
||||
targetLabel: '514',
|
||||
resultLabel: 'open or filtered',
|
||||
netIoLabel: '514: open or filtered',
|
||||
});
|
||||
expect(presentation?.toneClassName).toContain('amber');
|
||||
});
|
||||
|
||||
it('also reads availability evidence from platform data for live state rows', () => {
|
||||
vi.spyOn(Date, 'now').mockReturnValue(new Date('2026-05-06T13:00:20Z').getTime());
|
||||
|
||||
|
||||
@@ -31,8 +31,9 @@ export const getAvailabilityProbeMethodLabel = (
|
||||
): string => {
|
||||
const protocol = normalizeAvailabilityProtocol(availability?.protocol);
|
||||
if (protocol === 'icmp') return 'ICMP';
|
||||
if (protocol === 'tcp') {
|
||||
return availability?.port ? `TCP ${availability.port}` : 'TCP';
|
||||
if (protocol === 'tcp' || protocol === 'udp') {
|
||||
const label = protocol.toUpperCase();
|
||||
return availability?.port ? `${label} ${availability.port}` : label;
|
||||
}
|
||||
if (protocol === 'http' || protocol === 'https') {
|
||||
const path = (availability?.path ?? '').trim();
|
||||
@@ -45,7 +46,7 @@ export const getAvailabilityProbeTargetLabel = (
|
||||
availability?: ResourceAvailabilityMeta | null,
|
||||
): string | null => {
|
||||
const protocol = normalizeAvailabilityProtocol(availability?.protocol);
|
||||
if (protocol === 'tcp') {
|
||||
if (protocol === 'tcp' || protocol === 'udp') {
|
||||
const port = availability?.port;
|
||||
return typeof port === 'number' && Number.isFinite(port) && port > 0 ? String(port) : null;
|
||||
}
|
||||
@@ -98,6 +99,7 @@ const getAvailabilityProbeResultLabel = (
|
||||
): string => {
|
||||
const latency = availability.latencyMillis;
|
||||
const normalizedStatus = (resource.status ?? '').trim().toLowerCase();
|
||||
if (availability.probeOutcome === 'indeterminate') return 'open or filtered';
|
||||
if (availability.available === false || ['offline', 'degraded'].includes(normalizedStatus)) {
|
||||
return getAvailabilityProbeFailureLabel(availability);
|
||||
}
|
||||
@@ -116,6 +118,9 @@ const getAvailabilityProbeToneClassName = (
|
||||
freshnessLabel: AvailabilityProbePresentation['freshnessLabel'],
|
||||
): string => {
|
||||
const normalizedStatus = (resource.status ?? '').trim().toLowerCase();
|
||||
if (availability.probeOutcome === 'indeterminate') {
|
||||
return AVAILABILITY_PROBE_WARNING_CLASS;
|
||||
}
|
||||
if (availability.available === false || normalizedStatus === 'offline') {
|
||||
return AVAILABILITY_PROBE_ERROR_CLASS;
|
||||
}
|
||||
|
||||
@@ -117,6 +117,9 @@ func (m *Manager) SaveActiveAlerts() error {
|
||||
if err := os.Chmod(finalFile, alertsFilePerm); err != nil {
|
||||
return fmt.Errorf("failed to set active alerts file permissions: %w", err)
|
||||
}
|
||||
if err := m.saveIntentPendingSnapshot(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug().Int("count", len(alerts)).Msg("saved active alerts to disk")
|
||||
return nil
|
||||
@@ -132,7 +135,7 @@ func (m *Manager) LoadActiveAlerts() error {
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
log.Info().Msg("No active alerts file found, starting fresh")
|
||||
return nil
|
||||
return m.loadIntentPendingNoLock()
|
||||
}
|
||||
return fmt.Errorf("failed to read active alerts: %w", err)
|
||||
}
|
||||
@@ -260,6 +263,9 @@ func (m *Manager) LoadActiveAlerts() error {
|
||||
}(alertCopy)
|
||||
}
|
||||
}
|
||||
if err := m.loadIntentPendingNoLock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Int("restored", restoredCount).
|
||||
|
||||
@@ -26,6 +26,7 @@ type canonicalLifecycleAlertParams struct {
|
||||
AddToHistory bool
|
||||
RateLimit bool
|
||||
DispatchAsync bool
|
||||
IntentBackup BackupIntentContext
|
||||
}
|
||||
|
||||
type canonicalStatefulAlertParams struct {
|
||||
@@ -362,6 +363,9 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
|
||||
return alertspecs.EvaluationResult{}, false
|
||||
}
|
||||
|
||||
// Persist the canonical confirmation state before applying intent policy.
|
||||
// Grace and operator context are an additional gate; they must not prevent
|
||||
// the underlying evaluator from reaching its configured confirmation count.
|
||||
if params.Tracking != nil {
|
||||
if result.State.ConsecutiveMatches > 0 {
|
||||
params.Tracking[params.TrackingKey] = result.State.ConsecutiveMatches
|
||||
@@ -370,6 +374,33 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
|
||||
}
|
||||
}
|
||||
|
||||
intentSignal := ""
|
||||
if params.Spec.Kind == alertspecs.AlertSpecKindConnectivity || params.Spec.Kind == alertspecs.AlertSpecKindPoweredState {
|
||||
intentSignal = string(AlertIntentSignalOffline)
|
||||
}
|
||||
if intentSignal != "" && existing == nil {
|
||||
conditionActive := result.State.State == alertspecs.AlertStatePending || result.State.State == alertspecs.AlertStateFiring
|
||||
decision := m.evaluateIntentNoLock(params.Spec.ResourceID, string(params.Spec.ResourceType), intentSignal, storageKey, params.Evidence.ObservedAt, conditionActive, params.IntentBackup)
|
||||
if decision.StateChanged {
|
||||
m.saveActiveAlertsAsync("lifecycle intent state")
|
||||
}
|
||||
if decision.Effective.Explicit && conditionActive && !decision.ShouldActivate {
|
||||
result.State.State = alertspecs.AlertStatePending
|
||||
result.State.Reason = decision.Reason
|
||||
if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() {
|
||||
result.State.FirstMatchedAt = pending.FirstMatchedAt
|
||||
}
|
||||
return result, true
|
||||
}
|
||||
if decision.Effective.Explicit && result.State.State == alertspecs.AlertStateFiring {
|
||||
if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() {
|
||||
result.State.FirstMatchedAt = pending.FirstMatchedAt
|
||||
}
|
||||
delete(m.intentPending, storageKey)
|
||||
m.saveActiveAlertsAsync("lifecycle intent activated")
|
||||
}
|
||||
}
|
||||
|
||||
switch result.State.State {
|
||||
case alertspecs.AlertStatePending:
|
||||
return result, true
|
||||
@@ -400,6 +431,9 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert
|
||||
alert.Metadata["resourceType"] = string(params.Spec.ResourceType)
|
||||
}
|
||||
applyCanonicalIdentity(alert, params.Spec.ID, string(params.Spec.Kind))
|
||||
if !result.State.FirstMatchedAt.IsZero() {
|
||||
alert.StartTime = result.State.FirstMatchedAt
|
||||
}
|
||||
applyCanonicalOperationalEvidence(alert, params.Spec, params.Evidence, time.Now())
|
||||
m.preserveAlertState(storageKey, alert)
|
||||
m.setActiveAlertNoLock(storageKey, alert)
|
||||
|
||||
@@ -144,8 +144,21 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||
triggered := alertspecsMetricTriggered(spec.MetricThreshold, value)
|
||||
alertStartTime := observedAt
|
||||
if !exists && triggered {
|
||||
timeThreshold := m.getTimeThreshold(spec.ResourceID, resourceType, metricType)
|
||||
if timeThreshold > 0 {
|
||||
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||
if effectiveIntent.Explicit {
|
||||
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, true, BackupIntentContext{})
|
||||
if pending, ok := m.intentPending[trackingKey]; ok && !pending.FirstMatchedAt.IsZero() {
|
||||
alertStartTime = pending.FirstMatchedAt
|
||||
}
|
||||
if decision.StateChanged {
|
||||
m.saveActiveAlertsAsync("canonical metric intent pending state")
|
||||
}
|
||||
if !decision.ShouldActivate {
|
||||
return
|
||||
}
|
||||
delete(m.intentPending, trackingKey)
|
||||
m.saveActiveAlertsAsync("canonical metric intent activated")
|
||||
} else if timeThreshold := m.getTimeThreshold(spec.ResourceID, resourceType, metricType); timeThreshold > 0 {
|
||||
if pendingTime, isPending := m.pendingAlerts[trackingKey]; isPending {
|
||||
if time.Since(pendingTime) >= time.Duration(timeThreshold)*time.Second {
|
||||
delete(m.pendingAlerts, trackingKey)
|
||||
@@ -171,6 +184,10 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
|
||||
}
|
||||
|
||||
if !triggered {
|
||||
decision := m.evaluateIntentNoLock(spec.ResourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, observedAt, false, BackupIntentContext{})
|
||||
if decision.StateChanged {
|
||||
m.saveActiveAlertsAsync("canonical metric intent cleared")
|
||||
}
|
||||
delete(m.pendingAlerts, trackingKey)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
CurrentAlertIntentPolicySchemaVersion = 1
|
||||
maxAlertIntentGraceSeconds = 30 * 24 * 60 * 60
|
||||
)
|
||||
|
||||
// AlertIntentSignal is a stable policy key shared by configuration, preview,
|
||||
// and runtime evaluation. Metric signals use the metric.<name> namespace.
|
||||
type AlertIntentSignal string
|
||||
|
||||
const (
|
||||
AlertIntentSignalDefault AlertIntentSignal = "*"
|
||||
AlertIntentSignalOffline AlertIntentSignal = "state.offline"
|
||||
AlertIntentSignalAvailability AlertIntentSignal = "incident.availability"
|
||||
)
|
||||
|
||||
// BackupOfflineIntentPolicy extends an offline candidate while Pulse has fresh
|
||||
// evidence that a Proxmox backup is responsible for the transient state.
|
||||
type BackupOfflineIntentPolicy struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
PostGraceSeconds int `json:"postGraceSeconds,omitempty"`
|
||||
MaxDeferralSeconds int `json:"maxDeferralSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// AlertIntentRule is deliberately small and typed. Pointer fields distinguish
|
||||
// inheritance from an explicit zero/false override.
|
||||
type AlertIntentRule struct {
|
||||
GraceSeconds *int `json:"graceSeconds,omitempty"`
|
||||
HonorOperatorState *bool `json:"honorOperatorState,omitempty"`
|
||||
BackupOffline *BackupOfflineIntentPolicy `json:"backupOffline,omitempty"`
|
||||
}
|
||||
|
||||
// AlertIntentPolicyDocument contains alert-intent defaults and overrides.
|
||||
// Resources are keyed by canonical resource ID. ResourceTypes use canonical
|
||||
// alert resource-type keys. Signal maps accept "*", state.offline,
|
||||
// incident.availability, and metric.<name> keys.
|
||||
type AlertIntentPolicyDocument struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Revision int64 `json:"revision"`
|
||||
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||
Defaults map[string]AlertIntentRule `json:"defaults,omitempty"`
|
||||
ResourceTypes map[string]map[string]AlertIntentRule `json:"resourceTypes,omitempty"`
|
||||
Resources map[string]map[string]AlertIntentRule `json:"resources,omitempty"`
|
||||
}
|
||||
|
||||
func NewAlertIntentPolicyDocument() AlertIntentPolicyDocument {
|
||||
return AlertIntentPolicyDocument{
|
||||
SchemaVersion: CurrentAlertIntentPolicySchemaVersion,
|
||||
Defaults: make(map[string]AlertIntentRule),
|
||||
ResourceTypes: make(map[string]map[string]AlertIntentRule),
|
||||
Resources: make(map[string]map[string]AlertIntentRule),
|
||||
}
|
||||
}
|
||||
|
||||
func MetricAlertIntentSignal(metric string) string {
|
||||
metric = strings.ToLower(strings.TrimSpace(metric))
|
||||
if metric == "" {
|
||||
return ""
|
||||
}
|
||||
return "metric." + metric
|
||||
}
|
||||
|
||||
// NormalizeAlertIntentPolicyDocument returns a detached, canonical document.
|
||||
func NormalizeAlertIntentPolicyDocument(input AlertIntentPolicyDocument) AlertIntentPolicyDocument {
|
||||
out := NewAlertIntentPolicyDocument()
|
||||
if input.SchemaVersion > 0 {
|
||||
out.SchemaVersion = input.SchemaVersion
|
||||
}
|
||||
out.Revision = input.Revision
|
||||
if input.UpdatedAt != nil {
|
||||
updatedAt := input.UpdatedAt.UTC()
|
||||
out.UpdatedAt = &updatedAt
|
||||
}
|
||||
out.Defaults = normalizeIntentSignalRules(input.Defaults)
|
||||
out.ResourceTypes = normalizeIntentScopedRules(input.ResourceTypes, true)
|
||||
out.Resources = normalizeIntentScopedRules(input.Resources, false)
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeIntentScopedRules(input map[string]map[string]AlertIntentRule, lowerScope bool) map[string]map[string]AlertIntentRule {
|
||||
out := make(map[string]map[string]AlertIntentRule, len(input))
|
||||
for rawScope, rules := range input {
|
||||
scope := strings.TrimSpace(rawScope)
|
||||
if lowerScope {
|
||||
scope = CanonicalAlertResourceType(scope)
|
||||
}
|
||||
if scope == "" {
|
||||
continue
|
||||
}
|
||||
normalized := normalizeIntentSignalRules(rules)
|
||||
if len(normalized) > 0 {
|
||||
out[scope] = normalized
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeIntentSignalRules(input map[string]AlertIntentRule) map[string]AlertIntentRule {
|
||||
out := make(map[string]AlertIntentRule, len(input))
|
||||
for rawSignal, rule := range input {
|
||||
signal := strings.ToLower(strings.TrimSpace(rawSignal))
|
||||
if signal == "default" || signal == "_default" {
|
||||
signal = string(AlertIntentSignalDefault)
|
||||
}
|
||||
if signal == "" {
|
||||
continue
|
||||
}
|
||||
out[signal] = cloneAlertIntentRule(rule)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneAlertIntentRule(rule AlertIntentRule) AlertIntentRule {
|
||||
out := rule
|
||||
if rule.GraceSeconds != nil {
|
||||
value := *rule.GraceSeconds
|
||||
out.GraceSeconds = &value
|
||||
}
|
||||
if rule.HonorOperatorState != nil {
|
||||
value := *rule.HonorOperatorState
|
||||
out.HonorOperatorState = &value
|
||||
}
|
||||
if rule.BackupOffline != nil {
|
||||
value := *rule.BackupOffline
|
||||
out.BackupOffline = &value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func ValidateAlertIntentPolicyDocument(input AlertIntentPolicyDocument) error {
|
||||
doc := NormalizeAlertIntentPolicyDocument(input)
|
||||
if doc.SchemaVersion != CurrentAlertIntentPolicySchemaVersion {
|
||||
return fmt.Errorf("unsupported alert intent policy schema version %d", doc.SchemaVersion)
|
||||
}
|
||||
if doc.Revision < 0 {
|
||||
return fmt.Errorf("alert intent policy revision must be non-negative")
|
||||
}
|
||||
if err := validateIntentSignalRules("defaults", input.Defaults); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateIntentScopedRules("resource type", input.ResourceTypes, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateIntentScopedRules("resource", input.Resources, false); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIntentScopedRules(label string, scopes map[string]map[string]AlertIntentRule, resourceType bool) error {
|
||||
seen := make(map[string]string, len(scopes))
|
||||
for rawScope, rules := range scopes {
|
||||
scope := strings.TrimSpace(rawScope)
|
||||
if resourceType {
|
||||
scope = CanonicalAlertResourceType(scope)
|
||||
}
|
||||
if scope == "" {
|
||||
return fmt.Errorf("alert intent %s is required", label)
|
||||
}
|
||||
if previous, exists := seen[scope]; exists {
|
||||
return fmt.Errorf("alert intent %s keys %q and %q normalize to the same value", label, previous, rawScope)
|
||||
}
|
||||
seen[scope] = rawScope
|
||||
if err := validateIntentSignalRules(label+" "+scope, rules); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIntentSignalRules(scope string, rules map[string]AlertIntentRule) error {
|
||||
seen := make(map[string]string, len(rules))
|
||||
for rawSignal, rule := range rules {
|
||||
signal := strings.ToLower(strings.TrimSpace(rawSignal))
|
||||
if signal == "default" || signal == "_default" {
|
||||
signal = string(AlertIntentSignalDefault)
|
||||
}
|
||||
if signal == "" {
|
||||
return fmt.Errorf("%s has an empty alert intent signal", scope)
|
||||
}
|
||||
if previous, exists := seen[signal]; exists {
|
||||
return fmt.Errorf("%s signal keys %q and %q normalize to the same value", scope, previous, rawSignal)
|
||||
}
|
||||
seen[signal] = rawSignal
|
||||
if !validAlertIntentSignal(signal) {
|
||||
return fmt.Errorf("%s has unsupported alert intent signal %q", scope, signal)
|
||||
}
|
||||
if rule.GraceSeconds != nil && (*rule.GraceSeconds < 0 || *rule.GraceSeconds > maxAlertIntentGraceSeconds) {
|
||||
return fmt.Errorf("%s signal %q graceSeconds must be between 0 and %d", scope, signal, maxAlertIntentGraceSeconds)
|
||||
}
|
||||
if backup := rule.BackupOffline; backup != nil {
|
||||
if signal != string(AlertIntentSignalOffline) && signal != string(AlertIntentSignalDefault) {
|
||||
return fmt.Errorf("%s signal %q may not configure backupOffline", scope, signal)
|
||||
}
|
||||
if backup.PostGraceSeconds < 0 || backup.PostGraceSeconds > maxAlertIntentGraceSeconds {
|
||||
return fmt.Errorf("%s signal %q postGraceSeconds must be between 0 and %d", scope, signal, maxAlertIntentGraceSeconds)
|
||||
}
|
||||
if backup.MaxDeferralSeconds < 0 || backup.MaxDeferralSeconds > maxAlertIntentGraceSeconds {
|
||||
return fmt.Errorf("%s signal %q maxDeferralSeconds must be between 0 and %d", scope, signal, maxAlertIntentGraceSeconds)
|
||||
}
|
||||
if backup.Enabled && backup.MaxDeferralSeconds <= 0 {
|
||||
return fmt.Errorf("%s signal %q backupOffline maxDeferralSeconds must be positive when enabled", scope, signal)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validAlertIntentSignal(signal string) bool {
|
||||
if signal == string(AlertIntentSignalDefault) || signal == string(AlertIntentSignalOffline) || signal == string(AlertIntentSignalAvailability) {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(signal, "metric.") && strings.TrimPrefix(signal, "metric.") != ""
|
||||
}
|
||||
@@ -27,15 +27,39 @@ type SnapshotAlertConfig = alertconfig.SnapshotAlertConfig
|
||||
type BackupAlertConfig = alertconfig.BackupAlertConfig
|
||||
type GuestLookup = alertconfig.GuestLookup
|
||||
type AlertConfig = alertconfig.AlertConfig
|
||||
type AlertIntentSignal = alertconfig.AlertIntentSignal
|
||||
type BackupOfflineIntentPolicy = alertconfig.BackupOfflineIntentPolicy
|
||||
type AlertIntentRule = alertconfig.AlertIntentRule
|
||||
type AlertIntentPolicyDocument = alertconfig.AlertIntentPolicyDocument
|
||||
|
||||
const (
|
||||
AlertLevelWarning = alertconfig.AlertLevelWarning
|
||||
AlertLevelCritical = alertconfig.AlertLevelCritical
|
||||
ActivationPending = alertconfig.ActivationPending
|
||||
ActivationActive = alertconfig.ActivationActive
|
||||
ActivationSnoozed = alertconfig.ActivationSnoozed
|
||||
AlertLevelWarning = alertconfig.AlertLevelWarning
|
||||
AlertLevelCritical = alertconfig.AlertLevelCritical
|
||||
ActivationPending = alertconfig.ActivationPending
|
||||
ActivationActive = alertconfig.ActivationActive
|
||||
ActivationSnoozed = alertconfig.ActivationSnoozed
|
||||
CurrentAlertIntentPolicySchemaVersion = alertconfig.CurrentAlertIntentPolicySchemaVersion
|
||||
AlertIntentSignalDefault = alertconfig.AlertIntentSignalDefault
|
||||
AlertIntentSignalOffline = alertconfig.AlertIntentSignalOffline
|
||||
AlertIntentSignalAvailability = alertconfig.AlertIntentSignalAvailability
|
||||
)
|
||||
|
||||
func NewAlertIntentPolicyDocument() AlertIntentPolicyDocument {
|
||||
return alertconfig.NewAlertIntentPolicyDocument()
|
||||
}
|
||||
|
||||
func MetricAlertIntentSignal(metric string) string {
|
||||
return alertconfig.MetricAlertIntentSignal(metric)
|
||||
}
|
||||
|
||||
func NormalizeAlertIntentPolicyDocument(document AlertIntentPolicyDocument) AlertIntentPolicyDocument {
|
||||
return alertconfig.NormalizeAlertIntentPolicyDocument(document)
|
||||
}
|
||||
|
||||
func ValidateAlertIntentPolicyDocument(document AlertIntentPolicyDocument) error {
|
||||
return alertconfig.ValidateAlertIntentPolicyDocument(document)
|
||||
}
|
||||
|
||||
var ErrAlertNotFound = errors.New("alert not found")
|
||||
|
||||
func NormalizeAlertConfigAliases(config *AlertConfig) {
|
||||
|
||||
@@ -184,7 +184,22 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
|
||||
m.mu.RLock()
|
||||
thresholds := m.getGuestThresholds(guest, guestID)
|
||||
m.mu.RUnlock()
|
||||
m.checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType, thresholds, monitorOnly)
|
||||
backupContext := BackupIntentContext{
|
||||
Active: strings.EqualFold(strings.TrimSpace(snapshot.Lock), "backup"),
|
||||
ObservedAt: time.Now(),
|
||||
Evidence: "guest_lock",
|
||||
}
|
||||
if !backupContext.Active {
|
||||
m.mu.RLock()
|
||||
resolver := m.backupIntentResolver
|
||||
m.mu.RUnlock()
|
||||
if resolver != nil {
|
||||
if resolved, ok := resolver(guestID, instanceName, node, snapshot.VMID, backupContext.ObservedAt); ok {
|
||||
backupContext = resolved
|
||||
}
|
||||
}
|
||||
}
|
||||
m.checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, backupContext)
|
||||
}
|
||||
} else {
|
||||
// For paused/suspended, clear powered-off alert
|
||||
@@ -359,6 +374,10 @@ func (m *Manager) checkGuestPoweredOff(guestID, name, node, instanceName, guestT
|
||||
}
|
||||
|
||||
func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool) {
|
||||
m.checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType, thresholds, monitorOnly, BackupIntentContext{})
|
||||
}
|
||||
|
||||
func (m *Manager) checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, node, instanceName, guestType string, thresholds ThresholdConfig, monitorOnly bool, backupContext BackupIntentContext) {
|
||||
alertID := fmt.Sprintf("guest-powered-off-%s", guestID)
|
||||
severity := NormalizePoweredOffSeverity(thresholds.PoweredOffSeverity)
|
||||
resourceType := unifiedresources.ResourceTypeVM
|
||||
@@ -400,6 +419,7 @@ func (m *Manager) checkGuestPoweredOffWithThresholds(guestID, name, node, instan
|
||||
AddToRecent: true,
|
||||
AddToHistory: true,
|
||||
DispatchAsync: false,
|
||||
IntentBackup: backupContext,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -424,6 +444,9 @@ func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) {
|
||||
Msg("Guest is running, resetting powered-off confirmation count")
|
||||
delete(m.offlineConfirmations, guestID)
|
||||
}
|
||||
if decision := m.evaluateIntentNoLock(guestID, "", string(AlertIntentSignalOffline), alertID, time.Now(), false, BackupIntentContext{}); decision.StateChanged {
|
||||
m.saveActiveAlertsAsync("guest offline intent cleared")
|
||||
}
|
||||
|
||||
// Check if powered-off alert exists. A guest that moved nodes may still
|
||||
// hold the alert under its old node-scoped identity; re-home it so it
|
||||
|
||||
@@ -23,6 +23,7 @@ type guestSnapshot struct {
|
||||
Node string
|
||||
Instance string
|
||||
Status string
|
||||
Lock string
|
||||
|
||||
CPUPercent float64
|
||||
MemUsage float64
|
||||
@@ -98,6 +99,7 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot {
|
||||
Node: vm.Node,
|
||||
Instance: vm.Instance,
|
||||
Status: vm.Status,
|
||||
Lock: vm.Lock,
|
||||
CPUPercent: vm.CPU * 100,
|
||||
MemUsage: vm.Memory.Usage,
|
||||
DiskUsage: vm.Disk.Usage,
|
||||
@@ -120,6 +122,7 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
|
||||
Node: container.Node,
|
||||
Instance: container.Instance,
|
||||
Status: container.Status,
|
||||
Lock: container.Lock,
|
||||
CPUPercent: container.CPU * 100,
|
||||
MemUsage: container.Memory.Usage,
|
||||
DiskUsage: container.Disk.Usage,
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const intentPendingFileName = "intent-pending.json"
|
||||
|
||||
func (m *Manager) saveIntentPendingSnapshot() error {
|
||||
m.mu.RLock()
|
||||
states := make([]IntentPendingState, 0, len(m.intentPending))
|
||||
for _, state := range m.intentPending {
|
||||
states = append(states, state)
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
sort.Slice(states, func(i, j int) bool { return states[i].TrackingKey < states[j].TrackingKey })
|
||||
data, err := json.Marshal(states)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal alert intent pending state: %w", err)
|
||||
}
|
||||
alertsDir := m.getAlertsDir()
|
||||
if err := os.MkdirAll(alertsDir, alertsDirPerm); err != nil {
|
||||
return fmt.Errorf("create alert intent state directory: %w", err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(alertsDir, "intent-pending-*.json.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create alert intent state temp file: %w", err)
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer func() {
|
||||
if err := os.Remove(tmpName); err != nil && !os.IsNotExist(err) {
|
||||
log.Warn().Err(err).Str("file", tmpName).Msg("Failed to remove alert intent state temp file")
|
||||
}
|
||||
}()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("write alert intent state temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Chmod(alertsFilePerm); err != nil {
|
||||
tmp.Close()
|
||||
return fmt.Errorf("chmod alert intent state temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return fmt.Errorf("close alert intent state temp file: %w", err)
|
||||
}
|
||||
finalPath := filepath.Join(alertsDir, intentPendingFileName)
|
||||
if err := os.Rename(tmpName, finalPath); err != nil {
|
||||
return fmt.Errorf("persist alert intent pending state: %w", err)
|
||||
}
|
||||
if err := os.Chmod(finalPath, alertsFilePerm); err != nil {
|
||||
return fmt.Errorf("chmod alert intent pending state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadIntentPendingNoLock restores pending policy candidates. Caller holds m.mu.
|
||||
func (m *Manager) loadIntentPendingNoLock() error {
|
||||
path := filepath.Join(m.getAlertsDir(), intentPendingFileName)
|
||||
data, err := readLimitedRegularFile(path, maxActiveAlertsFileSizeBytes)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("read alert intent pending state: %w", err)
|
||||
}
|
||||
var states []IntentPendingState
|
||||
if err := json.Unmarshal(data, &states); err != nil {
|
||||
return fmt.Errorf("decode alert intent pending state: %w", err)
|
||||
}
|
||||
now := time.Now()
|
||||
if m.intentPending == nil {
|
||||
m.intentPending = make(map[string]IntentPendingState)
|
||||
}
|
||||
for _, state := range states {
|
||||
if state.TrackingKey == "" || state.ResourceID == "" || state.Signal == "" || state.FirstMatchedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if now.Sub(state.LastObservedAt) > 24*time.Hour || state.LastObservedAt.After(now.Add(time.Minute)) || state.FirstMatchedAt.After(now.Add(time.Minute)) {
|
||||
continue
|
||||
}
|
||||
m.intentPending[state.TrackingKey] = state
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrAlertIntentPolicyRevisionConflict = errors.New("alert_intent_policy_revision_conflict")
|
||||
|
||||
type OperatorIntentContext struct {
|
||||
IntentionallyOffline bool `json:"intentionallyOffline"`
|
||||
MaintenanceStartAt *time.Time `json:"maintenanceStartAt,omitempty"`
|
||||
MaintenanceEndAt *time.Time `json:"maintenanceEndAt,omitempty"`
|
||||
MaintenanceReason string `json:"maintenanceReason,omitempty"`
|
||||
}
|
||||
|
||||
func (c OperatorIntentContext) MaintenanceActiveAt(now time.Time) bool {
|
||||
return c.MaintenanceStartAt != nil && c.MaintenanceEndAt != nil &&
|
||||
!now.Before(*c.MaintenanceStartAt) && now.Before(*c.MaintenanceEndAt)
|
||||
}
|
||||
|
||||
type BackupIntentContext struct {
|
||||
Active bool `json:"active"`
|
||||
ObservedAt time.Time `json:"observedAt,omitempty"`
|
||||
Evidence string `json:"evidence,omitempty"`
|
||||
}
|
||||
|
||||
type OperatorIntentContextResolver func(resourceID string, now time.Time) (OperatorIntentContext, bool)
|
||||
type BackupIntentContextResolver func(resourceID, instance, node string, vmid int, now time.Time) (BackupIntentContext, bool)
|
||||
type ResourceIntentIdentityResolver func(resourceID string) (canonicalID string, found bool)
|
||||
|
||||
type IntentPendingState struct {
|
||||
TrackingKey string `json:"trackingKey"`
|
||||
ResourceID string `json:"resourceId"`
|
||||
ResourceType string `json:"resourceType"`
|
||||
Signal string `json:"signal"`
|
||||
FirstMatchedAt time.Time `json:"firstMatchedAt"`
|
||||
LastObservedAt time.Time `json:"lastObservedAt"`
|
||||
BackupActive bool `json:"backupActive,omitempty"`
|
||||
BackupEndedAt *time.Time `json:"backupEndedAt,omitempty"`
|
||||
BackupEvidence string `json:"backupEvidence,omitempty"`
|
||||
}
|
||||
|
||||
type EffectiveAlertIntentPolicy struct {
|
||||
GraceSeconds int `json:"graceSeconds"`
|
||||
HonorOperatorState bool `json:"honorOperatorState"`
|
||||
BackupOffline *BackupOfflineIntentPolicy `json:"backupOffline,omitempty"`
|
||||
Sources map[string]string `json:"sources"`
|
||||
Explicit bool `json:"explicit"`
|
||||
}
|
||||
|
||||
type AlertIntentPolicyPreviewRequest struct {
|
||||
ResourceID string `json:"resourceId"`
|
||||
ResourceType string `json:"resourceType"`
|
||||
Signal string `json:"signal"`
|
||||
ConditionActive bool `json:"conditionActive"`
|
||||
FirstMatchedAt *time.Time `json:"firstMatchedAt,omitempty"`
|
||||
BackupActive *bool `json:"backupActive,omitempty"`
|
||||
BackupObservedAt *time.Time `json:"backupObservedAt,omitempty"`
|
||||
}
|
||||
|
||||
type AlertIntentPolicyPreviewContext struct {
|
||||
Kind string `json:"kind"`
|
||||
Active bool `json:"active"`
|
||||
Evidence string `json:"evidence,omitempty"`
|
||||
ObservedAt *time.Time `json:"observedAt,omitempty"`
|
||||
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
|
||||
}
|
||||
|
||||
type AlertIntentPolicyPreview struct {
|
||||
ResourceID string `json:"resourceId"`
|
||||
ResourceType string `json:"resourceType"`
|
||||
Signal string `json:"signal"`
|
||||
Status string `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
Effective EffectiveAlertIntentPolicy `json:"effective"`
|
||||
FirstMatchedAt *time.Time `json:"firstMatchedAt,omitempty"`
|
||||
EligibleAt *time.Time `json:"eligibleAt,omitempty"`
|
||||
HardCapAt *time.Time `json:"hardCapAt,omitempty"`
|
||||
RemainingSecs int64 `json:"remainingSeconds,omitempty"`
|
||||
Contexts []AlertIntentPolicyPreviewContext `json:"contexts"`
|
||||
Warnings []string `json:"warnings"`
|
||||
}
|
||||
|
||||
type intentDecision struct {
|
||||
ShouldActivate bool
|
||||
Pending bool
|
||||
Suppressed bool
|
||||
Reason string
|
||||
EligibleAt time.Time
|
||||
HardCapAt time.Time
|
||||
StateChanged bool
|
||||
Effective EffectiveAlertIntentPolicy
|
||||
}
|
||||
|
||||
func (m *Manager) SetOperatorIntentContextResolver(resolver OperatorIntentContextResolver) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.operatorIntentResolver = resolver
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Manager) SetBackupIntentContextResolver(resolver BackupIntentContextResolver) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.backupIntentResolver = resolver
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Manager) SetResourceIntentIdentityResolver(resolver ResourceIntentIdentityResolver) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.resourceIntentResolver = resolver
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// LoadIntentPolicies installs a persisted policy document without changing its
|
||||
// revision. API writes should use UpdateIntentPolicies.
|
||||
func (m *Manager) LoadIntentPolicies(document AlertIntentPolicyDocument) error {
|
||||
if err := ValidateAlertIntentPolicyDocument(document); err != nil {
|
||||
return err
|
||||
}
|
||||
if m == nil {
|
||||
return errors.New("alert manager is nil")
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.intentPolicies = NormalizeAlertIntentPolicyDocument(document)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) GetIntentPolicies() AlertIntentPolicyDocument {
|
||||
if m == nil {
|
||||
return NewAlertIntentPolicyDocument()
|
||||
}
|
||||
m.mu.RLock()
|
||||
document := NormalizeAlertIntentPolicyDocument(m.intentPolicies)
|
||||
m.mu.RUnlock()
|
||||
return document
|
||||
}
|
||||
|
||||
func (m *Manager) UpdateIntentPolicies(document AlertIntentPolicyDocument) (AlertIntentPolicyDocument, error) {
|
||||
if err := ValidateAlertIntentPolicyDocument(document); err != nil {
|
||||
return AlertIntentPolicyDocument{}, err
|
||||
}
|
||||
if m == nil {
|
||||
return AlertIntentPolicyDocument{}, errors.New("alert manager is nil")
|
||||
}
|
||||
|
||||
now := m.policyNow().UTC()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if document.Revision != m.intentPolicies.Revision {
|
||||
return AlertIntentPolicyDocument{}, fmt.Errorf("%w: got %d want %d", ErrAlertIntentPolicyRevisionConflict, document.Revision, m.intentPolicies.Revision)
|
||||
}
|
||||
normalized := NormalizeAlertIntentPolicyDocument(document)
|
||||
normalized.Revision = m.intentPolicies.Revision + 1
|
||||
normalized.UpdatedAt = &now
|
||||
m.intentPolicies = normalized
|
||||
return NormalizeAlertIntentPolicyDocument(normalized), nil
|
||||
}
|
||||
|
||||
func (m *Manager) resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal string) EffectiveAlertIntentPolicy {
|
||||
resourceID = strings.TrimSpace(resourceID)
|
||||
resourceType = strings.ToLower(strings.TrimSpace(resourceType))
|
||||
signal = strings.ToLower(strings.TrimSpace(signal))
|
||||
effective := EffectiveAlertIntentPolicy{Sources: make(map[string]string)}
|
||||
|
||||
if strings.HasPrefix(signal, "metric.") {
|
||||
metric := strings.TrimPrefix(signal, "metric.")
|
||||
if delay, source, ok := m.getLegacyTimeThresholdWithSource(resourceType, metric); ok {
|
||||
effective.GraceSeconds = delay
|
||||
effective.Sources["graceSeconds"] = source
|
||||
}
|
||||
}
|
||||
|
||||
apply := func(rule AlertIntentRule, source string) {
|
||||
if rule.GraceSeconds != nil {
|
||||
effective.GraceSeconds = *rule.GraceSeconds
|
||||
effective.Sources["graceSeconds"] = source
|
||||
effective.Explicit = true
|
||||
}
|
||||
if rule.HonorOperatorState != nil {
|
||||
effective.HonorOperatorState = *rule.HonorOperatorState
|
||||
effective.Sources["honorOperatorState"] = source
|
||||
effective.Explicit = true
|
||||
}
|
||||
if rule.BackupOffline != nil {
|
||||
backup := *rule.BackupOffline
|
||||
effective.BackupOffline = &backup
|
||||
effective.Sources["backupOffline"] = source
|
||||
effective.Explicit = true
|
||||
}
|
||||
}
|
||||
applyRules := func(rules map[string]AlertIntentRule, source string) {
|
||||
if rule, ok := rules[string(AlertIntentSignalDefault)]; ok {
|
||||
apply(rule, source+".*")
|
||||
}
|
||||
if rule, ok := rules[signal]; ok {
|
||||
apply(rule, source+"."+signal)
|
||||
}
|
||||
}
|
||||
|
||||
applyRules(m.intentPolicies.Defaults, "defaults")
|
||||
for _, typeKey := range CanonicalResourceTypeKeys(resourceType) {
|
||||
if rules, ok := m.intentPolicies.ResourceTypes[typeKey]; ok {
|
||||
applyRules(rules, "resourceTypes."+typeKey)
|
||||
break
|
||||
}
|
||||
}
|
||||
resourceIDs := make([]string, 0, 2)
|
||||
if m.resourceIntentResolver != nil {
|
||||
if canonicalID, ok := m.resourceIntentResolver(resourceID); ok {
|
||||
canonicalID = strings.TrimSpace(canonicalID)
|
||||
if canonicalID != "" && canonicalID != resourceID {
|
||||
resourceIDs = append(resourceIDs, canonicalID)
|
||||
}
|
||||
}
|
||||
}
|
||||
resourceIDs = append(resourceIDs, resourceID)
|
||||
for _, candidateID := range resourceIDs {
|
||||
if rules, ok := m.intentPolicies.Resources[candidateID]; ok {
|
||||
applyRules(rules, "resources."+candidateID)
|
||||
}
|
||||
}
|
||||
if _, ok := effective.Sources["graceSeconds"]; !ok {
|
||||
effective.Sources["graceSeconds"] = "factory"
|
||||
}
|
||||
if _, ok := effective.Sources["honorOperatorState"]; !ok {
|
||||
effective.Sources["honorOperatorState"] = "factory"
|
||||
}
|
||||
return effective
|
||||
}
|
||||
|
||||
func (m *Manager) ResolveEffectiveIntentPolicy(resourceID, resourceType, signal string) EffectiveAlertIntentPolicy {
|
||||
if m == nil {
|
||||
return EffectiveAlertIntentPolicy{Sources: map[string]string{"graceSeconds": "factory", "honorOperatorState": "factory"}}
|
||||
}
|
||||
m.mu.RLock()
|
||||
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal)
|
||||
m.mu.RUnlock()
|
||||
return effective
|
||||
}
|
||||
|
||||
func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackingKey string, observedAt time.Time, conditionActive bool, backup BackupIntentContext) intentDecision {
|
||||
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal)
|
||||
decision := intentDecision{Effective: effective}
|
||||
if !conditionActive {
|
||||
if _, exists := m.intentPending[trackingKey]; exists {
|
||||
delete(m.intentPending, trackingKey)
|
||||
decision.StateChanged = true
|
||||
}
|
||||
decision.Reason = "condition_clear"
|
||||
return decision
|
||||
}
|
||||
if !effective.Explicit {
|
||||
if _, exists := m.intentPending[trackingKey]; exists {
|
||||
delete(m.intentPending, trackingKey)
|
||||
decision.StateChanged = true
|
||||
}
|
||||
decision.ShouldActivate = true
|
||||
return decision
|
||||
}
|
||||
if observedAt.IsZero() {
|
||||
observedAt = m.policyNow()
|
||||
}
|
||||
state, exists := m.intentPending[trackingKey]
|
||||
if !exists || state.FirstMatchedAt.IsZero() {
|
||||
state = IntentPendingState{
|
||||
TrackingKey: trackingKey, ResourceID: resourceID, ResourceType: resourceType,
|
||||
Signal: signal, FirstMatchedAt: observedAt,
|
||||
}
|
||||
decision.StateChanged = true
|
||||
}
|
||||
state.LastObservedAt = observedAt
|
||||
|
||||
if effective.HonorOperatorState && m.operatorIntentResolver != nil {
|
||||
if operator, ok := m.operatorIntentResolver(resourceID, observedAt); ok {
|
||||
if operator.MaintenanceActiveAt(observedAt) {
|
||||
m.intentPending[trackingKey] = state
|
||||
decision.Pending = true
|
||||
decision.Suppressed = true
|
||||
decision.Reason = "operator_maintenance"
|
||||
if operator.MaintenanceEndAt != nil {
|
||||
decision.EligibleAt = *operator.MaintenanceEndAt
|
||||
}
|
||||
return decision
|
||||
}
|
||||
if signal == string(AlertIntentSignalOffline) && operator.IntentionallyOffline {
|
||||
m.intentPending[trackingKey] = state
|
||||
decision.Pending = true
|
||||
decision.Suppressed = true
|
||||
decision.Reason = "operator_intentionally_offline"
|
||||
return decision
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eligibleAt := state.FirstMatchedAt.Add(time.Duration(effective.GraceSeconds) * time.Second)
|
||||
if effective.BackupOffline != nil && effective.BackupOffline.Enabled && signal == string(AlertIntentSignalOffline) {
|
||||
if backup.Active {
|
||||
if !state.BackupActive || state.BackupEvidence != backup.Evidence {
|
||||
decision.StateChanged = true
|
||||
}
|
||||
state.BackupActive = true
|
||||
state.BackupEndedAt = nil
|
||||
state.BackupEvidence = backup.Evidence
|
||||
} else if state.BackupActive {
|
||||
endedAt := observedAt
|
||||
state.BackupActive = false
|
||||
state.BackupEndedAt = &endedAt
|
||||
decision.StateChanged = true
|
||||
}
|
||||
|
||||
capAt := state.FirstMatchedAt.Add(time.Duration(effective.BackupOffline.MaxDeferralSeconds) * time.Second)
|
||||
decision.HardCapAt = capAt
|
||||
if state.BackupActive && observedAt.Before(capAt) {
|
||||
m.intentPending[trackingKey] = state
|
||||
decision.Pending = true
|
||||
decision.Suppressed = true
|
||||
decision.Reason = "backup_active"
|
||||
decision.EligibleAt = capAt
|
||||
return decision
|
||||
}
|
||||
if state.BackupEndedAt != nil {
|
||||
postEligible := state.BackupEndedAt.Add(time.Duration(effective.BackupOffline.PostGraceSeconds) * time.Second)
|
||||
if postEligible.After(eligibleAt) {
|
||||
eligibleAt = postEligible
|
||||
}
|
||||
}
|
||||
if eligibleAt.After(capAt) {
|
||||
eligibleAt = capAt
|
||||
}
|
||||
if !observedAt.Before(capAt) {
|
||||
decision.Reason = "backup_grace_cap_exceeded"
|
||||
}
|
||||
}
|
||||
|
||||
m.intentPending[trackingKey] = state
|
||||
decision.EligibleAt = eligibleAt
|
||||
if observedAt.Before(eligibleAt) {
|
||||
decision.Pending = true
|
||||
if decision.Reason == "" {
|
||||
decision.Reason = "grace_period"
|
||||
}
|
||||
return decision
|
||||
}
|
||||
decision.ShouldActivate = true
|
||||
if decision.Reason == "" {
|
||||
decision.Reason = "eligible"
|
||||
}
|
||||
return decision
|
||||
}
|
||||
|
||||
func (m *Manager) PreviewIntentPolicy(request AlertIntentPolicyPreviewRequest) (AlertIntentPolicyPreview, error) {
|
||||
request.ResourceID = strings.TrimSpace(request.ResourceID)
|
||||
request.ResourceType = strings.ToLower(strings.TrimSpace(request.ResourceType))
|
||||
request.Signal = strings.ToLower(strings.TrimSpace(request.Signal))
|
||||
if request.ResourceID == "" || request.ResourceType == "" || request.Signal == "" {
|
||||
return AlertIntentPolicyPreview{}, errors.New("resourceId, resourceType, and signal are required")
|
||||
}
|
||||
now := m.policyNow().UTC()
|
||||
trackingKey := "preview:" + request.ResourceID + ":" + request.Signal
|
||||
backup := BackupIntentContext{}
|
||||
if request.BackupActive != nil {
|
||||
backup.Active = *request.BackupActive
|
||||
backup.Evidence = "preview"
|
||||
if request.BackupObservedAt != nil {
|
||||
backup.ObservedAt = request.BackupObservedAt.UTC()
|
||||
}
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
previous, hadPrevious := m.intentPending[trackingKey]
|
||||
if request.FirstMatchedAt != nil {
|
||||
m.intentPending[trackingKey] = IntentPendingState{
|
||||
TrackingKey: trackingKey, ResourceID: request.ResourceID, ResourceType: request.ResourceType,
|
||||
Signal: request.Signal, FirstMatchedAt: request.FirstMatchedAt.UTC(),
|
||||
}
|
||||
}
|
||||
decision := m.evaluateIntentNoLock(request.ResourceID, request.ResourceType, request.Signal, trackingKey, now, request.ConditionActive, backup)
|
||||
state, stateExists := m.intentPending[trackingKey]
|
||||
operator := OperatorIntentContext{}
|
||||
operatorFound := false
|
||||
if decision.Effective.HonorOperatorState && m.operatorIntentResolver != nil {
|
||||
operator, operatorFound = m.operatorIntentResolver(request.ResourceID, now)
|
||||
}
|
||||
if hadPrevious {
|
||||
m.intentPending[trackingKey] = previous
|
||||
} else {
|
||||
delete(m.intentPending, trackingKey)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
preview := AlertIntentPolicyPreview{
|
||||
ResourceID: request.ResourceID, ResourceType: request.ResourceType, Signal: request.Signal,
|
||||
Effective: decision.Effective, Contexts: []AlertIntentPolicyPreviewContext{}, Warnings: []string{},
|
||||
}
|
||||
if stateExists && !state.FirstMatchedAt.IsZero() {
|
||||
first := state.FirstMatchedAt
|
||||
preview.FirstMatchedAt = &first
|
||||
}
|
||||
if !decision.EligibleAt.IsZero() {
|
||||
eligible := decision.EligibleAt
|
||||
preview.EligibleAt = &eligible
|
||||
if eligible.After(now) {
|
||||
preview.RemainingSecs = int64(eligible.Sub(now).Seconds())
|
||||
}
|
||||
}
|
||||
if !decision.HardCapAt.IsZero() {
|
||||
capAt := decision.HardCapAt
|
||||
preview.HardCapAt = &capAt
|
||||
}
|
||||
switch {
|
||||
case !request.ConditionActive:
|
||||
preview.Status, preview.Reason = "clear", "condition_clear"
|
||||
case decision.Suppressed:
|
||||
preview.Status, preview.Reason = "expected_transient", decision.Reason
|
||||
case decision.Pending:
|
||||
preview.Status, preview.Reason = "pending_grace", decision.Reason
|
||||
default:
|
||||
preview.Status, preview.Reason = "would_activate", decision.Reason
|
||||
}
|
||||
if request.BackupActive != nil {
|
||||
ctx := AlertIntentPolicyPreviewContext{Kind: "backup", Active: *request.BackupActive, Evidence: backup.Evidence}
|
||||
if !backup.ObservedAt.IsZero() {
|
||||
observedAt := backup.ObservedAt
|
||||
ctx.ObservedAt = &observedAt
|
||||
}
|
||||
preview.Contexts = append(preview.Contexts, ctx)
|
||||
}
|
||||
if operatorFound {
|
||||
active := operator.IntentionallyOffline || operator.MaintenanceActiveAt(now)
|
||||
ctx := AlertIntentPolicyPreviewContext{Kind: "operator_state", Active: active, Evidence: operator.MaintenanceReason}
|
||||
if operator.MaintenanceEndAt != nil {
|
||||
expiresAt := operator.MaintenanceEndAt.UTC()
|
||||
ctx.ExpiresAt = &expiresAt
|
||||
}
|
||||
preview.Contexts = append(preview.Contexts, ctx)
|
||||
}
|
||||
if request.Signal == string(AlertIntentSignalAvailability) {
|
||||
preview.Warnings = append(preview.Warnings, "Availability probe failure thresholds are evaluated before alert grace.")
|
||||
}
|
||||
return preview, nil
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
)
|
||||
|
||||
func intPointer(value int) *int { return &value }
|
||||
func boolPointer(value bool) *bool { return &value }
|
||||
|
||||
func TestAlertIntentPolicyResolutionPrecedenceIsFieldByField(t *testing.T) {
|
||||
m := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(m.Stop)
|
||||
document := NewAlertIntentPolicyDocument()
|
||||
document.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{
|
||||
GraceSeconds: intPointer(30),
|
||||
HonorOperatorState: boolPointer(true),
|
||||
BackupOffline: &BackupOfflineIntentPolicy{
|
||||
Enabled: true, PostGraceSeconds: 45, MaxDeferralSeconds: 600,
|
||||
},
|
||||
}
|
||||
document.ResourceTypes["vm"] = map[string]AlertIntentRule{
|
||||
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(60)},
|
||||
}
|
||||
document.Resources["vm:101"] = map[string]AlertIntentRule{
|
||||
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(0)},
|
||||
}
|
||||
if err := m.LoadIntentPolicies(document); err != nil {
|
||||
t.Fatalf("LoadIntentPolicies() error = %v", err)
|
||||
}
|
||||
|
||||
effective := m.ResolveEffectiveIntentPolicy("vm:101", "vm", string(AlertIntentSignalOffline))
|
||||
if effective.GraceSeconds != 0 || !effective.HonorOperatorState || effective.BackupOffline == nil || !effective.BackupOffline.Enabled {
|
||||
t.Fatalf("effective policy = %+v", effective)
|
||||
}
|
||||
if got := effective.Sources["graceSeconds"]; got != "resources.vm:101.state.offline" {
|
||||
t.Fatalf("grace source = %q", got)
|
||||
}
|
||||
if got := effective.Sources["honorOperatorState"]; got != "defaults.state.offline" {
|
||||
t.Fatalf("operator source = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertIntentPolicyResolvesCanonicalResourceFromSourceID(t *testing.T) {
|
||||
m := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(m.Stop)
|
||||
document := NewAlertIntentPolicyDocument()
|
||||
document.Resources["vm-canonical"] = map[string]AlertIntentRule{
|
||||
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(90)},
|
||||
}
|
||||
if err := m.LoadIntentPolicies(document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.SetResourceIntentIdentityResolver(func(resourceID string) (string, bool) {
|
||||
if resourceID == "cluster-a:node-a:101" {
|
||||
return "vm-canonical", true
|
||||
}
|
||||
return "", false
|
||||
})
|
||||
|
||||
effective := m.ResolveEffectiveIntentPolicy("cluster-a:node-a:101", "vm", string(AlertIntentSignalOffline))
|
||||
if effective.GraceSeconds != 90 || effective.Sources["graceSeconds"] != "resources.vm-canonical.state.offline" {
|
||||
t.Fatalf("effective policy = %+v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertIntentPolicyValidationRejectsAmbiguousNormalizedKeys(t *testing.T) {
|
||||
document := NewAlertIntentPolicyDocument()
|
||||
document.ResourceTypes["VM"] = map[string]AlertIntentRule{"default": {GraceSeconds: intPointer(10)}}
|
||||
document.ResourceTypes[" vm "] = map[string]AlertIntentRule{"*": {GraceSeconds: intPointer(20)}}
|
||||
if err := ValidateAlertIntentPolicyDocument(document); err == nil {
|
||||
t.Fatal("expected normalized resource type collision to fail validation")
|
||||
}
|
||||
|
||||
document = NewAlertIntentPolicyDocument()
|
||||
document.Defaults["default"] = AlertIntentRule{GraceSeconds: intPointer(10)}
|
||||
document.Defaults["*"] = AlertIntentRule{GraceSeconds: intPointer(20)}
|
||||
if err := ValidateAlertIntentPolicyDocument(document); err == nil {
|
||||
t.Fatal("expected normalized signal collision to fail validation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertIntentBackupDeferralEndsWithPostGraceAndHardCap(t *testing.T) {
|
||||
m := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(m.Stop)
|
||||
document := NewAlertIntentPolicyDocument()
|
||||
document.Resources["vm:101"] = map[string]AlertIntentRule{
|
||||
string(AlertIntentSignalOffline): {
|
||||
GraceSeconds: intPointer(30),
|
||||
BackupOffline: &BackupOfflineIntentPolicy{
|
||||
Enabled: true, PostGraceSeconds: 60, MaxDeferralSeconds: 300,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := m.LoadIntentPolicies(document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
|
||||
m.mu.Lock()
|
||||
active := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start, true, BackupIntentContext{Active: true, Evidence: "guest_lock"})
|
||||
ended := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(100*time.Second), true, BackupIntentContext{})
|
||||
pending := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(159*time.Second), true, BackupIntentContext{})
|
||||
eligible := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(160*time.Second), true, BackupIntentContext{})
|
||||
m.mu.Unlock()
|
||||
|
||||
if !active.Pending || active.Reason != "backup_active" {
|
||||
t.Fatalf("active backup decision = %+v", active)
|
||||
}
|
||||
if !ended.Pending || ended.EligibleAt != start.Add(160*time.Second) {
|
||||
t.Fatalf("ended backup decision = %+v", ended)
|
||||
}
|
||||
if !pending.Pending || pending.ShouldActivate {
|
||||
t.Fatalf("post-backup decision = %+v", pending)
|
||||
}
|
||||
if !eligible.ShouldActivate || eligible.HardCapAt != start.Add(300*time.Second) {
|
||||
t.Fatalf("eligible decision = %+v", eligible)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertIntentPreviewHonorsOperatorStateWithoutMutatingRuntime(t *testing.T) {
|
||||
m := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(m.Stop)
|
||||
document := NewAlertIntentPolicyDocument()
|
||||
document.Defaults[string(AlertIntentSignalOffline)] = AlertIntentRule{
|
||||
GraceSeconds: intPointer(10), HonorOperatorState: boolPointer(true),
|
||||
}
|
||||
if err := m.LoadIntentPolicies(document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 7, 13, 10, 0, 0, 0, time.UTC)
|
||||
m.now = func() time.Time { return now }
|
||||
end := now.Add(time.Hour)
|
||||
m.SetOperatorIntentContextResolver(func(resourceID string, observedAt time.Time) (OperatorIntentContext, bool) {
|
||||
return OperatorIntentContext{MaintenanceStartAt: &now, MaintenanceEndAt: &end, MaintenanceReason: "upgrade"}, true
|
||||
})
|
||||
|
||||
preview, err := m.PreviewIntentPolicy(AlertIntentPolicyPreviewRequest{
|
||||
ResourceID: "vm:101", ResourceType: "vm", Signal: string(AlertIntentSignalOffline), ConditionActive: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("PreviewIntentPolicy() error = %v", err)
|
||||
}
|
||||
if preview.Status != "expected_transient" || preview.Reason != "operator_maintenance" {
|
||||
t.Fatalf("preview = %+v", preview)
|
||||
}
|
||||
if len(preview.Contexts) != 1 || preview.Contexts[0].Kind != "operator_state" || !preview.Contexts[0].Active {
|
||||
t.Fatalf("preview contexts = %+v", preview.Contexts)
|
||||
}
|
||||
if len(m.intentPending) != 0 {
|
||||
t.Fatalf("preview leaked runtime state: %+v", m.intentPending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleAlertStartsAtFirstIntentMatch(t *testing.T) {
|
||||
m := NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(m.Stop)
|
||||
document := NewAlertIntentPolicyDocument()
|
||||
document.Resources["vm:101"] = map[string]AlertIntentRule{
|
||||
string(AlertIntentSignalOffline): {GraceSeconds: intPointer(60)},
|
||||
}
|
||||
if err := m.LoadIntentPolicies(document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
spec, err := buildCanonicalPoweredStateSpec("vm:101", "database", unifiedresources.ResourceTypeVM, AlertLevelWarning, 1, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tracking := make(map[string]int)
|
||||
start := time.Date(2026, 7, 13, 11, 0, 0, 0, time.UTC)
|
||||
params := canonicalLifecycleAlertParams{
|
||||
Spec: spec,
|
||||
Evidence: alertspecs.AlertEvidence{
|
||||
ObservedAt: start,
|
||||
PoweredState: &alertspecs.PoweredStateEvidence{
|
||||
Expected: alertspecs.PowerStateOn,
|
||||
Observed: alertspecs.PowerStateOff,
|
||||
},
|
||||
},
|
||||
Tracking: tracking, TrackingKey: "vm:101", AlertID: "guest-powered-off-vm:101",
|
||||
AlertType: "powered-off", ResourceID: "vm:101", ResourceName: "database",
|
||||
}
|
||||
if result, _ := m.evaluateCanonicalLifecycleAlert(params); result.State.State != alertspecs.AlertStatePending {
|
||||
t.Fatalf("initial state = %s, want pending", result.State.State)
|
||||
}
|
||||
params.Evidence.ObservedAt = start.Add(60 * time.Second)
|
||||
if result, _ := m.evaluateCanonicalLifecycleAlert(params); result.State.State != alertspecs.AlertStateFiring {
|
||||
t.Fatalf("eligible state = %s, want firing", result.State.State)
|
||||
}
|
||||
|
||||
m.mu.RLock()
|
||||
alert, ok := m.getActiveAlertNoLock(canonicalPoweredStateStateID("vm:101"))
|
||||
m.mu.RUnlock()
|
||||
if !ok || alert == nil {
|
||||
t.Fatal("expected powered-state alert")
|
||||
}
|
||||
if !alert.StartTime.Equal(start) {
|
||||
t.Fatalf("alert start = %v, want first match %v", alert.StartTime, start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntentPendingStatePersistsAcrossRestart(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
first := NewManagerWithDataDir(dataDir)
|
||||
t.Cleanup(first.Stop)
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
first.mu.Lock()
|
||||
first.intentPending["state:vm:101"] = IntentPendingState{
|
||||
TrackingKey: "state:vm:101", ResourceID: "vm:101", ResourceType: "vm",
|
||||
Signal: string(AlertIntentSignalOffline), FirstMatchedAt: now.Add(-time.Minute), LastObservedAt: now,
|
||||
}
|
||||
first.mu.Unlock()
|
||||
if err := first.SaveActiveAlerts(); err != nil {
|
||||
t.Fatalf("SaveActiveAlerts: %v", err)
|
||||
}
|
||||
|
||||
second := NewManagerWithDataDir(dataDir)
|
||||
t.Cleanup(second.Stop)
|
||||
if err := second.LoadActiveAlerts(); err != nil {
|
||||
t.Fatalf("LoadActiveAlerts: %v", err)
|
||||
}
|
||||
second.mu.RLock()
|
||||
restored, ok := second.intentPending["state:vm:101"]
|
||||
second.mu.RUnlock()
|
||||
if !ok || !restored.FirstMatchedAt.Equal(now.Add(-time.Minute)) {
|
||||
t.Fatalf("restored intent state = %+v, found %v", restored, ok)
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,14 @@ type Manager struct {
|
||||
resolvedMutex sync.RWMutex // Secondary lock - see Lock Ordering Documentation above
|
||||
// Time threshold tracking
|
||||
pendingAlerts map[string]time.Time // Track when thresholds were first exceeded
|
||||
// Intent-policy pending state retains wall-clock and transient-context
|
||||
// evidence for policy-enabled candidates. It is keyed by canonical alert
|
||||
// tracking key and persisted with the alert manager's transition state.
|
||||
intentPending map[string]IntentPendingState
|
||||
intentPolicies AlertIntentPolicyDocument
|
||||
operatorIntentResolver OperatorIntentContextResolver
|
||||
backupIntentResolver BackupIntentContextResolver
|
||||
resourceIntentResolver ResourceIntentIdentityResolver
|
||||
// Offline confirmation tracking
|
||||
nodeOfflineCount map[string]int // Track consecutive offline counts for nodes (legacy)
|
||||
connectionDegradedCount map[string]int // Track consecutive degraded counts for platform connections (pve/pbs/pmg/vmware/truenas)
|
||||
@@ -128,6 +136,8 @@ func NewManagerWithDataDir(dataDir string) *Manager {
|
||||
recentlyResolved: make(map[string]*ResolvedAlert),
|
||||
resolvedAlias: make(map[string]string),
|
||||
pendingAlerts: make(map[string]time.Time),
|
||||
intentPending: make(map[string]IntentPendingState),
|
||||
intentPolicies: NewAlertIntentPolicyDocument(),
|
||||
nodeOfflineCount: make(map[string]int),
|
||||
connectionDegradedCount: make(map[string]int),
|
||||
offlineConfirmations: make(map[string]int),
|
||||
|
||||
@@ -46,20 +46,30 @@ func getThresholdForMetricFromConfig(config ThresholdConfig, metricType string)
|
||||
}
|
||||
|
||||
// getTimeThreshold determines the delay to apply for a metric/resource combination.
|
||||
func (m *Manager) getTimeThreshold(_ string, resourceType, metricType string) int {
|
||||
func (m *Manager) getTimeThreshold(resourceID string, resourceType, metricType string) int {
|
||||
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||
return effective.GraceSeconds
|
||||
}
|
||||
|
||||
func (m *Manager) getLegacyTimeThresholdWithSource(resourceType, metricType string) (int, string, bool) {
|
||||
if delay, ok := m.getMetricTimeThreshold(resourceType, metricType); ok {
|
||||
return delay
|
||||
return delay, "legacy.metricTimeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)) + "." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
|
||||
base, hasTypeSpecific := m.getBaseTimeThreshold(resourceType)
|
||||
|
||||
if !hasTypeSpecific {
|
||||
if delay, ok := m.getGlobalMetricTimeThreshold(metricType); ok {
|
||||
return delay
|
||||
return delay, "legacy.metricTimeThresholds.all." + strings.ToLower(strings.TrimSpace(metricType)), true
|
||||
}
|
||||
}
|
||||
|
||||
return base
|
||||
if hasTypeSpecific {
|
||||
return base, "legacy.timeThresholds." + strings.ToLower(strings.TrimSpace(resourceType)), true
|
||||
}
|
||||
if base != 0 {
|
||||
return base, "legacy.timeThresholds.all", true
|
||||
}
|
||||
return 0, "", false
|
||||
}
|
||||
|
||||
// getMetricTimeThreshold returns a metric-specific delay if configured at the resource-type level.
|
||||
@@ -215,11 +225,21 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
if !exists {
|
||||
alertStartTime := time.Now()
|
||||
|
||||
// Determine the appropriate time threshold based on resource/metric type
|
||||
timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType)
|
||||
|
||||
// Check if we have a time threshold configured
|
||||
if timeThreshold > 0 {
|
||||
effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
|
||||
if effectiveIntent.Explicit {
|
||||
decision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, alertStartTime, true, BackupIntentContext{})
|
||||
if pending, ok := m.intentPending[trackingKey]; ok && !pending.FirstMatchedAt.IsZero() {
|
||||
alertStartTime = pending.FirstMatchedAt
|
||||
}
|
||||
if decision.StateChanged {
|
||||
m.saveActiveAlertsAsync("metric intent pending state")
|
||||
}
|
||||
if !decision.ShouldActivate {
|
||||
return
|
||||
}
|
||||
delete(m.intentPending, trackingKey)
|
||||
m.saveActiveAlertsAsync("metric intent activated")
|
||||
} else if timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType); timeThreshold > 0 {
|
||||
// Check if this threshold was already pending
|
||||
if pendingTime, isPending := m.pendingAlerts[trackingKey]; isPending {
|
||||
// Check if enough time has passed
|
||||
@@ -472,6 +492,10 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
|
||||
}
|
||||
} else {
|
||||
// Value is below trigger threshold
|
||||
intentDecision := m.evaluateIntentNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType), trackingKey, time.Now(), false, BackupIntentContext{})
|
||||
if intentDecision.StateChanged {
|
||||
m.saveActiveAlertsAsync("metric intent cleared")
|
||||
}
|
||||
// Clear any pending alert for this metric
|
||||
if _, isPending := m.pendingAlerts[trackingKey]; isPending {
|
||||
delete(m.pendingAlerts, trackingKey)
|
||||
|
||||
@@ -113,6 +113,53 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
// Availability failure thresholds establish that an incident exists;
|
||||
// intent grace is a second, operator-controlled gate before activation.
|
||||
// Track observed keys separately so a pending incident is not mistaken for
|
||||
// a cleared condition when it is intentionally removed from desired.
|
||||
observedAvailability := make(map[string]struct{})
|
||||
intentStateChanged := false
|
||||
for storageKey, alert := range desired {
|
||||
if !isAvailabilityIncidentAlert(alert) {
|
||||
continue
|
||||
}
|
||||
observedAvailability[storageKey] = struct{}{}
|
||||
if existing, exists := m.getActiveAlertNoLock(storageKey); exists && existing != nil {
|
||||
if _, pending := m.intentPending[storageKey]; pending {
|
||||
delete(m.intentPending, storageKey)
|
||||
intentStateChanged = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
decision := m.evaluateIntentNoLock(alert.ResourceID, alertMetadataString(alert, "resourceType"), string(AlertIntentSignalAvailability), storageKey, alert.LastSeen, true, BackupIntentContext{})
|
||||
intentStateChanged = intentStateChanged || decision.StateChanged
|
||||
if decision.Effective.Explicit && !decision.ShouldActivate {
|
||||
delete(desired, storageKey)
|
||||
continue
|
||||
}
|
||||
if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() {
|
||||
alert.StartTime = pending.FirstMatchedAt
|
||||
}
|
||||
if _, ok := m.intentPending[storageKey]; ok {
|
||||
delete(m.intentPending, storageKey)
|
||||
intentStateChanged = true
|
||||
}
|
||||
}
|
||||
for storageKey, pending := range m.intentPending {
|
||||
if pending.Signal != string(AlertIntentSignalAvailability) {
|
||||
continue
|
||||
}
|
||||
if _, observed := observedAvailability[storageKey]; observed {
|
||||
continue
|
||||
}
|
||||
delete(m.intentPending, storageKey)
|
||||
intentStateChanged = true
|
||||
}
|
||||
if intentStateChanged {
|
||||
m.saveActiveAlertsAsync("availability intent state")
|
||||
}
|
||||
|
||||
for storageKey, alert := range m.activeAlerts {
|
||||
if !strings.HasPrefix(storageKey, unifiedresources.CanonicalResourceID(alert.ResourceID)+canonicalStateSeparator+"alertspec:provider-incident:") {
|
||||
continue
|
||||
@@ -147,6 +194,18 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso
|
||||
}
|
||||
}
|
||||
|
||||
func isAvailabilityIncidentAlert(alert *Alert) bool {
|
||||
return alertMetadataString(alert, "incidentCode") == "availability_unreachable"
|
||||
}
|
||||
|
||||
func alertMetadataString(alert *Alert, key string) string {
|
||||
if alert == nil || alert.Metadata == nil {
|
||||
return ""
|
||||
}
|
||||
value, _ := alert.Metadata[key].(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func shouldSuppressCanonicalUnifiedIncidentSpec(spec alertspecs.ResourceAlertSpec, providerSpecsByResource map[string][]alertspecs.ResourceAlertSpec) bool {
|
||||
if len(spec.SuppressionKeys) == 0 {
|
||||
return false
|
||||
|
||||
@@ -3,6 +3,7 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -42,6 +43,17 @@ type ConfigPersistence interface {
|
||||
SaveAlertConfig(alerts.AlertConfig) error
|
||||
}
|
||||
|
||||
type alertIntentPolicyManager interface {
|
||||
GetIntentPolicies() alerts.AlertIntentPolicyDocument
|
||||
LoadIntentPolicies(alerts.AlertIntentPolicyDocument) error
|
||||
UpdateIntentPolicies(alerts.AlertIntentPolicyDocument) (alerts.AlertIntentPolicyDocument, error)
|
||||
PreviewIntentPolicy(alerts.AlertIntentPolicyPreviewRequest) (alerts.AlertIntentPolicyPreview, error)
|
||||
}
|
||||
|
||||
type alertIntentPolicyPersistence interface {
|
||||
SaveAlertIntentPolicies(alerts.AlertIntentPolicyDocument) error
|
||||
}
|
||||
|
||||
// AlertMonitor defines the interface for monitoring operations used by alert handlers.
|
||||
type AlertMonitor interface {
|
||||
GetAlertManager() AlertManager
|
||||
@@ -55,6 +67,7 @@ type AlertMonitor interface {
|
||||
// AlertHandlers handles alert-related HTTP endpoints
|
||||
type AlertHandlers struct {
|
||||
stateMu sync.RWMutex
|
||||
intentPolicyMu sync.Mutex
|
||||
mtMonitor *monitoring.MultiTenantMonitor
|
||||
defaultMonitor AlertMonitor
|
||||
wsHub *websocket.Hub
|
||||
@@ -297,6 +310,88 @@ func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
h.intentPolicyMu.Lock()
|
||||
defer h.intentPolicyMu.Unlock()
|
||||
manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
|
||||
if !ok {
|
||||
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if err := utils.WriteJSONResponse(w, manager.GetIntentPolicies()); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write alert intent policies response")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 256*1024)
|
||||
var document alerts.AlertIntentPolicyDocument
|
||||
if err := json.NewDecoder(r.Body).Decode(&document); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Treat revision check, in-memory install, and durable save as one API
|
||||
// transaction. This also prevents a failed earlier writer from rolling
|
||||
// back a later successful update.
|
||||
h.intentPolicyMu.Lock()
|
||||
defer h.intentPolicyMu.Unlock()
|
||||
monitor := h.getMonitor(r.Context())
|
||||
manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
|
||||
if !ok {
|
||||
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
persistence, ok := monitor.GetConfigPersistence().(alertIntentPolicyPersistence)
|
||||
if !ok {
|
||||
http.Error(w, "alert intent policy persistence is unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
previous := manager.GetIntentPolicies()
|
||||
updated, err := manager.UpdateIntentPolicies(document)
|
||||
if err != nil {
|
||||
if errors.Is(err, alerts.ErrAlertIntentPolicyRevisionConflict) {
|
||||
http.Error(w, err.Error(), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := persistence.SaveAlertIntentPolicies(updated); err != nil {
|
||||
if rollbackErr := manager.LoadIntentPolicies(previous); rollbackErr != nil {
|
||||
log.Error().Err(rollbackErr).Msg("Failed to roll back in-memory alert intent policies after persistence failure")
|
||||
}
|
||||
http.Error(w, fmt.Sprintf("Failed to save alert intent policies: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := utils.WriteJSONResponse(w, updated); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write updated alert intent policies response")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 64*1024)
|
||||
var request alerts.AlertIntentPolicyPreviewRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
h.intentPolicyMu.Lock()
|
||||
defer h.intentPolicyMu.Unlock()
|
||||
manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
|
||||
if !ok {
|
||||
http.Error(w, "alert intent policy preview is unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
preview, err := manager.PreviewIntentPolicy(request)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := utils.WriteJSONResponse(w, preview); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write alert intent policy preview response")
|
||||
}
|
||||
}
|
||||
|
||||
// GetAlertDeliveryDiagnosis explains current notification delivery policy for
|
||||
// one active alert without sending a notification or mutating delivery state.
|
||||
func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1049,6 +1144,21 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
h.UpdateAlertConfig(w, r)
|
||||
case path == "intent-policies" && r.Method == http.MethodGet:
|
||||
if !ensureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
}
|
||||
h.GetAlertIntentPolicies(w, r)
|
||||
case path == "intent-policies" && r.Method == http.MethodPut:
|
||||
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
||||
return
|
||||
}
|
||||
h.UpdateAlertIntentPolicies(w, r)
|
||||
case path == "intent-policies/preview" && r.Method == http.MethodPost:
|
||||
if !ensureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
}
|
||||
h.PreviewAlertIntentPolicy(w, r)
|
||||
case path == "activate" && r.Method == http.MethodPost:
|
||||
if !ensureScope(w, r, config.ScopeMonitoringWrite) {
|
||||
return
|
||||
|
||||
@@ -72,6 +72,82 @@ func TestAlertsEndpoints(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("AlertIntentPolicies", func(t *testing.T) {
|
||||
res, err := http.Get(srv.server.URL + "/api/alerts/intent-policies")
|
||||
if err != nil {
|
||||
t.Fatalf("get policies: %v", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
t.Fatalf("get policies status = %d, want %d", res.StatusCode, http.StatusOK)
|
||||
}
|
||||
var document alerts.AlertIntentPolicyDocument
|
||||
if err := json.NewDecoder(res.Body).Decode(&document); err != nil {
|
||||
t.Fatalf("decode policies: %v", err)
|
||||
}
|
||||
|
||||
grace := 45
|
||||
if document.Defaults == nil {
|
||||
document.Defaults = make(map[string]alerts.AlertIntentRule)
|
||||
}
|
||||
document.Defaults[string(alerts.AlertIntentSignalOffline)] = alerts.AlertIntentRule{GraceSeconds: &grace}
|
||||
body, err := json.Marshal(document)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal policies: %v", err)
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/intent-policies", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create policy update: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
updatedResponse, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("update policies: %v", err)
|
||||
}
|
||||
defer updatedResponse.Body.Close()
|
||||
if updatedResponse.StatusCode != http.StatusOK {
|
||||
t.Fatalf("update policies status = %d, want %d", updatedResponse.StatusCode, http.StatusOK)
|
||||
}
|
||||
var updated alerts.AlertIntentPolicyDocument
|
||||
if err := json.NewDecoder(updatedResponse.Body).Decode(&updated); err != nil {
|
||||
t.Fatalf("decode updated policies: %v", err)
|
||||
}
|
||||
if updated.Revision != document.Revision+1 {
|
||||
t.Fatalf("updated revision = %d, want %d", updated.Revision, document.Revision+1)
|
||||
}
|
||||
|
||||
staleReq, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/intent-policies", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create stale update: %v", err)
|
||||
}
|
||||
staleReq.Header.Set("Content-Type", "application/json")
|
||||
staleResponse, err := http.DefaultClient.Do(staleReq)
|
||||
if err != nil {
|
||||
t.Fatalf("stale update: %v", err)
|
||||
}
|
||||
defer staleResponse.Body.Close()
|
||||
if staleResponse.StatusCode != http.StatusConflict {
|
||||
t.Fatalf("stale update status = %d, want %d", staleResponse.StatusCode, http.StatusConflict)
|
||||
}
|
||||
|
||||
previewBody := []byte(`{"resourceId":"vm:101","resourceType":"vm","signal":"state.offline","conditionActive":true}`)
|
||||
previewResponse, err := http.Post(srv.server.URL+"/api/alerts/intent-policies/preview", "application/json", bytes.NewReader(previewBody))
|
||||
if err != nil {
|
||||
t.Fatalf("preview policy: %v", err)
|
||||
}
|
||||
defer previewResponse.Body.Close()
|
||||
if previewResponse.StatusCode != http.StatusOK {
|
||||
t.Fatalf("preview status = %d, want %d", previewResponse.StatusCode, http.StatusOK)
|
||||
}
|
||||
var preview alerts.AlertIntentPolicyPreview
|
||||
if err := json.NewDecoder(previewResponse.Body).Decode(&preview); err != nil {
|
||||
t.Fatalf("decode preview: %v", err)
|
||||
}
|
||||
if preview.Status != "pending_grace" || preview.Effective.GraceSeconds != grace {
|
||||
t.Fatalf("preview = %+v", preview)
|
||||
}
|
||||
})
|
||||
|
||||
// 3. Activate alerts
|
||||
t.Run("ActivateAlerts", func(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPost, srv.server.URL+"/api/alerts/activate", nil)
|
||||
|
||||
@@ -27,6 +27,7 @@ type availabilityTargetResponse struct {
|
||||
type availabilityTestResponse struct {
|
||||
Success bool `json:"success"`
|
||||
LatencyMillis int64 `json:"latencyMillis"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
@@ -250,7 +251,7 @@ func (h *AvailabilityHandlers) testTarget(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
err := monitoring.ProbeAvailabilityTarget(r.Context(), target)
|
||||
outcome, err := monitoring.ProbeAvailabilityTargetResult(r.Context(), target)
|
||||
latencyMs := time.Since(start).Milliseconds()
|
||||
if err == nil && latencyMs == 0 {
|
||||
latencyMs = 1
|
||||
@@ -258,6 +259,7 @@ func (h *AvailabilityHandlers) testTarget(w http.ResponseWriter, r *http.Request
|
||||
response := availabilityTestResponse{
|
||||
Success: err == nil,
|
||||
LatencyMillis: latencyMs,
|
||||
Outcome: string(outcome),
|
||||
}
|
||||
if err != nil {
|
||||
response.Error = err.Error()
|
||||
|
||||
@@ -22,10 +22,22 @@ const (
|
||||
AvailabilityProbeTCP AvailabilityProbeProtocol = "tcp"
|
||||
AvailabilityProbeHTTP AvailabilityProbeProtocol = "http"
|
||||
AvailabilityProbeHTTPS AvailabilityProbeProtocol = "https"
|
||||
AvailabilityProbeUDP AvailabilityProbeProtocol = "udp"
|
||||
|
||||
availabilityProbePingAlias AvailabilityProbeProtocol = "ping"
|
||||
)
|
||||
|
||||
type AvailabilityUDPMode string
|
||||
|
||||
const (
|
||||
// AvailabilityUDPResponseRequired requires a datagram response and treats
|
||||
// silence as failure. This is the safe default for alerting.
|
||||
AvailabilityUDPResponseRequired AvailabilityUDPMode = "response_required"
|
||||
// AvailabilityUDPOpenOrFiltered treats silence as an indeterminate,
|
||||
// non-failing result; an ICMP port-unreachable response still fails.
|
||||
AvailabilityUDPOpenOrFiltered AvailabilityUDPMode = "open_or_filtered"
|
||||
)
|
||||
|
||||
type AvailabilityTargetKind string
|
||||
|
||||
const (
|
||||
@@ -44,6 +56,9 @@ type AvailabilityTarget struct {
|
||||
Protocol AvailabilityProbeProtocol `json:"protocol"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
UDPMode AvailabilityUDPMode `json:"udpMode,omitempty"`
|
||||
UDPRequest string `json:"udpRequest,omitempty"`
|
||||
UDPExpected string `json:"udpExpectedResponse,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
PollIntervalSecs int `json:"pollIntervalSeconds,omitempty"`
|
||||
TimeoutMillis int `json:"timeoutMillis,omitempty"`
|
||||
@@ -88,6 +103,12 @@ func (t *AvailabilityTarget) ApplyDefaults() {
|
||||
if t.FailureThreshold <= 0 {
|
||||
t.FailureThreshold = DefaultAvailabilityFailureThreshold
|
||||
}
|
||||
if t.Protocol == AvailabilityProbeUDP {
|
||||
t.UDPMode = normalizeAvailabilityUDPMode(t.UDPMode)
|
||||
if t.UDPMode == "" {
|
||||
t.UDPMode = AvailabilityUDPResponseRequired
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t AvailabilityTarget) EffectivePollIntervalSecs() int {
|
||||
@@ -137,9 +158,9 @@ func (t AvailabilityTarget) Validate() error {
|
||||
if t.Port != 0 {
|
||||
return fmt.Errorf("icmp availability targets must not set a port")
|
||||
}
|
||||
case AvailabilityProbeTCP:
|
||||
case AvailabilityProbeTCP, AvailabilityProbeUDP:
|
||||
if t.Port <= 0 || t.Port > 65535 {
|
||||
return fmt.Errorf("tcp availability targets require a valid port")
|
||||
return fmt.Errorf("%s availability targets require a valid port", protocol)
|
||||
}
|
||||
case AvailabilityProbeHTTP, AvailabilityProbeHTTPS:
|
||||
if t.Port < 0 || t.Port > 65535 {
|
||||
@@ -148,6 +169,25 @@ func (t AvailabilityTarget) Validate() error {
|
||||
default:
|
||||
return fmt.Errorf("unsupported availability protocol %q", t.Protocol)
|
||||
}
|
||||
if protocol == AvailabilityProbeUDP {
|
||||
mode := normalizeAvailabilityUDPMode(t.UDPMode)
|
||||
switch mode {
|
||||
case AvailabilityUDPResponseRequired, AvailabilityUDPOpenOrFiltered:
|
||||
default:
|
||||
return fmt.Errorf("unsupported UDP availability mode %q", t.UDPMode)
|
||||
}
|
||||
if mode == AvailabilityUDPResponseRequired && len(t.UDPRequest) == 0 {
|
||||
return fmt.Errorf("UDP response-required checks require a request payload")
|
||||
}
|
||||
if len(t.UDPRequest) > 512 {
|
||||
return fmt.Errorf("UDP request payload must be 512 bytes or less")
|
||||
}
|
||||
if len(t.UDPExpected) > 4096 {
|
||||
return fmt.Errorf("UDP expected response must be 4096 bytes or less")
|
||||
}
|
||||
} else if t.UDPMode != "" || t.UDPRequest != "" || t.UDPExpected != "" {
|
||||
return fmt.Errorf("UDP settings may only be used with UDP availability targets")
|
||||
}
|
||||
if t.PollIntervalSecs > 0 && t.PollIntervalSecs < 10 {
|
||||
return fmt.Errorf("availability poll interval must be at least 10 seconds")
|
||||
}
|
||||
@@ -218,11 +258,21 @@ func NormalizeAvailabilityTarget(target AvailabilityTarget) AvailabilityTarget {
|
||||
target.Address = normalizeAvailabilityAddress(target.Address)
|
||||
}
|
||||
target.Path = strings.TrimSpace(target.Path)
|
||||
target.UDPMode = normalizeAvailabilityUDPMode(target.UDPMode)
|
||||
if target.Protocol != AvailabilityProbeUDP {
|
||||
target.UDPMode = ""
|
||||
target.UDPRequest = ""
|
||||
target.UDPExpected = ""
|
||||
}
|
||||
target.LinkedResourceID = strings.TrimSpace(target.LinkedResourceID)
|
||||
target.ApplyDefaults()
|
||||
return target
|
||||
}
|
||||
|
||||
func normalizeAvailabilityUDPMode(mode AvailabilityUDPMode) AvailabilityUDPMode {
|
||||
return AvailabilityUDPMode(strings.ToLower(strings.TrimSpace(string(mode))))
|
||||
}
|
||||
|
||||
func normalizeAvailabilityProbeProtocol(protocol AvailabilityProbeProtocol) AvailabilityProbeProtocol {
|
||||
normalized := AvailabilityProbeProtocol(strings.ToLower(strings.TrimSpace(string(protocol))))
|
||||
if normalized == availabilityProbePingAlias {
|
||||
|
||||
@@ -225,7 +225,7 @@ func TestBranchCovValidate(t *testing.T) {
|
||||
// --- Protocol switch: default arm. ---
|
||||
{
|
||||
name: "unsupported protocol rejected",
|
||||
target: AvailabilityTarget{Address: "device.local", TargetKind: AvailabilityTargetService, Protocol: AvailabilityProbeProtocol("udp")},
|
||||
target: AvailabilityTarget{Address: "device.local", TargetKind: AvailabilityTargetService, Protocol: AvailabilityProbeProtocol("smtp")},
|
||||
wantErr: true,
|
||||
wantErrContains: "unsupported availability protocol",
|
||||
},
|
||||
|
||||
+29
-13
@@ -24,17 +24,18 @@ const (
|
||||
|
||||
// ExportData contains all configuration data for export
|
||||
type ExportData struct {
|
||||
Version string `json:"version"`
|
||||
ExportedAt time.Time `json:"exportedAt"`
|
||||
Nodes NodesConfig `json:"nodes"`
|
||||
Alerts alerts.AlertConfig `json:"alerts"`
|
||||
Email notifications.EmailConfig `json:"email"`
|
||||
Webhooks []notifications.WebhookConfig `json:"webhooks"`
|
||||
Apprise notifications.AppriseConfig `json:"apprise"`
|
||||
System SystemSettings `json:"system"`
|
||||
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
|
||||
SSO *SSOConfig `json:"sso,omitempty"`
|
||||
APITokens []APITokenRecord `json:"apiTokens,omitempty"`
|
||||
Version string `json:"version"`
|
||||
ExportedAt time.Time `json:"exportedAt"`
|
||||
Nodes NodesConfig `json:"nodes"`
|
||||
Alerts alerts.AlertConfig `json:"alerts"`
|
||||
AlertIntent *alerts.AlertIntentPolicyDocument `json:"alertIntentPolicies,omitempty"`
|
||||
Email notifications.EmailConfig `json:"email"`
|
||||
Webhooks []notifications.WebhookConfig `json:"webhooks"`
|
||||
Apprise notifications.AppriseConfig `json:"apprise"`
|
||||
System SystemSettings `json:"system"`
|
||||
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
|
||||
SSO *SSOConfig `json:"sso,omitempty"`
|
||||
APITokens []APITokenRecord `json:"apiTokens,omitempty"`
|
||||
}
|
||||
|
||||
// ExportConfig exports all configuration with passphrase-based encryption
|
||||
@@ -55,6 +56,10 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to load alert config: %w", err)
|
||||
}
|
||||
alertIntent, err := c.LoadAlertIntentPolicies()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to load alert intent policies: %w", err)
|
||||
}
|
||||
|
||||
emailConfig, err := c.LoadEmailConfig()
|
||||
if err != nil {
|
||||
@@ -97,10 +102,11 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
|
||||
// Create export data
|
||||
exportData := ExportData{
|
||||
Version: "4.2",
|
||||
Version: "4.3",
|
||||
ExportedAt: time.Now(),
|
||||
Nodes: *nodes,
|
||||
Alerts: *alertConfig,
|
||||
AlertIntent: alertIntent,
|
||||
Email: *emailConfig,
|
||||
Webhooks: webhooks,
|
||||
Apprise: *appriseConfig,
|
||||
@@ -152,8 +158,10 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
|
||||
|
||||
// Check version compatibility (warn but don't fail)
|
||||
switch exportData.Version {
|
||||
case "4.2", "":
|
||||
case "4.3", "":
|
||||
// current version, nothing to do
|
||||
case "4.2":
|
||||
log.Info().Msg("Config was exported from version 4.2. Alert intent policies were not included in that format.")
|
||||
case "4.1":
|
||||
log.Info().Msg("Config was exported from version 4.1. SSO settings were not included in that format.")
|
||||
case "4.0":
|
||||
@@ -186,6 +194,14 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
|
||||
if err := c.SaveAlertConfig(exportData.Alerts); err != nil {
|
||||
return fmt.Errorf("failed to import alert config: %w", err)
|
||||
}
|
||||
intentPolicies := exportData.AlertIntent
|
||||
if intentPolicies == nil {
|
||||
defaults := alerts.NewAlertIntentPolicyDocument()
|
||||
intentPolicies = &defaults
|
||||
}
|
||||
if err := c.SaveAlertIntentPolicies(*intentPolicies); err != nil {
|
||||
return fmt.Errorf("failed to import alert intent policies: %w", err)
|
||||
}
|
||||
|
||||
if err := c.SaveEmailConfig(exportData.Email); err != nil {
|
||||
return fmt.Errorf("failed to import email config: %w", err)
|
||||
|
||||
@@ -29,6 +29,7 @@ type ConfigPersistence struct {
|
||||
tx *importTransaction
|
||||
configDir string
|
||||
alertFile string
|
||||
alertIntentFile string
|
||||
emailFile string
|
||||
webhookFile string
|
||||
appriseFile string
|
||||
@@ -107,6 +108,7 @@ type alertScheduleGroupingPresence struct {
|
||||
|
||||
type resolvedConfigPersistencePaths struct {
|
||||
alertFile string
|
||||
alertIntentFile string
|
||||
emailFile string
|
||||
webhookFile string
|
||||
appriseFile string
|
||||
@@ -146,6 +148,10 @@ func resolveConfigPersistencePaths(configDir string) (string, resolvedConfigPers
|
||||
if err != nil {
|
||||
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve alerts.json: %w", err)
|
||||
}
|
||||
alertIntentFile, err := resolveLeaf("alert_intent_policies.json")
|
||||
if err != nil {
|
||||
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve alert_intent_policies.json: %w", err)
|
||||
}
|
||||
emailFile, err := resolveLeaf("email.enc")
|
||||
if err != nil {
|
||||
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve email.enc: %w", err)
|
||||
@@ -241,6 +247,7 @@ func resolveConfigPersistencePaths(configDir string) (string, resolvedConfigPers
|
||||
|
||||
return normalizedConfigDir, resolvedConfigPersistencePaths{
|
||||
alertFile: alertFile,
|
||||
alertIntentFile: alertIntentFile,
|
||||
emailFile: emailFile,
|
||||
webhookFile: webhookFile,
|
||||
appriseFile: appriseFile,
|
||||
@@ -296,6 +303,7 @@ func newConfigPersistence(configDir string) (*ConfigPersistence, error) {
|
||||
cp := &ConfigPersistence{
|
||||
configDir: resolvedConfigDir,
|
||||
alertFile: resolvedPaths.alertFile,
|
||||
alertIntentFile: resolvedPaths.alertIntentFile,
|
||||
emailFile: resolvedPaths.emailFile,
|
||||
webhookFile: resolvedPaths.webhookFile,
|
||||
appriseFile: resolvedPaths.appriseFile,
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
)
|
||||
|
||||
func (c *ConfigPersistence) SaveAlertIntentPolicies(document alerts.AlertIntentPolicyDocument) error {
|
||||
if err := alerts.ValidateAlertIntentPolicyDocument(document); err != nil {
|
||||
return err
|
||||
}
|
||||
document = alerts.NormalizeAlertIntentPolicyDocument(document)
|
||||
return saveJSON(c, c.alertIntentFile, document, false)
|
||||
}
|
||||
|
||||
func (c *ConfigPersistence) LoadAlertIntentPolicies() (*alerts.AlertIntentPolicyDocument, error) {
|
||||
document := alerts.NewAlertIntentPolicyDocument()
|
||||
if err := loadJSON(c, c.alertIntentFile, false, &document); err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return &document, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
document = alerts.NormalizeAlertIntentPolicyDocument(document)
|
||||
if err := alerts.ValidateAlertIntentPolicyDocument(document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &document, nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
)
|
||||
|
||||
func TestAlertIntentPolicyPersistenceMissingAndRoundTrip(t *testing.T) {
|
||||
cp := config.NewConfigPersistence(t.TempDir())
|
||||
if err := cp.EnsureConfigDir(); err != nil {
|
||||
t.Fatalf("EnsureConfigDir: %v", err)
|
||||
}
|
||||
|
||||
missing, err := cp.LoadAlertIntentPolicies()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAlertIntentPolicies missing: %v", err)
|
||||
}
|
||||
if missing.SchemaVersion != alerts.CurrentAlertIntentPolicySchemaVersion || missing.Revision != 0 {
|
||||
t.Fatalf("missing policy document = %+v", missing)
|
||||
}
|
||||
|
||||
grace := 75
|
||||
document := alerts.NewAlertIntentPolicyDocument()
|
||||
document.Revision = 4
|
||||
document.Resources["vm:101"] = map[string]alerts.AlertIntentRule{
|
||||
string(alerts.AlertIntentSignalOffline): {
|
||||
GraceSeconds: &grace,
|
||||
BackupOffline: &alerts.BackupOfflineIntentPolicy{
|
||||
Enabled: true, PostGraceSeconds: 30, MaxDeferralSeconds: 900,
|
||||
},
|
||||
},
|
||||
}
|
||||
if err := cp.SaveAlertIntentPolicies(document); err != nil {
|
||||
t.Fatalf("SaveAlertIntentPolicies: %v", err)
|
||||
}
|
||||
loaded, err := cp.LoadAlertIntentPolicies()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAlertIntentPolicies: %v", err)
|
||||
}
|
||||
want := alerts.NormalizeAlertIntentPolicyDocument(document)
|
||||
if !reflect.DeepEqual(*loaded, want) {
|
||||
t.Fatalf("loaded policy = %#v, want %#v", *loaded, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportImportIncludesAlertIntentPolicies(t *testing.T) {
|
||||
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
||||
source := config.NewConfigPersistence(t.TempDir())
|
||||
if err := source.EnsureConfigDir(); err != nil {
|
||||
t.Fatalf("EnsureConfigDir source: %v", err)
|
||||
}
|
||||
|
||||
grace := 120
|
||||
document := alerts.NewAlertIntentPolicyDocument()
|
||||
document.Resources["service:dns"] = map[string]alerts.AlertIntentRule{
|
||||
string(alerts.AlertIntentSignalAvailability): {GraceSeconds: &grace},
|
||||
}
|
||||
if err := source.SaveAlertIntentPolicies(document); err != nil {
|
||||
t.Fatalf("SaveAlertIntentPolicies: %v", err)
|
||||
}
|
||||
|
||||
const passphrase = "intent-policy-round-trip"
|
||||
bundle, err := source.ExportConfig(passphrase)
|
||||
if err != nil {
|
||||
t.Fatalf("ExportConfig: %v", err)
|
||||
}
|
||||
decoded := mustDecodeExport(t, bundle, passphrase)
|
||||
if decoded.Version != "4.3" || decoded.AlertIntent == nil {
|
||||
t.Fatalf("export metadata = version %q intent %#v", decoded.Version, decoded.AlertIntent)
|
||||
}
|
||||
|
||||
destination := config.NewConfigPersistence(t.TempDir())
|
||||
if err := destination.EnsureConfigDir(); err != nil {
|
||||
t.Fatalf("EnsureConfigDir destination: %v", err)
|
||||
}
|
||||
if err := destination.ImportConfig(bundle, passphrase); err != nil {
|
||||
t.Fatalf("ImportConfig: %v", err)
|
||||
}
|
||||
loaded, err := destination.LoadAlertIntentPolicies()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAlertIntentPolicies: %v", err)
|
||||
}
|
||||
want := alerts.NormalizeAlertIntentPolicyDocument(document)
|
||||
if !reflect.DeepEqual(*loaded, want) {
|
||||
t.Fatalf("imported policy = %#v, want %#v", *loaded, want)
|
||||
}
|
||||
}
|
||||
@@ -589,8 +589,8 @@ func TestExportConfigIncludesAPITokens(t *testing.T) {
|
||||
|
||||
decoded := mustDecodeExport(t, exported, passphrase)
|
||||
|
||||
if decoded.Version != "4.2" {
|
||||
t.Fatalf("expected export version 4.2, got %q", decoded.Version)
|
||||
if decoded.Version != "4.3" {
|
||||
t.Fatalf("expected export version 4.3, got %q", decoded.Version)
|
||||
}
|
||||
|
||||
assertJSONEqual(t, decoded.APITokens, tokens, "api tokens")
|
||||
|
||||
+11
-10
@@ -2944,16 +2944,17 @@ func (b PVEBackups) NormalizeCollections() PVEBackups {
|
||||
|
||||
// BackupTask represents a PVE backup task
|
||||
type BackupTask struct {
|
||||
ID string `json:"id"`
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"` // Unique instance identifier
|
||||
Type string `json:"type"`
|
||||
VMID int `json:"vmid"`
|
||||
Status string `json:"status"`
|
||||
StartTime time.Time `json:"startTime"`
|
||||
EndTime time.Time `json:"endTime,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Node string `json:"node"`
|
||||
Instance string `json:"instance"` // Unique instance identifier
|
||||
Type string `json:"type"`
|
||||
VMID int `json:"vmid"`
|
||||
Status string `json:"status"`
|
||||
StartTime time.Time `json:"startTime"`
|
||||
EndTime time.Time `json:"endTime,omitempty"`
|
||||
ObservedAt time.Time `json:"observedAt,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// StorageBackup represents a backup file in storage
|
||||
|
||||
@@ -29,6 +29,7 @@ type AvailabilityProbeStatus struct {
|
||||
TargetKind string `json:"targetKind,omitempty"`
|
||||
Address string `json:"address"`
|
||||
Protocol string `json:"protocol"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Available bool `json:"available"`
|
||||
LastChecked time.Time `json:"lastChecked,omitempty"`
|
||||
@@ -39,6 +40,14 @@ type AvailabilityProbeStatus struct {
|
||||
FailureThreshold int `json:"failureThreshold,omitempty"`
|
||||
}
|
||||
|
||||
type AvailabilityProbeOutcome string
|
||||
|
||||
const (
|
||||
AvailabilityProbeReachable AvailabilityProbeOutcome = "reachable"
|
||||
AvailabilityProbeUnreachable AvailabilityProbeOutcome = "unreachable"
|
||||
AvailabilityProbeIndeterminate AvailabilityProbeOutcome = "indeterminate"
|
||||
)
|
||||
|
||||
type availabilityPollProvider struct{}
|
||||
|
||||
func newAvailabilityPollProvider() PollProvider {
|
||||
@@ -255,10 +264,10 @@ func (m *Monitor) RefreshAvailabilityTargets() {
|
||||
func (m *Monitor) pollAvailabilityTarget(ctx context.Context, target config.AvailabilityTarget) {
|
||||
target = config.NormalizeAvailabilityTarget(target)
|
||||
start := time.Now()
|
||||
err := ProbeAvailabilityTarget(ctx, target)
|
||||
outcome, err := ProbeAvailabilityTargetResult(ctx, target)
|
||||
latency := time.Since(start)
|
||||
checkedAt := time.Now().UTC()
|
||||
m.setAvailabilityStatus(target, checkedAt, latency, err)
|
||||
m.setAvailabilityStatus(target, checkedAt, latency, outcome, err)
|
||||
|
||||
if err == nil {
|
||||
if m.stalenessTracker != nil {
|
||||
@@ -275,11 +284,12 @@ func (m *Monitor) pollAvailabilityTarget(ctx context.Context, target config.Avai
|
||||
m.updateResourceStore(m.GetState())
|
||||
}
|
||||
|
||||
func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checkedAt time.Time, latency time.Duration, probeErr error) {
|
||||
func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checkedAt time.Time, latency time.Duration, outcome AvailabilityProbeOutcome, probeErr error) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
status := availabilityStatusFromTarget(target)
|
||||
status.Outcome = string(outcome)
|
||||
status.LastChecked = checkedAt
|
||||
latencyMs := latency.Milliseconds()
|
||||
if probeErr == nil && latencyMs == 0 {
|
||||
@@ -287,8 +297,13 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke
|
||||
}
|
||||
status.LatencyMillis = latencyMs
|
||||
if probeErr == nil {
|
||||
status.Available = true
|
||||
status.LastSuccess = checkedAt
|
||||
// An open-or-filtered UDP timeout is healthy probe execution but does
|
||||
// not prove endpoint reachability. Keep it non-failing without claiming
|
||||
// the endpoint is available.
|
||||
status.Available = outcome == AvailabilityProbeReachable
|
||||
if outcome == AvailabilityProbeReachable {
|
||||
status.LastSuccess = checkedAt
|
||||
}
|
||||
} else {
|
||||
status.Available = false
|
||||
status.LastError = probeErr.Error()
|
||||
@@ -303,7 +318,9 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke
|
||||
if probeErr == nil {
|
||||
status.ConsecutiveFailures = 0
|
||||
status.LastError = ""
|
||||
status.LastSuccess = checkedAt
|
||||
if outcome == AvailabilityProbeReachable {
|
||||
status.LastSuccess = checkedAt
|
||||
}
|
||||
} else {
|
||||
status.ConsecutiveFailures = previous.ConsecutiveFailures + 1
|
||||
}
|
||||
@@ -316,9 +333,16 @@ func (m *Monitor) setAvailabilityStatus(target config.AvailabilityTarget, checke
|
||||
|
||||
// ProbeAvailabilityTarget executes one agentless availability check.
|
||||
func ProbeAvailabilityTarget(ctx context.Context, target config.AvailabilityTarget) error {
|
||||
_, err := ProbeAvailabilityTargetResult(ctx, target)
|
||||
return err
|
||||
}
|
||||
|
||||
// ProbeAvailabilityTargetResult preserves UDP's open-or-filtered state rather
|
||||
// than incorrectly claiming that a silent UDP endpoint was proven reachable.
|
||||
func ProbeAvailabilityTargetResult(ctx context.Context, target config.AvailabilityTarget) (AvailabilityProbeOutcome, error) {
|
||||
target = config.NormalizeAvailabilityTarget(target)
|
||||
if err := target.Validate(); err != nil {
|
||||
return err
|
||||
return AvailabilityProbeUnreachable, err
|
||||
}
|
||||
|
||||
timeout := time.Duration(target.EffectiveTimeoutMillis()) * time.Millisecond
|
||||
@@ -330,16 +354,87 @@ func ProbeAvailabilityTarget(ctx context.Context, target config.AvailabilityTarg
|
||||
|
||||
switch target.Protocol {
|
||||
case config.AvailabilityProbeICMP:
|
||||
return probeICMP(probeCtx, target)
|
||||
return outcomeFromProbeError(probeICMP(probeCtx, target))
|
||||
case config.AvailabilityProbeTCP:
|
||||
return probeTCP(probeCtx, target)
|
||||
return outcomeFromProbeError(probeTCP(probeCtx, target))
|
||||
case config.AvailabilityProbeUDP:
|
||||
return probeUDP(probeCtx, target)
|
||||
case config.AvailabilityProbeHTTP, config.AvailabilityProbeHTTPS:
|
||||
return probeHTTP(probeCtx, target, timeout)
|
||||
return outcomeFromProbeError(probeHTTP(probeCtx, target, timeout))
|
||||
default:
|
||||
return fmt.Errorf("unsupported availability protocol %q", target.Protocol)
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("unsupported availability protocol %q", target.Protocol)
|
||||
}
|
||||
}
|
||||
|
||||
func outcomeFromProbeError(err error) (AvailabilityProbeOutcome, error) {
|
||||
if err != nil {
|
||||
return AvailabilityProbeUnreachable, err
|
||||
}
|
||||
return AvailabilityProbeReachable, nil
|
||||
}
|
||||
|
||||
func probeUDP(ctx context.Context, target config.AvailabilityTarget) (AvailabilityProbeOutcome, error) {
|
||||
host := target.ProbeAddress()
|
||||
addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil {
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("resolve UDP availability target: %w", err)
|
||||
}
|
||||
var selected net.IP
|
||||
for _, address := range addresses {
|
||||
if address.IP == nil || address.IP.IsUnspecified() || address.IP.IsMulticast() || address.IP.Equal(net.IPv4bcast) {
|
||||
continue
|
||||
}
|
||||
selected = address.IP
|
||||
break
|
||||
}
|
||||
if selected == nil {
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("UDP availability target did not resolve to an allowed unicast address")
|
||||
}
|
||||
|
||||
dialer := net.Dialer{}
|
||||
conn, err := dialer.DialContext(ctx, "udp", net.JoinHostPort(selected.String(), strconv.Itoa(target.Port)))
|
||||
if err != nil {
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe dial failed: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
if err := conn.SetDeadline(deadline); err != nil {
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("set UDP probe deadline: %w", err)
|
||||
}
|
||||
}
|
||||
payload := []byte(target.UDPRequest)
|
||||
if len(payload) == 0 {
|
||||
// A one-byte datagram gives the kernel an opportunity to surface an
|
||||
// ICMP port-unreachable result in open-or-filtered mode.
|
||||
payload = []byte{0}
|
||||
}
|
||||
if _, err := conn.Write(payload); err != nil {
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe write failed: %w", err)
|
||||
}
|
||||
|
||||
response := make([]byte, 4096)
|
||||
n, err := conn.Read(response)
|
||||
if err == nil {
|
||||
if target.UDPExpected != "" && string(response[:n]) != target.UDPExpected {
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("UDP response did not match the expected payload")
|
||||
}
|
||||
return AvailabilityProbeReachable, nil
|
||||
}
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
if target.UDPMode == config.AvailabilityUDPOpenOrFiltered && ctxErr == context.DeadlineExceeded {
|
||||
return AvailabilityProbeIndeterminate, nil
|
||||
}
|
||||
return AvailabilityProbeUnreachable, ctxErr
|
||||
}
|
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
|
||||
if target.UDPMode == config.AvailabilityUDPOpenOrFiltered {
|
||||
return AvailabilityProbeIndeterminate, nil
|
||||
}
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe timed out waiting for a response")
|
||||
}
|
||||
return AvailabilityProbeUnreachable, fmt.Errorf("UDP probe failed: %w", err)
|
||||
}
|
||||
|
||||
func probeICMP(ctx context.Context, target config.AvailabilityTarget) error {
|
||||
host := target.ProbeAddress()
|
||||
if host == "" {
|
||||
@@ -522,6 +617,8 @@ func availabilityResourceFromTarget(target config.AvailabilityTarget, status Ava
|
||||
TargetKind: string(target.TargetKind),
|
||||
Address: target.Address,
|
||||
Protocol: string(target.Protocol),
|
||||
ProbeOutcome: status.Outcome,
|
||||
UDPMode: string(target.UDPMode),
|
||||
Port: target.Port,
|
||||
Path: target.Path,
|
||||
Enabled: target.Enabled,
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
)
|
||||
|
||||
func udpTestTarget(address string, port int) config.AvailabilityTarget {
|
||||
return config.NormalizeAvailabilityTarget(config.AvailabilityTarget{
|
||||
ID: "udp-test", Name: "UDP test", TargetKind: config.AvailabilityTargetService,
|
||||
Address: address, Protocol: config.AvailabilityProbeUDP, Port: port, Enabled: true,
|
||||
TimeoutMillis: 250, UDPMode: config.AvailabilityUDPResponseRequired, UDPRequest: "PING",
|
||||
})
|
||||
}
|
||||
|
||||
func TestProbeAvailabilityTargetUDPResponseRequired(t *testing.T) {
|
||||
listener, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
go func() {
|
||||
buffer := make([]byte, 32)
|
||||
_, remote, readErr := listener.ReadFromUDP(buffer)
|
||||
if readErr == nil {
|
||||
_, _ = listener.WriteToUDP([]byte("PONG"), remote)
|
||||
}
|
||||
}()
|
||||
|
||||
port := listener.LocalAddr().(*net.UDPAddr).Port
|
||||
target := udpTestTarget("127.0.0.1", port)
|
||||
target.UDPExpected = "PONG"
|
||||
outcome, err := ProbeAvailabilityTargetResult(context.Background(), target)
|
||||
if err != nil || outcome != AvailabilityProbeReachable {
|
||||
t.Fatalf("UDP outcome = %q, error = %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProbeAvailabilityTargetUDPOpenOrFilteredIsIndeterminate(t *testing.T) {
|
||||
listener, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
go func() {
|
||||
buffer := make([]byte, 32)
|
||||
_, _, _ = listener.ReadFromUDP(buffer)
|
||||
}()
|
||||
|
||||
port := listener.LocalAddr().(*net.UDPAddr).Port
|
||||
target := udpTestTarget("127.0.0.1", port)
|
||||
target.UDPMode = config.AvailabilityUDPOpenOrFiltered
|
||||
target.UDPRequest = ""
|
||||
started := time.Now()
|
||||
outcome, err := ProbeAvailabilityTargetResult(context.Background(), target)
|
||||
if err != nil || outcome != AvailabilityProbeIndeterminate {
|
||||
t.Fatalf("UDP outcome = %q, error = %v", outcome, err)
|
||||
}
|
||||
if time.Since(started) < 200*time.Millisecond {
|
||||
t.Fatalf("indeterminate result returned before the response deadline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAvailabilityUDPValidationRequiresPayloadInResponseMode(t *testing.T) {
|
||||
target := udpTestTarget("127.0.0.1", 53)
|
||||
target.UDPRequest = ""
|
||||
if err := target.Validate(); err == nil {
|
||||
t.Fatal("Validate() error = nil, want response-required payload error")
|
||||
}
|
||||
target.UDPMode = config.AvailabilityUDPOpenOrFiltered
|
||||
if err := target.Validate(); err != nil {
|
||||
t.Fatalf("open-or-filtered Validate() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAvailabilityUDPIndeterminateDoesNotClaimReachabilityOrAccumulateFailures(t *testing.T) {
|
||||
target := udpTestTarget("127.0.0.1", 53)
|
||||
target.UDPMode = config.AvailabilityUDPOpenOrFiltered
|
||||
target.UDPRequest = ""
|
||||
monitor := &Monitor{availabilityStatuses: map[string]AvailabilityProbeStatus{
|
||||
target.ID: {TargetID: target.ID, ConsecutiveFailures: 2},
|
||||
}}
|
||||
|
||||
checkedAt := time.Now().UTC()
|
||||
monitor.setAvailabilityStatus(target, checkedAt, 250*time.Millisecond, AvailabilityProbeIndeterminate, nil)
|
||||
status := monitor.AvailabilityStatusSnapshot()[target.ID]
|
||||
if status.Available {
|
||||
t.Fatal("indeterminate UDP result must not be reported as reachable")
|
||||
}
|
||||
if status.Outcome != string(AvailabilityProbeIndeterminate) {
|
||||
t.Fatalf("outcome = %q, want indeterminate", status.Outcome)
|
||||
}
|
||||
if status.ConsecutiveFailures != 0 {
|
||||
t.Fatalf("consecutive failures = %d, want 0", status.ConsecutiveFailures)
|
||||
}
|
||||
if got := availabilityResourceStatus(target, status); got != "warning" {
|
||||
t.Fatalf("resource status = %q, want warning", got)
|
||||
}
|
||||
if incident := availabilityIncident(target, status, checkedAt); incident != nil {
|
||||
t.Fatalf("indeterminate UDP result created an incident: %#v", incident)
|
||||
}
|
||||
}
|
||||
@@ -1716,6 +1716,7 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||
}
|
||||
|
||||
m.executor = newRealExecutor(m)
|
||||
m.alertManager.SetBackupIntentContextResolver(m.resolveBackupIntentContext)
|
||||
m.registerBuiltInPollProviders()
|
||||
m.buildInstanceInfoCache(cfg)
|
||||
|
||||
@@ -1741,6 +1742,13 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||
} else {
|
||||
log.Warn().Err(err).Msg("failed to load alert configuration")
|
||||
}
|
||||
if intentPolicies, err := m.configPersist.LoadAlertIntentPolicies(); err == nil {
|
||||
if err := m.alertManager.LoadIntentPolicies(*intentPolicies); err != nil {
|
||||
log.Warn().Err(err).Msg("failed to install alert intent policies")
|
||||
}
|
||||
} else {
|
||||
log.Warn().Err(err).Msg("failed to load alert intent policies")
|
||||
}
|
||||
|
||||
if emailConfig, err := m.configPersist.LoadEmailConfig(); err == nil {
|
||||
m.notificationMgr.SetEmailConfig(*emailConfig)
|
||||
@@ -4281,6 +4289,7 @@ func (m *Monitor) SetResourceStore(store ResourceStoreInterface) {
|
||||
m.resourceStore = store
|
||||
incidentStore := m.incidentStore
|
||||
m.mu.Unlock()
|
||||
m.installOperatorIntentResolver(store)
|
||||
log.Info().Msg("resource store set for polling optimization")
|
||||
|
||||
if incidentStore != nil {
|
||||
|
||||
@@ -295,6 +295,7 @@ func TestMonitor_HandleAlertResolved_SendsRecoveryForGuestPoweredOffState(t *tes
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
notifMgr := notifications.NewNotificationManagerWithDataDir("http://pulse.example", t.TempDir())
|
||||
t.Cleanup(notifMgr.Stop)
|
||||
if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
|
||||
t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
|
||||
}
|
||||
@@ -308,7 +309,8 @@ func TestMonitor_HandleAlertResolved_SendsRecoveryForGuestPoweredOffState(t *tes
|
||||
notifMgr.SetNotifyOnResolve(true)
|
||||
notifMgr.SetGroupingWindow(0)
|
||||
|
||||
alertMgr := alerts.NewManager()
|
||||
alertMgr := alerts.NewManagerWithDataDir(t.TempDir())
|
||||
t.Cleanup(alertMgr.Stop)
|
||||
cfg := alertMgr.GetConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.ActivationState = alerts.ActivationActive
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const backupIntentEvidenceMaxAge = 5 * time.Minute
|
||||
|
||||
type resourceOperatorIntentReader interface {
|
||||
GetResourceOperatorState(canonicalID string) (unifiedresources.ResourceOperatorState, bool, error)
|
||||
}
|
||||
|
||||
type resourceIntentIdentityReader interface {
|
||||
ResolveCanonicalResourceID(ref string) (string, bool)
|
||||
}
|
||||
|
||||
func (m *Monitor) installOperatorIntentResolver(store ResourceStoreInterface) {
|
||||
if m == nil || m.alertManager == nil {
|
||||
return
|
||||
}
|
||||
identityReader, hasIdentityReader := store.(resourceIntentIdentityReader)
|
||||
if hasIdentityReader && identityReader != nil {
|
||||
m.alertManager.SetResourceIntentIdentityResolver(identityReader.ResolveCanonicalResourceID)
|
||||
} else {
|
||||
m.alertManager.SetResourceIntentIdentityResolver(nil)
|
||||
}
|
||||
reader, ok := store.(resourceOperatorIntentReader)
|
||||
if !ok || reader == nil {
|
||||
m.alertManager.SetOperatorIntentContextResolver(nil)
|
||||
return
|
||||
}
|
||||
m.alertManager.SetOperatorIntentContextResolver(func(resourceID string, _ time.Time) (alerts.OperatorIntentContext, bool) {
|
||||
if hasIdentityReader {
|
||||
if canonicalID, found := identityReader.ResolveCanonicalResourceID(resourceID); found {
|
||||
resourceID = canonicalID
|
||||
}
|
||||
}
|
||||
state, found, err := reader.GetResourceOperatorState(resourceID)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("resourceID", resourceID).Msg("Failed to read operator state for alert intent")
|
||||
return alerts.OperatorIntentContext{}, false
|
||||
}
|
||||
if !found {
|
||||
return alerts.OperatorIntentContext{}, false
|
||||
}
|
||||
return alerts.OperatorIntentContext{
|
||||
IntentionallyOffline: state.IntentionallyOffline,
|
||||
MaintenanceStartAt: state.MaintenanceStartAt,
|
||||
MaintenanceEndAt: state.MaintenanceEndAt,
|
||||
MaintenanceReason: state.MaintenanceReason,
|
||||
}, true
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Monitor) resolveBackupIntentContext(_ string, instance, node string, vmid int, now time.Time) (alerts.BackupIntentContext, bool) {
|
||||
if m == nil || m.state == nil || vmid <= 0 {
|
||||
return alerts.BackupIntentContext{}, false
|
||||
}
|
||||
instance = strings.TrimSpace(instance)
|
||||
node = strings.TrimSpace(node)
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
|
||||
for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {
|
||||
if task.VMID != vmid || (instance != "" && task.Instance != instance) || (node != "" && task.Node != "" && task.Node != node) {
|
||||
continue
|
||||
}
|
||||
if task.ObservedAt.IsZero() || now.Sub(task.ObservedAt) > backupIntentEvidenceMaxAge || task.ObservedAt.After(now.Add(time.Minute)) {
|
||||
continue
|
||||
}
|
||||
status := strings.ToLower(strings.TrimSpace(task.Status))
|
||||
if !task.EndTime.IsZero() || status == "ok" || status == "stopped" || status == "error" || status == "warning" {
|
||||
continue
|
||||
}
|
||||
return alerts.BackupIntentContext{
|
||||
Active: true,
|
||||
ObservedAt: task.ObservedAt,
|
||||
Evidence: "pve_vzdump_task:" + task.ID,
|
||||
}, true
|
||||
}
|
||||
return alerts.BackupIntentContext{}, false
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
||||
func TestResolveBackupIntentContextRequiresFreshActiveMatchingEvidence(t *testing.T) {
|
||||
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
|
||||
state := models.NewState()
|
||||
state.UpdateBackupTasksForInstance("pve-a", []models.BackupTask{
|
||||
{
|
||||
ID: "active-101",
|
||||
Instance: "pve-a",
|
||||
Node: "node-a",
|
||||
VMID: 101,
|
||||
Status: "running",
|
||||
ObservedAt: now.Add(-time.Minute),
|
||||
},
|
||||
{
|
||||
ID: "stale-102",
|
||||
Instance: "pve-a",
|
||||
Node: "node-a",
|
||||
VMID: 102,
|
||||
Status: "running",
|
||||
ObservedAt: now.Add(-backupIntentEvidenceMaxAge - time.Second),
|
||||
},
|
||||
{
|
||||
ID: "finished-103",
|
||||
Instance: "pve-a",
|
||||
Node: "node-a",
|
||||
VMID: 103,
|
||||
Status: "OK",
|
||||
ObservedAt: now.Add(-time.Minute),
|
||||
EndTime: now.Add(-30 * time.Second),
|
||||
},
|
||||
})
|
||||
|
||||
monitor := &Monitor{state: state}
|
||||
context, found := monitor.resolveBackupIntentContext("", "pve-a", "node-a", 101, now)
|
||||
if !found || !context.Active {
|
||||
t.Fatalf("fresh active task did not resolve: found=%v context=%+v", found, context)
|
||||
}
|
||||
if context.ObservedAt != now.Add(-time.Minute) {
|
||||
t.Fatalf("observedAt = %v, want %v", context.ObservedAt, now.Add(-time.Minute))
|
||||
}
|
||||
if context.Evidence != "pve_vzdump_task:active-101" {
|
||||
t.Fatalf("evidence = %q, want active task identity", context.Evidence)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
instance string
|
||||
node string
|
||||
vmid int
|
||||
}{
|
||||
{name: "wrong instance", instance: "pve-b", node: "node-a", vmid: 101},
|
||||
{name: "wrong node", instance: "pve-a", node: "node-b", vmid: 101},
|
||||
{name: "stale", instance: "pve-a", node: "node-a", vmid: 102},
|
||||
{name: "finished", instance: "pve-a", node: "node-a", vmid: 103},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got, ok := monitor.resolveBackupIntentContext("", tc.instance, tc.node, tc.vmid, now); ok {
|
||||
t.Fatalf("unexpected backup intent context: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1993,13 +1993,14 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
|
||||
taskID := fmt.Sprintf("%s-%s", instanceName, task.UPID)
|
||||
|
||||
backupTask := models.BackupTask{
|
||||
ID: taskID,
|
||||
Node: task.Node,
|
||||
Instance: instanceName,
|
||||
Type: task.Type,
|
||||
VMID: vmid,
|
||||
Status: task.Status,
|
||||
StartTime: time.Unix(task.StartTime, 0),
|
||||
ID: taskID,
|
||||
Node: task.Node,
|
||||
Instance: instanceName,
|
||||
Type: task.Type,
|
||||
VMID: vmid,
|
||||
Status: task.Status,
|
||||
StartTime: time.Unix(task.StartTime, 0),
|
||||
ObservedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
if task.EndTime > 0 {
|
||||
|
||||
@@ -763,6 +763,9 @@ func TestMonitor_PollBackupAndReplication(t *testing.T) {
|
||||
if len(state.PVEBackups.BackupTasks) != 1 {
|
||||
t.Errorf("Expected 1 backup task, got %d", len(state.PVEBackups.BackupTasks))
|
||||
}
|
||||
if len(state.PVEBackups.BackupTasks) == 1 && state.PVEBackups.BackupTasks[0].ObservedAt.IsZero() {
|
||||
t.Error("Expected polled backup task to carry producer observation time")
|
||||
}
|
||||
|
||||
m.pollReplicationStatus(context.Background(), "pve-test", client, []models.VM{{VMID: 101, Name: "vm1"}})
|
||||
state = m.state.GetSnapshot()
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
)
|
||||
|
||||
func TestResolvedJobsRequireReceiptForSameOccurrenceAndDestination(t *testing.T) {
|
||||
manager := NewNotificationManagerWithDataDir("", t.TempDir())
|
||||
defer manager.Stop()
|
||||
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
|
||||
alert := &alerts.Alert{ID: "vm-offline-101", StartTime: start}
|
||||
first := WebhookConfig{ID: "first", URL: "https://first.example.test", Enabled: true}
|
||||
second := WebhookConfig{ID: "second", URL: "https://second.example.test", Enabled: true}
|
||||
firingJob := notificationDeliveryJob{Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &first}
|
||||
manager.recordSuccessfulDelivery(firingJob, start.Add(time.Second))
|
||||
|
||||
resolvedJobs := buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{first, second}, AppriseConfig{}, []*alerts.Alert{alert}, eventResolved, start.Add(time.Minute))
|
||||
filtered := manager.filterResolvedJobsByDeliveryReceipt(resolvedJobs)
|
||||
if len(filtered) != 1 || filtered[0].WebhookConfig == nil || filtered[0].WebhookConfig.ID != "first" {
|
||||
t.Fatalf("filtered jobs = %+v", filtered)
|
||||
}
|
||||
|
||||
newOccurrence := alert.Clone()
|
||||
newOccurrence.StartTime = start.Add(2 * time.Hour)
|
||||
resolvedJobs = buildNotificationDeliveryJobs(EmailConfig{}, []WebhookConfig{first}, AppriseConfig{}, []*alerts.Alert{newOccurrence}, eventResolved, start.Add(3*time.Hour))
|
||||
if got := manager.filterResolvedJobsByDeliveryReceipt(resolvedJobs); len(got) != 0 {
|
||||
t.Fatalf("new occurrence inherited old receipt: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedJobsDoNotReuseReceiptAfterWebhookURLChanges(t *testing.T) {
|
||||
manager := NewNotificationManagerWithDataDir("", t.TempDir())
|
||||
defer manager.Stop()
|
||||
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
|
||||
alert := &alerts.Alert{ID: "vm-offline-102", StartTime: start}
|
||||
original := WebhookConfig{ID: "ops", URL: "https://first.example.test", Enabled: true}
|
||||
reconfigured := WebhookConfig{ID: "ops", URL: "https://second.example.test", Enabled: true}
|
||||
manager.recordSuccessfulDelivery(notificationDeliveryJob{
|
||||
Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &original,
|
||||
}, start.Add(time.Second))
|
||||
|
||||
resolvedJobs := buildNotificationDeliveryJobs(
|
||||
EmailConfig{}, []WebhookConfig{reconfigured}, AppriseConfig{}, []*alerts.Alert{alert}, eventResolved, start.Add(time.Minute),
|
||||
)
|
||||
if got := manager.filterResolvedJobsByDeliveryReceipt(resolvedJobs); len(got) != 0 {
|
||||
t.Fatalf("reconfigured webhook inherited receipt for old URL: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryReceiptPersistsAndIsClearedAfterRecovery(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC)
|
||||
alert := &alerts.Alert{ID: "disk-critical-1", StartTime: start}
|
||||
webhook := WebhookConfig{ID: "ops", URL: "https://ops.example.test", Enabled: true}
|
||||
|
||||
first := NewNotificationManagerWithDataDir("", dataDir)
|
||||
first.recordSuccessfulDelivery(notificationDeliveryJob{Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook}, start.Add(time.Second))
|
||||
first.Stop()
|
||||
|
||||
second := NewNotificationManagerWithDataDir("", dataDir)
|
||||
defer second.Stop()
|
||||
resolved := notificationDeliveryJob{Type: "webhook", Event: eventResolved, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook}
|
||||
if got := second.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 1 {
|
||||
t.Fatalf("persisted receipt not found: %+v", got)
|
||||
}
|
||||
second.recordSuccessfulDelivery(resolved, start.Add(time.Minute))
|
||||
if got := second.filterResolvedJobsByDeliveryReceipt([]notificationDeliveryJob{resolved}); len(got) != 0 {
|
||||
t.Fatalf("receipt remained after recovery delivery: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -238,6 +238,7 @@ type NotificationManager struct {
|
||||
cooldown time.Duration
|
||||
notifyOnResolve bool
|
||||
lastNotified map[string]notificationRecord
|
||||
deliveryReceipts map[string]struct{}
|
||||
groupWindow time.Duration
|
||||
pendingAlerts []*alerts.Alert
|
||||
groupTimer *time.Timer
|
||||
@@ -686,11 +687,12 @@ func NewNotificationManagerWithDataDir(publicURL string, dataDir string) *Notifi
|
||||
}
|
||||
|
||||
nm := &NotificationManager{
|
||||
enabled: true,
|
||||
cooldown: 5 * time.Minute,
|
||||
notifyOnResolve: true,
|
||||
lastNotified: make(map[string]notificationRecord),
|
||||
webhooks: []WebhookConfig{},
|
||||
enabled: true,
|
||||
cooldown: 5 * time.Minute,
|
||||
notifyOnResolve: true,
|
||||
lastNotified: make(map[string]notificationRecord),
|
||||
deliveryReceipts: make(map[string]struct{}),
|
||||
webhooks: []WebhookConfig{},
|
||||
appriseConfig: AppriseConfig{
|
||||
Enabled: false,
|
||||
Mode: AppriseModeCLI,
|
||||
@@ -1075,13 +1077,14 @@ func (n *NotificationManager) sendAlert(alert *alerts.Alert, options alertSendOp
|
||||
time.Time{},
|
||||
options.target,
|
||||
)
|
||||
if len(jobs) == 0 {
|
||||
n.markAlertsNotified(alertsToSend, time.Now())
|
||||
return
|
||||
}
|
||||
if queue != nil {
|
||||
if anyFailed := n.enqueueNotificationJobs(queue, jobs); anyFailed {
|
||||
n.markAlertsNotified(alertsToSend, time.Now())
|
||||
}
|
||||
n.enqueueNotificationJobs(queue, jobs)
|
||||
} else {
|
||||
n.dispatchNotificationJobsAsync(jobs)
|
||||
n.markAlertsNotified(alertsToSend, time.Now())
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1128,6 +1131,155 @@ func (n *NotificationManager) markAlertsNotified(alertsToSend []*alerts.Alert, s
|
||||
n.mu.Unlock()
|
||||
}
|
||||
|
||||
func notificationDeliveryDestinationKey(job notificationDeliveryJob) string {
|
||||
var identity string
|
||||
switch job.Type {
|
||||
case "email":
|
||||
if job.EmailConfig == nil {
|
||||
return ""
|
||||
}
|
||||
recipients := append([]string(nil), job.EmailConfig.To...)
|
||||
if len(recipients) == 0 && strings.TrimSpace(job.EmailConfig.From) != "" {
|
||||
recipients = append(recipients, job.EmailConfig.From)
|
||||
}
|
||||
for i := range recipients {
|
||||
recipients[i] = strings.ToLower(strings.TrimSpace(recipients[i]))
|
||||
}
|
||||
sort.Strings(recipients)
|
||||
identity = strings.Join(recipients, "\x1f")
|
||||
case "webhook":
|
||||
if job.WebhookConfig == nil {
|
||||
return ""
|
||||
}
|
||||
// Include both the logical configuration ID and the actual endpoint.
|
||||
// Reusing an ID after changing its URL must not send a recovery to an
|
||||
// endpoint that never received the firing occurrence.
|
||||
webhookID := strings.TrimSpace(job.WebhookConfig.ID)
|
||||
webhookURL := strings.TrimSpace(job.WebhookConfig.URL)
|
||||
if webhookID == "" && webhookURL == "" {
|
||||
return ""
|
||||
}
|
||||
identity = strings.Join([]string{webhookID, webhookURL}, "\x1f")
|
||||
case "apprise":
|
||||
if job.AppriseConfig == nil {
|
||||
return ""
|
||||
}
|
||||
targets := append([]string(nil), job.AppriseConfig.Targets...)
|
||||
for i := range targets {
|
||||
targets[i] = strings.TrimSpace(targets[i])
|
||||
}
|
||||
sort.Strings(targets)
|
||||
identity = strings.Join([]string{
|
||||
string(job.AppriseConfig.Mode),
|
||||
strings.TrimSpace(job.AppriseConfig.ServerURL),
|
||||
strings.TrimSpace(job.AppriseConfig.ConfigKey),
|
||||
strings.Join(targets, "\x1f"),
|
||||
}, "\x1e")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
if identity == "" {
|
||||
return ""
|
||||
}
|
||||
digest := sha256.Sum256([]byte(job.Type + "\x00" + identity))
|
||||
return job.Type + ":" + hex.EncodeToString(digest[:16])
|
||||
}
|
||||
|
||||
func notificationDeliveryReceiptKey(alert *alerts.Alert, destinationKey string) string {
|
||||
if alert == nil || alert.ID == "" || alert.StartTime.IsZero() || destinationKey == "" {
|
||||
return ""
|
||||
}
|
||||
return alert.ID + "\x00" + strconv.FormatInt(alert.StartTime.UnixNano(), 10) + "\x00" + destinationKey
|
||||
}
|
||||
|
||||
func (n *NotificationManager) recordSuccessfulDelivery(job notificationDeliveryJob, deliveredAt time.Time) {
|
||||
destinationKey := notificationDeliveryDestinationKey(job)
|
||||
if destinationKey == "" {
|
||||
return
|
||||
}
|
||||
queue := n.GetQueue()
|
||||
for _, alert := range job.Alerts {
|
||||
if alert == nil || alert.StartTime.IsZero() {
|
||||
continue
|
||||
}
|
||||
if queue != nil {
|
||||
var err error
|
||||
if job.Event == eventResolved {
|
||||
err = queue.DeleteDeliveryReceipt(alert.ID, alert.StartTime, destinationKey)
|
||||
} else {
|
||||
err = queue.RecordDeliveryReceipt(alert.ID, alert.StartTime, destinationKey, deliveredAt)
|
||||
}
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Failed to update notification delivery receipt")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
key := notificationDeliveryReceiptKey(alert, destinationKey)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
n.mu.Lock()
|
||||
if n.deliveryReceipts == nil {
|
||||
n.deliveryReceipts = make(map[string]struct{})
|
||||
}
|
||||
if job.Event == eventResolved {
|
||||
delete(n.deliveryReceipts, key)
|
||||
} else {
|
||||
n.deliveryReceipts[key] = struct{}{}
|
||||
}
|
||||
n.mu.Unlock()
|
||||
}
|
||||
if job.Event == eventAlert {
|
||||
// Publish the cooldown/delivered marker only after the per-destination
|
||||
// receipts are durable. Resolution can race the delivery goroutine and
|
||||
// uses this marker as proof that receipt recording has completed.
|
||||
n.markAlertsNotified(job.Alerts, deliveredAt)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *NotificationManager) filterResolvedJobsByDeliveryReceipt(jobs []notificationDeliveryJob) []notificationDeliveryJob {
|
||||
queue := n.GetQueue()
|
||||
filtered := make([]notificationDeliveryJob, 0, len(jobs))
|
||||
for _, job := range jobs {
|
||||
destinationKey := notificationDeliveryDestinationKey(job)
|
||||
if destinationKey == "" {
|
||||
continue
|
||||
}
|
||||
eligibleAlerts := make([]*alerts.Alert, 0, len(job.Alerts))
|
||||
for _, alert := range job.Alerts {
|
||||
if alert == nil || alert.StartTime.IsZero() {
|
||||
continue
|
||||
}
|
||||
eligible := false
|
||||
if queue != nil {
|
||||
var err error
|
||||
eligible, err = queue.HasDeliveryReceipt(alert.ID, alert.StartTime, destinationKey)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Failed to read notification delivery receipt; suppressing recovery")
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
key := notificationDeliveryReceiptKey(alert, destinationKey)
|
||||
n.mu.RLock()
|
||||
_, eligible = n.deliveryReceipts[key]
|
||||
n.mu.RUnlock()
|
||||
}
|
||||
if eligible {
|
||||
eligibleAlerts = append(eligibleAlerts, alert)
|
||||
continue
|
||||
}
|
||||
log.Debug().Str("alertID", alert.ID).Str("destination", destinationKey).Msg("Resolved notification suppressed because this destination did not receive the firing occurrence")
|
||||
}
|
||||
if len(eligibleAlerts) == 0 {
|
||||
continue
|
||||
}
|
||||
job.Alerts = eligibleAlerts
|
||||
filtered = append(filtered, job)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
// SendResolvedAlert delivers notifications for a resolved alert immediately.
|
||||
func (n *NotificationManager) SendResolvedAlert(resolved *alerts.ResolvedAlert) {
|
||||
if resolved == nil || resolved.Alert == nil {
|
||||
@@ -1163,6 +1315,7 @@ func (n *NotificationManager) SendResolvedAlert(resolved *alerts.ResolvedAlert)
|
||||
|
||||
alertsToSend := []*alerts.Alert{alertCopy}
|
||||
jobs := buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertsToSend, eventResolved, resolvedAt)
|
||||
jobs = n.filterResolvedJobsByDeliveryReceipt(jobs)
|
||||
|
||||
if queue != nil {
|
||||
n.enqueueNotificationJobs(queue, jobs)
|
||||
@@ -1265,17 +1418,20 @@ func (n *NotificationManager) sendGroupedAlerts() {
|
||||
n.mu.Unlock()
|
||||
|
||||
jobs := buildNotificationDeliveryJobs(emailConfig, webhooks, appriseConfig, alertsToSend, eventAlert, time.Time{})
|
||||
if len(jobs) == 0 {
|
||||
// Preserve cooldown semantics when notifications are globally enabled
|
||||
// but no destination is configured. Delivery receipts remain empty, so
|
||||
// a later recovery is still correctly suppressed.
|
||||
n.markAlertsNotified(alertsToSend, time.Now())
|
||||
return
|
||||
}
|
||||
|
||||
// Use persistent queue if available, otherwise send directly
|
||||
if queue != nil {
|
||||
if anyFailed := n.enqueueNotificationJobs(queue, jobs); anyFailed {
|
||||
n.markAlertsNotified(alertsToSend, time.Now())
|
||||
}
|
||||
n.enqueueNotificationJobs(queue, jobs)
|
||||
// Note: Cooldown will be marked after successful dequeue and send
|
||||
} else {
|
||||
n.dispatchNotificationJobsAsync(jobs)
|
||||
// For direct sends, mark cooldown immediately (fire-and-forget)
|
||||
n.markAlertsNotified(alertsToSend, time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1512,7 +1668,9 @@ func (n *NotificationManager) dispatchNotificationJobAsync(job notificationDeliv
|
||||
spawnAsync(func() {
|
||||
if err := n.deliverNotificationJob(job); err != nil {
|
||||
n.logNotificationJobError(job, err, failureMessage)
|
||||
return
|
||||
}
|
||||
n.recordSuccessfulDelivery(job, time.Now())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3469,7 +3627,10 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Int("alertCount", len(notif.Alerts)).
|
||||
Msg("processing queued notification")
|
||||
|
||||
var err error
|
||||
var (
|
||||
err error
|
||||
deliveredJob notificationDeliveryJob
|
||||
)
|
||||
switch baseType {
|
||||
case "email":
|
||||
var emailConfig EmailConfig
|
||||
@@ -3485,13 +3646,14 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Msg("skipping queued email notification because email delivery is disabled")
|
||||
return nil
|
||||
}
|
||||
err = n.deliverNotificationJob(notificationDeliveryJob{
|
||||
deliveredJob = notificationDeliveryJob{
|
||||
Type: "email",
|
||||
Event: event,
|
||||
Alerts: notif.Alerts,
|
||||
ResolvedAt: resolvedTimeFromAlerts(notif.Alerts),
|
||||
EmailConfig: &emailConfig,
|
||||
})
|
||||
}
|
||||
err = n.deliverNotificationJob(deliveredJob)
|
||||
|
||||
case "webhook":
|
||||
var webhookConfig WebhookConfig
|
||||
@@ -3507,13 +3669,14 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Msg("skipping queued webhook notification because delivery is disabled")
|
||||
return nil
|
||||
}
|
||||
err = n.deliverNotificationJob(notificationDeliveryJob{
|
||||
deliveredJob = notificationDeliveryJob{
|
||||
Type: "webhook",
|
||||
Event: event,
|
||||
Alerts: notif.Alerts,
|
||||
ResolvedAt: resolvedTimeFromAlerts(notif.Alerts),
|
||||
WebhookConfig: &webhookConfig,
|
||||
})
|
||||
}
|
||||
err = n.deliverNotificationJob(deliveredJob)
|
||||
|
||||
case "apprise":
|
||||
var appriseConfig AppriseConfig
|
||||
@@ -3529,21 +3692,21 @@ func (n *NotificationManager) ProcessQueuedNotification(notif *QueuedNotificatio
|
||||
Msg("skipping queued Apprise notification because delivery is disabled")
|
||||
return nil
|
||||
}
|
||||
err = n.deliverNotificationJob(notificationDeliveryJob{
|
||||
deliveredJob = notificationDeliveryJob{
|
||||
Type: "apprise",
|
||||
Event: event,
|
||||
Alerts: notif.Alerts,
|
||||
ResolvedAt: resolvedTimeFromAlerts(notif.Alerts),
|
||||
AppriseConfig: &appriseConfig,
|
||||
})
|
||||
}
|
||||
err = n.deliverNotificationJob(deliveredJob)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown notification type: %s", baseType)
|
||||
}
|
||||
|
||||
// Mark cooldown after successful send for active alerts only
|
||||
if err == nil && event == eventAlert {
|
||||
n.markAlertsNotified(notif.Alerts, time.Now())
|
||||
if err == nil {
|
||||
n.recordSuccessfulDelivery(deliveredJob, time.Now())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -3927,12 +3927,13 @@ func TestSendResolvedAlert(t *testing.T) {
|
||||
defer server.Close()
|
||||
|
||||
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
|
||||
nm.AddWebhook(WebhookConfig{
|
||||
webhook := WebhookConfig{
|
||||
ID: "test-webhook",
|
||||
Name: "Test",
|
||||
URL: server.URL,
|
||||
Enabled: true,
|
||||
})
|
||||
}
|
||||
nm.AddWebhook(webhook)
|
||||
|
||||
nm.SetNotifyOnResolve(true)
|
||||
|
||||
@@ -3948,6 +3949,9 @@ func TestSendResolvedAlert(t *testing.T) {
|
||||
Alert: alert,
|
||||
ResolvedTime: time.Now(),
|
||||
}
|
||||
nm.recordSuccessfulDelivery(notificationDeliveryJob{
|
||||
Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook,
|
||||
}, time.Now())
|
||||
|
||||
// This should send immediately (no grouping)
|
||||
nm.SendResolvedAlert(resolved)
|
||||
@@ -4770,8 +4774,8 @@ func TestSendGroupedAlertsQueueEnqueueFailureDoesNotDeadlock(t *testing.T) {
|
||||
nm.mu.RLock()
|
||||
_, notified := nm.lastNotified[alert.ID]
|
||||
nm.mu.RUnlock()
|
||||
if !notified {
|
||||
t.Fatal("expected cooldown record to be marked for fallback direct send")
|
||||
if notified {
|
||||
t.Fatal("did not expect a cooldown record when fallback delivery never ran")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4815,21 +4819,21 @@ func TestSendResolvedAlertQueueEnqueueFailureFallsBackToSharedDeliveryExecutor(t
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
nm := NewNotificationManager("https://pulse.local")
|
||||
nm := NewNotificationManagerWithDataDir("https://pulse.local", t.TempDir())
|
||||
defer nm.Stop()
|
||||
_ = nm.UpdateAllowedPrivateCIDRs("127.0.0.1")
|
||||
nm.AddWebhook(WebhookConfig{
|
||||
webhook := WebhookConfig{
|
||||
ID: "resolved-fallback-hook",
|
||||
Name: "resolved-fallback",
|
||||
URL: server.URL + "/hook",
|
||||
Enabled: true,
|
||||
})
|
||||
}
|
||||
nm.AddWebhook(webhook)
|
||||
|
||||
queue := nm.GetQueue()
|
||||
if queue == nil {
|
||||
t.Skip("queue unavailable in this environment")
|
||||
}
|
||||
_ = queue.Stop()
|
||||
|
||||
alert := &alerts.Alert{
|
||||
ID: "resolved-fallback-1",
|
||||
@@ -4843,6 +4847,15 @@ func TestSendResolvedAlertQueueEnqueueFailureFallsBackToSharedDeliveryExecutor(t
|
||||
StartTime: time.Now().Add(-5 * time.Minute),
|
||||
}
|
||||
resolvedAt := time.Now()
|
||||
nm.recordSuccessfulDelivery(notificationDeliveryJob{
|
||||
Type: "webhook", Event: eventAlert, Alerts: []*alerts.Alert{alert}, WebhookConfig: &webhook,
|
||||
}, resolvedAt.Add(-time.Minute))
|
||||
// Keep the receipt store available while forcing only queue insertion to
|
||||
// fail. Recovery delivery must remain fail-closed if receipt lookup itself
|
||||
// is unavailable.
|
||||
if _, err := queue.db.Exec(`DROP TABLE notification_queue`); err != nil {
|
||||
t.Fatalf("force queue enqueue failure: %v", err)
|
||||
}
|
||||
|
||||
nm.SendResolvedAlert(&alerts.ResolvedAlert{
|
||||
Alert: alert,
|
||||
|
||||
+269
-27
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -65,6 +66,8 @@ type QueuedNotification struct {
|
||||
// NotificationQueue manages persistent notification delivery with retries and DLQ
|
||||
type NotificationQueue struct {
|
||||
mu sync.RWMutex
|
||||
deliveryGateMu sync.Mutex
|
||||
deliveryGates map[string]*notificationDeliveryGate
|
||||
stopOnce sync.Once
|
||||
stopErr error
|
||||
db *sql.DB
|
||||
@@ -78,6 +81,14 @@ type NotificationQueue struct {
|
||||
workerSem chan struct{} // Semaphore for limiting concurrent workers
|
||||
}
|
||||
|
||||
// notificationDeliveryGate orders delivery and cancellation for a single
|
||||
// alert identifier. References are counted so high-cardinality alert IDs do
|
||||
// not accumulate in the queue for the lifetime of the process.
|
||||
type notificationDeliveryGate struct {
|
||||
mu sync.RWMutex
|
||||
refs int
|
||||
}
|
||||
|
||||
func queueSQLiteDSN(path string) string {
|
||||
separator := "?"
|
||||
if strings.Contains(path, "?") {
|
||||
@@ -131,6 +142,7 @@ func newNotificationQueueFromDSN(dbPath, dsn string) (*NotificationQueue, error)
|
||||
nq := &NotificationQueue{
|
||||
db: db,
|
||||
dbPath: dbPath,
|
||||
deliveryGates: make(map[string]*notificationDeliveryGate),
|
||||
stopChan: make(chan struct{}),
|
||||
processorTicker: time.NewTicker(5 * time.Second),
|
||||
cleanupTicker: time.NewTicker(1 * time.Hour),
|
||||
@@ -189,6 +201,86 @@ func NewInMemoryNotificationQueue() (*NotificationQueue, error) {
|
||||
return newNotificationQueueFromDSN(":memory:", queueSQLiteDSN("file::memory:?mode=memory"))
|
||||
}
|
||||
|
||||
func normalizeAlertIdentifiers(alertIdentifiers []string) []string {
|
||||
seen := make(map[string]struct{}, len(alertIdentifiers))
|
||||
normalized := make([]string, 0, len(alertIdentifiers))
|
||||
for _, identifier := range alertIdentifiers {
|
||||
identifier = strings.TrimSpace(identifier)
|
||||
if identifier == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[identifier]; exists {
|
||||
continue
|
||||
}
|
||||
seen[identifier] = struct{}{}
|
||||
normalized = append(normalized, identifier)
|
||||
}
|
||||
sort.Strings(normalized)
|
||||
return normalized
|
||||
}
|
||||
|
||||
func alertIdentifiersFromAlerts(alertList []*alerts.Alert) []string {
|
||||
identifiers := make([]string, 0, len(alertList))
|
||||
for _, alert := range alertList {
|
||||
if alert != nil {
|
||||
identifiers = append(identifiers, alert.ID)
|
||||
}
|
||||
}
|
||||
return normalizeAlertIdentifiers(identifiers)
|
||||
}
|
||||
|
||||
// acquireAlertDeliveryGates coordinates a firing delivery with resolution
|
||||
// cancellation for only the alert IDs involved in that operation. Acquiring
|
||||
// IDs in sorted order prevents grouped notifications from deadlocking when
|
||||
// different alerts resolve concurrently.
|
||||
func (nq *NotificationQueue) acquireAlertDeliveryGates(alertIdentifiers []string, write bool) func() {
|
||||
identifiers := normalizeAlertIdentifiers(alertIdentifiers)
|
||||
if len(identifiers) == 0 {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
nq.deliveryGateMu.Lock()
|
||||
gates := make([]*notificationDeliveryGate, 0, len(identifiers))
|
||||
for _, identifier := range identifiers {
|
||||
gate := nq.deliveryGates[identifier]
|
||||
if gate == nil {
|
||||
gate = ¬ificationDeliveryGate{}
|
||||
nq.deliveryGates[identifier] = gate
|
||||
}
|
||||
gate.refs++
|
||||
gates = append(gates, gate)
|
||||
}
|
||||
nq.deliveryGateMu.Unlock()
|
||||
|
||||
for _, gate := range gates {
|
||||
if write {
|
||||
gate.mu.Lock()
|
||||
} else {
|
||||
gate.mu.RLock()
|
||||
}
|
||||
}
|
||||
|
||||
return func() {
|
||||
for i := len(gates) - 1; i >= 0; i-- {
|
||||
if write {
|
||||
gates[i].mu.Unlock()
|
||||
} else {
|
||||
gates[i].mu.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
nq.deliveryGateMu.Lock()
|
||||
for i, identifier := range identifiers {
|
||||
gate := gates[i]
|
||||
gate.refs--
|
||||
if gate.refs == 0 && nq.deliveryGates[identifier] == gate {
|
||||
delete(nq.deliveryGates, identifier)
|
||||
}
|
||||
}
|
||||
nq.deliveryGateMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// initSchema creates the database tables
|
||||
func (nq *NotificationQueue) initSchema() error {
|
||||
schema := `
|
||||
@@ -235,6 +327,17 @@ func (nq *NotificationQueue) initSchema() error {
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON notification_audit(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_notification_id ON notification_audit(notification_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_status ON notification_audit(status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS notification_delivery_receipts (
|
||||
alert_id TEXT NOT NULL,
|
||||
alert_start_nanos INTEGER NOT NULL,
|
||||
destination_key TEXT NOT NULL,
|
||||
delivered_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (alert_id, alert_start_nanos, destination_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_delivery_receipts_delivered_at
|
||||
ON notification_delivery_receipts(delivered_at);
|
||||
`
|
||||
|
||||
if _, err := nq.db.Exec(schema); err != nil {
|
||||
@@ -256,6 +359,63 @@ func (nq *NotificationQueue) initSchema() error {
|
||||
)
|
||||
}
|
||||
|
||||
func (nq *NotificationQueue) RecordDeliveryReceipt(alertID string, alertStart time.Time, destinationKey string, deliveredAt time.Time) error {
|
||||
if nq == nil || strings.TrimSpace(alertID) == "" || strings.TrimSpace(destinationKey) == "" || alertStart.IsZero() {
|
||||
return fmt.Errorf("alert occurrence and destination are required for a delivery receipt")
|
||||
}
|
||||
if deliveredAt.IsZero() {
|
||||
deliveredAt = time.Now()
|
||||
}
|
||||
nq.mu.Lock()
|
||||
defer nq.mu.Unlock()
|
||||
_, err := nq.db.Exec(`
|
||||
INSERT INTO notification_delivery_receipts
|
||||
(alert_id, alert_start_nanos, destination_key, delivered_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(alert_id, alert_start_nanos, destination_key)
|
||||
DO UPDATE SET delivered_at = excluded.delivered_at
|
||||
`, alertID, alertStart.UnixNano(), destinationKey, deliveredAt.Unix())
|
||||
if err != nil {
|
||||
return fmt.Errorf("record notification delivery receipt: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nq *NotificationQueue) HasDeliveryReceipt(alertID string, alertStart time.Time, destinationKey string) (bool, error) {
|
||||
if nq == nil || strings.TrimSpace(alertID) == "" || strings.TrimSpace(destinationKey) == "" || alertStart.IsZero() {
|
||||
return false, nil
|
||||
}
|
||||
nq.mu.RLock()
|
||||
defer nq.mu.RUnlock()
|
||||
var marker int
|
||||
err := nq.db.QueryRow(`
|
||||
SELECT 1 FROM notification_delivery_receipts
|
||||
WHERE alert_id = ? AND alert_start_nanos = ? AND destination_key = ?
|
||||
`, alertID, alertStart.UnixNano(), destinationKey).Scan(&marker)
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read notification delivery receipt: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (nq *NotificationQueue) DeleteDeliveryReceipt(alertID string, alertStart time.Time, destinationKey string) error {
|
||||
if nq == nil || strings.TrimSpace(alertID) == "" || strings.TrimSpace(destinationKey) == "" || alertStart.IsZero() {
|
||||
return nil
|
||||
}
|
||||
nq.mu.Lock()
|
||||
defer nq.mu.Unlock()
|
||||
if _, err := nq.db.Exec(`
|
||||
DELETE FROM notification_delivery_receipts
|
||||
WHERE alert_id = ? AND alert_start_nanos = ? AND destination_key = ?
|
||||
`, alertID, alertStart.UnixNano(), destinationKey); err != nil {
|
||||
return fmt.Errorf("delete notification delivery receipt: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (nq *NotificationQueue) migrateAlertIdentifierColumns() error {
|
||||
columns, err := nq.tableColumns("notification_audit")
|
||||
if err != nil {
|
||||
@@ -698,7 +858,10 @@ func (nq *NotificationQueue) IncrementAttempt(id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// IncrementAttemptAndSetStatus atomically increments attempt counter and sets status in a single operation
|
||||
// IncrementAttemptAndSetStatus updates a queue row and its operational links
|
||||
// atomically. Delivery workers use claimPendingForDelivery so a concurrent
|
||||
// resolution can cancel a pending row before it is claimed; this method
|
||||
// remains available for explicit lifecycle transitions and compatibility.
|
||||
func (nq *NotificationQueue) IncrementAttemptAndSetStatus(id string, status NotificationQueueStatus) error {
|
||||
nq.mu.Lock()
|
||||
defer nq.mu.Unlock()
|
||||
@@ -725,13 +888,75 @@ func (nq *NotificationQueue) IncrementAttemptAndSetStatus(id string, status Noti
|
||||
operational_links = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
_, err = nq.db.Exec(query, status, now.Unix(), string(linksJSON), id)
|
||||
if err != nil {
|
||||
if _, err := nq.db.Exec(query, status, now.Unix(), string(linksJSON), id); err != nil {
|
||||
return fmt.Errorf("failed to increment attempt and set status: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// claimPendingForDelivery atomically claims a pending queue row and refreshes
|
||||
// its alert list. The refresh is required because resolution cancellation may
|
||||
// have rewritten a grouped row after GetPending returned its snapshot.
|
||||
func (nq *NotificationQueue) claimPendingForDelivery(notif *QueuedNotification) (bool, error) {
|
||||
nq.mu.Lock()
|
||||
defer nq.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
links, err := nq.readNotificationLinksNoLock(notif.ID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
links = transitionNotificationLinks(
|
||||
links,
|
||||
notificationDeliveryStateForQueueStatus(QueueStatusSending),
|
||||
now,
|
||||
)
|
||||
linksJSON, err := json.Marshal(links)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("marshal notification links for attempt: %w", err)
|
||||
}
|
||||
query := `
|
||||
UPDATE notification_queue
|
||||
SET attempts = attempts + 1,
|
||||
status = ?,
|
||||
last_attempt = ?,
|
||||
operational_links = ?
|
||||
WHERE id = ? AND status = 'pending'
|
||||
`
|
||||
result, err := nq.db.Exec(
|
||||
query,
|
||||
QueueStatusSending,
|
||||
now.Unix(),
|
||||
string(linksJSON),
|
||||
notif.ID,
|
||||
)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to claim pending notification: %w", err)
|
||||
}
|
||||
rows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read claimed notification row count: %w", err)
|
||||
}
|
||||
if rows == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var alertsJSON []byte
|
||||
if err := nq.db.QueryRow(`SELECT alerts FROM notification_queue WHERE id = ?`, notif.ID).Scan(&alertsJSON); err != nil {
|
||||
return false, fmt.Errorf("reload claimed notification alerts: %w", err)
|
||||
}
|
||||
var refreshedAlerts []*alerts.Alert
|
||||
if err := json.Unmarshal(alertsJSON, &refreshedAlerts); err != nil {
|
||||
return false, fmt.Errorf("decode claimed notification alerts: %w", err)
|
||||
}
|
||||
notif.Alerts = refreshedAlerts
|
||||
notif.Attempts++
|
||||
notif.Status = QueueStatusSending
|
||||
notif.LastAttempt = &now
|
||||
notif.Links = links
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// GetPending returns notifications ready for processing
|
||||
func (nq *NotificationQueue) GetPending(limit int) ([]*QueuedNotification, error) {
|
||||
nq.mu.RLock()
|
||||
@@ -1106,9 +1331,13 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
|
||||
Msg("Skipping cancelled notification")
|
||||
return
|
||||
}
|
||||
releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiersFromAlerts(notif.Alerts), false)
|
||||
defer releaseDeliveryGates()
|
||||
|
||||
// Atomically increment attempt counter and set status to sending
|
||||
if err := nq.IncrementAttemptAndSetStatus(notif.ID, QueueStatusSending); err != nil {
|
||||
// Atomically claim the pending row. A concurrent resolution may have
|
||||
// cancelled it while it was waiting for its per-alert delivery gate.
|
||||
claimed, err := nq.claimPendingForDelivery(notif)
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Err(err).
|
||||
Str("component", "notification_queue").
|
||||
@@ -1117,19 +1346,18 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
|
||||
Str("type", notif.Type).
|
||||
Int("attempt", notif.Attempts+1).
|
||||
Int("maxAttempts", notif.MaxAttempts).
|
||||
Msg("Failed to increment attempt and set status")
|
||||
Msg("Failed to claim pending notification")
|
||||
return
|
||||
}
|
||||
if !claimed {
|
||||
log.Debug().
|
||||
Str("component", "notification_queue").
|
||||
Str("action", "skip_unclaimed").
|
||||
Str("id", notif.ID).
|
||||
Str("type", notif.Type).
|
||||
Msg("Skipping notification because it is no longer pending")
|
||||
return
|
||||
}
|
||||
attemptedAt := time.Now()
|
||||
notif.Attempts++
|
||||
notif.Status = QueueStatusSending
|
||||
notif.LastAttempt = &attemptedAt
|
||||
notif.Links = transitionNotificationLinks(
|
||||
notif.Links,
|
||||
operationaltrust.NotificationDelivering,
|
||||
attemptedAt,
|
||||
)
|
||||
|
||||
// Call processor if set
|
||||
nq.mu.RLock()
|
||||
processor := nq.processor
|
||||
@@ -1138,7 +1366,7 @@ func (nq *NotificationQueue) processNotification(notif *QueuedNotification) {
|
||||
return
|
||||
}
|
||||
|
||||
err := processor(notif)
|
||||
err = processor(notif)
|
||||
|
||||
success := err == nil
|
||||
errorMsg := ""
|
||||
@@ -1285,6 +1513,26 @@ func (nq *NotificationQueue) performCleanup() {
|
||||
// Keep completed/failed for 7 days, DLQ for 30 days
|
||||
completedCutoff := time.Now().Add(-7 * 24 * time.Hour).Unix()
|
||||
dlqCutoff := time.Now().Add(-30 * 24 * time.Hour).Unix()
|
||||
receiptCutoff := dlqCutoff
|
||||
|
||||
// Receipts are only needed to correlate a later recovery with a firing
|
||||
// occurrence. Bound their lifetime so installations with recovery delivery
|
||||
// disabled do not grow this side table indefinitely.
|
||||
if result, err := nq.db.Exec(`DELETE FROM notification_delivery_receipts WHERE delivered_at < ?`, receiptCutoff); err != nil {
|
||||
log.Error().
|
||||
Err(err).
|
||||
Str("component", "notification_queue").
|
||||
Str("action", "cleanup_delivery_receipts").
|
||||
Int64("receiptCutoff", receiptCutoff).
|
||||
Msg("Failed to cleanup old notification delivery receipts")
|
||||
} else if rows, _ := result.RowsAffected(); rows > 0 {
|
||||
log.Debug().
|
||||
Str("component", "notification_queue").
|
||||
Str("action", "cleanup_delivery_receipts").
|
||||
Int64("count", rows).
|
||||
Int64("receiptCutoff", receiptCutoff).
|
||||
Msg("Cleaned up old notification delivery receipts")
|
||||
}
|
||||
|
||||
// Delete audit records for notifications about to be cleaned up (FK constraint)
|
||||
auditCleanup := `DELETE FROM notification_audit WHERE notification_id IN (
|
||||
@@ -1419,9 +1667,12 @@ func calculateBackoff(attempt int) time.Duration {
|
||||
// mid-send ('sending') are cancelled best-effort but not counted, because
|
||||
// their delivery may still complete.
|
||||
func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string) (int, error) {
|
||||
alertIdentifiers = normalizeAlertIdentifiers(alertIdentifiers)
|
||||
if len(alertIdentifiers) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
releaseDeliveryGates := nq.acquireAlertDeliveryGates(alertIdentifiers, true)
|
||||
defer releaseDeliveryGates()
|
||||
|
||||
nq.mu.Lock()
|
||||
defer nq.mu.Unlock()
|
||||
@@ -1439,16 +1690,7 @@ func (nq *NotificationQueue) CancelByAlertIdentifiers(alertIdentifiers []string)
|
||||
|
||||
alertIdentifierSet := make(map[string]struct{})
|
||||
for _, id := range alertIdentifiers {
|
||||
id = strings.TrimSpace(id)
|
||||
if id != "" {
|
||||
alertIdentifierSet[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(alertIdentifierSet) == 0 {
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, fmt.Errorf("failed to close cancellation query: %w", err)
|
||||
}
|
||||
return 0, nil
|
||||
alertIdentifierSet[id] = struct{}{}
|
||||
}
|
||||
|
||||
type queuedAlertCancellation struct {
|
||||
|
||||
@@ -750,6 +750,126 @@ func TestCancelByAlertIdentifiers_MultipleAlertsPartialMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessNotificationReloadsAlertsAfterCancellationRewrite(t *testing.T) {
|
||||
nq, err := NewNotificationQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewNotificationQueue: %v", err)
|
||||
}
|
||||
defer func() { _ = nq.Stop() }()
|
||||
|
||||
futureRetry := time.Now().Add(time.Hour)
|
||||
queued := &QueuedNotification{
|
||||
ID: "notif-rewritten-before-claim", Type: "webhook", Status: QueueStatusPending,
|
||||
MaxAttempts: 3, Config: []byte(`{}`), NextRetryAt: &futureRetry,
|
||||
Alerts: []*alerts.Alert{{ID: "alert-1"}, {ID: "alert-2"}},
|
||||
}
|
||||
if err := nq.Enqueue(queued); err != nil {
|
||||
t.Fatalf("Enqueue: %v", err)
|
||||
}
|
||||
|
||||
// Model a worker snapshot returned by GetPending before resolution rewrites
|
||||
// the durable grouped row.
|
||||
workerSnapshot := *queued
|
||||
workerSnapshot.Alerts = append([]*alerts.Alert(nil), queued.Alerts...)
|
||||
if count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"}); err != nil {
|
||||
t.Fatalf("CancelByAlertIdentifiers: %v", err)
|
||||
} else if count != 1 {
|
||||
t.Fatalf("cancelled alert count = %d, want 1", count)
|
||||
}
|
||||
|
||||
var delivered []string
|
||||
nq.SetProcessor(func(notif *QueuedNotification) error {
|
||||
for _, alert := range notif.Alerts {
|
||||
if alert != nil {
|
||||
delivered = append(delivered, alert.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
nq.processNotification(&workerSnapshot)
|
||||
|
||||
if len(delivered) != 1 || delivered[0] != "alert-2" {
|
||||
t.Fatalf("delivered alerts = %v, want only alert-2", delivered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelByAlertIdentifiersWaitsForInFlightDelivery(t *testing.T) {
|
||||
nq, err := NewNotificationQueue(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewNotificationQueue: %v", err)
|
||||
}
|
||||
defer func() { _ = nq.Stop() }()
|
||||
|
||||
futureRetry := time.Now().Add(time.Hour)
|
||||
queued := &QueuedNotification{
|
||||
ID: "notif-in-flight", Type: "webhook", Status: QueueStatusPending,
|
||||
MaxAttempts: 3, Config: []byte(`{}`), NextRetryAt: &futureRetry,
|
||||
Alerts: []*alerts.Alert{{ID: "alert-1"}},
|
||||
}
|
||||
if err := nq.Enqueue(queued); err != nil {
|
||||
t.Fatalf("Enqueue: %v", err)
|
||||
}
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
nq.SetProcessor(func(*QueuedNotification) error {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
})
|
||||
processed := make(chan struct{})
|
||||
go func() {
|
||||
nq.processNotification(queued)
|
||||
close(processed)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("delivery did not start")
|
||||
}
|
||||
|
||||
cancelled := make(chan int, 1)
|
||||
cancelErr := make(chan error, 1)
|
||||
go func() {
|
||||
count, err := nq.CancelByAlertIdentifiers([]string{"alert-1"})
|
||||
cancelled <- count
|
||||
cancelErr <- err
|
||||
}()
|
||||
select {
|
||||
case count := <-cancelled:
|
||||
close(release)
|
||||
t.Fatalf("cancellation returned before in-flight delivery completed: %d", count)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case <-processed:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("delivery did not finish")
|
||||
}
|
||||
select {
|
||||
case err := <-cancelErr:
|
||||
if err != nil {
|
||||
t.Fatalf("CancelByAlertIdentifiers: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("cancellation did not finish")
|
||||
}
|
||||
if count := <-cancelled; count != 0 {
|
||||
t.Fatalf("cancelled alert count = %d, want 0 after delivery", count)
|
||||
}
|
||||
|
||||
var status string
|
||||
if err := nq.db.QueryRow(`SELECT status FROM notification_queue WHERE id = ?`, queued.ID).Scan(&status); err != nil {
|
||||
t.Fatalf("read notification status: %v", err)
|
||||
}
|
||||
if status != string(QueueStatusSent) {
|
||||
t.Fatalf("status = %q, want %q", status, QueueStatusSent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelByAlertIdentifiers_SendingRowsCancelledButNotCounted(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
nq, err := NewNotificationQueue(tempDir)
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package unifiedresources
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAvailabilityDataPreservesThreeStateUDPWireEvidence(t *testing.T) {
|
||||
resource := Resource{
|
||||
ID: "availability:syslog",
|
||||
Type: ResourceTypeNetworkEndpoint,
|
||||
Availability: &AvailabilityData{
|
||||
Protocol: "udp",
|
||||
ProbeOutcome: "indeterminate",
|
||||
UDPMode: "open_or_filtered",
|
||||
Available: false,
|
||||
},
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(resource)
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal() error = %v", err)
|
||||
}
|
||||
wire := string(payload)
|
||||
for _, expected := range []string{
|
||||
`"protocol":"udp"`,
|
||||
`"probeOutcome":"indeterminate"`,
|
||||
`"udpMode":"open_or_filtered"`,
|
||||
} {
|
||||
if !strings.Contains(wire, expected) {
|
||||
t.Fatalf("wire payload %s does not contain %s", wire, expected)
|
||||
}
|
||||
}
|
||||
if strings.Contains(wire, `"available":true`) {
|
||||
t.Fatalf("indeterminate UDP evidence must not claim availability: %s", wire)
|
||||
}
|
||||
}
|
||||
@@ -118,6 +118,28 @@ func (a *MonitorAdapter) GetRecentChanges(canonicalID string, since time.Time, l
|
||||
return registry.store.GetRecentChanges(canonicalID, since, limit)
|
||||
}
|
||||
|
||||
// GetResourceOperatorState exposes the durable operator-intent record through
|
||||
// the monitor adapter without widening monitoring's core resource-store
|
||||
// interface. Alert policy resolution discovers this capability optionally.
|
||||
func (a *MonitorAdapter) GetResourceOperatorState(canonicalID string) (ResourceOperatorState, bool, error) {
|
||||
registry := a.currentRegistry()
|
||||
if registry == nil || registry.store == nil {
|
||||
return ResourceOperatorState{}, false, nil
|
||||
}
|
||||
return registry.store.GetResourceOperatorState(canonicalID)
|
||||
}
|
||||
|
||||
// ResolveCanonicalResourceID bridges source-native alert identifiers to the
|
||||
// canonical resource identity used by durable operator state and policy UI.
|
||||
func (a *MonitorAdapter) ResolveCanonicalResourceID(ref string) (string, bool) {
|
||||
registry := a.currentRegistry()
|
||||
if registry == nil {
|
||||
return "", false
|
||||
}
|
||||
_, canonicalID, ok := registry.GetByReference(ref)
|
||||
return canonicalID, ok
|
||||
}
|
||||
|
||||
func (a *MonitorAdapter) replaceRegistry(snapshot models.StateSnapshot, recordsBySource map[DataSource][]IngestRecord) {
|
||||
registry := a.currentRegistry()
|
||||
if registry == nil {
|
||||
|
||||
@@ -99,6 +99,39 @@ func TestMonitorAdapterReadStateForwardsToRegistry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorAdapterResolvesCanonicalOperatorIntentCapabilities(t *testing.T) {
|
||||
registry := NewRegistry(NewMemoryStore())
|
||||
adapter := NewMonitorAdapter(registry)
|
||||
adapter.PopulateSupplementalRecords(SourceProxmox, []IngestRecord{{
|
||||
SourceID: "pve-a:vm:101",
|
||||
Resource: Resource{
|
||||
ID: "vm:pve-a:101",
|
||||
Type: ResourceTypeVM,
|
||||
Name: "vm-101",
|
||||
},
|
||||
}})
|
||||
|
||||
canonicalID, found := adapter.ResolveCanonicalResourceID("pve-a:vm:101")
|
||||
if !found || canonicalID == "" {
|
||||
t.Fatalf("ResolveCanonicalResourceID() = %q, %v", canonicalID, found)
|
||||
}
|
||||
operatorState := ResourceOperatorState{
|
||||
CanonicalID: canonicalID,
|
||||
IntentionallyOffline: true,
|
||||
MaintenanceReason: "planned hardware work",
|
||||
}
|
||||
if err := registry.store.SetResourceOperatorState(operatorState); err != nil {
|
||||
t.Fatalf("SetResourceOperatorState() error = %v", err)
|
||||
}
|
||||
got, found, err := adapter.GetResourceOperatorState(canonicalID)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("GetResourceOperatorState() found=%v error=%v", found, err)
|
||||
}
|
||||
if !got.IntentionallyOffline || got.MaintenanceReason != operatorState.MaintenanceReason {
|
||||
t.Fatalf("operator state = %+v, want persisted intent", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorAdapterPhysicalDiskReadStateRetainsProxmoxIdentityAfterSMARTMerge(t *testing.T) {
|
||||
adapter := NewMonitorAdapter(NewRegistry(nil))
|
||||
now := time.Date(2026, 7, 7, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
@@ -1639,6 +1639,8 @@ type AvailabilityData struct {
|
||||
TargetKind string `json:"targetKind,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
Protocol string `json:"protocol,omitempty"`
|
||||
ProbeOutcome string `json:"probeOutcome,omitempty"`
|
||||
UDPMode string `json:"udpMode,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
|
||||
@@ -247,8 +247,10 @@ class CanonicalCompletionGuardTest(unittest.TestCase):
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"internal/monitoring/availability_poller_test.go",
|
||||
"internal/monitoring/availability_udp_test.go",
|
||||
"internal/monitoring/canonical_guardrails_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/monitor_alert_intent_test.go",
|
||||
"internal/monitoring/monitor_alert_override_migration_test.go",
|
||||
"internal/monitoring/monitor_backups_readstate_test.go",
|
||||
"internal/monitoring/monitor_host_agents_test.go",
|
||||
@@ -2152,9 +2154,11 @@ None yet.
|
||||
"frontend-modern/src/components/Alerts/__tests__/BulkEditDialog.test.tsx",
|
||||
"frontend-modern/src/components/Alerts/__tests__/InvestigateAlertButton.test.tsx",
|
||||
"frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.emptystate.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.timelineerror.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.total24h.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/alertsConfigurationModel.test.ts",
|
||||
"frontend-modern/src/features/alerts/__tests__/helpers.test.ts",
|
||||
"frontend-modern/src/features/alerts/identity.test.ts",
|
||||
|
||||
@@ -81,12 +81,22 @@ class RegistryAuditTest(unittest.TestCase):
|
||||
"internal/api/ai_handlers_patrol_actions_additional_test.go",
|
||||
},
|
||||
("alerts", "alerts-frontend-surface"): {
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
|
||||
"frontend-modern/src/features/alerts/thresholds/hooks/__tests__/truenasThresholdPersistence.test.tsx",
|
||||
},
|
||||
("alerts", "canonical-alert-runtime"): {
|
||||
"internal/alerts/intent_policy_test.go",
|
||||
},
|
||||
("alerts", "alerts-runtime-support"): {
|
||||
"internal/alerts/canonical_override_migration_test.go",
|
||||
"internal/alerts/intent_policy_test.go",
|
||||
"internal/monitoring/monitor_alert_override_migration_test.go",
|
||||
},
|
||||
("api-contracts", "alert-intent-policy-api"): {
|
||||
"frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts",
|
||||
"internal/api/alerts_endpoints_test.go",
|
||||
},
|
||||
("api-contracts", "backend-payload-contracts"): {
|
||||
"internal/api/ai_handlers_patrol_actions_additional_test.go",
|
||||
},
|
||||
@@ -107,9 +117,16 @@ class RegistryAuditTest(unittest.TestCase):
|
||||
},
|
||||
("monitoring", "runtime-report-model"): {
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/monitor_full_coverage_test.go",
|
||||
},
|
||||
("monitoring", "pbs-protection-evidence-runtime"): {
|
||||
"internal/monitoring/monitor_alert_intent_test.go",
|
||||
"internal/monitoring/monitor_full_coverage_test.go",
|
||||
},
|
||||
("monitoring", "monitoring-runtime"): {
|
||||
"internal/monitoring/availability_udp_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/monitor_alert_intent_test.go",
|
||||
"internal/monitoring/monitor_alert_override_migration_test.go",
|
||||
},
|
||||
("monitoring", "agent-fleet-diagnostics-runtime"): {
|
||||
@@ -138,6 +155,7 @@ class RegistryAuditTest(unittest.TestCase):
|
||||
"internal/hostagent/issue1595_sas_collection_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/truenas/contract_test.go",
|
||||
"internal/unifiedresources/availability_projection_test.go",
|
||||
"pkg/diskinventory/identity_test.go",
|
||||
"pkg/diskinventory/status_test.go",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user