mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
feat(alerts): add external dead-man monitoring
This commit is contained in:
@@ -615,6 +615,15 @@ Alert configuration and history (requires `monitoring:read`/`monitoring:write`).
|
||||
|
||||
- `GET /api/alerts/config`
|
||||
- `PUT /api/alerts/config`
|
||||
- `GET /api/alerts/deadman/config` — returns only whether an external watchdog
|
||||
is configured; `pingUrl` is `***REDACTED***` when present and never returns
|
||||
the credential-bearing URL
|
||||
- `PUT /api/alerts/deadman/config` — body `{ "pingUrl": "..." }`; accepts a
|
||||
healthchecks-compatible base success URL, `***REDACTED***` to preserve the
|
||||
saved value, or an empty string to remove it
|
||||
- `GET /api/alerts/deadman/status` — live watchdog health, monitor-loop
|
||||
progress, sanitized delivery failure state, and the most recent restart
|
||||
interruption; never includes the URL or endpoint fingerprint
|
||||
- `POST /api/alerts/activate`
|
||||
- `GET /api/alerts/active`
|
||||
- `GET /api/alerts/delivery-diagnosis?alertIdentifier=<alert-id>` (omit `alertIdentifier` to get the diagnosis array for every active alert)
|
||||
|
||||
@@ -6655,3 +6655,13 @@ in the live process or regain a supposedly revoked credential after restart.
|
||||
The lifecycle proof deletes one exact token from a three-token inventory and
|
||||
forces a failed persistence commit in
|
||||
`internal/api/security_tokens_lifecycle_test.go`.
|
||||
|
||||
### External watchdog wiring does not widen agent lifecycle authority
|
||||
|
||||
`internal/monitoring/monitor.go` now starts and stops the external dead-man
|
||||
worker beside the canonical monitor loop. That adjacency grants the watchdog
|
||||
no enrollment, report admission, token, profile, update, command, tombstone,
|
||||
or re-enrollment authority. Its liveness marker observes only whether the
|
||||
monitor select loop is progressing; agent report success or failure cannot
|
||||
independently assert that Pulse is healthy. Agent-lifecycle behavior and proof
|
||||
routes remain unchanged.
|
||||
|
||||
@@ -288,6 +288,7 @@ default construction path still restores.
|
||||
81. `internal/alerts/history_migration.go`
|
||||
82. `internal/alerts/active_state_bootstrap.go`
|
||||
83. `internal/alerts/eventlog/active_state.go`
|
||||
84. `frontend-modern/src/features/alerts/AlertDeadManDestinationSection.tsx`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
@@ -2140,3 +2141,23 @@ and `useAlertWebhookDestinationsState`. Delivery evidence remains
|
||||
notification truth: the delivery log card must not resolve, suppress, or
|
||||
re-evaluate alerts, and it does not alter the `AlertConfig.enabled` versus
|
||||
`activationState` ownership boundary above.
|
||||
|
||||
### External watchdog is Pulse-availability evidence, not notification delivery
|
||||
|
||||
The Alerts destinations surface owns the operator contract for an external
|
||||
dead-man watchdog that detects loss of Pulse itself. It presents masked
|
||||
configuration, live heartbeat state, canonical-monitor progress, consecutive
|
||||
delivery failures, and the last restart interruption. The credential-bearing
|
||||
ping URL must never return to the browser after save; the configured state uses
|
||||
an explicit replacement placeholder and removal action. The watchdog remains
|
||||
active independently of alert activation, snooze, quiet hours, and notification
|
||||
delivery pause because those policies must not disable observation of Pulse.
|
||||
|
||||
Watchdog transport and monitoring progress remain notifications- and
|
||||
monitoring-owned respectively. Alerts owns the system-alert projection:
|
||||
delivery failure, canonical-loop stall, restart interruption, and durable-state
|
||||
failure use stable system-alert identities, normal lifecycle/event history, and
|
||||
idempotent fingerprints. A restart interruption is raised into history even
|
||||
when the first successful external report immediately resolves it, so recovery
|
||||
does not erase the outage record. The watchdog must never invent resource
|
||||
identity or become a second lifecycle for infrastructure alerts.
|
||||
|
||||
@@ -9920,3 +9920,21 @@ and returns `500`; it must not report a revocation that will reverse on restart.
|
||||
`TestSecurityTokensDeleteRollsBackWhenPersistenceFails` in
|
||||
`internal/api/security_tokens_lifecycle_test.go` pin both the multi-token
|
||||
identity boundary and the failed-commit response.
|
||||
|
||||
### External watchdog configuration and status are separate secret boundaries
|
||||
|
||||
`GET /api/alerts/deadman/config` requires monitoring-read scope and returns
|
||||
only `{pingUrl, configured}`. A configured URL is always the literal
|
||||
`***REDACTED***` sentinel; the credential-bearing value must never cross the
|
||||
response boundary. `PUT /api/alerts/deadman/config` requires monitoring-write
|
||||
scope, accepts the sentinel as preserve, an empty string as explicit removal,
|
||||
or a validated replacement URL, and reports success only after encrypted
|
||||
persistence commits. Validation and persistence errors never echo the URL.
|
||||
|
||||
`GET /api/alerts/deadman/status` requires monitoring-read scope and returns the
|
||||
tenant-scoped operational read model: configured flag, state, heartbeat and
|
||||
recommended-grace seconds, monitoring progress, attempt/success timestamps,
|
||||
failure count, sanitized error, and last interruption. It never returns the
|
||||
URL or endpoint fingerprint. A saved configuration that cannot be decrypted is
|
||||
`configuration_unavailable`, while explicit removal is immediately `disabled`
|
||||
even if cancellation of an older request is still unwinding.
|
||||
|
||||
@@ -6648,3 +6648,15 @@ tokens, labels the observation `Stale`, and exposes the last successful refresh
|
||||
age in its title. Freshness presentation remains in the pure storage row model;
|
||||
the table component must not infer age from render time or restyle retained
|
||||
capacity independently.
|
||||
|
||||
### Credential-bearing destination panels use replacement semantics
|
||||
|
||||
The feature-owned external-watchdog panel composes `SettingsPanel` without
|
||||
creating a second settings shell. A stored credential-bearing URL renders as
|
||||
an empty password input with a configured replacement placeholder and an
|
||||
explicit `Remove` action; it must not be inserted into the DOM, tooltip, status
|
||||
copy, or client logs. Only a newly entered value may be revealed with the
|
||||
panel-local Show/Hide control. Status badges use shared theme tokens, error and
|
||||
unavailable states remain textually distinct, and the four-part status grid
|
||||
collapses without horizontal overflow at phone widths. This pattern is the
|
||||
required primitive composition for future secret-bearing destination panels.
|
||||
|
||||
@@ -525,6 +525,7 @@ cleanup so readers cannot retain orphaned runtime or alert projections.
|
||||
62a. `internal/monitoring/monitor_xcpng.go`
|
||||
63. `internal/monitoring/metadata_stores.go`
|
||||
64. `internal/monitoring/system_alerts.go`
|
||||
65. `internal/monitoring/deadman.go`
|
||||
63a. `internal/config/docker_metadata.go`
|
||||
63b. `internal/config/guest_metadata.go`
|
||||
|
||||
@@ -3247,3 +3248,23 @@ until the next window instead of starting it again on every 30-second report
|
||||
(#1729). `TestCollectStorageUsageDecouplesFullDaemonScanFromLiveTelemetry` and
|
||||
`TestCollectStorageUsageThrottlesInitialTransientFailureWithoutRetry` pin the
|
||||
cadence, stale-result continuity, and no-retry boundary.
|
||||
|
||||
### External dead-man proves canonical loop liveness across restarts
|
||||
|
||||
`internal/monitoring/deadman.go` owns the monitoring side of the external
|
||||
watchdog. A separate worker emits one success signal per minute, while a
|
||||
15-second marker written only by the canonical `Monitor.Start` select loop
|
||||
proves that the polling scheduler itself is still progressing. A stale marker
|
||||
causes the worker to send the provider's `/fail` signal and raise a critical
|
||||
system alert; the worker's own timer can never count as monitoring progress.
|
||||
|
||||
The runtime persists a privacy-minimized `alerts/deadman-state.json` record
|
||||
through fsync, atomic replacement, and directory sync. It stores endpoint
|
||||
fingerprint and timing only, never the ping URL. On startup, a gap of at least
|
||||
two minutes for the same configured endpoint is reported in the first healthy
|
||||
POST and recorded through the alerts-owned system lifecycle, distinguishing a
|
||||
clean stop from an unexpected one. Stop and configuration changes cancel an
|
||||
in-flight request before durable stop or replacement state is published, so a
|
||||
revoked endpoint receives no trailing heartbeat and a removed destination
|
||||
becomes disabled immediately. State corruption and write failure are visible
|
||||
system conditions rather than silent loss of future outage evidence.
|
||||
|
||||
@@ -51,6 +51,7 @@ or displayed a notification.
|
||||
8. `internal/api/alerting/notification_queue.go`
|
||||
9. `internal/notifications/tag_routing.go`
|
||||
10. `internal/notifications/delivery_health.go`
|
||||
11. `internal/notifications/deadman_config.go`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
@@ -431,3 +432,24 @@ alerts-owned lifecycle.
|
||||
`internal/notifications/notifications_test.go`, and
|
||||
`internal/notifications/queue_test.go` prove occurrence and destination
|
||||
isolation, persistence, cleanup, and cancellation/claim ordering.
|
||||
|
||||
### External watchdog transport is a credential-bearing destination boundary
|
||||
|
||||
`internal/notifications/deadman_config.go` owns normalization and validation
|
||||
for healthchecks-compatible success-ping URLs. Possession of the URL can forge
|
||||
healthy state, so configuration is encrypted at rest, exported only inside the
|
||||
passphrase-encrypted configuration bundle, and represented to API clients by a
|
||||
redacted sentinel. URLs are bounded to HTTP(S), exclude userinfo, fragments,
|
||||
localhost, loopback, unspecified, and link-local targets, and must name the
|
||||
base success endpoint rather than `/start`, `/fail`, or `/log`.
|
||||
|
||||
The monitoring-owned sender uses a dedicated transport that bypasses ambient
|
||||
HTTP proxies, revalidates DNS answers at dial time, never follows redirects,
|
||||
and returns sanitized error classes that cannot disclose the URL or token.
|
||||
Private LAN watchdogs remain valid when separately hosted. Network and 5xx
|
||||
failures receive two bounded retries; permanent response failures do not. A
|
||||
healthy signal is GET, while canonical-loop stall and restart-gap diagnostics
|
||||
are bounded text POSTs containing Pulse health and UTC timing only—never alert
|
||||
content, infrastructure names, destination credentials, or tenant data. This
|
||||
watchdog path is deliberately independent of notification queue activation,
|
||||
quiet hours, grouping, and escalation routing.
|
||||
|
||||
@@ -2376,6 +2376,7 @@
|
||||
"frontend-modern/src/components/Alerts/Thresholds/hooks/__tests__/useCollapsedSections.test.ts",
|
||||
"frontend-modern/src/components/Alerts/Thresholds/sections/__tests__/CollapsibleSection.test.tsx",
|
||||
"frontend-modern/src/components/Alerts/WebhookConfig.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertDeadManDestinationSection.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/alertsConfigurationModel.test.ts",
|
||||
"frontend-modern/src/features/alerts/__tests__/helpers.test.ts",
|
||||
@@ -2383,6 +2384,7 @@
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.timelineerror.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.total24h.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/useAlertDestinationsState.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/useAlertOverridesState.test.tsx",
|
||||
"frontend-modern/src/features/alerts/AlertDeliveryHealthCard.test.tsx",
|
||||
@@ -2562,6 +2564,7 @@
|
||||
"internal/alerts/threshold_resolution_shared_test.go",
|
||||
"internal/alerts/unified_incident_confirmation_test.go",
|
||||
"internal/alerts/update_alerts_test.go",
|
||||
"internal/monitoring/deadman_test.go",
|
||||
"internal/monitoring/monitor_alert_override_migration_test.go"
|
||||
]
|
||||
},
|
||||
@@ -2992,6 +2995,7 @@
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts",
|
||||
"internal/api/alerting/deadman_handlers_test.go",
|
||||
"internal/api/alerts_endpoints_test.go"
|
||||
]
|
||||
},
|
||||
@@ -6015,6 +6019,7 @@
|
||||
"internal/monitoring/availability_udp_test.go",
|
||||
"internal/monitoring/canonical_guardrails_test.go",
|
||||
"internal/monitoring/ceph_test.go",
|
||||
"internal/monitoring/deadman_test.go",
|
||||
"internal/monitoring/issue1485_unraid_lifecycle_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/issue1613_contract_test.go",
|
||||
|
||||
@@ -5626,3 +5626,14 @@ restore, or recovery authority; it prevents a successful API response from
|
||||
describing credential state that the next restart would undo. The exact-token
|
||||
and forced-write-failure proofs live in
|
||||
`internal/api/security_tokens_lifecycle_test.go`.
|
||||
|
||||
### Dead-man persistence is availability evidence, not recovery authority
|
||||
|
||||
The shared configuration persistence and alert API now store the encrypted
|
||||
external-watchdog URL and a monitoring-owned restart marker. These records are
|
||||
not backup inventory, recovery points, retention state, restore evidence, or
|
||||
storage action authority. The restart marker is a bounded availability journal
|
||||
whose atomic/fsync discipline ensures a later process can report a Pulse
|
||||
monitoring gap; it contains endpoint fingerprint and timing only. Configuration
|
||||
export/import carries the watchdog URL solely inside the existing
|
||||
passphrase-encrypted bundle and advances that bundle contract to version 4.4.
|
||||
|
||||
@@ -1,17 +1,27 @@
|
||||
{
|
||||
"version": 1,
|
||||
"base_sha": "699143b7b8411ad671af9ba60c74aa7f87d57758",
|
||||
"verified_at": "2026-08-27T14:13:29Z",
|
||||
"base_sha": "f6773f262a7ace6418fc3ab6f890307290a3de99",
|
||||
"verified_at": "2026-08-27T15:02:27Z",
|
||||
"result": "passed",
|
||||
"changed_paths": [
|
||||
"frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx",
|
||||
"frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx"
|
||||
"frontend-modern/src/api/alerts.ts",
|
||||
"frontend-modern/src/types/alerts.ts",
|
||||
"frontend-modern/src/features/alerts/AlertDeadManDestinationSection.tsx",
|
||||
"frontend-modern/src/features/alerts/AlertsConfigurationSurface.tsx",
|
||||
"frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx",
|
||||
"frontend-modern/src/features/alerts/useAlertDestinationsState.ts",
|
||||
"frontend-modern/src/features/alerts/useAlertsConfigurationState.ts"
|
||||
],
|
||||
"content_sha256": {
|
||||
"frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx": "58fe2feaf93d6f37d0fd2811c829ada884941bf7de77d8d386f723d97ce32c33",
|
||||
"frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "67a7463ba44627ef48423bdcaf689153fe0d3d75c0b3b0aeeeeb6def953fe92c"
|
||||
"frontend-modern/src/api/alerts.ts": "0158e753c8d590e62551069458659ff95eee07bb48f5fb95d43f395cae3d5b48",
|
||||
"frontend-modern/src/types/alerts.ts": "1b63d2b690ad43267a23271f0d3244fcb8a51ceaab85b9266182f8033c242495",
|
||||
"frontend-modern/src/features/alerts/AlertDeadManDestinationSection.tsx": "4f7e355bfaa025b2645e74ec43a81489d3709fa5aae6c44c86d33c3946ae03c7",
|
||||
"frontend-modern/src/features/alerts/AlertsConfigurationSurface.tsx": "6ec0e78c6383ce62a1f55b1adac341d6cae0667e4a052ee276cc4a6c68015bf3",
|
||||
"frontend-modern/src/features/alerts/tabs/DestinationsTab.tsx": "0f70dd5cbf6cb962fe2474d3932b7e79c7d868ea3aad97280aae8d03a1b3c4fa",
|
||||
"frontend-modern/src/features/alerts/useAlertDestinationsState.ts": "28fd0ab1312e2d599bad73dc8a269aebc9de2a8a85e91b188c8566fb68586dbb",
|
||||
"frontend-modern/src/features/alerts/useAlertsConfigurationState.ts": "0d81fabff6da2f2d044f74d60c69f02a17989d241015ac7d134bd6840a8db34d"
|
||||
},
|
||||
"routes": ["/proxmox/overview"],
|
||||
"routes": ["/alerts/notifications"],
|
||||
"viewports": [
|
||||
{
|
||||
"width": 1280,
|
||||
@@ -23,14 +33,16 @@
|
||||
}
|
||||
],
|
||||
"states": [
|
||||
"mock-backed Proxmox Overview with PVE nodes, standalone PBS service rows, and guests",
|
||||
"Overview PBS health table omits backup artifact counts while retaining service and datastore health",
|
||||
"expanded canonical PBS resource detail exposes its History tab at desktop and narrow widths"
|
||||
"unconfigured external watchdog with one-minute heartbeat and three-minute grace guidance",
|
||||
"configured external watchdog with credential-masked replacement placeholder and explicit removal control",
|
||||
"unreachable external watchdog with delivery-failure status, monitoring progress, and sanitized error",
|
||||
"removed external watchdog returning atomically to the unconfigured state"
|
||||
],
|
||||
"interactions": [
|
||||
"opened Proxmox Overview at 1280x800 and confirmed the PBS table is positioned between Nodes and Guests",
|
||||
"expanded the first PBS row with its detail toggle and confirmed the shared resource detail and History tab",
|
||||
"reloaded at 390x844, expanded the first PBS row through the whole-row interaction, and confirmed the same History tab",
|
||||
"confirmed the narrow document scroll width did not exceed its client width"
|
||||
"opened Alerts Notifications and confirmed the external watchdog panel is part of the destination workflow",
|
||||
"entered a healthchecks-compatible URL, toggled reveal and conceal, saved, reloaded, and confirmed the credential was replaced by a masked placeholder",
|
||||
"confirmed an unreachable watchdog reports delivery failure without exposing its URL or token",
|
||||
"verified the panel layout and status grid at 390x844 with no horizontal control overflow",
|
||||
"removed the destination, saved, reloaded, and confirmed Not configured state, normal placeholder, and no browser errors"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -615,6 +615,15 @@ Alert configuration and history (requires `monitoring:read`/`monitoring:write`).
|
||||
|
||||
- `GET /api/alerts/config`
|
||||
- `PUT /api/alerts/config`
|
||||
- `GET /api/alerts/deadman/config` — returns only whether an external watchdog
|
||||
is configured; `pingUrl` is `***REDACTED***` when present and never returns
|
||||
the credential-bearing URL
|
||||
- `PUT /api/alerts/deadman/config` — body `{ "pingUrl": "..." }`; accepts a
|
||||
healthchecks-compatible base success URL, `***REDACTED***` to preserve the
|
||||
saved value, or an empty string to remove it
|
||||
- `GET /api/alerts/deadman/status` — live watchdog health, monitor-loop
|
||||
progress, sanitized delivery failure state, and the most recent restart
|
||||
interruption; never includes the URL or endpoint fingerprint
|
||||
- `POST /api/alerts/activate`
|
||||
- `GET /api/alerts/active`
|
||||
- `GET /api/alerts/delivery-diagnosis?alertIdentifier=<alert-id>` (omit `alertIdentifier` to get the diagnosis array for every active alert)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { AlertsAPI } from '@/api/alerts';
|
||||
import { apiFetchJSON } from '@/utils/apiClient';
|
||||
|
||||
vi.mock('@/utils/apiClient', () => ({
|
||||
apiFetchJSON: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockedApiFetchJSON = vi.mocked(apiFetchJSON);
|
||||
|
||||
describe('AlertsAPI external watchdog', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('uses the canonical masked configuration and operational status routes', async () => {
|
||||
mockedApiFetchJSON.mockResolvedValueOnce({
|
||||
pingUrl: '***REDACTED***',
|
||||
configured: true,
|
||||
});
|
||||
await expect(AlertsAPI.getDeadManConfig()).resolves.toEqual({
|
||||
pingUrl: '***REDACTED***',
|
||||
configured: true,
|
||||
});
|
||||
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/deadman/config');
|
||||
|
||||
mockedApiFetchJSON.mockResolvedValueOnce({
|
||||
configured: true,
|
||||
state: 'healthy',
|
||||
heartbeatIntervalSeconds: 60,
|
||||
recommendedGraceSeconds: 180,
|
||||
consecutiveFailures: 0,
|
||||
});
|
||||
await expect(AlertsAPI.getDeadManStatus()).resolves.toMatchObject({ state: 'healthy' });
|
||||
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/deadman/status');
|
||||
|
||||
mockedApiFetchJSON.mockResolvedValueOnce({ success: true, configured: true });
|
||||
await expect(
|
||||
AlertsAPI.updateDeadManConfig('https://watchdog.example.com/ping/replacement-token'),
|
||||
).resolves.toEqual({ success: true, configured: true });
|
||||
expect(mockedApiFetchJSON).toHaveBeenLastCalledWith('/api/alerts/deadman/config', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({
|
||||
pingUrl: 'https://watchdog.example.com/ping/replacement-token',
|
||||
}),
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Alert, AlertDeliveryDiagnosis, AlertEvent, Incident } from '@/types/api';
|
||||
import type { AlertConfig } from '@/types/alerts';
|
||||
import type { AlertConfig, DeadManStatus } from '@/types/alerts';
|
||||
import { apiFetchJSON } from '@/utils/apiClient';
|
||||
import { arrayOrEmpty } from './responseUtils';
|
||||
|
||||
@@ -121,6 +121,27 @@ export class AlertsAPI {
|
||||
});
|
||||
}
|
||||
|
||||
static async getDeadManStatus(): Promise<DeadManStatus> {
|
||||
return apiFetchJSON(`${this.baseUrl}/deadman/status`) as Promise<DeadManStatus>;
|
||||
}
|
||||
|
||||
static async getDeadManConfig(): Promise<{ pingUrl: string; configured: boolean }> {
|
||||
return apiFetchJSON(`${this.baseUrl}/deadman/config`) as Promise<{
|
||||
pingUrl: string;
|
||||
configured: boolean;
|
||||
}>;
|
||||
}
|
||||
|
||||
static async updateDeadManConfig(pingUrl: string): Promise<{
|
||||
success: boolean;
|
||||
configured: boolean;
|
||||
}> {
|
||||
return apiFetchJSON(`${this.baseUrl}/deadman/config`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ pingUrl }),
|
||||
}) as Promise<{ success: boolean; configured: boolean }>;
|
||||
}
|
||||
|
||||
static async activate(): Promise<{ success: boolean; state: string; activationTime?: string }> {
|
||||
return apiFetchJSON(`${this.baseUrl}/activate`, {
|
||||
method: 'POST',
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import { createSignal, createUniqueId, onMount, Show, type Accessor } from 'solid-js';
|
||||
|
||||
import { AlertsAPI } from '@/api/alerts';
|
||||
import { SettingsPanel } from '@/components/shared/SettingsPanel';
|
||||
import type { DeadManStatus } from '@/types/alerts';
|
||||
import { formatRelativeTime } from '@/utils/format';
|
||||
import { logger } from '@/utils/logger';
|
||||
|
||||
interface AlertDeadManDestinationSectionProps {
|
||||
pingUrl: Accessor<string>;
|
||||
setPingUrl: (value: string) => void;
|
||||
setHasUnsavedChanges: (value: boolean) => void;
|
||||
}
|
||||
|
||||
const REDACTED_PING_URL = '***REDACTED***';
|
||||
|
||||
const statusPresentation: Record<DeadManStatus['state'], { label: string; class: string }> = {
|
||||
disabled: { label: 'Not configured', class: 'bg-base text-muted' },
|
||||
starting: {
|
||||
label: 'Starting',
|
||||
class: 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-100',
|
||||
},
|
||||
healthy: {
|
||||
label: 'Heartbeat healthy',
|
||||
class: 'bg-emerald-100 text-emerald-800 dark:bg-emerald-900 dark:text-emerald-100',
|
||||
},
|
||||
delivery_failed: {
|
||||
label: 'Delivery failing',
|
||||
class: 'bg-amber-100 text-amber-900 dark:bg-amber-900 dark:text-amber-100',
|
||||
},
|
||||
monitor_stalled: {
|
||||
label: 'Monitoring stalled',
|
||||
class: 'bg-red-100 text-red-900 dark:bg-red-900 dark:text-red-100',
|
||||
},
|
||||
misconfigured: {
|
||||
label: 'Configuration invalid',
|
||||
class: 'bg-red-100 text-red-900 dark:bg-red-900 dark:text-red-100',
|
||||
},
|
||||
configuration_unavailable: {
|
||||
label: 'Configuration unavailable',
|
||||
class: 'bg-red-100 text-red-900 dark:bg-red-900 dark:text-red-100',
|
||||
},
|
||||
};
|
||||
|
||||
export function AlertDeadManDestinationSection(props: AlertDeadManDestinationSectionProps) {
|
||||
const inputId = `alert-deadman-url-${createUniqueId()}`;
|
||||
const titleId = `${inputId}-title`;
|
||||
const [status, setStatus] = createSignal<DeadManStatus | null>(null);
|
||||
const [loading, setLoading] = createSignal(false);
|
||||
const [unavailable, setUnavailable] = createSignal(false);
|
||||
const [showUrl, setShowUrl] = createSignal(false);
|
||||
|
||||
const loadStatus = async () => {
|
||||
if (loading()) return;
|
||||
setLoading(true);
|
||||
try {
|
||||
setStatus(await AlertsAPI.getDeadManStatus());
|
||||
setUnavailable(false);
|
||||
} catch (error) {
|
||||
logger.error('Failed to load external watchdog status', error);
|
||||
setUnavailable(true);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
onMount(() => void loadStatus());
|
||||
|
||||
const presentation = () => {
|
||||
const current = status();
|
||||
return current ? statusPresentation[current.state] : statusPresentation.disabled;
|
||||
};
|
||||
const hasStoredUrl = () => props.pingUrl() === REDACTED_PING_URL;
|
||||
const inputValue = () => (hasStoredUrl() ? '' : props.pingUrl());
|
||||
|
||||
return (
|
||||
<SettingsPanel
|
||||
titleId={titleId}
|
||||
title="External watchdog"
|
||||
description="Detect when Pulse itself stops monitoring by pinging a watchdog on a different host."
|
||||
action={
|
||||
<div class="flex items-center gap-2">
|
||||
<span class={`rounded-full px-2.5 py-1 text-xs font-semibold ${presentation().class}`}>
|
||||
{unavailable() ? 'Status unavailable' : presentation().label}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-md border border-border px-2.5 py-1 text-xs font-medium text-base-content hover:bg-surface-hover disabled:opacity-50"
|
||||
disabled={loading()}
|
||||
onClick={() => void loadStatus()}
|
||||
>
|
||||
{loading() ? 'Refreshing…' : 'Refresh'}
|
||||
</button>
|
||||
</div>
|
||||
}
|
||||
class="min-w-0"
|
||||
>
|
||||
<div class="space-y-4">
|
||||
<div>
|
||||
<label for={inputId} class="mb-1.5 block text-sm font-medium text-base-content">
|
||||
Healthchecks-compatible success ping URL
|
||||
</label>
|
||||
<div class="flex min-w-0 gap-2">
|
||||
<input
|
||||
id={inputId}
|
||||
type={showUrl() ? 'text' : 'password'}
|
||||
value={inputValue()}
|
||||
maxLength={2048}
|
||||
spellcheck={false}
|
||||
autocomplete="off"
|
||||
placeholder={
|
||||
hasStoredUrl() ? 'Configured — enter a new URL to replace' : 'https://hc-ping.com/…'
|
||||
}
|
||||
aria-describedby={`${inputId}-help`}
|
||||
class="min-w-0 flex-1 rounded-md border border-border bg-surface px-3 py-2 font-mono text-sm text-base-content shadow-sm focus:border-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-500/20"
|
||||
onInput={(event) => {
|
||||
props.setPingUrl(event.currentTarget.value);
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
/>
|
||||
<Show
|
||||
when={!hasStoredUrl()}
|
||||
fallback={
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-md border border-red-300 px-3 py-2 text-sm font-medium text-red-700 hover:bg-red-50 dark:border-red-800 dark:text-red-300 dark:hover:bg-red-950"
|
||||
onClick={() => {
|
||||
props.setPingUrl('');
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
>
|
||||
Remove
|
||||
</button>
|
||||
}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-md border border-border px-3 py-2 text-sm font-medium text-base-content hover:bg-surface-hover"
|
||||
aria-pressed={showUrl()}
|
||||
onClick={() => setShowUrl((value) => !value)}
|
||||
>
|
||||
{showUrl() ? 'Hide' : 'Show'}
|
||||
</button>
|
||||
</Show>
|
||||
</div>
|
||||
<p id={`${inputId}-help`} class="mt-2 text-xs leading-5 text-muted">
|
||||
Pulse sends a success signal every minute and a <code>/fail</code> signal if its
|
||||
monitoring loop stalls. Configure a three-minute grace period at the watchdog. Use a
|
||||
service on another machine or network. A watchdog on this Pulse host cannot detect host
|
||||
failure. Clearing this field does not pause the remote check.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<Show when={status()}>
|
||||
{(current) => (
|
||||
<div class="grid gap-3 rounded-md border border-border bg-base p-3 text-xs sm:grid-cols-2 lg:grid-cols-4">
|
||||
<div>
|
||||
<div class="font-medium text-muted">Last success</div>
|
||||
<div class="mt-1 text-base-content">
|
||||
{current().lastSuccessAt
|
||||
? formatRelativeTime(current().lastSuccessAt, { emptyText: 'Never' })
|
||||
: 'Never'}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-muted">Monitor progress</div>
|
||||
<div class="mt-1 text-base-content">
|
||||
{current().lastMonitoringProgress
|
||||
? formatRelativeTime(current().lastMonitoringProgress, { emptyText: 'Waiting' })
|
||||
: 'Waiting'}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-muted">Consecutive failures</div>
|
||||
<div class="mt-1 text-base-content">{current().consecutiveFailures}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-muted">Last interruption</div>
|
||||
<div class="mt-1 text-base-content">
|
||||
{current().lastInterruption
|
||||
? `${Math.max(1, Math.round(current().lastInterruption!.durationSeconds / 60))} min (${current().lastInterruption!.cleanShutdown ? 'clean restart' : 'unexpected stop'})`
|
||||
: 'None recorded'}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Show>
|
||||
|
||||
<Show when={status()?.lastError}>
|
||||
<p class="rounded-md border border-amber-300 bg-amber-50 px-3 py-2 text-xs text-amber-900 dark:border-amber-700 dark:bg-amber-950 dark:text-amber-100">
|
||||
{status()!.lastError}
|
||||
</p>
|
||||
</Show>
|
||||
|
||||
<p class="text-xs leading-5 text-muted">
|
||||
After a restart, Pulse reports any monitoring gap of two minutes or longer to the watchdog
|
||||
and records it in alert history. The ping contains only Pulse health and UTC
|
||||
timestamps—never infrastructure names, alert contents, or credentials.
|
||||
</p>
|
||||
</div>
|
||||
</SettingsPanel>
|
||||
);
|
||||
}
|
||||
@@ -212,6 +212,8 @@ export function AlertsConfigurationSurface(props: AlertsConfigurationSurfaceProp
|
||||
<Show when={props.activeTab() === 'destinations'}>
|
||||
<DestinationsTab
|
||||
setHasUnsavedChanges={state.guardedSetHasUnsavedChanges}
|
||||
deadManPingUrl={state.deadManPingUrl}
|
||||
setDeadManPingUrl={state.setDeadManPingUrl}
|
||||
emailConfig={state.emailConfig}
|
||||
setEmailConfig={state.setEmailConfig}
|
||||
appriseConfig={state.appriseConfig}
|
||||
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from '@solidjs/testing-library';
|
||||
import { createSignal } from 'solid-js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { AlertsAPI } from '@/api/alerts';
|
||||
|
||||
import { AlertDeadManDestinationSection } from '../AlertDeadManDestinationSection';
|
||||
|
||||
vi.mock('@/api/alerts', () => ({
|
||||
AlertsAPI: {
|
||||
getDeadManStatus: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@/utils/logger', () => ({
|
||||
logger: { error: vi.fn() },
|
||||
}));
|
||||
|
||||
describe('AlertDeadManDestinationSection', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(AlertsAPI.getDeadManStatus).mockReset();
|
||||
vi.mocked(AlertsAPI.getDeadManStatus).mockResolvedValue({
|
||||
configured: true,
|
||||
state: 'healthy',
|
||||
heartbeatIntervalSeconds: 60,
|
||||
recommendedGraceSeconds: 180,
|
||||
lastMonitoringProgress: '2026-08-27T11:59:55Z',
|
||||
lastSuccessAt: '2026-08-27T12:00:00Z',
|
||||
consecutiveFailures: 0,
|
||||
lastInterruption: {
|
||||
from: '2026-08-27T11:55:00Z',
|
||||
to: '2026-08-27T12:00:00Z',
|
||||
durationSeconds: 300,
|
||||
cleanShutdown: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
it('never places the stored credential in the DOM and makes removal explicit', async () => {
|
||||
const [pingUrl, setPingUrl] = createSignal('***REDACTED***');
|
||||
const setHasUnsavedChanges = vi.fn();
|
||||
const { container } = render(() => (
|
||||
<AlertDeadManDestinationSection
|
||||
pingUrl={pingUrl}
|
||||
setPingUrl={setPingUrl}
|
||||
setHasUnsavedChanges={setHasUnsavedChanges}
|
||||
/>
|
||||
));
|
||||
|
||||
expect(container.textContent).not.toContain('credential-token');
|
||||
const input = screen.getByLabelText('Healthchecks-compatible success ping URL');
|
||||
expect(input).toHaveAttribute('type', 'password');
|
||||
expect(input).toHaveValue('');
|
||||
expect(input).toHaveAttribute('placeholder', 'Configured — enter a new URL to replace');
|
||||
|
||||
await waitFor(() => expect(screen.getByText('Heartbeat healthy')).toBeInTheDocument());
|
||||
expect(screen.getByText('5 min (unexpected stop)')).toBeInTheDocument();
|
||||
expect(screen.getByText('0')).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Remove' }));
|
||||
expect(pingUrl()).toBe('');
|
||||
expect(setHasUnsavedChanges).toHaveBeenCalledWith(true);
|
||||
});
|
||||
|
||||
it('supports replacing and revealing only a newly entered URL', async () => {
|
||||
const [pingUrl, setPingUrl] = createSignal('***REDACTED***');
|
||||
const { container } = render(() => (
|
||||
<AlertDeadManDestinationSection
|
||||
pingUrl={pingUrl}
|
||||
setPingUrl={setPingUrl}
|
||||
setHasUnsavedChanges={vi.fn()}
|
||||
/>
|
||||
));
|
||||
const input = screen.getByLabelText('Healthchecks-compatible success ping URL');
|
||||
fireEvent.input(input, {
|
||||
target: { value: 'https://watchdog.example.test/ping/new-token' },
|
||||
});
|
||||
|
||||
expect(input).toHaveValue('https://watchdog.example.test/ping/new-token');
|
||||
expect(container.textContent).not.toContain('***REDACTED***');
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Show' }));
|
||||
expect(input).toHaveAttribute('type', 'text');
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Hide' }));
|
||||
expect(input).toHaveAttribute('type', 'password');
|
||||
});
|
||||
|
||||
it('surfaces encrypted configuration failures as an actionable state', async () => {
|
||||
vi.mocked(AlertsAPI.getDeadManStatus).mockResolvedValue({
|
||||
configured: true,
|
||||
state: 'configuration_unavailable',
|
||||
heartbeatIntervalSeconds: 60,
|
||||
recommendedGraceSeconds: 180,
|
||||
consecutiveFailures: 0,
|
||||
lastError: 'Saved external watchdog configuration could not be read',
|
||||
});
|
||||
const [pingUrl, setPingUrl] = createSignal('***REDACTED***');
|
||||
render(() => (
|
||||
<AlertDeadManDestinationSection
|
||||
pingUrl={pingUrl}
|
||||
setPingUrl={setPingUrl}
|
||||
setHasUnsavedChanges={vi.fn()}
|
||||
/>
|
||||
));
|
||||
|
||||
await waitFor(() => expect(screen.getByText('Configuration unavailable')).toBeInTheDocument());
|
||||
expect(
|
||||
screen.getByText('Saved external watchdog configuration could not be read'),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { createSignal } from 'solid-js';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { NotificationsAPI } from '@/api/notifications';
|
||||
import { AlertsAPI } from '@/api/alerts';
|
||||
|
||||
import { useAlertDestinationsState } from '../useAlertDestinationsState';
|
||||
|
||||
@@ -15,6 +16,13 @@ vi.mock('@/api/notifications', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@/api/alerts', () => ({
|
||||
AlertsAPI: {
|
||||
getDeadManConfig: vi.fn(),
|
||||
updateDeadManConfig: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@/utils/logger', () => ({
|
||||
logger: {
|
||||
error: vi.fn(),
|
||||
@@ -27,6 +35,8 @@ describe('useAlertDestinationsState', () => {
|
||||
vi.mocked(NotificationsAPI.getAppriseConfig).mockReset();
|
||||
vi.mocked(NotificationsAPI.updateEmailConfig).mockReset();
|
||||
vi.mocked(NotificationsAPI.updateAppriseConfig).mockReset();
|
||||
vi.mocked(AlertsAPI.getDeadManConfig).mockReset();
|
||||
vi.mocked(AlertsAPI.updateDeadManConfig).mockReset();
|
||||
});
|
||||
|
||||
it('owns alert destinations reload and save behavior separately from alert policy config', async () => {
|
||||
@@ -63,20 +73,31 @@ describe('useAlertDestinationsState', () => {
|
||||
timeoutSeconds: 30,
|
||||
skipTlsVerify: false,
|
||||
} as any);
|
||||
vi.mocked(AlertsAPI.getDeadManConfig).mockResolvedValue({
|
||||
pingUrl: '***REDACTED***',
|
||||
configured: true,
|
||||
});
|
||||
vi.mocked(AlertsAPI.updateDeadManConfig).mockResolvedValue({
|
||||
success: true,
|
||||
configured: true,
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useAlertDestinationsState({ activeTab }));
|
||||
|
||||
await result.loadDestinations();
|
||||
expect(NotificationsAPI.getEmailConfig).toHaveBeenCalledTimes(1);
|
||||
expect(NotificationsAPI.getAppriseConfig).toHaveBeenCalledTimes(1);
|
||||
expect(AlertsAPI.getDeadManConfig).toHaveBeenCalledTimes(1);
|
||||
expect(result.emailConfig().server).toBe('smtp.example.com');
|
||||
expect(result.appriseConfig().targetsText).toContain('mailto://ops@example.com');
|
||||
expect(result.deadManPingUrl()).toBe('***REDACTED***');
|
||||
|
||||
setActiveTab('destinations');
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(NotificationsAPI.getEmailConfig).toHaveBeenCalledTimes(2);
|
||||
expect(NotificationsAPI.getAppriseConfig).toHaveBeenCalledTimes(2);
|
||||
expect(AlertsAPI.getDeadManConfig).toHaveBeenCalledTimes(2);
|
||||
|
||||
setActiveTab('overview');
|
||||
await Promise.resolve();
|
||||
@@ -91,6 +112,7 @@ describe('useAlertDestinationsState', () => {
|
||||
serverUrl: 'https://apprise.internal',
|
||||
targetsText: 'https://notify.internal',
|
||||
});
|
||||
result.setDeadManPingUrl('https://watchdog.example.test/ping/replacement-token');
|
||||
|
||||
await result.saveDestinations();
|
||||
|
||||
@@ -104,6 +126,9 @@ describe('useAlertDestinationsState', () => {
|
||||
targets: ['https://notify.internal'],
|
||||
}),
|
||||
);
|
||||
expect(AlertsAPI.updateDeadManConfig).toHaveBeenCalledWith(
|
||||
'https://watchdog.example.test/ping/replacement-token',
|
||||
);
|
||||
expect(result.appriseConfig().mode).toBe('http');
|
||||
expect(result.appriseConfig().serverUrl).toBe('https://apprise.example.test');
|
||||
|
||||
@@ -111,5 +136,6 @@ describe('useAlertDestinationsState', () => {
|
||||
expect(result.destConfigLoadError()).toBeNull();
|
||||
expect(result.emailConfig().enabled).toBe(false);
|
||||
expect(result.appriseConfig().enabled).toBe(false);
|
||||
expect(result.deadManPingUrl()).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,7 @@ import { AlertAppriseDestinationsSection } from '../AlertAppriseDestinationsSect
|
||||
import { AlertDeliveryHealthCard } from '../AlertDeliveryHealthCard';
|
||||
import { AlertDeliveryLogCard } from '../AlertDeliveryLogCard';
|
||||
import { AlertDeliveryPausedCard } from '../AlertDeliveryPausedCard';
|
||||
import { AlertDeadManDestinationSection } from '../AlertDeadManDestinationSection';
|
||||
import { AlertDestinationsLoadErrorCard } from '../AlertDestinationsLoadErrorCard';
|
||||
import { AlertDestinationsLoadingState } from '../AlertDestinationsLoadingState';
|
||||
import { AlertEmailDestinationsSection } from '../AlertEmailDestinationsSection';
|
||||
@@ -23,6 +24,8 @@ import {
|
||||
export interface DestinationsTabProps extends AlertDestinationsTabStateProps {
|
||||
setHasUnsavedChanges: (value: boolean) => void;
|
||||
setEmailConfig: (config: ReturnType<AlertDestinationsTabStateProps['emailConfig']>) => void;
|
||||
deadManPingUrl: () => string;
|
||||
setDeadManPingUrl: (value: string) => void;
|
||||
}
|
||||
|
||||
export function DestinationsTab(props: DestinationsTabProps) {
|
||||
@@ -120,6 +123,12 @@ export function DestinationsTab(props: DestinationsTabProps) {
|
||||
testingWebhook={state.testingWebhook()}
|
||||
/>
|
||||
|
||||
<AlertDeadManDestinationSection
|
||||
pingUrl={props.deadManPingUrl}
|
||||
setPingUrl={props.setDeadManPingUrl}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
/>
|
||||
|
||||
<AlertPushDestinationsSection
|
||||
relayLicensed={hasFeature('relay')}
|
||||
showUpgradePrompts={!presentationPolicyHidesUpgradePrompts()}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createEffect, createSignal } from 'solid-js';
|
||||
import type { Accessor } from 'solid-js';
|
||||
|
||||
import { AlertsAPI } from '@/api/alerts';
|
||||
import { NotificationsAPI } from '@/api/notifications';
|
||||
import { getAlertDestinationsConfigLoadError } from '@/utils/alertDestinationsPresentation';
|
||||
import { logger } from '@/utils/logger';
|
||||
@@ -25,6 +26,7 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
|
||||
const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||
createDefaultAppriseConfig(),
|
||||
);
|
||||
const [deadManPingUrl, setDeadManPingUrl] = createSignal('');
|
||||
|
||||
let reloadVersion = 0;
|
||||
let lastActiveTab: AlertTab | null = null;
|
||||
@@ -33,6 +35,7 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
|
||||
setDestConfigLoadError(null);
|
||||
setEmailConfig(createDefaultEmailConfig());
|
||||
setAppriseConfig(createDefaultAppriseConfig());
|
||||
setDeadManPingUrl('');
|
||||
};
|
||||
|
||||
const loadDestinations = async (options: { indicateLoading?: boolean } = {}) => {
|
||||
@@ -46,13 +49,14 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
|
||||
const results = await Promise.allSettled([
|
||||
NotificationsAPI.getEmailConfig(),
|
||||
NotificationsAPI.getAppriseConfig(),
|
||||
AlertsAPI.getDeadManConfig(),
|
||||
]);
|
||||
|
||||
if (thisVersion !== reloadVersion) {
|
||||
return;
|
||||
}
|
||||
|
||||
const [emailResult, appriseResult] = results;
|
||||
const [emailResult, appriseResult, deadManResult] = results;
|
||||
|
||||
if (emailResult.status === 'fulfilled') {
|
||||
setEmailConfig(normalizeEmailConfigFromAPI(emailResult.value));
|
||||
@@ -62,6 +66,10 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
|
||||
setAppriseConfig(normalizeAppriseConfig(appriseResult.value));
|
||||
}
|
||||
|
||||
if (deadManResult.status === 'fulfilled') {
|
||||
setDeadManPingUrl(deadManResult.value.pingUrl || '');
|
||||
}
|
||||
|
||||
const failures = results.filter(
|
||||
(result): result is PromiseRejectedResult => result.status === 'rejected',
|
||||
);
|
||||
@@ -87,6 +95,8 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
|
||||
buildAppriseConfigPayload(appriseConfig()),
|
||||
);
|
||||
|
||||
await AlertsAPI.updateDeadManConfig(deadManPingUrl());
|
||||
|
||||
setAppriseConfig(normalizeAppriseConfig(updatedApprise));
|
||||
};
|
||||
|
||||
@@ -109,6 +119,8 @@ export function useAlertDestinationsState(options: AlertDestinationsStateOptions
|
||||
setEmailConfig,
|
||||
appriseConfig,
|
||||
setAppriseConfig,
|
||||
deadManPingUrl,
|
||||
setDeadManPingUrl,
|
||||
resetDestinations,
|
||||
loadDestinations,
|
||||
saveDestinations,
|
||||
|
||||
@@ -137,6 +137,8 @@ export function useAlertsConfigurationState(props: AlertsConfigurationSurfacePro
|
||||
setEmailConfig: destinationsState.setEmailConfig,
|
||||
appriseConfig: destinationsState.appriseConfig,
|
||||
setAppriseConfig: destinationsState.setAppriseConfig,
|
||||
deadManPingUrl: destinationsState.deadManPingUrl,
|
||||
setDeadManPingUrl: destinationsState.setDeadManPingUrl,
|
||||
...configurationSnapshotState,
|
||||
allGuests: overridesState.allGuests,
|
||||
agentResources: overridesState.agentResources,
|
||||
|
||||
@@ -261,6 +261,33 @@ export interface AlertConfig {
|
||||
disableAllDockerHostsOffline?: boolean;
|
||||
}
|
||||
|
||||
export interface DeadManInterruption {
|
||||
from: string;
|
||||
to: string;
|
||||
durationSeconds: number;
|
||||
cleanShutdown: boolean;
|
||||
}
|
||||
|
||||
export interface DeadManStatus {
|
||||
configured: boolean;
|
||||
state:
|
||||
| 'disabled'
|
||||
| 'starting'
|
||||
| 'healthy'
|
||||
| 'delivery_failed'
|
||||
| 'monitor_stalled'
|
||||
| 'misconfigured'
|
||||
| 'configuration_unavailable';
|
||||
heartbeatIntervalSeconds: number;
|
||||
recommendedGraceSeconds: number;
|
||||
lastMonitoringProgress?: string;
|
||||
lastAttemptAt?: string;
|
||||
lastSuccessAt?: string;
|
||||
consecutiveFailures: number;
|
||||
lastError?: string;
|
||||
lastInterruption?: DeadManInterruption;
|
||||
}
|
||||
|
||||
// Priority levels:
|
||||
// 0: Global defaults
|
||||
// 1-99: Reserved for system rules
|
||||
|
||||
@@ -212,6 +212,21 @@ describe('docsLinks', () => {
|
||||
expect(apiReference).toContain('append-only alert event log');
|
||||
});
|
||||
|
||||
it('ships the credential-safe external watchdog API contract', () => {
|
||||
const apiReference = readFileSync(path.join(repoRoot, 'docs', 'API.md'), 'utf8');
|
||||
const shippedAPIReference = readFileSync(
|
||||
path.join(frontendRoot, 'public', 'docs', 'API.md'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
expect(shippedAPIReference).toBe(apiReference);
|
||||
expect(apiReference).toContain('`GET /api/alerts/deadman/config`');
|
||||
expect(apiReference).toContain('`PUT /api/alerts/deadman/config`');
|
||||
expect(apiReference).toContain('`GET /api/alerts/deadman/status`');
|
||||
expect(apiReference).toContain('`pingUrl` is `***REDACTED***` when present');
|
||||
expect(apiReference).toContain('never includes the URL or endpoint fingerprint');
|
||||
});
|
||||
|
||||
it('ships the truthful Patrol objective API contract', () => {
|
||||
const apiReference = readFileSync(path.join(repoRoot, 'docs', 'API.md'), 'utf8');
|
||||
const shippedAPIReference = readFileSync(
|
||||
|
||||
@@ -27,6 +27,18 @@ const (
|
||||
// NotificationDeliveryAlertType is the first system alert: configured
|
||||
// notification destinations are not delivering.
|
||||
NotificationDeliveryAlertType = "notification-delivery"
|
||||
|
||||
// DeadManDeliveryAlertType reports that Pulse is healthy but cannot reach
|
||||
// the configured external watchdog.
|
||||
DeadManDeliveryAlertType = "deadman-delivery"
|
||||
// DeadManMonitoringStalledAlertType reports that the watchdog worker is
|
||||
// alive but the canonical monitoring loop has stopped making progress.
|
||||
DeadManMonitoringStalledAlertType = "deadman-monitoring-stalled"
|
||||
// DeadManInterruptionAlertType records a monitoring availability gap found
|
||||
// when Pulse restarts.
|
||||
DeadManInterruptionAlertType = "deadman-interruption"
|
||||
// DeadManStateAlertType reports loss of the durable restart-gap record.
|
||||
DeadManStateAlertType = "deadman-state"
|
||||
)
|
||||
|
||||
// SystemAlertInput describes a system-scoped condition. Type is required and
|
||||
|
||||
@@ -66,6 +66,9 @@ type AlertMonitor interface {
|
||||
GetConfigPersistence() ConfigPersistence
|
||||
GetIncidentStore() *memory.IncidentStore
|
||||
GetNotificationManager() *notifications.NotificationManager
|
||||
DeadManConfig() notifications.DeadManConfig
|
||||
UpdateDeadManConfig(notifications.DeadManConfig) error
|
||||
DeadManStatus() monitoring.DeadManStatus
|
||||
SyncAlertState()
|
||||
BuildFrontendState() models.StateFrontend
|
||||
}
|
||||
@@ -205,6 +208,80 @@ func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// GetDeadManStatus returns the live external-watchdog read model without
|
||||
// echoing the secret-bearing ping URL.
|
||||
func (h *AlertHandlers) GetDeadManStatus(w http.ResponseWriter, r *http.Request) {
|
||||
monitor := h.getMonitor(r.Context())
|
||||
if monitor == nil {
|
||||
http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
if err := utils.WriteJSONResponse(w, monitor.DeadManStatus()); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write dead-man status response")
|
||||
}
|
||||
}
|
||||
|
||||
const deadManRedactedPingURL = "***REDACTED***"
|
||||
|
||||
type deadManConfigView struct {
|
||||
PingURL string `json:"pingUrl"`
|
||||
Configured bool `json:"configured"`
|
||||
}
|
||||
|
||||
// GetDeadManConfig returns only a presence marker for the credential-bearing
|
||||
// ping URL. The sentinel supports an edit round-trip without disclosing it.
|
||||
func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request) {
|
||||
monitor := h.getMonitor(r.Context())
|
||||
if monitor == nil {
|
||||
http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
configured := strings.TrimSpace(monitor.DeadManConfig().PingURL) != ""
|
||||
view := deadManConfigView{Configured: configured}
|
||||
if configured {
|
||||
view.PingURL = deadManRedactedPingURL
|
||||
}
|
||||
if err := utils.WriteJSONResponse(w, view); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write dead-man config response")
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateDeadManConfig validates and durably encrypts the watchdog destination
|
||||
// before its live worker is reconfigured.
|
||||
func (h *AlertHandlers) UpdateDeadManConfig(w http.ResponseWriter, r *http.Request) {
|
||||
monitor := h.getMonitor(r.Context())
|
||||
if monitor == nil {
|
||||
http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, notifications.MaxDeadManPingURLLength+1024)
|
||||
var request deadManConfigView
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
http.Error(w, "Invalid request body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
pingURL := strings.TrimSpace(request.PingURL)
|
||||
if pingURL == deadManRedactedPingURL {
|
||||
pingURL = monitor.DeadManConfig().PingURL
|
||||
}
|
||||
config := notifications.DeadManConfig{PingURL: pingURL}
|
||||
if err := notifications.ValidateDeadManPingURL(config.PingURL); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := monitor.UpdateDeadManConfig(config); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to update dead-man configuration")
|
||||
http.Error(w, "Failed to save external watchdog configuration", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := utils.WriteJSONResponse(w, map[string]interface{}{
|
||||
"success": true,
|
||||
"configured": config.PingURL != "",
|
||||
}); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write dead-man config update response")
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateAlertConfig updates the alert configuration
|
||||
func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request) {
|
||||
// Config size scales with the fleet: every per-resource toggle adds an
|
||||
@@ -217,7 +294,6 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
h.getMonitor(r.Context()).GetAlertManager().UpdateConfig(config)
|
||||
updatedConfig := h.getMonitor(r.Context()).GetAlertManager().GetConfig()
|
||||
|
||||
@@ -1253,6 +1329,21 @@ func (h *AlertHandlers) HandleAlerts(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
h.UpdateAlertConfig(w, r)
|
||||
case path == "deadman/status" && r.Method == http.MethodGet:
|
||||
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
}
|
||||
h.GetDeadManStatus(w, r)
|
||||
case path == "deadman/config" && r.Method == http.MethodGet:
|
||||
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
}
|
||||
h.GetDeadManConfig(w, r)
|
||||
case path == "deadman/config" && r.Method == http.MethodPut:
|
||||
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringWrite) {
|
||||
return
|
||||
}
|
||||
h.UpdateDeadManConfig(w, r)
|
||||
case path == "intent-policies" && r.Method == http.MethodGet:
|
||||
if !apihttp.EnsureScope(w, r, config.ScopeMonitoringRead) {
|
||||
return
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -153,6 +154,26 @@ func (m *MockAlertMonitor) GetNotificationManager() *notifications.NotificationM
|
||||
return args.Get(0).(*notifications.NotificationManager)
|
||||
}
|
||||
|
||||
func (m *MockAlertMonitor) DeadManStatus() monitoring.DeadManStatus {
|
||||
args := m.Called()
|
||||
if value := args.Get(0); value != nil {
|
||||
return value.(monitoring.DeadManStatus)
|
||||
}
|
||||
return monitoring.DeadManStatus{}
|
||||
}
|
||||
|
||||
func (m *MockAlertMonitor) DeadManConfig() notifications.DeadManConfig {
|
||||
args := m.Called()
|
||||
if value := args.Get(0); value != nil {
|
||||
return value.(notifications.DeadManConfig)
|
||||
}
|
||||
return notifications.DeadManConfig{}
|
||||
}
|
||||
|
||||
func (m *MockAlertMonitor) UpdateDeadManConfig(config notifications.DeadManConfig) error {
|
||||
return m.Called(config).Error(0)
|
||||
}
|
||||
|
||||
func (m *MockAlertMonitor) SyncAlertState() {
|
||||
m.Called()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
func TestDeadManConfigurationHandlersKeepCredentialSecretAndUpdatesAtomic(t *testing.T) {
|
||||
const secretURL = "https://watchdog.example.com/ping/credential-token"
|
||||
|
||||
t.Run("GET masks configured URL", func(t *testing.T) {
|
||||
monitor := new(MockAlertMonitor)
|
||||
monitor.On("DeadManConfig").Return(notifications.DeadManConfig{PingURL: secretURL}).Once()
|
||||
handler := NewAlertHandlers(nil, monitor, nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.GetDeadManConfig(response, httptest.NewRequest(http.MethodGet, "/api/alerts/deadman/config", nil))
|
||||
|
||||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
assert.NotContains(t, response.Body.String(), secretURL)
|
||||
assert.NotContains(t, response.Body.String(), "credential-token")
|
||||
assert.Contains(t, response.Body.String(), deadManRedactedPingURL)
|
||||
assert.Contains(t, response.Body.String(), `"configured":true`)
|
||||
monitor.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("redacted sentinel preserves stored URL", func(t *testing.T) {
|
||||
monitor := new(MockAlertMonitor)
|
||||
monitor.On("DeadManConfig").Return(notifications.DeadManConfig{PingURL: secretURL}).Once()
|
||||
monitor.On("UpdateDeadManConfig", notifications.DeadManConfig{PingURL: secretURL}).Return(nil).Once()
|
||||
handler := NewAlertHandlers(nil, monitor, nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.UpdateDeadManConfig(response, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/api/alerts/deadman/config",
|
||||
strings.NewReader(`{"pingUrl":"***REDACTED***"}`),
|
||||
))
|
||||
|
||||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
assert.NotContains(t, response.Body.String(), secretURL)
|
||||
monitor.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("empty URL explicitly removes destination", func(t *testing.T) {
|
||||
monitor := new(MockAlertMonitor)
|
||||
monitor.On("UpdateDeadManConfig", notifications.DeadManConfig{}).Return(nil).Once()
|
||||
handler := NewAlertHandlers(nil, monitor, nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.UpdateDeadManConfig(response, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/api/alerts/deadman/config",
|
||||
strings.NewReader(`{"pingUrl":""}`),
|
||||
))
|
||||
|
||||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
assert.Contains(t, response.Body.String(), `"configured":false`)
|
||||
monitor.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("invalid same-host URL never reaches persistence", func(t *testing.T) {
|
||||
monitor := new(MockAlertMonitor)
|
||||
handler := NewAlertHandlers(nil, monitor, nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.UpdateDeadManConfig(response, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/api/alerts/deadman/config",
|
||||
strings.NewReader(`{"pingUrl":"http://127.0.0.1/ping/token"}`),
|
||||
))
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, response.Code)
|
||||
monitor.AssertNotCalled(t, "UpdateDeadManConfig", mock.Anything)
|
||||
})
|
||||
|
||||
t.Run("persistence errors do not echo URL", func(t *testing.T) {
|
||||
monitor := new(MockAlertMonitor)
|
||||
monitor.On("UpdateDeadManConfig", notifications.DeadManConfig{PingURL: secretURL}).Return(errors.New("disk full near credential-token")).Once()
|
||||
handler := NewAlertHandlers(nil, monitor, nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.UpdateDeadManConfig(response, httptest.NewRequest(
|
||||
http.MethodPut,
|
||||
"/api/alerts/deadman/config",
|
||||
strings.NewReader(`{"pingUrl":"https://watchdog.example.com/ping/credential-token"}`),
|
||||
))
|
||||
|
||||
assert.Equal(t, http.StatusInternalServerError, response.Code)
|
||||
assert.NotContains(t, response.Body.String(), "credential-token")
|
||||
monitor.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeadManStatusHandlerReturnsOnlyOperationalReadModel(t *testing.T) {
|
||||
now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
|
||||
monitor := new(MockAlertMonitor)
|
||||
monitor.On("DeadManStatus").Return(monitoring.DeadManStatus{
|
||||
Configured: true,
|
||||
State: "healthy",
|
||||
HeartbeatIntervalSecs: 60,
|
||||
RecommendedGraceSecs: 180,
|
||||
LastMonitoringProgress: &now,
|
||||
LastSuccessAt: &now,
|
||||
}).Once()
|
||||
handler := NewAlertHandlers(nil, monitor, nil)
|
||||
response := httptest.NewRecorder()
|
||||
|
||||
handler.GetDeadManStatus(response, httptest.NewRequest(http.MethodGet, "/api/alerts/deadman/status", nil))
|
||||
|
||||
assert.Equal(t, http.StatusOK, response.Code)
|
||||
assert.Contains(t, response.Body.String(), `"state":"healthy"`)
|
||||
assert.NotContains(t, response.Body.String(), "pingUrl")
|
||||
assert.NotContains(t, response.Body.String(), "fingerprint")
|
||||
monitor.AssertExpectations(t)
|
||||
}
|
||||
@@ -36,6 +36,18 @@ func (w *AlertMonitorWrapper) GetNotificationManager() *notifications.Notificati
|
||||
return w.m.GetNotificationManager()
|
||||
}
|
||||
|
||||
func (w *AlertMonitorWrapper) DeadManStatus() monitoring.DeadManStatus {
|
||||
return w.m.DeadManStatus()
|
||||
}
|
||||
|
||||
func (w *AlertMonitorWrapper) DeadManConfig() notifications.DeadManConfig {
|
||||
return w.m.DeadManConfig()
|
||||
}
|
||||
|
||||
func (w *AlertMonitorWrapper) UpdateDeadManConfig(config notifications.DeadManConfig) error {
|
||||
return w.m.UpdateDeadManConfig(config)
|
||||
}
|
||||
|
||||
func (w *AlertMonitorWrapper) SyncAlertState() {
|
||||
w.m.SyncAlertState()
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ type ExportData struct {
|
||||
Email notifications.EmailConfig `json:"email"`
|
||||
Webhooks []notifications.WebhookConfig `json:"webhooks"`
|
||||
Apprise notifications.AppriseConfig `json:"apprise"`
|
||||
DeadMan notifications.DeadManConfig `json:"deadMan"`
|
||||
System SystemSettings `json:"system"`
|
||||
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
|
||||
SSO *SSOConfig `json:"sso,omitempty"`
|
||||
@@ -70,6 +71,10 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to load Apprise config: %w", err)
|
||||
}
|
||||
deadManConfig, err := c.LoadDeadManConfig()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to load dead-man config: %w", err)
|
||||
}
|
||||
|
||||
webhooks, err := c.LoadWebhooks()
|
||||
if err != nil {
|
||||
@@ -102,7 +107,7 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
|
||||
// Create export data
|
||||
exportData := ExportData{
|
||||
Version: "4.3",
|
||||
Version: "4.4",
|
||||
ExportedAt: time.Now(),
|
||||
Nodes: *nodes,
|
||||
Alerts: *alertConfig,
|
||||
@@ -110,6 +115,7 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
Email: *emailConfig,
|
||||
Webhooks: webhooks,
|
||||
Apprise: *appriseConfig,
|
||||
DeadMan: *deadManConfig,
|
||||
System: *systemSettings,
|
||||
GuestMetadata: guestMetadata,
|
||||
SSO: ssoConfig,
|
||||
@@ -158,8 +164,10 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
|
||||
|
||||
// Check version compatibility (warn but don't fail)
|
||||
switch exportData.Version {
|
||||
case "4.3", "":
|
||||
case "4.4", "":
|
||||
// current version, nothing to do
|
||||
case "4.3":
|
||||
log.Info().Msg("Config was exported from version 4.3. External watchdog settings were not included in that format.")
|
||||
case "4.2":
|
||||
log.Info().Msg("Config was exported from version 4.2. Alert intent policies were not included in that format.")
|
||||
case "4.1":
|
||||
@@ -211,6 +219,10 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
|
||||
return fmt.Errorf("failed to import Apprise config: %w", err)
|
||||
}
|
||||
|
||||
if err := c.SaveDeadManConfig(exportData.DeadMan); err != nil {
|
||||
return fmt.Errorf("failed to import dead-man config: %w", err)
|
||||
}
|
||||
|
||||
if err := c.SaveWebhooks(exportData.Webhooks); err != nil {
|
||||
return fmt.Errorf("failed to import webhooks: %w", err)
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ type ConfigPersistence struct {
|
||||
emailFile string
|
||||
webhookFile string
|
||||
appriseFile string
|
||||
deadManFile string
|
||||
reportSchedulesFile string
|
||||
nodesFile string
|
||||
trueNASFile string
|
||||
@@ -114,6 +115,7 @@ type resolvedConfigPersistencePaths struct {
|
||||
emailFile string
|
||||
webhookFile string
|
||||
appriseFile string
|
||||
deadManFile string
|
||||
reportSchedulesFile string
|
||||
nodesFile string
|
||||
trueNASFile string
|
||||
@@ -167,6 +169,10 @@ func resolveConfigPersistencePaths(configDir string) (string, resolvedConfigPers
|
||||
if err != nil {
|
||||
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve apprise.enc: %w", err)
|
||||
}
|
||||
deadManFile, err := resolveLeaf("deadman.enc")
|
||||
if err != nil {
|
||||
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve deadman.enc: %w", err)
|
||||
}
|
||||
reportSchedulesFile, err := resolveLeaf("report_schedules.json")
|
||||
if err != nil {
|
||||
return "", resolvedConfigPersistencePaths{}, fmt.Errorf("resolve report_schedules.json: %w", err)
|
||||
@@ -258,6 +264,7 @@ func resolveConfigPersistencePaths(configDir string) (string, resolvedConfigPers
|
||||
emailFile: emailFile,
|
||||
webhookFile: webhookFile,
|
||||
appriseFile: appriseFile,
|
||||
deadManFile: deadManFile,
|
||||
reportSchedulesFile: reportSchedulesFile,
|
||||
nodesFile: nodesFile,
|
||||
trueNASFile: trueNASFile,
|
||||
@@ -315,6 +322,7 @@ func newConfigPersistence(configDir string) (*ConfigPersistence, error) {
|
||||
emailFile: resolvedPaths.emailFile,
|
||||
webhookFile: resolvedPaths.webhookFile,
|
||||
appriseFile: resolvedPaths.appriseFile,
|
||||
deadManFile: resolvedPaths.deadManFile,
|
||||
reportSchedulesFile: resolvedPaths.reportSchedulesFile,
|
||||
nodesFile: resolvedPaths.nodesFile,
|
||||
trueNASFile: resolvedPaths.trueNASFile,
|
||||
@@ -1301,6 +1309,38 @@ func (c *ConfigPersistence) LoadAppriseConfig() (*notifications.AppriseConfig, e
|
||||
return &normalized, nil
|
||||
}
|
||||
|
||||
// SaveDeadManConfig stores the credential-bearing external watchdog URL in
|
||||
// the same encrypted destination boundary as email, Apprise, and webhooks.
|
||||
func (c *ConfigPersistence) SaveDeadManConfig(config notifications.DeadManConfig) error {
|
||||
config = notifications.NormalizeDeadManConfig(config)
|
||||
if err := notifications.ValidateDeadManPingURL(config.PingURL); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := saveJSON(c, c.deadManFile, config, true); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info().Str("file", c.deadManFile).Bool("configured", config.PingURL != "").Msg("Dead-man configuration saved")
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadDeadManConfig loads and, when necessary, migrates the encrypted
|
||||
// external watchdog configuration.
|
||||
func (c *ConfigPersistence) LoadDeadManConfig() (*notifications.DeadManConfig, error) {
|
||||
var config notifications.DeadManConfig
|
||||
if err := loadJSON(c, c.deadManFile, true, &config); err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return ¬ifications.DeadManConfig{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
config = notifications.NormalizeDeadManConfig(config)
|
||||
if err := notifications.ValidateDeadManPingURL(config.PingURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Info().Str("file", c.deadManFile).Bool("configured", config.PingURL != "").Msg("Dead-man configuration loaded")
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
// SaveWebhooks saves webhook configurations to file
|
||||
func (c *ConfigPersistence) SaveWebhooks(webhooks []notifications.WebhookConfig) error {
|
||||
c.mu.Lock()
|
||||
|
||||
@@ -68,7 +68,7 @@ func TestExportImportIncludesAlertIntentPolicies(t *testing.T) {
|
||||
t.Fatalf("ExportConfig: %v", err)
|
||||
}
|
||||
decoded := mustDecodeExport(t, bundle, passphrase)
|
||||
if decoded.Version != "4.3" || decoded.AlertIntent == nil {
|
||||
if decoded.Version != "4.4" || decoded.AlertIntent == nil {
|
||||
t.Fatalf("export metadata = version %q intent %#v", decoded.Version, decoded.AlertIntent)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package config_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
)
|
||||
|
||||
func TestDeadManConfigEncryptedRoundTripAndRemoval(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cp := config.NewConfigPersistence(dir)
|
||||
const endpoint = "https://watchdog.example.com/ping/credential-token"
|
||||
|
||||
if err := cp.SaveDeadManConfig(notifications.DeadManConfig{PingURL: endpoint}); err != nil {
|
||||
t.Fatalf("SaveDeadManConfig: %v", err)
|
||||
}
|
||||
loaded, err := cp.LoadDeadManConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadDeadManConfig: %v", err)
|
||||
}
|
||||
if loaded.PingURL != endpoint {
|
||||
t.Fatalf("loaded ping URL = %q, want %q", loaded.PingURL, endpoint)
|
||||
}
|
||||
|
||||
stored, err := os.ReadFile(filepath.Join(dir, "deadman.enc"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile deadman.enc: %v", err)
|
||||
}
|
||||
if bytes.Contains(stored, []byte(endpoint)) || bytes.Contains(stored, []byte("credential-token")) {
|
||||
t.Fatal("deadman.enc exposes the credential-bearing ping URL")
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(dir, "deadman.enc"))
|
||||
if err != nil {
|
||||
t.Fatalf("Stat deadman.enc: %v", err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("deadman.enc permissions = %o, want 600", info.Mode().Perm())
|
||||
}
|
||||
|
||||
if err := cp.SaveDeadManConfig(notifications.DeadManConfig{}); err != nil {
|
||||
t.Fatalf("clear dead-man config: %v", err)
|
||||
}
|
||||
cleared, err := cp.LoadDeadManConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("load cleared dead-man config: %v", err)
|
||||
}
|
||||
if cleared.PingURL != "" {
|
||||
t.Fatalf("cleared ping URL = %q", cleared.PingURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManConfigMigratesPlaintextStorage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cp := config.NewConfigPersistence(dir)
|
||||
const endpoint = "https://watchdog.example.com/ping/plaintext-token"
|
||||
plain, err := json.Marshal(notifications.DeadManConfig{PingURL: endpoint})
|
||||
if err != nil {
|
||||
t.Fatalf("Marshal: %v", err)
|
||||
}
|
||||
path := filepath.Join(dir, "deadman.enc")
|
||||
if err := os.WriteFile(path, plain, 0o600); err != nil {
|
||||
t.Fatalf("WriteFile plaintext deadman.enc: %v", err)
|
||||
}
|
||||
|
||||
loaded, err := cp.LoadDeadManConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadDeadManConfig: %v", err)
|
||||
}
|
||||
if loaded.PingURL != endpoint {
|
||||
t.Fatalf("loaded ping URL = %q, want %q", loaded.PingURL, endpoint)
|
||||
}
|
||||
rewritten, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile rewritten deadman.enc: %v", err)
|
||||
}
|
||||
if bytes.Equal(rewritten, plain) || bytes.Contains(rewritten, []byte("plaintext-token")) {
|
||||
t.Fatal("plaintext dead-man configuration was not rewritten encrypted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManConfigRejectsInvalidDestinationWithoutReplacingStoredValue(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cp := config.NewConfigPersistence(dir)
|
||||
const endpoint = "https://watchdog.example.com/ping/original-token"
|
||||
if err := cp.SaveDeadManConfig(notifications.DeadManConfig{PingURL: endpoint}); err != nil {
|
||||
t.Fatalf("SaveDeadManConfig: %v", err)
|
||||
}
|
||||
if err := cp.SaveDeadManConfig(notifications.DeadManConfig{PingURL: "http://127.0.0.1/ping/token"}); err == nil {
|
||||
t.Fatal("expected same-host dead-man destination to be rejected")
|
||||
}
|
||||
loaded, err := cp.LoadDeadManConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadDeadManConfig: %v", err)
|
||||
}
|
||||
if loaded.PingURL != endpoint {
|
||||
t.Fatalf("invalid update replaced stored value with %q", loaded.PingURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportImportIncludesDeadManConfig(t *testing.T) {
|
||||
sourceDir := t.TempDir()
|
||||
t.Setenv("PULSE_DATA_DIR", sourceDir)
|
||||
source := config.NewConfigPersistence(sourceDir)
|
||||
const endpoint = "https://watchdog.example.com/ping/export-token"
|
||||
if err := source.SaveDeadManConfig(notifications.DeadManConfig{PingURL: endpoint}); err != nil {
|
||||
t.Fatalf("SaveDeadManConfig: %v", err)
|
||||
}
|
||||
|
||||
const passphrase = "dead-man-export-round-trip"
|
||||
bundle, err := source.ExportConfig(passphrase)
|
||||
if err != nil {
|
||||
t.Fatalf("ExportConfig: %v", err)
|
||||
}
|
||||
decoded := mustDecodeExport(t, bundle, passphrase)
|
||||
if decoded.Version != "4.4" || decoded.DeadMan.PingURL != endpoint {
|
||||
t.Fatalf("export metadata = version %q deadMan %#v", decoded.Version, decoded.DeadMan)
|
||||
}
|
||||
|
||||
destinationDir := t.TempDir()
|
||||
destination := config.NewConfigPersistence(destinationDir)
|
||||
if err := destination.ImportConfig(bundle, passphrase); err != nil {
|
||||
t.Fatalf("ImportConfig: %v", err)
|
||||
}
|
||||
loaded, err := destination.LoadDeadManConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadDeadManConfig imported: %v", err)
|
||||
}
|
||||
if loaded.PingURL != endpoint {
|
||||
t.Fatalf("imported ping URL = %q, want %q", loaded.PingURL, endpoint)
|
||||
}
|
||||
}
|
||||
@@ -589,8 +589,8 @@ func TestExportConfigIncludesAPITokens(t *testing.T) {
|
||||
|
||||
decoded := mustDecodeExport(t, exported, passphrase)
|
||||
|
||||
if decoded.Version != "4.3" {
|
||||
t.Fatalf("expected export version 4.3, got %q", decoded.Version)
|
||||
if decoded.Version != "4.4" {
|
||||
t.Fatalf("expected export version 4.4, got %q", decoded.Version)
|
||||
}
|
||||
|
||||
assertJSONEqual(t, decoded.APITokens, tokens, "api tokens")
|
||||
|
||||
@@ -0,0 +1,843 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
deadManHeartbeatInterval = time.Minute
|
||||
deadManMonitoringFreshness = 45 * time.Second
|
||||
deadManRestartReportThreshold = 2 * time.Minute
|
||||
deadManRequestTimeout = 10 * time.Second
|
||||
deadManDeliveryAlertThreshold = 3
|
||||
deadManStateSchemaVersion = 1
|
||||
deadManStateMaxBytes int64 = 64 << 10
|
||||
deadManStateDirPerm = 0o700
|
||||
deadManStateFilePerm = 0o600
|
||||
)
|
||||
|
||||
// DeadManInterruption is the durable, privacy-preserving record of the most
|
||||
// recent monitoring availability gap observed across a Pulse restart.
|
||||
type DeadManInterruption struct {
|
||||
From time.Time `json:"from"`
|
||||
To time.Time `json:"to"`
|
||||
DurationSecs int64 `json:"durationSeconds"`
|
||||
CleanShutdown bool `json:"cleanShutdown"`
|
||||
}
|
||||
|
||||
// DeadManStatus is the tenant-scoped read model for the Alerts destination UI.
|
||||
// It deliberately never returns the ping URL or its fingerprint.
|
||||
type DeadManStatus struct {
|
||||
Configured bool `json:"configured"`
|
||||
State string `json:"state"`
|
||||
HeartbeatIntervalSecs int64 `json:"heartbeatIntervalSeconds"`
|
||||
RecommendedGraceSecs int64 `json:"recommendedGraceSeconds"`
|
||||
LastMonitoringProgress *time.Time `json:"lastMonitoringProgress,omitempty"`
|
||||
LastAttemptAt *time.Time `json:"lastAttemptAt,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
|
||||
ConsecutiveFailures int `json:"consecutiveFailures"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
LastInterruption *DeadManInterruption `json:"lastInterruption,omitempty"`
|
||||
}
|
||||
|
||||
type deadManPersistedState struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
EndpointFingerprint string `json:"endpointFingerprint"`
|
||||
Enabled bool `json:"enabled"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
LastHealthyAt time.Time `json:"lastHealthyAt,omitempty"`
|
||||
LastSuccessfulPing time.Time `json:"lastSuccessfulPing,omitempty"`
|
||||
StoppedAt time.Time `json:"stoppedAt,omitempty"`
|
||||
LastInterruption *DeadManInterruption `json:"lastInterruption,omitempty"`
|
||||
}
|
||||
|
||||
type deadManSignalError struct {
|
||||
message string
|
||||
retryable bool
|
||||
}
|
||||
|
||||
func (e *deadManSignalError) Error() string { return e.message }
|
||||
|
||||
type deadManRuntime struct {
|
||||
mu sync.RWMutex
|
||||
persistMu sync.Mutex
|
||||
|
||||
statePath string
|
||||
startupAt time.Time
|
||||
previous deadManPersistedState
|
||||
persisted deadManPersistedState
|
||||
initialized bool
|
||||
pendingGap *DeadManInterruption
|
||||
loadError error
|
||||
consecutiveFail int
|
||||
stopping bool
|
||||
status DeadManStatus
|
||||
wake chan struct{}
|
||||
signalGeneration uint64
|
||||
activeSignalCancel context.CancelFunc
|
||||
|
||||
client *http.Client
|
||||
now func() time.Time
|
||||
interval time.Duration
|
||||
progressFreshness time.Duration
|
||||
restartThreshold time.Duration
|
||||
retryDelays []time.Duration
|
||||
}
|
||||
|
||||
func newDeadManRuntime(dataDir string) *deadManRuntime {
|
||||
now := time.Now().UTC()
|
||||
d := &deadManRuntime{
|
||||
statePath: filepath.Join(dataDir, "alerts", "deadman-state.json"),
|
||||
startupAt: now,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
interval: deadManHeartbeatInterval,
|
||||
progressFreshness: deadManMonitoringFreshness,
|
||||
restartThreshold: deadManRestartReportThreshold,
|
||||
retryDelays: []time.Duration{2 * time.Second, 5 * time.Second},
|
||||
status: DeadManStatus{
|
||||
State: "disabled",
|
||||
HeartbeatIntervalSecs: int64(deadManHeartbeatInterval / time.Second),
|
||||
RecommendedGraceSecs: int64((3 * deadManHeartbeatInterval) / time.Second),
|
||||
},
|
||||
wake: make(chan struct{}, 1),
|
||||
}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
// A credential-bearing health URL must not inherit a process-wide proxy:
|
||||
// doing so would disclose the full secret path to that proxy. Resolve the
|
||||
// endpoint at dial time as well as at configuration time so a hostname that
|
||||
// changes to a loopback or link-local address fails closed.
|
||||
transport.Proxy = nil
|
||||
transport.DialContext = deadManDialContext
|
||||
d.client = &http.Client{
|
||||
Timeout: deadManRequestTimeout,
|
||||
Transport: transport,
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
}
|
||||
|
||||
state, err := loadDeadManState(d.statePath)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
d.loadError = err
|
||||
log.Error().Err(err).Msg("failed to load durable dead-man state")
|
||||
} else if err == nil {
|
||||
d.previous = state
|
||||
d.persisted = state
|
||||
d.status.LastInterruption = cloneDeadManInterruption(state.LastInterruption)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func cloneDeadManInterruption(value *DeadManInterruption) *DeadManInterruption {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
copyValue := *value
|
||||
return ©Value
|
||||
}
|
||||
|
||||
func deadManEndpointFingerprint(endpoint string) string {
|
||||
digest := sha256.Sum256([]byte(endpoint))
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) run(
|
||||
ctx context.Context,
|
||||
configURL func() string,
|
||||
monitorProgress func() time.Time,
|
||||
manager *alerts.Manager,
|
||||
) {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
d.runCycle(ctx, configURL, monitorProgress, manager)
|
||||
|
||||
ticker := time.NewTicker(d.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
d.runCycle(ctx, configURL, monitorProgress, manager)
|
||||
case <-d.wake:
|
||||
d.runCycle(ctx, configURL, monitorProgress, manager)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) runCycle(
|
||||
ctx context.Context,
|
||||
configURL func() string,
|
||||
monitorProgress func() time.Time,
|
||||
manager *alerts.Manager,
|
||||
) {
|
||||
now := d.now()
|
||||
d.mu.RLock()
|
||||
stopping := d.stopping
|
||||
d.mu.RUnlock()
|
||||
if stopping {
|
||||
return
|
||||
}
|
||||
endpoint := ""
|
||||
if configURL != nil {
|
||||
endpoint = strings.TrimSpace(configURL())
|
||||
}
|
||||
if endpoint == "" {
|
||||
d.disable(now, manager)
|
||||
return
|
||||
}
|
||||
|
||||
if err := notifications.ValidateDeadManPingURL(endpoint); err != nil {
|
||||
d.setMisconfigured(err.Error(), manager)
|
||||
return
|
||||
}
|
||||
fingerprint := deadManEndpointFingerprint(endpoint)
|
||||
d.ensureEndpoint(now, fingerprint, manager)
|
||||
|
||||
progress := time.Time{}
|
||||
if monitorProgress != nil {
|
||||
progress = monitorProgress().UTC()
|
||||
}
|
||||
fresh := !progress.IsZero() && !progress.After(now.Add(5*time.Second)) && now.Sub(progress) <= d.progressFreshness
|
||||
d.setMonitoringProgress(progress)
|
||||
if fresh {
|
||||
d.recordHealthyProgress(progress, manager)
|
||||
d.clearSystemAlert(manager, alerts.DeadManMonitoringStalledAlertType)
|
||||
} else {
|
||||
d.setState("monitor_stalled", "Pulse monitoring loop has stopped making progress")
|
||||
d.raiseMonitoringStalled(manager, progress)
|
||||
}
|
||||
|
||||
message := ""
|
||||
signalURL := endpoint
|
||||
if !fresh {
|
||||
signalURL = deadManFailureURL(endpoint)
|
||||
message = "Pulse watchdog is alive, but the monitoring loop has stopped making progress."
|
||||
} else if gap := d.pendingInterruption(); gap != nil {
|
||||
message = deadManInterruptionMessage(gap)
|
||||
}
|
||||
|
||||
attemptAt := d.now()
|
||||
d.setAttempt(attemptAt)
|
||||
signalCtx, generation, ok := d.beginSignal(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer d.finishSignal(generation)
|
||||
err := d.sendWithRetry(signalCtx, signalURL, message)
|
||||
if signalCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
d.recordSignalFailure(err.Error(), manager)
|
||||
return
|
||||
}
|
||||
d.recordSignalSuccess(d.now(), fresh, manager)
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) ensureEndpoint(now time.Time, fingerprint string, manager *alerts.Manager) {
|
||||
d.mu.Lock()
|
||||
if d.initialized && d.persisted.EndpointFingerprint == fingerprint && d.persisted.Enabled {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
var gap *DeadManInterruption
|
||||
previous := d.previous
|
||||
if previous.Enabled && previous.EndpointFingerprint == fingerprint {
|
||||
from := previous.LastHealthyAt
|
||||
clean := false
|
||||
if !previous.StoppedAt.IsZero() {
|
||||
from = previous.StoppedAt
|
||||
clean = true
|
||||
}
|
||||
if !from.IsZero() && d.startupAt.After(from) && d.startupAt.Sub(from) >= d.restartThreshold {
|
||||
gap = &DeadManInterruption{
|
||||
From: from,
|
||||
To: d.startupAt,
|
||||
DurationSecs: int64(d.startupAt.Sub(from).Round(time.Second) / time.Second),
|
||||
CleanShutdown: clean,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
d.persisted = deadManPersistedState{
|
||||
SchemaVersion: deadManStateSchemaVersion,
|
||||
EndpointFingerprint: fingerprint,
|
||||
Enabled: true,
|
||||
StartedAt: d.startupAt,
|
||||
LastInterruption: cloneDeadManInterruption(previous.LastInterruption),
|
||||
}
|
||||
d.previous = d.persisted
|
||||
if gap != nil {
|
||||
d.persisted.LastInterruption = cloneDeadManInterruption(gap)
|
||||
d.pendingGap = cloneDeadManInterruption(gap)
|
||||
d.status.LastInterruption = cloneDeadManInterruption(gap)
|
||||
} else {
|
||||
d.pendingGap = nil
|
||||
}
|
||||
d.initialized = true
|
||||
d.consecutiveFail = 0
|
||||
d.status.Configured = true
|
||||
d.status.State = "starting"
|
||||
d.status.LastError = ""
|
||||
d.status.ConsecutiveFailures = 0
|
||||
loadErr := d.loadError
|
||||
d.loadError = nil
|
||||
d.mu.Unlock()
|
||||
|
||||
if loadErr != nil && manager != nil {
|
||||
manager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManStateAlertType,
|
||||
Level: alerts.AlertLevelWarning,
|
||||
Message: "Pulse could not read its previous dead-man restart record. Future outage reporting has restarted from a clean baseline.",
|
||||
Fingerprint: "state-load-failed",
|
||||
})
|
||||
}
|
||||
if err := d.persistCurrentState(); err != nil {
|
||||
d.handlePersistenceError(err, manager)
|
||||
} else {
|
||||
d.clearSystemAlert(manager, alerts.DeadManStateAlertType)
|
||||
}
|
||||
if gap != nil && manager != nil {
|
||||
level := alerts.AlertLevelWarning
|
||||
if !gap.CleanShutdown {
|
||||
level = alerts.AlertLevelCritical
|
||||
}
|
||||
manager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManInterruptionAlertType,
|
||||
Level: level,
|
||||
Message: deadManInterruptionAlertMessage(gap),
|
||||
Fingerprint: gap.From.Format(time.RFC3339Nano) + "/" + gap.To.Format(time.RFC3339Nano),
|
||||
Metadata: map[string]interface{}{
|
||||
"interruptionFrom": gap.From,
|
||||
"interruptionTo": gap.To,
|
||||
"durationSeconds": gap.DurationSecs,
|
||||
"cleanShutdown": gap.CleanShutdown,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) disable(now time.Time, manager *alerts.Manager) {
|
||||
d.mu.Lock()
|
||||
wasEnabled := d.initialized && d.persisted.Enabled
|
||||
if wasEnabled {
|
||||
d.persisted.Enabled = false
|
||||
d.persisted.StoppedAt = now
|
||||
}
|
||||
d.previous = d.persisted
|
||||
d.initialized = false
|
||||
d.pendingGap = nil
|
||||
d.consecutiveFail = 0
|
||||
d.status.Configured = false
|
||||
d.status.State = "disabled"
|
||||
d.status.ConsecutiveFailures = 0
|
||||
d.status.LastError = ""
|
||||
d.mu.Unlock()
|
||||
|
||||
if wasEnabled {
|
||||
if err := d.persistCurrentState(); err != nil {
|
||||
d.handlePersistenceError(err, manager)
|
||||
}
|
||||
}
|
||||
for _, alertType := range []string{
|
||||
alerts.DeadManDeliveryAlertType,
|
||||
alerts.DeadManMonitoringStalledAlertType,
|
||||
alerts.DeadManInterruptionAlertType,
|
||||
} {
|
||||
d.clearSystemAlert(manager, alertType)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) stop(now time.Time, manager *alerts.Manager) {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.stopping = true
|
||||
cancel := d.activeSignalCancel
|
||||
d.activeSignalCancel = nil
|
||||
if !d.initialized || !d.persisted.Enabled {
|
||||
d.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
return
|
||||
}
|
||||
d.persisted.StoppedAt = now.UTC()
|
||||
d.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
if err := d.persistCurrentState(); err != nil {
|
||||
d.handlePersistenceError(err, manager)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) statusSnapshot() DeadManStatus {
|
||||
if d == nil {
|
||||
return DeadManStatus{
|
||||
State: "disabled",
|
||||
HeartbeatIntervalSecs: int64(deadManHeartbeatInterval / time.Second),
|
||||
RecommendedGraceSecs: int64((3 * deadManHeartbeatInterval) / time.Second),
|
||||
}
|
||||
}
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
status := d.status
|
||||
status.LastInterruption = cloneDeadManInterruption(d.status.LastInterruption)
|
||||
return status
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) setMonitoringProgress(progress time.Time) {
|
||||
d.mu.Lock()
|
||||
if progress.IsZero() {
|
||||
d.status.LastMonitoringProgress = nil
|
||||
} else {
|
||||
copyTime := progress
|
||||
d.status.LastMonitoringProgress = ©Time
|
||||
}
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) recordHealthyProgress(progress time.Time, manager *alerts.Manager) {
|
||||
d.mu.Lock()
|
||||
if d.stopping {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
d.persisted.LastHealthyAt = progress
|
||||
d.persisted.StoppedAt = time.Time{}
|
||||
d.mu.Unlock()
|
||||
if err := d.persistCurrentState(); err != nil {
|
||||
d.handlePersistenceError(err, manager)
|
||||
} else {
|
||||
d.clearSystemAlert(manager, alerts.DeadManStateAlertType)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) setAttempt(at time.Time) {
|
||||
d.mu.Lock()
|
||||
d.status.LastAttemptAt = &at
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) setState(state, message string) {
|
||||
d.mu.Lock()
|
||||
d.status.State = state
|
||||
d.status.LastError = message
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) pendingInterruption() *DeadManInterruption {
|
||||
d.mu.RLock()
|
||||
defer d.mu.RUnlock()
|
||||
return cloneDeadManInterruption(d.pendingGap)
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) setMisconfigured(message string, manager *alerts.Manager) {
|
||||
d.mu.Lock()
|
||||
d.status.Configured = true
|
||||
d.status.State = "misconfigured"
|
||||
d.status.LastError = message
|
||||
d.status.ConsecutiveFailures = 0
|
||||
d.mu.Unlock()
|
||||
if manager != nil {
|
||||
manager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManDeliveryAlertType,
|
||||
Level: alerts.AlertLevelWarning,
|
||||
Message: "The external dead-man destination is invalid and Pulse cannot send watchdog signals.",
|
||||
Fingerprint: "misconfigured",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) recordSignalFailure(message string, manager *alerts.Manager) {
|
||||
d.mu.Lock()
|
||||
if d.stopping {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
d.consecutiveFail++
|
||||
d.status.ConsecutiveFailures = d.consecutiveFail
|
||||
if d.status.State != "monitor_stalled" {
|
||||
d.status.State = "delivery_failed"
|
||||
}
|
||||
d.status.LastError = message
|
||||
failures := d.consecutiveFail
|
||||
d.mu.Unlock()
|
||||
|
||||
if failures >= deadManDeliveryAlertThreshold && manager != nil {
|
||||
manager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManDeliveryAlertType,
|
||||
Level: alerts.AlertLevelWarning,
|
||||
Message: fmt.Sprintf("Pulse monitoring is running, but the external dead-man destination has failed %d consecutive heartbeat cycles.", failures),
|
||||
Fingerprint: "delivery-failing",
|
||||
Metadata: map[string]interface{}{
|
||||
"consecutiveFailures": failures,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) recordSignalSuccess(at time.Time, monitoringFresh bool, manager *alerts.Manager) {
|
||||
d.mu.Lock()
|
||||
if d.stopping {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
d.consecutiveFail = 0
|
||||
d.status.ConsecutiveFailures = 0
|
||||
d.status.LastError = ""
|
||||
if monitoringFresh {
|
||||
d.status.LastSuccessAt = &at
|
||||
d.status.State = "healthy"
|
||||
d.pendingGap = nil
|
||||
d.persisted.LastSuccessfulPing = at
|
||||
} else {
|
||||
d.status.State = "monitor_stalled"
|
||||
d.status.LastError = "Pulse monitoring loop has stopped making progress"
|
||||
}
|
||||
d.mu.Unlock()
|
||||
|
||||
if err := d.persistCurrentState(); err != nil {
|
||||
d.handlePersistenceError(err, manager)
|
||||
}
|
||||
d.clearSystemAlert(manager, alerts.DeadManDeliveryAlertType)
|
||||
if monitoringFresh {
|
||||
d.clearSystemAlert(manager, alerts.DeadManInterruptionAlertType)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) notifyConfigChanged() {
|
||||
if d == nil {
|
||||
return
|
||||
}
|
||||
d.mu.Lock()
|
||||
cancel := d.activeSignalCancel
|
||||
d.activeSignalCancel = nil
|
||||
d.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
select {
|
||||
case d.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) beginSignal(ctx context.Context) (context.Context, uint64, bool) {
|
||||
signalCtx, cancel := context.WithCancel(ctx)
|
||||
d.mu.Lock()
|
||||
if d.stopping {
|
||||
d.mu.Unlock()
|
||||
cancel()
|
||||
return signalCtx, 0, false
|
||||
}
|
||||
d.signalGeneration++
|
||||
generation := d.signalGeneration
|
||||
d.activeSignalCancel = cancel
|
||||
d.mu.Unlock()
|
||||
return signalCtx, generation, true
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) finishSignal(generation uint64) {
|
||||
d.mu.Lock()
|
||||
var cancel context.CancelFunc
|
||||
if d.signalGeneration == generation {
|
||||
cancel = d.activeSignalCancel
|
||||
d.activeSignalCancel = nil
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) persistCurrentState() error {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
d.persistMu.Lock()
|
||||
defer d.persistMu.Unlock()
|
||||
d.mu.RLock()
|
||||
state := d.persisted
|
||||
d.mu.RUnlock()
|
||||
return persistDeadManState(d.statePath, state)
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) raiseMonitoringStalled(manager *alerts.Manager, progress time.Time) {
|
||||
if manager == nil {
|
||||
return
|
||||
}
|
||||
message := "Pulse's external watchdog is running, but the monitoring loop has stopped making progress."
|
||||
metadata := map[string]interface{}{}
|
||||
if !progress.IsZero() {
|
||||
metadata["lastMonitoringProgress"] = progress
|
||||
}
|
||||
manager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManMonitoringStalledAlertType,
|
||||
Level: alerts.AlertLevelCritical,
|
||||
Message: message,
|
||||
Fingerprint: "monitor-stalled",
|
||||
Metadata: metadata,
|
||||
})
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) handlePersistenceError(err error, manager *alerts.Manager) {
|
||||
log.Error().Err(err).Msg("failed to persist dead-man restart state")
|
||||
if manager != nil {
|
||||
manager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManStateAlertType,
|
||||
Level: alerts.AlertLevelWarning,
|
||||
Message: "Pulse cannot persist its dead-man restart record, so a later restart may not report the full monitoring gap.",
|
||||
Fingerprint: "state-write-failed",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) clearSystemAlert(manager *alerts.Manager, alertType string) {
|
||||
if manager != nil {
|
||||
manager.ClearSystemAlert(alertType)
|
||||
}
|
||||
}
|
||||
|
||||
func deadManFailureURL(endpoint string) string {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return endpoint
|
||||
}
|
||||
parsed.Path = strings.TrimSuffix(parsed.Path, "/") + "/fail"
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
func deadManInterruptionMessage(gap *DeadManInterruption) string {
|
||||
if gap == nil {
|
||||
return ""
|
||||
}
|
||||
kind := "unexpected shutdown"
|
||||
if gap.CleanShutdown {
|
||||
kind = "clean shutdown"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"Pulse monitoring resumed at %s after an interruption from %s to %s (%s; previous %s).",
|
||||
gap.To.Format(time.RFC3339),
|
||||
gap.From.Format(time.RFC3339),
|
||||
gap.To.Format(time.RFC3339),
|
||||
time.Duration(gap.DurationSecs)*time.Second,
|
||||
kind,
|
||||
)
|
||||
}
|
||||
|
||||
func deadManInterruptionAlertMessage(gap *DeadManInterruption) string {
|
||||
if gap == nil {
|
||||
return "Pulse monitoring resumed after an interruption."
|
||||
}
|
||||
shutdown := "unexpectedly"
|
||||
if gap.CleanShutdown {
|
||||
shutdown = "cleanly"
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"Pulse monitoring was unavailable for %s, from %s until %s. The previous process stopped %s.",
|
||||
time.Duration(gap.DurationSecs)*time.Second,
|
||||
gap.From.Format(time.RFC3339),
|
||||
gap.To.Format(time.RFC3339),
|
||||
shutdown,
|
||||
)
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) sendWithRetry(ctx context.Context, endpoint, message string) error {
|
||||
var lastErr error
|
||||
for attempt := 0; ; attempt++ {
|
||||
err := d.sendSignal(ctx, endpoint, message)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
signalErr, retryable := err.(*deadManSignalError)
|
||||
if !retryable || !signalErr.retryable || attempt >= len(d.retryDelays) {
|
||||
return lastErr
|
||||
}
|
||||
timer := time.NewTimer(d.retryDelays[attempt])
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return &deadManSignalError{message: "heartbeat cancelled", retryable: false}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *deadManRuntime) sendSignal(ctx context.Context, endpoint, message string) error {
|
||||
method := http.MethodGet
|
||||
var body io.Reader
|
||||
if message != "" {
|
||||
method = http.MethodPost
|
||||
body = strings.NewReader(message)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, endpoint, body)
|
||||
if err != nil {
|
||||
return &deadManSignalError{message: "could not create heartbeat request", retryable: false}
|
||||
}
|
||||
request.Header.Set("User-Agent", "Pulse dead-man monitor")
|
||||
if message != "" {
|
||||
request.Header.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
}
|
||||
|
||||
response, err := d.client.Do(request)
|
||||
if err != nil {
|
||||
return sanitizeDeadManRequestError(err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
responseBody, readErr := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
if readErr != nil {
|
||||
return &deadManSignalError{message: "could not read heartbeat response", retryable: true}
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return &deadManSignalError{
|
||||
message: fmt.Sprintf("heartbeat endpoint returned HTTP %d", response.StatusCode),
|
||||
retryable: response.StatusCode >= 500,
|
||||
}
|
||||
}
|
||||
normalizedBody := strings.ToLower(strings.TrimSpace(string(responseBody)))
|
||||
if strings.Contains(normalizedBody, "not found") || strings.Contains(normalizedBody, "rate limit") {
|
||||
return &deadManSignalError{message: "heartbeat endpoint rejected the signal", retryable: false}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeDeadManRequestError(err error) error {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return &deadManSignalError{message: "heartbeat request timed out", retryable: true}
|
||||
}
|
||||
var urlErr *url.Error
|
||||
if errors.As(err, &urlErr) {
|
||||
err = urlErr.Err
|
||||
}
|
||||
var networkErr net.Error
|
||||
if errors.As(err, &networkErr) {
|
||||
return &deadManSignalError{message: "heartbeat network request failed", retryable: true}
|
||||
}
|
||||
return &deadManSignalError{message: "heartbeat request failed", retryable: true}
|
||||
}
|
||||
|
||||
func deadManDialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid watchdog address")
|
||||
}
|
||||
addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||||
if err != nil || len(addresses) == 0 {
|
||||
return nil, fmt.Errorf("resolve watchdog host")
|
||||
}
|
||||
for _, candidate := range addresses {
|
||||
ip := candidate.IP
|
||||
if ip == nil || ip.IsLoopback() || ip.IsUnspecified() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
|
||||
return nil, fmt.Errorf("watchdog host resolved to a same-host address")
|
||||
}
|
||||
}
|
||||
|
||||
dialer := &net.Dialer{}
|
||||
var lastErr error
|
||||
for _, candidate := range addresses {
|
||||
connection, dialErr := dialer.DialContext(ctx, network, net.JoinHostPort(candidate.IP.String(), port))
|
||||
if dialErr == nil {
|
||||
return connection, nil
|
||||
}
|
||||
lastErr = dialErr
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
return nil, fmt.Errorf("connect to watchdog host")
|
||||
}
|
||||
|
||||
func loadDeadManState(path string) (deadManPersistedState, error) {
|
||||
data, err := securityutil.ReadSecureStorageFile(path, deadManStateMaxBytes)
|
||||
if err != nil {
|
||||
return deadManPersistedState{}, err
|
||||
}
|
||||
var state deadManPersistedState
|
||||
if err := json.Unmarshal(data, &state); err != nil {
|
||||
return deadManPersistedState{}, fmt.Errorf("decode dead-man state: %w", err)
|
||||
}
|
||||
if state.SchemaVersion != deadManStateSchemaVersion {
|
||||
return deadManPersistedState{}, fmt.Errorf("unsupported dead-man state schema %d", state.SchemaVersion)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func persistDeadManState(path string, state deadManPersistedState) error {
|
||||
state.SchemaVersion = deadManStateSchemaVersion
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dead-man state: %w", err)
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
if err := securityutil.EnsureSecureStorageDir(dir, deadManStateDirPerm); err != nil {
|
||||
return fmt.Errorf("prepare dead-man state directory: %w", err)
|
||||
}
|
||||
temp, err := os.CreateTemp(dir, ".deadman-state-*.json.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create dead-man state temp file: %w", err)
|
||||
}
|
||||
tempPath := temp.Name()
|
||||
cleanup := true
|
||||
defer func() {
|
||||
if cleanup {
|
||||
_ = os.Remove(tempPath)
|
||||
}
|
||||
}()
|
||||
if err := temp.Chmod(deadManStateFilePerm); err != nil {
|
||||
_ = temp.Close()
|
||||
return fmt.Errorf("set dead-man state permissions: %w", err)
|
||||
}
|
||||
if _, err := temp.Write(data); err != nil {
|
||||
_ = temp.Close()
|
||||
return fmt.Errorf("write dead-man state: %w", err)
|
||||
}
|
||||
if err := temp.Sync(); err != nil {
|
||||
_ = temp.Close()
|
||||
return fmt.Errorf("sync dead-man state: %w", err)
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return fmt.Errorf("close dead-man state: %w", err)
|
||||
}
|
||||
if err := replaceDeadManStateFile(tempPath, path); err != nil {
|
||||
return err
|
||||
}
|
||||
cleanup = false
|
||||
if err := os.Chmod(path, deadManStateFilePerm); err != nil {
|
||||
return fmt.Errorf("harden dead-man state permissions: %w", err)
|
||||
}
|
||||
if err := syncDeadManStateDirectory(dir); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
//go:build !windows
|
||||
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func replaceDeadManStateFile(from, to string) error {
|
||||
if err := os.Rename(from, to); err != nil {
|
||||
return fmt.Errorf("replace dead-man state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncDeadManStateDirectory(path string) error {
|
||||
dir, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open dead-man state directory for sync: %w", err)
|
||||
}
|
||||
if err := dir.Sync(); err != nil {
|
||||
_ = dir.Close()
|
||||
return fmt.Errorf("sync dead-man state directory: %w", err)
|
||||
}
|
||||
if err := dir.Close(); err != nil {
|
||||
return fmt.Errorf("close dead-man state directory: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
//go:build windows
|
||||
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func replaceDeadManStateFile(from, to string) error {
|
||||
fromPath, err := windows.UTF16PtrFromString(from)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dead-man state source path: %w", err)
|
||||
}
|
||||
toPath, err := windows.UTF16PtrFromString(to)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode dead-man state destination path: %w", err)
|
||||
}
|
||||
if err := windows.MoveFileEx(
|
||||
fromPath,
|
||||
toPath,
|
||||
windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH,
|
||||
); err != nil {
|
||||
return fmt.Errorf("replace dead-man state with write-through: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncDeadManStateDirectory(string) error { return nil }
|
||||
@@ -0,0 +1,290 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
)
|
||||
|
||||
type deadManRoundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (fn deadManRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return fn(request)
|
||||
}
|
||||
|
||||
func deadManResponse(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func newDeadManTestRuntime(t *testing.T, now time.Time, transport http.RoundTripper) *deadManRuntime {
|
||||
t.Helper()
|
||||
runtime := newDeadManRuntime(t.TempDir())
|
||||
runtime.startupAt = now
|
||||
runtime.now = func() time.Time { return now }
|
||||
runtime.retryDelays = nil
|
||||
runtime.client = &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: time.Second,
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
}
|
||||
return runtime
|
||||
}
|
||||
|
||||
func TestDeadManRunCycleSendsHealthySignalAndPersistsProgress(t *testing.T) {
|
||||
now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
|
||||
var request *http.Request
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(got *http.Request) (*http.Response, error) {
|
||||
request = got
|
||||
return deadManResponse(http.StatusOK, "OK"), nil
|
||||
}))
|
||||
|
||||
runtime.runCycle(
|
||||
context.Background(),
|
||||
func() string { return "https://watchdog.example.com/ping/secret-token" },
|
||||
func() time.Time { return now.Add(-5 * time.Second) },
|
||||
nil,
|
||||
)
|
||||
|
||||
if request == nil || request.Method != http.MethodGet || request.URL.String() != "https://watchdog.example.com/ping/secret-token" {
|
||||
t.Fatalf("healthy request = %#v", request)
|
||||
}
|
||||
status := runtime.statusSnapshot()
|
||||
if status.State != "healthy" || status.LastSuccessAt == nil || status.ConsecutiveFailures != 0 {
|
||||
t.Fatalf("healthy status = %+v", status)
|
||||
}
|
||||
persisted, err := loadDeadManState(runtime.statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("loadDeadManState: %v", err)
|
||||
}
|
||||
if !persisted.Enabled || persisted.LastHealthyAt.IsZero() || persisted.LastSuccessfulPing.IsZero() {
|
||||
t.Fatalf("persisted healthy state = %+v", persisted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManRunCycleSignalsFailureWhenCanonicalMonitorStalls(t *testing.T) {
|
||||
now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
|
||||
var method, path, body string
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
method = request.Method
|
||||
path = request.URL.Path
|
||||
data, _ := io.ReadAll(request.Body)
|
||||
body = string(data)
|
||||
return deadManResponse(http.StatusOK, "OK"), nil
|
||||
}))
|
||||
|
||||
runtime.runCycle(
|
||||
context.Background(),
|
||||
func() string { return "https://watchdog.example.com/ping/secret-token" },
|
||||
func() time.Time { return now.Add(-2 * time.Minute) },
|
||||
nil,
|
||||
)
|
||||
|
||||
if method != http.MethodPost || path != "/ping/secret-token/fail" || !strings.Contains(body, "monitoring loop has stopped") {
|
||||
t.Fatalf("stalled signal = method %q path %q body %q", method, path, body)
|
||||
}
|
||||
status := runtime.statusSnapshot()
|
||||
if status.State != "monitor_stalled" || status.LastSuccessAt != nil {
|
||||
t.Fatalf("stalled status = %+v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManRetriesTransientResponsesButNotPermanentRejections(t *testing.T) {
|
||||
now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
|
||||
t.Run("retries server failures", func(t *testing.T) {
|
||||
attempts := 0
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(_ *http.Request) (*http.Response, error) {
|
||||
attempts++
|
||||
if attempts < 3 {
|
||||
return deadManResponse(http.StatusServiceUnavailable, "unavailable"), nil
|
||||
}
|
||||
return deadManResponse(http.StatusOK, "OK"), nil
|
||||
}))
|
||||
runtime.retryDelays = []time.Duration{0, 0}
|
||||
runtime.runCycle(context.Background(), func() string {
|
||||
return "https://watchdog.example.com/ping/retry-token"
|
||||
}, func() time.Time { return now }, nil)
|
||||
if attempts != 3 || runtime.statusSnapshot().State != "healthy" {
|
||||
t.Fatalf("attempts = %d status = %+v", attempts, runtime.statusSnapshot())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("does not follow redirects", func(t *testing.T) {
|
||||
attempts := 0
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(_ *http.Request) (*http.Response, error) {
|
||||
attempts++
|
||||
response := deadManResponse(http.StatusFound, "redirect")
|
||||
response.Header.Set("Location", "https://collector.example.net/capture")
|
||||
return response, nil
|
||||
}))
|
||||
runtime.retryDelays = []time.Duration{0, 0}
|
||||
runtime.runCycle(context.Background(), func() string {
|
||||
return "https://watchdog.example.com/ping/never-forward-this-token"
|
||||
}, func() time.Time { return now }, nil)
|
||||
status := runtime.statusSnapshot()
|
||||
if attempts != 1 || status.State != "delivery_failed" || strings.Contains(status.LastError, "never-forward-this-token") {
|
||||
t.Fatalf("attempts = %d status = %+v", attempts, status)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects deceptive success body", func(t *testing.T) {
|
||||
attempts := 0
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(_ *http.Request) (*http.Response, error) {
|
||||
attempts++
|
||||
return deadManResponse(http.StatusOK, "Ping not found"), nil
|
||||
}))
|
||||
runtime.retryDelays = []time.Duration{0, 0}
|
||||
runtime.runCycle(context.Background(), func() string {
|
||||
return "https://watchdog.example.com/ping/missing-token"
|
||||
}, func() time.Time { return now }, nil)
|
||||
if attempts != 1 || runtime.statusSnapshot().State != "delivery_failed" {
|
||||
t.Fatalf("attempts = %d status = %+v", attempts, runtime.statusSnapshot())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeadManRestartGapIsReportedExternallyAndRecordedInAlertHistory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
startup := time.Now().UTC().Truncate(time.Second)
|
||||
endpoint := "https://watchdog.example.com/ping/restart-token"
|
||||
previous := deadManPersistedState{
|
||||
SchemaVersion: deadManStateSchemaVersion,
|
||||
EndpointFingerprint: deadManEndpointFingerprint(endpoint),
|
||||
Enabled: true,
|
||||
StartedAt: startup.Add(-time.Hour),
|
||||
LastHealthyAt: startup.Add(-5 * time.Minute),
|
||||
}
|
||||
statePath := dir + "/alerts/deadman-state.json"
|
||||
if err := persistDeadManState(statePath, previous); err != nil {
|
||||
t.Fatalf("persist previous state: %v", err)
|
||||
}
|
||||
|
||||
var method, body string
|
||||
runtime := newDeadManRuntime(dir)
|
||||
runtime.startupAt = startup
|
||||
runtime.now = func() time.Time { return startup }
|
||||
runtime.retryDelays = nil
|
||||
runtime.client = &http.Client{Transport: deadManRoundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
method = request.Method
|
||||
data, _ := io.ReadAll(request.Body)
|
||||
body = string(data)
|
||||
return deadManResponse(http.StatusOK, "OK"), nil
|
||||
})}
|
||||
manager := alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
|
||||
t.Cleanup(manager.Stop)
|
||||
|
||||
runtime.runCycle(context.Background(), func() string { return endpoint }, func() time.Time { return startup }, manager)
|
||||
|
||||
if method != http.MethodPost || !strings.Contains(body, "unexpected shutdown") || !strings.Contains(body, "5m0s") {
|
||||
t.Fatalf("restart report = method %q body %q", method, body)
|
||||
}
|
||||
status := runtime.statusSnapshot()
|
||||
if status.LastInterruption == nil || status.LastInterruption.CleanShutdown || status.LastInterruption.DurationSecs != 300 {
|
||||
t.Fatalf("restart status = %+v", status)
|
||||
}
|
||||
history := manager.GetAlertHistory(0)
|
||||
found := false
|
||||
for _, alert := range history {
|
||||
if alert.ID == alerts.SystemAlertID(alerts.DeadManInterruptionAlertType) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("restart interruption missing from alert history: %+v", history)
|
||||
}
|
||||
stored, err := loadDeadManState(statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("load restart state: %v", err)
|
||||
}
|
||||
if stored.LastInterruption == nil || stored.LastInterruption.DurationSecs != 300 {
|
||||
t.Fatalf("stored interruption = %+v", stored.LastInterruption)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManStopWinsAgainstLaterHeartbeatWrites(t *testing.T) {
|
||||
now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
|
||||
var mu sync.Mutex
|
||||
requests := 0
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(_ *http.Request) (*http.Response, error) {
|
||||
mu.Lock()
|
||||
requests++
|
||||
mu.Unlock()
|
||||
return deadManResponse(http.StatusOK, "OK"), nil
|
||||
}))
|
||||
endpoint := "https://watchdog.example.com/ping/stop-token"
|
||||
runtime.runCycle(context.Background(), func() string { return endpoint }, func() time.Time { return now }, nil)
|
||||
stoppedAt := now.Add(30 * time.Second)
|
||||
runtime.stop(stoppedAt, nil)
|
||||
runtime.runCycle(context.Background(), func() string { return endpoint }, func() time.Time { return stoppedAt }, nil)
|
||||
|
||||
mu.Lock()
|
||||
requestCount := requests
|
||||
mu.Unlock()
|
||||
if requestCount != 1 {
|
||||
t.Fatalf("requests after stop = %d, want 1", requestCount)
|
||||
}
|
||||
stored, err := loadDeadManState(runtime.statePath)
|
||||
if err != nil {
|
||||
t.Fatalf("load stopped state: %v", err)
|
||||
}
|
||||
if !stored.StoppedAt.Equal(stoppedAt) {
|
||||
t.Fatalf("stoppedAt = %s, want %s", stored.StoppedAt, stoppedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManConfigurationChangeCancelsActiveSignalWithoutRecordingFailure(t *testing.T) {
|
||||
now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
|
||||
started := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
runtime := newDeadManTestRuntime(t, now, deadManRoundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
close(started)
|
||||
<-request.Context().Done()
|
||||
return nil, request.Context().Err()
|
||||
}))
|
||||
|
||||
go func() {
|
||||
defer close(finished)
|
||||
runtime.runCycle(context.Background(), func() string {
|
||||
return "https://watchdog.example.com/ping/replaced-token"
|
||||
}, func() time.Time { return now }, nil)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("heartbeat request did not start")
|
||||
}
|
||||
runtime.notifyConfigChanged()
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("configuration change did not cancel heartbeat request")
|
||||
}
|
||||
status := runtime.statusSnapshot()
|
||||
if status.ConsecutiveFailures != 0 || status.State == "delivery_failed" {
|
||||
t.Fatalf("cancelled replacement recorded a delivery failure: %+v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeadManDialRejectsLoopbackResolution(t *testing.T) {
|
||||
connection, err := deadManDialContext(context.Background(), "tcp", "localhost:80")
|
||||
if connection != nil {
|
||||
_ = connection.Close()
|
||||
t.Fatal("loopback watchdog dial unexpectedly succeeded")
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("loopback watchdog dial unexpectedly returned no error")
|
||||
}
|
||||
}
|
||||
@@ -1110,6 +1110,11 @@ type Monitor struct {
|
||||
connectionsSnapshotLister func() []alerts.ConnectionSnapshot // returns platform connection snapshots for the connection-degraded check
|
||||
incidentStore *memory.IncidentStore
|
||||
notificationMgr *notifications.NotificationManager
|
||||
deadMan *deadManRuntime
|
||||
deadManProgressUnixNano atomic.Int64
|
||||
deadManConfigMu sync.RWMutex
|
||||
deadManConfig notifications.DeadManConfig
|
||||
deadManConfigLoadErr error
|
||||
lastDeliveryHealthCheck time.Time // throttles the notification-delivery system alert evaluation; guarded by mu
|
||||
configPersist *config.ConfigPersistence
|
||||
discoveryService *discovery.Service // Background discovery service
|
||||
@@ -1689,6 +1694,7 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||
alertManager: alerts.NewManagerWithDataDir(cfg.DataPath, alertManagerRestoreOptions()...),
|
||||
incidentStore: incidentStore,
|
||||
notificationMgr: notifications.NewNotificationManagerWithDataDir(cfg.PublicURL, cfg.DataPath),
|
||||
deadMan: newDeadManRuntime(config.ResolveRuntimeDataDir(cfg.DataPath)),
|
||||
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
||||
discoveryService: nil, // Will be initialized in Start()
|
||||
authFailures: make(map[string]int),
|
||||
@@ -1813,6 +1819,12 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||
} else {
|
||||
log.Warn().Err(err).Msg("failed to load Apprise configuration")
|
||||
}
|
||||
if deadManConfig, err := m.configPersist.LoadDeadManConfig(); err == nil {
|
||||
m.deadManConfig = *deadManConfig
|
||||
} else {
|
||||
m.deadManConfigLoadErr = err
|
||||
log.Warn().Err(err).Msg("failed to load dead-man configuration")
|
||||
}
|
||||
|
||||
// Migrate webhooks if needed (from unencrypted to encrypted)
|
||||
if err := m.configPersist.MigrateWebhooksIfNeeded(); err != nil {
|
||||
@@ -1999,6 +2011,28 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||
})
|
||||
m.replayAlertLifecycleProjections()
|
||||
m.reconcileActiveAlertTimelines()
|
||||
m.markDeadManMonitoringProgress(time.Now().UTC())
|
||||
if err := m.deadManConfigurationLoadError(); err != nil {
|
||||
m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{
|
||||
Type: alerts.DeadManStateAlertType,
|
||||
Level: alerts.AlertLevelWarning,
|
||||
Message: "Pulse could not read the encrypted external watchdog configuration. Watchdog monitoring is unavailable until the destination is saved again.",
|
||||
Fingerprint: "configuration-load-failed",
|
||||
})
|
||||
}
|
||||
if m.deadMan != nil {
|
||||
go m.deadMan.run(
|
||||
ctx,
|
||||
func() string {
|
||||
if m.alertManager == nil {
|
||||
return ""
|
||||
}
|
||||
return m.deadManConfigSnapshot().PingURL
|
||||
},
|
||||
m.deadManMonitoringProgress,
|
||||
m.alertManager,
|
||||
)
|
||||
}
|
||||
|
||||
// Create separate tickers for polling and broadcasting using the configured cadence
|
||||
|
||||
@@ -2010,6 +2044,8 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||
|
||||
broadcastTicker := time.NewTicker(pollingInterval)
|
||||
defer broadcastTicker.Stop()
|
||||
deadManProgressTicker := time.NewTicker(15 * time.Second)
|
||||
defer deadManProgressTicker.Stop()
|
||||
|
||||
keepRealPolling := keepRealPollingInMockMode()
|
||||
|
||||
@@ -2035,6 +2071,11 @@ func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
||||
|
||||
for {
|
||||
select {
|
||||
case now := <-deadManProgressTicker.C:
|
||||
// This tick runs on the canonical monitor loop itself. A separate
|
||||
// heartbeat goroutine can therefore prove that scheduling remains
|
||||
// responsive instead of merely proving its own timer is alive.
|
||||
m.markDeadManMonitoringProgress(now.UTC())
|
||||
case <-pollTicker.C:
|
||||
now := time.Now()
|
||||
m.evaluateDockerAgents(now)
|
||||
@@ -7061,6 +7102,10 @@ const guestMetadataDrainTimeout = 2 * time.Second
|
||||
func (m *Monitor) Stop() {
|
||||
log.Info().Msg("stopping monitor")
|
||||
|
||||
if m.deadMan != nil {
|
||||
m.deadMan.stop(time.Now().UTC(), m.alertManager)
|
||||
}
|
||||
|
||||
// Stop the alert manager to save history
|
||||
if m.alertManager != nil {
|
||||
m.alertManager.Stop()
|
||||
|
||||
@@ -2,6 +2,7 @@ package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
|
||||
"github.com/rs/zerolog/log"
|
||||
@@ -29,6 +31,96 @@ func (m *Monitor) GetIncidentStore() *memory.IncidentStore {
|
||||
return m.incidentStore
|
||||
}
|
||||
|
||||
// DeadManStatus returns the external watchdog state without exposing the
|
||||
// configured secret-bearing ping URL.
|
||||
func (m *Monitor) DeadManStatus() DeadManStatus {
|
||||
if m == nil || m.deadMan == nil {
|
||||
return (*deadManRuntime)(nil).statusSnapshot()
|
||||
}
|
||||
status := m.deadMan.statusSnapshot()
|
||||
if m.deadManConfigurationLoadError() != nil {
|
||||
status.Configured = true
|
||||
status.State = "configuration_unavailable"
|
||||
status.LastError = "Saved external watchdog configuration could not be read"
|
||||
} else if strings.TrimSpace(m.deadManConfigSnapshot().PingURL) == "" {
|
||||
status.Configured = false
|
||||
status.State = "disabled"
|
||||
status.LastAttemptAt = nil
|
||||
status.LastSuccessAt = nil
|
||||
status.ConsecutiveFailures = 0
|
||||
status.LastError = ""
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// DeadManConfig returns the in-memory encrypted-destination configuration.
|
||||
// API callers must mask PingURL before returning it to a client.
|
||||
func (m *Monitor) DeadManConfig() notifications.DeadManConfig {
|
||||
return m.deadManConfigSnapshot()
|
||||
}
|
||||
|
||||
func (m *Monitor) deadManConfigSnapshot() notifications.DeadManConfig {
|
||||
if m == nil {
|
||||
return notifications.DeadManConfig{}
|
||||
}
|
||||
m.deadManConfigMu.RLock()
|
||||
defer m.deadManConfigMu.RUnlock()
|
||||
return m.deadManConfig
|
||||
}
|
||||
|
||||
func (m *Monitor) deadManConfigurationLoadError() error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
m.deadManConfigMu.RLock()
|
||||
defer m.deadManConfigMu.RUnlock()
|
||||
return m.deadManConfigLoadErr
|
||||
}
|
||||
|
||||
// UpdateDeadManConfig persists the secret before changing live behavior, so a
|
||||
// failed encrypted write can never create a runtime-only watchdog setting.
|
||||
func (m *Monitor) UpdateDeadManConfig(config notifications.DeadManConfig) error {
|
||||
if m == nil || m.configPersist == nil {
|
||||
return fmt.Errorf("dead-man configuration persistence unavailable")
|
||||
}
|
||||
config = notifications.NormalizeDeadManConfig(config)
|
||||
if err := notifications.ValidateDeadManPingURL(config.PingURL); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.configPersist.SaveDeadManConfig(config); err != nil {
|
||||
return err
|
||||
}
|
||||
m.deadManConfigMu.Lock()
|
||||
m.deadManConfig = config
|
||||
m.deadManConfigLoadErr = nil
|
||||
m.deadManConfigMu.Unlock()
|
||||
if m.alertManager != nil {
|
||||
m.alertManager.ClearSystemAlert(alerts.DeadManStateAlertType)
|
||||
}
|
||||
if m.deadMan != nil {
|
||||
m.deadMan.notifyConfigChanged()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Monitor) markDeadManMonitoringProgress(at time.Time) {
|
||||
if m == nil || at.IsZero() {
|
||||
return
|
||||
}
|
||||
m.deadManProgressUnixNano.Store(at.UTC().UnixNano())
|
||||
}
|
||||
|
||||
func (m *Monitor) deadManMonitoringProgress() time.Time {
|
||||
if m == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
value := m.deadManProgressUnixNano.Load()
|
||||
if value <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(0, value).UTC()
|
||||
}
|
||||
|
||||
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
|
||||
// This enables token-efficient, real-time AI insights on specific resources
|
||||
// SetAlertTriggeredAICallback sets an additional callback for AI analysis when alerts fire
|
||||
|
||||
@@ -3827,6 +3827,8 @@ func TestRemoveHostAgent_BlocksFutureReportsUntilAllowed(t *testing.T) {
|
||||
config: &config.Config{},
|
||||
}
|
||||
t.Cleanup(func() { monitor.alertManager.Stop() })
|
||||
canonicalProgress := time.Now().Add(-time.Minute).UTC()
|
||||
monitor.markDeadManMonitoringProgress(canonicalProgress)
|
||||
|
||||
hostID := "host-blocked"
|
||||
monitor.state.UpsertHost(models.Host{
|
||||
@@ -3858,6 +3860,14 @@ func TestRemoveHostAgent_BlocksFutureReportsUntilAllowed(t *testing.T) {
|
||||
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
|
||||
t.Fatalf("expected host report after allow reenroll, got %v", err)
|
||||
}
|
||||
|
||||
// Host-agent deletion, rejected reports, and explicit re-enrollment are
|
||||
// important lifecycle activity, but none prove that the canonical monitor
|
||||
// loop is still scheduling. The independent watchdog must only advance from
|
||||
// Monitor.Start's own select loop.
|
||||
if got := monitor.deadManMonitoringProgress(); !got.Equal(canonicalProgress) {
|
||||
t.Fatalf("host-agent lifecycle advanced canonical monitor progress: got %s, want %s", got, canonicalProgress)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyHostReport_FreshTokenClearsRemovalBlock pins the #1581 re-enroll
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const MaxDeadManPingURLLength = 2048
|
||||
|
||||
// DeadManConfig is encrypted destination configuration. PingURL is
|
||||
// credential-like because possession lets a caller forge healthy signals.
|
||||
type DeadManConfig struct {
|
||||
PingURL string `json:"pingUrl,omitempty"`
|
||||
}
|
||||
|
||||
func NormalizeDeadManConfig(config DeadManConfig) DeadManConfig {
|
||||
config.PingURL = strings.TrimSpace(config.PingURL)
|
||||
return config
|
||||
}
|
||||
|
||||
// ValidateDeadManPingURL validates a healthchecks-compatible base ping URL.
|
||||
// Private-network destinations remain supported for separately hosted LAN
|
||||
// watchdogs, but same-host addresses are rejected because they cannot detect
|
||||
// loss of the Pulse host. The runtime rejects redirects so the secret URL path
|
||||
// cannot be forwarded to another origin.
|
||||
func ValidateDeadManPingURL(value string) error {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
if len(value) > MaxDeadManPingURLLength {
|
||||
return fmt.Errorf("dead-man ping URL exceeds %d characters", MaxDeadManPingURLLength)
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dead-man ping URL is invalid")
|
||||
}
|
||||
if !parsed.IsAbs() || parsed.Opaque != "" {
|
||||
return fmt.Errorf("dead-man ping URL is invalid")
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("dead-man ping URL must use http or https")
|
||||
}
|
||||
if parsed.Hostname() == "" {
|
||||
return fmt.Errorf("dead-man ping URL must include a host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return fmt.Errorf("dead-man ping URL must not contain user credentials")
|
||||
}
|
||||
if parsed.Fragment != "" {
|
||||
return fmt.Errorf("dead-man ping URL must not contain a fragment")
|
||||
}
|
||||
|
||||
hostname := strings.TrimSuffix(strings.ToLower(parsed.Hostname()), ".")
|
||||
if hostname == "localhost" || strings.HasSuffix(hostname, ".localhost") {
|
||||
return fmt.Errorf("dead-man monitoring must use a different host from Pulse")
|
||||
}
|
||||
if ip := net.ParseIP(hostname); ip != nil && (ip.IsLoopback() || ip.IsUnspecified() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast()) {
|
||||
return fmt.Errorf("dead-man monitoring must use a different host from Pulse")
|
||||
}
|
||||
|
||||
path := strings.TrimSuffix(strings.ToLower(parsed.EscapedPath()), "/")
|
||||
for _, suffix := range []string{"/start", "/fail", "/log"} {
|
||||
if strings.HasSuffix(path, suffix) {
|
||||
return fmt.Errorf("dead-man ping URL must be the base success URL, without %s", suffix)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateDeadManPingURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
value string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "disabled", value: ""},
|
||||
{name: "hosted HTTPS", value: "https://hc.example.com/ping/secret-token"},
|
||||
{name: "separate LAN host", value: "http://192.168.50.12:8000/ping/secret-token"},
|
||||
{name: "IPv6 LAN host", value: "http://[fd00::12]/ping/secret-token"},
|
||||
{name: "query token", value: "https://watchdog.example.com/ping?id=secret-token"},
|
||||
{name: "non HTTP scheme", value: "ftp://watchdog.example.com/ping/token", wantErr: true},
|
||||
{name: "relative URL", value: "/ping/token", wantErr: true},
|
||||
{name: "userinfo", value: "https://admin:secret@watchdog.example.com/ping/token", wantErr: true},
|
||||
{name: "fragment", value: "https://watchdog.example.com/ping/token#secret", wantErr: true},
|
||||
{name: "localhost", value: "http://localhost:8000/ping/token", wantErr: true},
|
||||
{name: "localhost subdomain", value: "http://pulse.localhost/ping/token", wantErr: true},
|
||||
{name: "IPv4 loopback", value: "http://127.0.0.1/ping/token", wantErr: true},
|
||||
{name: "IPv6 loopback", value: "http://[::1]/ping/token", wantErr: true},
|
||||
{name: "unspecified", value: "http://0.0.0.0/ping/token", wantErr: true},
|
||||
{name: "link local", value: "http://169.254.20.10/ping/token", wantErr: true},
|
||||
{name: "failure suffix", value: "https://watchdog.example.com/ping/token/fail", wantErr: true},
|
||||
{name: "failure suffix slash", value: "https://watchdog.example.com/ping/token/fail/", wantErr: true},
|
||||
{name: "start suffix", value: "https://watchdog.example.com/ping/token/start", wantErr: true},
|
||||
{name: "log suffix", value: "https://watchdog.example.com/ping/token/log", wantErr: true},
|
||||
{name: "oversized", value: "https://watchdog.example.com/ping/" + strings.Repeat("x", MaxDeadManPingURLLength), wantErr: true},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := ValidateDeadManPingURL(test.value)
|
||||
if test.wantErr && err == nil {
|
||||
t.Fatalf("ValidateDeadManPingURL(%q) unexpectedly succeeded", test.value)
|
||||
}
|
||||
if !test.wantErr && err != nil {
|
||||
t.Fatalf("ValidateDeadManPingURL(%q): %v", test.value, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDeadManConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := NormalizeDeadManConfig(DeadManConfig{PingURL: " https://watchdog.example.com/ping/token "})
|
||||
if got.PingURL != "https://watchdog.example.com/ping/token" {
|
||||
t.Fatalf("normalized URL = %q", got.PingURL)
|
||||
}
|
||||
}
|
||||
@@ -297,6 +297,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase):
|
||||
"internal/monitoring/availability_udp_test.go",
|
||||
"internal/monitoring/canonical_guardrails_test.go",
|
||||
"internal/monitoring/ceph_test.go",
|
||||
"internal/monitoring/deadman_test.go",
|
||||
"internal/monitoring/issue1485_unraid_lifecycle_test.go",
|
||||
"internal/monitoring/issue1595_collection_trust_test.go",
|
||||
"internal/monitoring/issue1613_contract_test.go",
|
||||
@@ -2384,6 +2385,7 @@ None yet.
|
||||
"frontend-modern/src/components/Alerts/__tests__/InvestigateAlertButton.test.tsx",
|
||||
"frontend-modern/src/components/Alerts/__tests__/ThresholdsTable.test.tsx",
|
||||
"frontend-modern/src/features/alerts/AlertDeliveryHealthCard.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertDeadManDestinationSection.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.emptystate.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/OverviewTab.timelineerror.test.tsx",
|
||||
@@ -2391,6 +2393,7 @@ None yet.
|
||||
"frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/alertsConfigurationModel.test.ts",
|
||||
"frontend-modern/src/features/alerts/__tests__/helpers.test.ts",
|
||||
"frontend-modern/src/features/alerts/__tests__/useAlertDestinationsState.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/useAlertDestinationsTabState.test.tsx",
|
||||
"frontend-modern/src/features/alerts/__tests__/useAlertOverridesState.test.tsx",
|
||||
"frontend-modern/src/features/alerts/identity.test.ts",
|
||||
|
||||
Reference in New Issue
Block a user