From 08e785eb8d475a6906bc5135d6379c0ced78054b Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:37:23 +0100 Subject: [PATCH] Show the live reading of threshold alerts held below trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A threshold alert stays open after its reading drops below the trigger: it holds until the reading reaches the clear level, then must stay there for the recovery delay. While it held, the evaluator returned without touching the alert, so Value, Message and LastSeen stayed at the last breach. On 2026-10-06 minipc's row and guests said "Node temperature at 80.0°C" while the node table read a green 72°C, with nothing explaining why the warning was still there (also #2068: a disk alert that stayed open after usage dropped, with no sign of why). The evaluator now records a volatile MetricStatus on every evaluation: the value it compared (the average for rolling-average rules, with the latest sample beside it), the phase (breaching, latched or recovering), the trigger and clear levels, the recovery delay and the reducer's monotonic recovery progress. Value, Message and LastSeen keep meaning the last breach, and the hold branch still never notifies. The status is stripped from the active-state checkpoint, the recovery mirror and the degraded recovery projection, so a held alert adds no durable writes. It reaches the websocket state through the active-alert projection. Stale-alert cleanup and the retention TTLs now use the latest observation. Before, an alert held below its trigger for over a day had an old LastSeen and was auto-resolved as "resource no longer being monitored" while it was still being evaluated. Surfaces format the status through one helper and never re-derive the phase. Node, guest and Mail Gateway drawer rows lead with the reading now and say what clears the alert, with the last breach on hover. The Proxmox TEMP cell keeps its own open temperature alert's tone and explains it on hover instead of reading green beside a warning row. Storage rows stop saying "Over N%" when usage is back under the limit, and the Assistant handoff no longer labels the breach value as current. Refs #2068 --- .../v6/internal/subsystems/agent-lifecycle.md | 8 + .../v6/internal/subsystems/alerts.md | 17 ++ .../v6/internal/subsystems/api-contracts.md | 13 ++ .../subsystems/frontend-primitives.md | 8 + .../v6/internal/subsystems/monitoring.md | 8 + .../subsystems/performance-and-scalability.md | 7 + .../internal/subsystems/storage-recovery.md | 8 + .../internal/subsystems/unified-resources.md | 8 + .../4766668c29936861.json | 62 ++++++ .../alertAssistantHandoffModel.test.ts | 31 +++ .../Alerts/alertAssistantHandoffModel.ts | 45 ++++- .../components/Workloads/GuestDrawer.test.tsx | 40 ++++ .../Workloads/GuestDrawerOverview.tsx | 4 +- .../Workloads/NodeDrawerOverview.tsx | 3 +- .../components/Workloads/guestDrawerModel.ts | 13 +- .../shared/DrawerAttentionSection.tsx | 9 +- .../SharedPrimitives.guardrails.test.ts | 15 ++ .../components/shared/TemperatureGauge.tsx | 10 +- .../features/alerts/__tests__/helpers.test.ts | 153 ++++++++++++++- .../alerts/metricAlertPresentation.ts | 157 +++++++++++++++ .../proxmox/ProxmoxMailGatewayDrawer.tsx | 3 +- .../features/proxmox/ProxmoxNodesTable.tsx | 23 +++ .../__tests__/ProxmoxNodesTable.test.tsx | 94 ++++++++- .../storageRowAlertPresentation.test.ts | 25 +++ .../storageRowAlertPresentation.ts | 12 ++ frontend-modern/src/types/api.ts | 26 +++ .../src/utils/__tests__/temperature.test.ts | 16 ++ frontend-modern/src/utils/temperature.ts | 22 ++- internal/alerts/active_cleanup.go | 2 +- internal/alerts/active_persistence.go | 5 + internal/alerts/alerts_test.go | 54 +++++ internal/alerts/canonical_metric.go | 22 ++- internal/alerts/metric_status.go | 79 ++++++++ internal/alerts/model.go | 20 ++ internal/alerts/tracking_cleanup.go | 4 +- internal/alerts/unified_eval_test.go | 185 ++++++++++++++++++ internal/models/metrics_types_test.go | 18 ++ internal/models/models.go | 52 +++++ .../monitoring/monitor_alert_handling_test.go | 36 ++++ internal/monitoring/monitor_alert_sync.go | 6 +- 40 files changed, 1297 insertions(+), 26 deletions(-) create mode 100644 frontend-modern/browser-verification/4766668c29936861.json create mode 100644 frontend-modern/src/features/alerts/metricAlertPresentation.ts create mode 100644 internal/alerts/metric_status.go diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b51b2d57dc..408160343f 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -15,6 +15,14 @@ ## Purpose +### Alert projections carry a volatile metric status — issue #2068 + +`models.Alert` gains an optional `metricStatus` (`models.MetricAlertStatus`) +beside the alert-engine metadata. It is a live evaluation snapshot owned by +the alerts subsystem. Agent report admission, host continuity and +re-enrollment never read or write it, and it is not persisted with agent or +state records. + ### Source-record lookup preserves report admission Host and Docker report admission still performs canonical host-view matching diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index cb03c572e2..1f4f14d152 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,23 @@ ## Purpose +### Open threshold alerts carry a volatile live status — issue #2068 + +`evaluateCanonicalMetricAlert` builds `MetricStatus` from the evaluator's own +inputs on every evaluation, including the hysteresis hold and the recovery +run, where `Value`, `Message` and `LastSeen` stay at the last breach and no +branch may notify. The phase comes from the trigger, the clear level handed to +the reducer (a value at or below it counts toward recovery) and the reducer's +recovery run; progress uses that run's monotonic elapsed time. The status is +stripped from the active-state checkpoint, the recovery mirror and the +degraded recovery projection, so a held alert adds no durable writes, and +after a restart it is absent until the next evaluation. Resolved records carry +no live status. Stale-alert cleanup and the retention TTLs use the latest +observation (`lastObservedAt`), so an alert held below its trigger for over a +day is not auto-resolved as unmonitored. Frontend surfaces format the status +through `features/alerts/metricAlertPresentation.ts` and never re-derive the +phase or recovery timing. + ### Retained guest observations are not alert evidence Guest memory marked `last-known`, `unavailable` or an unknown observation state diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e410966406..6c38e23650 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -20,6 +20,19 @@ ## Purpose +### Open threshold alerts expose their live evaluation — issue #2068 + +`Alert.metricStatus` (`/api/alerts/active`, the websocket state and +`/api/state`) is optional and additive. It carries `phase` (`breaching`, +`latched` or `recovering`), `value` (the value compared with the rule, the +average for a rolling-average rule) with `rawValue` and +`evaluationWindowSeconds` when windowed, `unit`, `observedAt`, `trigger`, +`recovery` (the alert clears at or below it), `recoveryDelaySeconds`, and, +while recovering, `recoveryStartedAt` and `recoveryElapsedSeconds`. Legacy +`value`, `message` and `lastSeen` keep meaning the last reading that met the +trigger. The field is absent for non-threshold alerts and until the first +evaluation after a restart, and clients then fall back to `message`. + ### Organisation deletion retains data after incomplete monitoring shutdown Authenticated organisation-owner deletion waits for tenant-loop exit and sealed diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 308d1470e2..1d8ce207c1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -22,6 +22,14 @@ All drawer History fallbacks must distinguish current observations from retained, unavailable or freshness-unknown guest readings, using original memory evidence and filesystem read reasons. +### Drawer attention detail line and temperature alert floor — issue #2068 + +`DrawerAttentionSection` items accept an optional `detail` (a muted second +line) and `title` (hover text). `TemperatureGauge` accepts `alertSeverity` +and `title`, and `getTemperatureTextClass` treats an open alert's severity as +a floor: a reading that has dipped under an open alert's trigger keeps the +alert's tone, while the reading still wins when it is worse. + ### Canonical drawer History preserves guest read provenance The shared resource drawer passes selected memory observation state/source/time diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index f02a9c2224..2becd58456 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,14 @@ ## Purpose +### Active-alert state projection carries the live metric status — issue #2068 + +`activeAlertsSnapshot` copies the alert manager's volatile `MetricStatus` +into `models.Alert.metricStatus`, so the websocket and `/api/state` alert +lists carry the reading being evaluated now for a threshold alert held below +its trigger, while `value` and `message` keep the last breach. The projection +adds no copy of its own: `GetActiveAlerts` already returns deep clones. + ### Fresh broadcast projection owns one resource/target capture Canonical live, fallback and standalone-host continuity broadcast reads prefer diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 50344121e9..44f18cda41 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -17,6 +17,13 @@ Drawer observation labels derive locally from the selected snapshot without renewing timestamps, adding reads, remounting charts or changing stored metric coordinates. +### Guest drawer alert copy formats backend status only + +Guest drawer attention rows format `alert.metricStatus` through +`getGuestDrawerAlertAttention` and the shared `getAlertAttentionCopy`. No +threshold comparison, recovery timing or resource-metric lookup runs in the +drawer, and the host-capacity memory comparison stays a suffix on that copy. + ### Drawer provenance stays local to snapshot presentation Guest and canonical resource drawer memory presentation share the same pure diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index ebbc81311a..75882a9d08 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -21,6 +21,14 @@ ## Purpose +### Storage usage headline follows the alert phase — issue #2068 + +`describeStorageAlertHeadline` says "Over N% usage limit" only while a usage +alert is breaching. When `metricStatus.phase` is `latched` or `recovering`, +the row says usage is under the limit and names the clear level, so the +headline never contradicts the usage bar beside it. Fill-forecast headlines +are unchanged. + ### Guest-memory read state stays independent of backup disk state The existing guest drawer retains Core's selected memory provenance through diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index a22df85cf1..2bd755d13a 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -23,6 +23,14 @@ and sort the complete canonical change table while startup and ingestion wait. ## Purpose +### Proxmox node temperature cell follows its own open alert — issue #2068 + +`ProxmoxNodesTable` finds the node's own temperature alert through +`hostOverrideIdCandidates` and passes its severity (unless acknowledged) and +the shared live-status copy to `TemperatureGauge`. Other node alerts never +recolour the cell. The Mail Gateway drawer formats its attention rows through +`getAlertAttentionCopy`. + ### Canonical Discovery honours VM guest-read deferrals The canonical resource drawer uses the same fixed PVE VM lock, backup and diff --git a/frontend-modern/browser-verification/4766668c29936861.json b/frontend-modern/browser-verification/4766668c29936861.json new file mode 100644 index 0000000000..af31ef6f50 --- /dev/null +++ b/frontend-modern/browser-verification/4766668c29936861.json @@ -0,0 +1,62 @@ +{ + "version": 1, + "base_sha": "92d946d057ddeefcae4d247a87e88f2245abca4a", + "verified_at": "2026-10-06T12:36:53Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts", + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx", + "frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx", + "frontend-modern/src/components/Workloads/guestDrawerModel.ts", + "frontend-modern/src/components/shared/DrawerAttentionSection.tsx", + "frontend-modern/src/components/shared/TemperatureGauge.tsx", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts", + "frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx", + "frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts", + "frontend-modern/src/types/api.ts", + "frontend-modern/src/utils/temperature.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts": "3c6f6f01df4a13b9fb3084b398be093473b63cdb71bbc01db1dbf41531e15edd", + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx": "667a9b9e6fd4385a23b950f18be16fb71cbd6eccc2a4aafb57d2c4532b3af6f1", + "frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx": "5adf11bfc948cae2cca66fde5e2132c66bbb01388efdbc23cbf022b449ae3b0c", + "frontend-modern/src/components/Workloads/guestDrawerModel.ts": "a922c4c26813a676be12e06376bdb40df2aca8bfe5fc850b76009a504dada663", + "frontend-modern/src/components/shared/DrawerAttentionSection.tsx": "2a4e15e781a6158e253b6b934198d7aed71275416eb12c07ade68d5fa80c0332", + "frontend-modern/src/components/shared/TemperatureGauge.tsx": "612172a594698216b8a5fe247e5fa7361946056e02eaba9e71cfe827d0dc5023", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts": "2c875f102207c36d21d2868fa6bf05be7a5ae3e1cfb0351f54c2e89d448259bc", + "frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx": "c979737f3825f12da80b25f50083b15960f7217525905c01d27edc4b26e1fbc3", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx": "b3ebf1259281b7d590c4cbd0eee9ed54ad060ed0e6fd1d80f740d44c95cd3107", + "frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts": "ad5c973c1fbe64a94ad98d8f89d2ac3e51dacad1b1d06af82f77d0d32462ea16", + "frontend-modern/src/types/api.ts": "0db85005fb024e72434e35fdb7eb9ce13d70361569563579365a877fac714f79", + "frontend-modern/src/utils/temperature.ts": "3381a7948f19ae52bf0c72d6fc011ded3691aeac9fdbc9d795dc298af68c4690" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack built from this change (sidecar backend on 7721, Vite on 5241); pve2's CPU temperature driven through 88, 78 and 72 degrees C by a local-only, uncommitted mock overlay", + "Breaching (88C, trigger 80/clear 75): node drawer Needs attention reads 'Temperature 88°C, above the 80°C alert level' with 'Clears once it drops to 75°C or lower and stays there for 5 minutes.'", + "Held (78C): drawer reads 'Temperature 78°C now, back under the 80°C alert level' / 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.'; legacy alert message still 'Node temperature at 88.0°C' in the API; TEMP cell amber with the same explanation as hover text", + "Recovering (72C, below the 75C warning colour): drawer reads 'Temperature 72°C now, recovering' / 'Clears after 5 minutes at 75°C or lower, 1 minute so far.'; TEMP cell 72°C stays amber (text-amber-600) while 68°C on the next node is green", + "Resolved after 5 minutes at 72C: alert closed with 'Resolved: Node temperature at 72.0°C', TEMP cell green with no hover text, Needs attention section gone", + "Narrow 375px: drawer attention copy wraps inside the card, TEMP column hidden as before", + "Guest drawer, Mail Gateway drawer, storage headline and Assistant handoff copy covered by component and model tests, not exercised live" + ], + "interactions": [ + "Expanded the West Production B node row to open its inline drawer at desktop and phone widths", + "Read the TEMP cell colour class and title text through the DOM in each phase", + "Reloaded the page mid-recovery to confirm the cell and drawer render from fresh state", + "Resized between 1440x900 and 375x812" + ] +} diff --git a/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts b/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts index 9f23bdbe9a..1d9d579301 100644 --- a/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts +++ b/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts @@ -157,4 +157,35 @@ describe('alertAssistantHandoffModel', () => { const briefing = handoff.context.briefing as { detailLines: string[] }; expect(briefing.detailLines).toContain('Current value 92.5% · threshold 80.0%'); }); + + it('hands the assistant the live reading, not the breach an open alert still holds', () => { + const handoff = buildAlertAssistantHandoff({ + alert: makeAlert({ + type: 'temperature', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: '2026-05-07T10:04:50.000Z', + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + recoveryElapsedSeconds: 120, + }, + }), + now: new Date('2026-05-07T10:05:00.000Z'), + }); + + const briefing = handoff.context.briefing as { detailLines: string[] }; + expect(briefing.detailLines[0]).toContain('72.0°C'); + expect(briefing.detailLines.join('\n')).not.toContain('Node temperature at 80.0°C'); + expect(handoff.context.handoffContext).toContain('Current Value: 72.0°C'); + expect(handoff.context.handoffContext).toContain('Alert Phase: recovering'); + expect(handoff.context.handoffContext).toContain('Clears At: 75.0°C or lower for 300s'); + expect(handoff.context.handoffContext).toContain('Recovery Progress: 120s of 300s'); + expect(handoff.context.handoffContext).toContain('Last Reading At Or Above Threshold: 80.0°C'); + }); }); diff --git a/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts b/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts index 3a6a08b9c7..4ca70859cb 100644 --- a/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts +++ b/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts @@ -2,6 +2,7 @@ import type { Alert } from '@/types/api'; import type { AIChatContext } from '@/stores/aiChat'; import { DEFAULT_LOCALE, t, type SupportedLocale } from '@/i18n'; import { getCanonicalAlertId } from '@/features/alerts/identity'; +import { getMetricAlertPresentation } from '@/features/alerts/metricAlertPresentation'; import { formatAlertValue } from '@/utils/alertFormatters'; import { isMetricAlertType } from '@/utils/alerts'; import { isPulseSystemAlert } from '@/utils/alertScope'; @@ -35,7 +36,16 @@ export function buildAlertAssistantHandoff({ // to convey what's wrong. const systemScoped = isPulseSystemAlert(alert); const hasMetricValues = !systemScoped && isMetricAlertType(alert.type); - const currentValue = hasMetricValues ? formatAlertValue(alert.value, alert.type) : ''; + // An open threshold alert keeps value/message at its last breach while it + // holds; the live status is what the metric reads now. + const metricStatus = hasMetricValues ? alert.metricStatus : undefined; + const presentation = metricStatus ? getMetricAlertPresentation(alert, now.getTime()) : null; + const currentValue = hasMetricValues + ? formatAlertValue(metricStatus ? metricStatus.value : alert.value, alert.type) + : ''; + const briefingMessage = presentation + ? `${presentation.summary}. ${presentation.detail}` + : alert.message; const thresholdValue = hasMetricValues ? formatAlertValue(alert.threshold, alert.type) : ''; const nodeLabel = !systemScoped && alert.node ? alert.nodeDisplayName || alert.node : ''; const levelLabel = formatAlertLevel(alert.level); @@ -98,8 +108,8 @@ export function buildAlertAssistantHandoff({ }) : undefined, nodeLabel ? t('alerts.assistant.detail.node', { node: nodeLabel }) : undefined, - alert.message - ? t('alerts.assistant.detail.message', { message: alert.message }) + briefingMessage + ? t('alerts.assistant.detail.message', { message: briefingMessage }) : undefined, ].filter((line): line is string => Boolean(line)), actionLabel: t('alerts.assistant.action.investigate', { alertIdentifier }), @@ -149,6 +159,7 @@ function buildAlertAssistantModelContext({ systemScoped ? undefined : formatContextLine('Resource ID', alert.resourceId), formatContextLine('Current Value', currentValue), formatContextLine('Threshold', thresholdValue), + ...metricStatusContextLines(alert), formatContextLine('Duration', durationText), formatContextLine('Node', nodeLabel), formatContextLine('Message', alert.message), @@ -158,6 +169,34 @@ function buildAlertAssistantModelContext({ .join('\n'); } +const METRIC_PHASE_CONTEXT: Record = { + breaching: 'breaching (at or above the threshold)', + latched: 'holding (below the threshold, not yet down to the clear level)', + recovering: 'recovering (at or below the clear level, waiting out the recovery delay)', +}; + +function metricStatusContextLines(alert: Alert): Array { + const status = isMetricAlertType(alert.type) ? alert.metricStatus : undefined; + if (!status) return []; + const delay = status.recoveryDelaySeconds ?? 0; + return [ + formatContextLine('Alert Phase', METRIC_PHASE_CONTEXT[status.phase] ?? status.phase), + formatContextLine( + 'Clears At', + `${formatAlertValue(status.recovery, alert.type)} or lower${delay > 0 ? ` for ${delay}s` : ''}`, + ), + status.phase === 'recovering' && status.recoveryElapsedSeconds + ? formatContextLine('Recovery Progress', `${status.recoveryElapsedSeconds}s of ${delay}s`) + : undefined, + status.phase === 'breaching' + ? undefined + : formatContextLine( + 'Last Reading At Or Above Threshold', + formatAlertValue(alert.value, alert.type), + ), + ]; +} + function formatContextLine(label: string, value?: string | number | null): string | undefined { if (value === undefined || value === null) return undefined; const text = String(value).trim(); diff --git a/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx b/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx index 443cbbd5d5..0438d3f0a8 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx @@ -528,6 +528,46 @@ describe('GuestDrawer', () => { ).toBeInTheDocument(); }); + it('leads with the live reading when an open alert holds below its trigger', () => { + solidRender(() => ( + + )); + + expect(screen.getByText('Memory 87% now, back under the 90% alert level')).toBeInTheDocument(); + expect( + screen.getByText('Stays open until it reaches 85% or lower and stays there for 5 minutes.'), + ).toBeInTheDocument(); + expect(screen.queryByText('VM memory at 91.0%')).not.toBeInTheDocument(); + }); + it('keeps generic Assistant and context-copy actions out of the drawer header', () => { render(() => ); diff --git a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx index 8ad1d0c370..de41cfe2e4 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx @@ -21,7 +21,7 @@ import { import { AvailabilityProbeSuggestionCard } from './AvailabilityProbeSuggestionCard'; import { buildWorkloadsDiskPresentation } from './diskListModel'; import { - getGuestDrawerAlertMessage, + getGuestDrawerAlertAttention, getGuestDrawerMemoryRows, getGuestDrawerMemoryReading, getGuestDrawerGuestReadPresentation, @@ -297,7 +297,7 @@ export function GuestDrawerOverview(props: GuestDrawerOverviewProps) { ({ id: alert.id, - message: getGuestDrawerAlertMessage(alert, { + ...getGuestDrawerAlertAttention(alert, { guest: props.guest, memoryDisplayBasis: props.memoryDisplayBasis, parentMemoryTotal: props.parentMemoryTotal, diff --git a/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx b/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx index 50957eac70..218cd127ba 100644 --- a/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx @@ -9,6 +9,7 @@ import { } from '@/components/shared/DetailSectionTable'; import type { Alert, Disk, HostGPUSensor, Node, Temperature } from '@/types/api'; import { alertTypeDisplayLabel } from '@/features/alerts/helpers'; +import { getAlertAttentionCopy } from '@/features/alerts/metricAlertPresentation'; import { formatBytes, normalizeDiskArray } from '@/utils/format'; import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { getProxmoxUpdateEvidencePresentation } from '@/utils/proxmoxUpdateEvidence'; @@ -370,7 +371,7 @@ export function NodeDrawerOverview(props: NodeDrawerOverviewProps) { ({ id: alert.id, - message: alert.message, + ...getAlertAttentionCopy(alert), subject: cleanText(alert.resourceName) || cleanText(alert.nodeDisplayName) || diff --git a/frontend-modern/src/components/Workloads/guestDrawerModel.ts b/frontend-modern/src/components/Workloads/guestDrawerModel.ts index f78290928c..27fa776ad3 100644 --- a/frontend-modern/src/components/Workloads/guestDrawerModel.ts +++ b/frontend-modern/src/components/Workloads/guestDrawerModel.ts @@ -7,6 +7,10 @@ import type { } from '@/api/charts'; import { formatHistoryChartTooltipValue } from '@/components/shared/historyChartModel'; +import { + getAlertAttentionCopy, + type AlertAttentionCopy, +} from '@/features/alerts/metricAlertPresentation'; import { formatBytes, formatPercent, getBackupInfo, type BackupThresholds } from '@/utils/format'; import { getWorkloadGuestDiskStatusMessage, @@ -82,12 +86,13 @@ export interface GuestDrawerBackupPresentation { export const isGuestDrawerVM = (guest: Guest): guest is VM => resolveWorkloadType(guest) === 'vm'; -export const getGuestDrawerAlertMessage = ( +export const getGuestDrawerAlertAttention = ( alert: Alert, context: Pick, -): string => { +): AlertAttentionCopy => { + const copy = getAlertAttentionCopy(alert); if (context.memoryDisplayBasis !== 'host' || alert.type.trim().toLowerCase() !== 'memory') { - return alert.message; + return copy; } const used = context.guest.memory?.used; @@ -104,7 +109,7 @@ export const getGuestDrawerAlertMessage = ( const comparison = hostShare ? `guest allocation · ${hostShare} of host capacity` : 'guest allocation · row uses host capacity'; - return `${alert.message} (${comparison})`; + return { ...copy, message: `${copy.message} (${comparison})` }; }; const getGuestDrawerDiskUsage = (guest: Guest): number | undefined => { diff --git a/frontend-modern/src/components/shared/DrawerAttentionSection.tsx b/frontend-modern/src/components/shared/DrawerAttentionSection.tsx index c0d8fd86d1..fc3d5c6b5f 100644 --- a/frontend-modern/src/components/shared/DrawerAttentionSection.tsx +++ b/frontend-modern/src/components/shared/DrawerAttentionSection.tsx @@ -10,6 +10,10 @@ import { Button } from './Button'; export interface DrawerAttentionItem { id: string; message: string; + /** Secondary line, e.g. what clears an alert whose reading has dropped. */ + detail?: string; + /** Hover text when it should say more than the message. */ + title?: string; subject?: string; metric?: string; severity?: string; @@ -73,10 +77,13 @@ export const DrawerAttentionSection: Component = (p

{item.message}

+ +

{item.detail}

+ { expect(drawerAttentionSectionSource).toContain('Show fewer alerts'); expect(drawerAttentionSectionSource).toContain('aria-expanded={expanded()}'); expect(drawerAttentionSectionSource).not.toContain('DetailSectionTable'); + // A threshold alert held under its trigger leads with the live reading and + // a second line saying what clears it, never only its stale breach text. + expect(drawerAttentionSectionSource).toContain('item.detail'); + expect(drawerAttentionSectionSource).toContain('item.title ?? item.message'); + for (const liveAlertConsumer of [ + 'src/components/Workloads/NodeDrawerOverview.tsx', + 'src/features/proxmox/ProxmoxMailGatewayDrawer.tsx', + ]) { + const source = readFrontendSource(liveAlertConsumer); + expect(source).toContain('...getAlertAttentionCopy(alert)'); + expect(source).not.toContain('message: alert.message'); + } + expect(readFrontendSource('src/components/Workloads/guestDrawerModel.ts')).toContain( + 'getAlertAttentionCopy(alert)', + ); const overviewConsumers = [ 'src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx', diff --git a/frontend-modern/src/components/shared/TemperatureGauge.tsx b/frontend-modern/src/components/shared/TemperatureGauge.tsx index 01e0c179bc..f5622ab88b 100644 --- a/frontend-modern/src/components/shared/TemperatureGauge.tsx +++ b/frontend-modern/src/components/shared/TemperatureGauge.tsx @@ -14,6 +14,9 @@ interface TemperatureGaugeProps { warning?: number; thresholds?: MetricDisplayThresholds | null; metric?: TemperatureDisplayMetric; + /** Severity of an open alert on this reading; keeps its tone while the alert holds. */ + alertSeverity?: 'warning' | 'critical' | null; + title?: string; class?: string; } @@ -30,11 +33,14 @@ export const TemperatureGauge: Component = (props) => { }); const textColorClass = createMemo(() => - getTemperatureTextClass(props.value, explicitThresholds(), props.metric), + getTemperatureTextClass(props.value, explicitThresholds(), props.metric, props.alertSeverity), ); return ( - + {formatTemperature(props.value)} ); diff --git a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts index 160cc8da7a..676038b14f 100644 --- a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts +++ b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Alert, MetricAlertStatus } from '@/types/api'; import { clampMaxAlertsPerHour, fallbackMaxAlertsPerHour, @@ -12,6 +13,11 @@ import { alertTypeDisplayLabel, clampEscalationDelayMinutes, } from '@/features/alerts/helpers'; +import { + METRIC_ALERT_STATUS_STALE_MS, + getAlertAttentionCopy, + getMetricAlertPresentation, +} from '@/features/alerts/metricAlertPresentation'; describe('alerts helpers', () => { describe('clampEscalationDelayMinutes', () => { @@ -229,3 +235,148 @@ describe('alerts helpers', () => { }); }); }); + +const NOW = Date.parse('2026-10-06T10:52:00Z'); + +// minipc on 2026-10-06: the alert opened at 80°C, the node then read 71-76°C, +// and the drawer kept saying "Node temperature at 80.0°C". +const minipcAlert = (status?: Partial): Alert => ({ + id: 'homelab-minipc::metric-threshold:temperature', + type: 'temperature', + level: 'warning', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + node: 'minipc', + instance: 'homelab', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + startTime: '2026-10-06T07:57:44Z', + lastSeen: '2026-10-06T10:48:52Z', + acknowledged: false, + metricStatus: status + ? { + phase: 'latched', + value: 76, + unit: '°C', + observedAt: new Date(NOW - 10_000).toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + ...status, + } + : undefined, +}); + +describe('getMetricAlertPresentation', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('leads with the reading now and what clears an alert held under its trigger', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ phase: 'latched', value: 76 }), + NOW, + ); + expect(presentation).toMatchObject({ + phase: 'latched', + stale: false, + summary: 'Temperature 76°C now, back under the 80°C alert level', + detail: 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.', + phaseLabel: 'Alert still open', + }); + expect(presentation?.lastBreach).toMatch(/^Last reading at or above 80°C: 80°C, /); + }); + + it('says a reading at the clear level is recovering, with progress once it has some', () => { + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 0 }), + NOW, + ), + ).toMatchObject({ + summary: 'Temperature 72°C now, recovering', + detail: 'Clears after 5 minutes at 75°C or lower.', + phaseLabel: 'Recovering', + }); + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 120 }), + NOW, + )?.detail, + ).toBe('Clears after 5 minutes at 75°C or lower, 2 minutes so far.'); + }); + + it('names the alert level while the reading is still over it', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ phase: 'breaching', value: 82 }), + NOW, + ); + expect(presentation).toMatchObject({ + summary: 'Temperature 82°C, above the 80°C alert level', + detail: 'Clears once it drops to 75°C or lower and stays there for 5 minutes.', + phaseLabel: 'Above 80°C', + }); + expect(presentation?.lastBreach).toBeUndefined(); + }); + + it('stops calling an old evaluation "now"', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ + phase: 'latched', + value: 76, + observedAt: new Date(NOW - METRIC_ALERT_STATUS_STALE_MS - 60_000).toISOString(), + }), + NOW, + ); + expect(presentation?.stale).toBe(true); + expect(presentation?.summary).toMatch(/^Last reading: Temperature 76°C, /); + expect(presentation?.summary).not.toContain('now'); + expect(presentation?.phaseLabel).toBe('No recent reading'); + }); + + it('shows the averaged value the rule follows beside the latest sample', () => { + const alert: Alert = { + ...minipcAlert(), + type: 'cpu', + metricStatus: { + phase: 'latched', + value: 84.2, + rawValue: 61, + evaluationWindowSeconds: 600, + unit: '%', + observedAt: new Date(NOW).toISOString(), + trigger: 90, + recovery: 80, + }, + }; + expect(getMetricAlertPresentation(alert, NOW)).toMatchObject({ + summary: 'CPU averaged 84% over 10 minutes, latest 61% now, back under the 90% alert level', + detail: 'Stays open until it reaches 80% or lower.', + }); + }); + + it('returns null without a live status so callers keep the alert message', () => { + expect(getMetricAlertPresentation(minipcAlert(), NOW)).toBeNull(); + expect(getAlertAttentionCopy(minipcAlert(), NOW)).toEqual({ + message: 'Node temperature at 80.0°C', + }); + }); +}); + +describe('getAlertAttentionCopy', () => { + it('replaces the stale breach message and keeps the breach in the hover text', () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const copy = getAlertAttentionCopy(minipcAlert({ phase: 'recovering', value: 72 }), NOW); + vi.useRealTimers(); + expect(copy.message).toBe('Temperature 72°C now, recovering'); + expect(copy.detail).toBe('Clears after 5 minutes at 75°C or lower.'); + expect(copy.title).toContain('Last reading at or above 80°C: 80°C'); + expect(copy.message).not.toContain('80'); + }); +}); diff --git a/frontend-modern/src/features/alerts/metricAlertPresentation.ts b/frontend-modern/src/features/alerts/metricAlertPresentation.ts new file mode 100644 index 0000000000..8d72d7c312 --- /dev/null +++ b/frontend-modern/src/features/alerts/metricAlertPresentation.ts @@ -0,0 +1,157 @@ +import type { Alert, MetricAlertPhase, MetricAlertStatus } from '@/types/api'; +import { formatRelativeTime } from '@/utils/format'; +import { formatTemperature } from '@/utils/temperature'; +import { alertTypeDisplayLabel } from './helpers'; + +/** + * A live status older than this is no longer "now": the resource stopped + * reporting, or Pulse stopped evaluating it. Generous enough for the slowest + * poll intervals. + */ +export const METRIC_ALERT_STATUS_STALE_MS = 10 * 60 * 1000; + +export interface MetricAlertPresentation { + phase: MetricAlertPhase; + /** The live status is too old to describe as the current reading. */ + stale: boolean; + /** Leads with the reading Pulse is evaluating now and where it stands. */ + summary: string; + /** What clears the alert from here. */ + detail: string; + /** Short state for compact cells and badges. */ + phaseLabel: string; + /** The last reading that met the trigger, once the live one differs. */ + lastBreach?: string; +} + +const formatMetricValue = (value: number, unit: string | undefined): string => { + if (unit === '°C') return formatTemperature(value); + if (unit === '%') { + return `${Math.abs(value) >= 10 ? Math.round(value) : Number(value.toFixed(1))}%`; + } + const rounded = Number(value.toFixed(1)); + return unit ? `${rounded} ${unit}` : String(rounded); +}; + +const formatSeconds = (seconds: number): string => { + if (seconds < 60) return `${seconds} ${seconds === 1 ? 'second' : 'seconds'}`; + if (seconds < 3600) { + const minutes = Math.round(seconds / 60); + return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`; + } + const hours = Number((seconds / 3600).toFixed(1)); + return `${hours} ${hours === 1 ? 'hour' : 'hours'}`; +}; + +const describeReading = (label: string, status: MetricAlertStatus): string => { + const value = formatMetricValue(status.value, status.unit); + const window = status.evaluationWindowSeconds ?? 0; + if (window <= 0) return `${label} ${value}`; + const latest = + typeof status.rawValue === 'number' && Number.isFinite(status.rawValue) + ? `, latest ${formatMetricValue(status.rawValue, status.unit)}` + : ''; + return `${label} averaged ${value} over ${formatSeconds(window)}${latest}`; +}; + +const describeClearRule = (status: MetricAlertStatus, lead: string): string => { + const recovery = formatMetricValue(status.recovery, status.unit); + const delay = status.recoveryDelaySeconds ?? 0; + return delay > 0 + ? `${lead} ${recovery} or lower and stays there for ${formatSeconds(delay)}.` + : `${lead} ${recovery} or lower.`; +}; + +const isUsableStatus = (status: MetricAlertStatus | undefined): status is MetricAlertStatus => + !!status && + Number.isFinite(status.value) && + Number.isFinite(status.trigger) && + Number.isFinite(status.recovery) && + (status.phase === 'breaching' || status.phase === 'latched' || status.phase === 'recovering'); + +/** + * Describes an open threshold alert from the backend's live evaluation, so + * every surface says what the reading is now and why the alert is still open + * instead of repeating the last breach. Returns null when the alert carries + * no live status (non-threshold alerts, or the moments after a restart). + */ +export function getMetricAlertPresentation( + alert: Pick, + now: number = Date.now(), +): MetricAlertPresentation | null { + const status = alert.metricStatus; + if (!isUsableStatus(status)) return null; + + const label = alertTypeDisplayLabel(alert.type); + const reading = describeReading(label, status); + const trigger = formatMetricValue(status.trigger, status.unit); + const observedAt = Date.parse(status.observedAt); + const stale = !Number.isFinite(observedAt) || now - observedAt > METRIC_ALERT_STATUS_STALE_MS; + + let summary: string; + let detail: string; + let phaseLabel: string; + switch (status.phase) { + case 'breaching': + summary = `${reading}, above the ${trigger} alert level`; + detail = describeClearRule(status, 'Clears once it drops to'); + phaseLabel = `Above ${trigger}`; + break; + case 'latched': + summary = `${reading} now, back under the ${trigger} alert level`; + detail = describeClearRule(status, 'Stays open until it reaches'); + phaseLabel = 'Alert still open'; + break; + case 'recovering': { + const delay = status.recoveryDelaySeconds ?? 0; + const elapsed = status.recoveryElapsedSeconds ?? 0; + const recovery = formatMetricValue(status.recovery, status.unit); + summary = `${reading} now, recovering`; + detail = + delay > 0 + ? `Clears after ${formatSeconds(delay)} at ${recovery} or lower${ + elapsed > 0 ? `, ${formatSeconds(Math.min(elapsed, delay))} so far` : '' + }.` + : `Clears at ${recovery} or lower.`; + phaseLabel = 'Recovering'; + break; + } + } + + if (stale) { + summary = `Last reading: ${reading}${ + Number.isFinite(observedAt) ? `, ${formatRelativeTime(observedAt)}` : '' + }`; + phaseLabel = 'No recent reading'; + } + + let lastBreach: string | undefined; + if (status.phase !== 'breaching' && Number.isFinite(alert.value)) { + const when = alert.lastSeen ? `, ${formatRelativeTime(alert.lastSeen)}` : ''; + lastBreach = `Last reading at or above ${trigger}: ${formatMetricValue(alert.value, status.unit)}${when}`; + } + + return { phase: status.phase, stale, summary, detail, phaseLabel, lastBreach }; +} + +export interface AlertAttentionCopy { + message: string; + detail?: string; + title?: string; +} + +/** Copy for an alert row in a drawer's "Needs attention" list. */ +export function getAlertAttentionCopy( + alert: Pick, + now: number = Date.now(), +): AlertAttentionCopy { + const presentation = getMetricAlertPresentation(alert, now); + if (!presentation) return { message: alert.message }; + return { + message: presentation.summary, + detail: presentation.detail, + title: [presentation.summary, presentation.detail, presentation.lastBreach] + .filter(Boolean) + .join('\n'), + }; +} diff --git a/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx b/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx index bca5931740..6211981044 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx @@ -1,6 +1,7 @@ import { For, Show, createMemo, createResource, type Component } from 'solid-js'; import { DrawerAttentionSection } from '@/components/shared/DrawerAttentionSection'; import { alertTypeDisplayLabel } from '@/features/alerts/helpers'; +import { getAlertAttentionCopy } from '@/features/alerts/metricAlertPresentation'; import type { Alert } from '@/types/api'; import { Card } from '@/components/shared/Card'; import { DrawerSubjectHeading } from '@/components/shared/DrawerSubjectHeading'; @@ -351,7 +352,7 @@ export const ProxmoxMailGatewayDrawer: Component<{ ({ id: alert.id, - message: alert.message, + ...getAlertAttentionCopy(alert), subject: name(), metric: alertTypeDisplayLabel(alert.type), severity: alert.level, diff --git a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx index 932f0767e3..191809f431 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx @@ -21,6 +21,7 @@ import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; import { MetricMiniSparkline } from '@/components/Workloads/MetricMiniSparkline'; import { TemperatureGauge } from '@/components/shared/TemperatureGauge'; import { hostOverrideIdCandidates } from '@/features/alerts/alertOverridesModel'; +import { getMetricAlertPresentation } from '@/features/alerts/metricAlertPresentation'; import { useBreakpoint } from '@/hooks/useBreakpoint'; import { TableCell, TableRow } from '@/components/shared/Table'; import { SUMMARY_ROW_ACTION_BUTTON_FOCUS_CLASS } from '@/components/shared/summaryInteractionA11y'; @@ -451,6 +452,26 @@ export const ProxmoxNodesTable: Component<{ const alertResourceIds = () => hostOverrideIdCandidates(node); const temperatureThresholds = () => alertsActivation.getMetricThresholds('node', 'temperature', alertResourceIds()); + // The node's own temperature alert holds after the reading dips + // under its trigger; the cell keeps the alert's tone and says why + // instead of reading green beside a warning row. + const temperatureAlert = createMemo(() => + getAlertsForResource(alertResourceIds(), activeAlerts, alertsEnabled()).find( + (alert) => alert.type === 'temperature', + ), + ); + const temperatureAlertSeverity = () => { + const alert = temperatureAlert(); + if (!alert || alert.acknowledged) return null; + return alert.level === 'critical' ? 'critical' : 'warning'; + }; + const temperatureAlertTitle = () => { + const alert = temperatureAlert(); + const presentation = alert ? getMetricAlertPresentation(alert) : null; + return presentation + ? `${presentation.summary}. ${presentation.detail}` + : alert?.message; + }; const cpuThresholds = () => alertsActivation.getMetricThresholds('node', 'cpu', alertResourceIds()); const memoryThresholds = () => @@ -729,6 +750,8 @@ export const ProxmoxNodesTable: Component<{ diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx index e985df726d..fc4e8b9ae9 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx @@ -46,8 +46,18 @@ vi.mock('@/components/Workloads/MetricMiniSparkline', () => ({ })); vi.mock('@/components/shared/TemperatureGauge', () => ({ - TemperatureGauge: (props: { value: number; thresholds?: unknown }) => { - temperatureGaugeMock({ value: props.value, thresholds: props.thresholds }); + TemperatureGauge: (props: { + value: number; + thresholds?: unknown; + alertSeverity?: string | null; + title?: string; + }) => { + temperatureGaugeMock({ + value: props.value, + thresholds: props.thresholds, + alertSeverity: props.alertSeverity, + title: props.title, + }); return
; }, })); @@ -358,6 +368,86 @@ describe('ProxmoxNodesTable', () => { ); }); + it('keeps an open temperature alert visible on a reading that dipped under its trigger', () => { + // minipc on 2026-10-06: the row warned "80.0°C" while the cell read a green 72°C. + activeAlertsMock.value = { + 'cpu-alert': { + id: 'cpu-alert', + resourceId: 'agent:pve-node-1', + type: 'cpu', + level: 'critical', + message: 'Node cpu at 97.0%', + value: 97, + acknowledged: false, + }, + 'temperature-alert': { + id: 'temperature-alert', + resourceId: 'agent:pve-node-1', + type: 'temperature', + level: 'warning', + message: 'Node temperature at 80.0°C', + value: 80, + acknowledged: false, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: new Date().toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + }, + }, + }; + + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + // The node's own temperature alert sets the tone; its CPU alert does not. + expect(temperatureGaugeMock).toHaveBeenLastCalledWith( + expect.objectContaining({ + value: 72, + alertSeverity: 'warning', + title: 'Temperature 72°C now, recovering. Clears after 5 minutes at 75°C or lower.', + }), + ); + }); + + it('leaves the temperature tone to the reading once the alert is acknowledged', () => { + activeAlertsMock.value = { + 'temperature-alert': { + id: 'temperature-alert', + resourceId: 'agent:pve-node-1', + type: 'temperature', + level: 'warning', + message: 'Node temperature at 80.0°C', + value: 80, + acknowledged: true, + }, + }; + + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + expect(temperatureGaugeMock).toHaveBeenLastCalledWith( + expect.objectContaining({ alertSeverity: null, title: 'Node temperature at 80.0°C' }), + ); + }); + it('keeps the Proxmox cache-inclusive memory comparison label on node rows', () => { render(() => ( { expect(result.dataAlertState).toBe('acknowledged'); }); + it('does not claim usage is over the limit while the alert holds below it', () => { + const holding = (phase: 'latched' | 'recovering') => + makeAlert({ + metricStatus: { + phase, + value: phase === 'latched' ? 83 : 79, + unit: '%', + observedAt: '2026-10-04T12:30:00Z', + trigger: 85, + recovery: 80, + recoveryDelaySeconds: 300, + }, + }); + expect(describeStorageAlertHeadline(holding('latched'))).toBe( + 'Under 85% limit, clears at 80% or lower', + ); + expect(describeStorageAlertHeadline(holding('latched'), { compact: true })).toBe('Clears ≤80%'); + expect(describeStorageAlertHeadline(holding('recovering'))).toBe( + 'Recovering, clears at 80% or lower', + ); + expect(describeStorageAlertHeadline(holding('recovering'), { compact: true })).toBe( + 'Recovering', + ); + }); + it('explains a highlighted row with a fill forecast, a crossed limit or the alert text', () => { const forecast = (days: number) => makeAlert({ threshold: 100, metadata: { forecastDaysToFull: days } }); diff --git a/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts b/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts index 57c6e34cff..30d7a3aa1b 100644 --- a/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts +++ b/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts @@ -78,6 +78,18 @@ export const describeStorageAlertHeadline = ( if (options.compact) return `Full in ~${rounded}d`; return `Full in ~${rounded} ${rounded === 1 ? 'day' : 'days'}`; } + const status = alert.type === 'usage' ? alert.metricStatus : undefined; + if (status && status.phase !== 'breaching') { + // Usage has dropped back under the limit but the alert holds until it + // reaches the clear level, so "Over" would contradict the bar beside it. + const recovery = `${Math.round(status.recovery)}%`; + if (status.phase === 'recovering') { + return options.compact ? 'Recovering' : `Recovering, clears at ${recovery} or lower`; + } + return options.compact + ? `Clears ≤${recovery}` + : `Under ${Math.round(status.trigger)}% limit, clears at ${recovery} or lower`; + } if (alert.type === 'usage' && alert.threshold > 0 && alert.threshold < 100) { return options.compact ? `Over ${alert.threshold}%` : `Over ${alert.threshold}% usage limit`; } diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5e7da6d6de..a455f1c4a2 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -1380,6 +1380,32 @@ export interface Alert { latestTransition?: LifecycleTransition; transitions?: LifecycleTransition[]; evidence?: EvidenceEnvelope[]; + /** + * Live evaluation behind an open threshold alert. `value`, `message` and + * `lastSeen` keep the last reading that met the trigger; this is the + * reading being evaluated now and why the alert has not cleared. + */ + metricStatus?: MetricAlertStatus; +} + +export type MetricAlertPhase = 'breaching' | 'latched' | 'recovering'; + +export interface MetricAlertStatus { + phase: MetricAlertPhase; + /** Value compared with the rule (the average for a rolling-average rule). */ + value: number; + /** Latest sample when the rule evaluates a rolling average. */ + rawValue?: number; + evaluationWindowSeconds?: number; + unit?: string; + observedAt: string; + /** Opens at or above this value. */ + trigger: number; + /** Clears after staying at or below this value for recoveryDelaySeconds. */ + recovery: number; + recoveryDelaySeconds?: number; + recoveryStartedAt?: string; + recoveryElapsedSeconds?: number; } export interface ResolvedAlert extends Alert { diff --git a/frontend-modern/src/utils/__tests__/temperature.test.ts b/frontend-modern/src/utils/__tests__/temperature.test.ts index ec4824f67f..5224735935 100644 --- a/frontend-modern/src/utils/__tests__/temperature.test.ts +++ b/frontend-modern/src/utils/__tests__/temperature.test.ts @@ -138,6 +138,22 @@ describe('temperature', () => { expect(getTemperatureTextClass(85, thresholds)).toContain('text-red-600'); }); + it("keeps an open alert's tone while the reading holds under its trigger", () => { + const thresholds = { warning: 75, critical: 80 }; + // minipc: alert open at 80°C, now reading 72°C on its way down. + expect(getTemperatureTextClass(72, thresholds)).toContain('text-green-600'); + expect(getTemperatureTextClass(72, thresholds, 'temperature', 'warning')).toContain( + 'text-amber-600', + ); + expect(getTemperatureTextClass(72, thresholds, 'temperature', 'critical')).toContain( + 'text-red-600', + ); + // The reading still wins when it is worse than the alert. + expect(getTemperatureTextClass(86, thresholds, 'temperature', 'warning')).toContain( + 'text-red-600', + ); + }); + it('uses disk temperature defaults for disk temperature displays', () => { expect(getTemperatureTextClass(49, undefined, 'diskTemperature')).toContain('text-green-600'); expect(getTemperatureTextClass(50, undefined, 'diskTemperature')).toContain('text-amber-600'); diff --git a/frontend-modern/src/utils/temperature.ts b/frontend-modern/src/utils/temperature.ts index f8b822504f..6b993af762 100644 --- a/frontend-modern/src/utils/temperature.ts +++ b/frontend-modern/src/utils/temperature.ts @@ -109,15 +109,31 @@ export type TemperatureDisplayMetric = Extract< 'temperature' | 'diskTemperature' >; +const TEMPERATURE_SEVERITY_RANK = { normal: 0, warning: 1, critical: 2 } as const; + +/** + * Colour for a temperature reading. An open alert on the reading sets a floor: + * the alert holds until the reading falls to its clear level, so a reading + * that has dipped under the trigger must not look healthy beside the alert. + */ export const getTemperatureTextClass = ( celsius: number | null | undefined, thresholds?: MetricDisplayThresholds | null, metric: TemperatureDisplayMetric = 'temperature', + openAlertSeverity?: 'warning' | 'critical' | null, ): string => { if (celsius === null || celsius === undefined || !Number.isFinite(celsius)) { return 'text-muted'; } - return TEMPERATURE_TEXT_CLASSES[ - getMetricSeverity(celsius, metric, thresholds ?? getDefaultDisplayMetricThresholds(metric)) - ]; + const readingSeverity = getMetricSeverity( + celsius, + metric, + thresholds ?? getDefaultDisplayMetricThresholds(metric), + ); + const severity = + openAlertSeverity && + TEMPERATURE_SEVERITY_RANK[openAlertSeverity] > TEMPERATURE_SEVERITY_RANK[readingSeverity] + ? openAlertSeverity + : readingSeverity; + return TEMPERATURE_TEXT_CLASSES[severity]; }; diff --git a/internal/alerts/active_cleanup.go b/internal/alerts/active_cleanup.go index 9d0a19af68..e8d24899db 100644 --- a/internal/alerts/active_cleanup.go +++ b/internal/alerts/active_cleanup.go @@ -19,7 +19,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) { if alert == nil { return true } - lastSeen := alert.LastSeen + lastSeen := alert.lastObservedAt() if lastSeen.IsZero() { lastSeen = alert.StartTime } diff --git a/internal/alerts/active_persistence.go b/internal/alerts/active_persistence.go index d5c0c122a1..a1a094767c 100644 --- a/internal/alerts/active_persistence.go +++ b/internal/alerts/active_persistence.go @@ -180,6 +180,10 @@ func (m *Manager) snapshotActiveAlerts() []*Alert { clone := alert.Clone() backfillCanonicalIdentity(clone) clone.ID = exportedAlertID(clone, clone.ID) + // The live status changes on every poll while a threshold alert + // holds; keeping it out of the checkpoint keeps an unchanged alert + // set byte-identical, so the recovery mirror is not rewritten. + clone.MetricStatus = nil alerts = append(alerts, clone) } m.mu.RUnlock() @@ -308,6 +312,7 @@ func (m *Manager) setActiveRecoveryAlert(alert *Alert, fallback string) { clone := alert.Clone() backfillCanonicalIdentity(clone) clone.ID = exportedAlertID(clone, clone.ID) + clone.MetricStatus = nil storageKey := activeAlertStorageKey(clone, fallback) if storageKey == "" { return diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 68daa1b65c..f95e034cbf 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -21689,3 +21689,57 @@ func TestCleanupRetentionRequiresObservationInactivity(t *testing.T) { }) } } + +func TestMetricStatusStaysOutOfDurableAlertSnapshots(t *testing.T) { + m, clock := newMetricStatusTestManager(t) + fireNodeTemperatureAlert(t, m, clock, 85) + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 78) + + live := m.GetActiveAlerts() + if len(live) != 1 || live[0].MetricStatus == nil || live[0].MetricStatus.Value != 78 { + t.Fatalf("live alerts do not carry the current reading: %+v", live) + } + for _, alert := range m.snapshotActiveAlerts() { + if alert.MetricStatus != nil { + t.Fatalf("durable checkpoint carries volatile status for %s: %+v", alert.ID, alert.MetricStatus) + } + } + for _, alert := range m.activeRecoverySnapshot() { + if alert.MetricStatus != nil { + t.Fatalf("recovery projection carries volatile status for %s: %+v", alert.ID, alert.MetricStatus) + } + } +} + +func TestStaleCleanupKeepsThresholdAlertStillBeingEvaluated(t *testing.T) { + m := newTestManager(t) + now := time.Now() + breachedLongAgo := now.Add(-StaleTrackingThreshold - time.Hour) + + held := &Alert{ + ID: "held::metric-threshold:usage", Type: "usage", ResourceID: "held", + StartTime: breachedLongAgo, LastSeen: breachedLongAgo, + MetricStatus: &models.MetricAlertStatus{Phase: models.MetricAlertPhaseLatched, ObservedAt: now}, + } + abandoned := &Alert{ + ID: "gone::metric-threshold:usage", Type: "usage", ResourceID: "gone", + StartTime: breachedLongAgo, LastSeen: breachedLongAgo, + MetricStatus: &models.MetricAlertStatus{Phase: models.MetricAlertPhaseLatched, ObservedAt: breachedLongAgo}, + } + m.mu.Lock() + m.activeAlerts[held.ID] = held + m.activeAlerts[abandoned.ID] = abandoned + m.mu.Unlock() + + m.cleanupStaleMaps() + + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.activeAlerts[held.ID]; !ok { + t.Fatal("cleanup resolved a held alert whose condition is still being evaluated") + } + if _, ok := m.activeAlerts[abandoned.ID]; ok { + t.Fatal("cleanup kept an alert nobody has evaluated for over a day") + } +} diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index fcfa75baa5..4d56b7ca48 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -261,6 +261,17 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec alertMetadata[k] = v } } + metricStatus := buildMetricAlertStatus(metricStatusInput{ + value: value, + triggered: triggered, + trigger: spec.MetricThreshold.Trigger, + clear: clearThreshold, + recoveryDelaySeconds: stability.RecoveryDelaySeconds, + unit: metricStatusUnit(metricType), + window: windowed, + incident: incident, + observedAt: observedAt, + }) if !exists { alert := &Alert{ @@ -278,6 +289,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec StartTime: alertStartTime, LastSeen: observedAt, Metadata: alertMetadata, + MetricStatus: metricStatus, } applyCanonicalIdentity(alert, spec.ID, string(spec.Kind)) @@ -331,12 +343,16 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec } if !triggered && primary == "" { - // Hysteresis latch: below trigger but above recovery — hold - // without refreshing, as the pre-cutover engine did. + // Hysteresis hold or recovery run: Value, Message and LastSeen + // keep the last breach, and nothing here may notify. Only the + // volatile live status moves, so surfaces can say what the + // reading is now and why the alert has not cleared. + existingAlert.MetricStatus = metricStatus return } oldLevel := existingAlert.Level + existingAlert.MetricStatus = metricStatus existingAlert.LastSeen = observedAt existingAlert.Value = value existingAlert.Threshold = spec.MetricThreshold.Trigger @@ -390,6 +406,8 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec existingAlert = existingAlert.Clone() existingAlert.Value = value existingAlert.LastSeen = observedAt + // A resolved record is history; the live status describes open alerts only. + existingAlert.MetricStatus = nil message, _ := metricAlertMessage(resourceType, metricType, value, opts) existingAlert.Message = "Resolved: " + message diff --git a/internal/alerts/metric_status.go b/internal/alerts/metric_status.go new file mode 100644 index 0000000000..7bc19204b1 --- /dev/null +++ b/internal/alerts/metric_status.go @@ -0,0 +1,79 @@ +package alerts + +import ( + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +// metricStatusInput is what the canonical metric evaluator compared on one +// observation: the value after any rolling-average window, the exact rule +// handed to the reducer, and the incident the reducer holds afterwards. +type metricStatusInput struct { + value float64 + triggered bool + trigger float64 + clear float64 + recoveryDelaySeconds int + unit string + window metricWindowObservation + incident reducer.Incident + observedAt time.Time +} + +// metricStatusUnit is the unit of an evaluated metric value. CPU and memory +// share the "%" of usage even though their legacy messages omit a unit. +func metricStatusUnit(metricType string) string { + switch metricType { + case "temperature", "disk_temperature", "diskTemperature": + return "°C" + case "diskRead", "diskWrite", "networkIn", "networkOut": + return "MB/s" + default: + return "%" + } +} + +// buildMetricAlertStatus derives the live status of an open threshold alert +// from the evaluator's own inputs, so surfaces never re-derive the phase or +// the recovery timer from thresholds and a resource reading of their own. +func buildMetricAlertStatus(input metricStatusInput) *models.MetricAlertStatus { + // The reducer treats a non-positive clear as "clear at the trigger". + recovery := input.clear + if recovery <= 0 { + recovery = input.trigger + } + status := &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseBreaching, + Value: input.value, + Unit: input.unit, + ObservedAt: input.observedAt, + Trigger: input.trigger, + Recovery: recovery, + RecoveryDelaySeconds: input.recoveryDelaySeconds, + } + if input.window.WindowSeconds > 0 { + raw := input.window.CurrentValue + status.RawValue = &raw + status.EvaluationWindowSeconds = input.window.WindowSeconds + } + if input.triggered { + return status + } + if input.value > recovery || input.incident.RecoverySince.IsZero() { + status.Phase = models.MetricAlertPhaseLatched + return status + } + status.Phase = models.MetricAlertPhaseRecovering + startedAt := input.incident.RecoverySince + status.RecoveryStartedAt = &startedAt + elapsed := input.observedAt.Sub(input.incident.RecoverySince) + if input.incident.RecoveryTicksSupplied { + elapsed = input.incident.RecoveryElapsed + } + if elapsed > 0 { + status.RecoveryElapsedSeconds = int(elapsed / time.Second) + } + return status +} diff --git a/internal/alerts/model.go b/internal/alerts/model.go index 4b6324a2fd..66eecd2a29 100644 --- a/internal/alerts/model.go +++ b/internal/alerts/model.go @@ -4,6 +4,7 @@ import ( "strings" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) @@ -144,6 +145,23 @@ type Alert struct { LatestTransition *operationaltrust.LifecycleTransition `json:"latestTransition,omitempty"` Transitions []operationaltrust.LifecycleTransition `json:"transitions,omitempty"` Evidence []operationaltrust.EvidenceEnvelope `json:"evidence,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. It + // is volatile and stripped from durable snapshots. + MetricStatus *models.MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// lastObservedAt is when the alert's condition was last evaluated. A +// threshold alert held below its trigger keeps LastSeen at the last breach, +// so its live status carries the newer observation. +func (a *Alert) lastObservedAt() time.Time { + if a == nil { + return time.Time{} + } + observed := a.LastSeen + if a.MetricStatus != nil && a.MetricStatus.ObservedAt.After(observed) { + observed = a.MetricStatus.ObservedAt + } + return observed } // Clone returns a deep copy of the alert so it can be safely shared across goroutines. @@ -176,6 +194,8 @@ func (a *Alert) Clone() *Alert { clone.Metadata = cloneMetadata(a.Metadata) } + clone.MetricStatus = a.MetricStatus.Clone() + if a.OperationalRecord != nil { value := a.OperationalRecord.Clone() clone.OperationalRecord = &value diff --git a/internal/alerts/tracking_cleanup.go b/internal/alerts/tracking_cleanup.go index a328600064..9e5882c2a1 100644 --- a/internal/alerts/tracking_cleanup.go +++ b/internal/alerts/tracking_cleanup.go @@ -156,7 +156,9 @@ func (m *Manager) cleanupStaleMaps() { staleAlerts := make([]string, 0) for storageKey, alert := range m.activeAlerts { alertID := effectiveAlertID(alert, storageKey) - if alert != nil && now.Sub(alert.LastSeen) > staleThreshold { + // A threshold alert holding below its trigger is still observed + // even though LastSeen stays at its last breach. + if alert != nil && now.Sub(alert.lastObservedAt()) > staleThreshold { staleAlerts = append(staleAlerts, alertID) } } diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index de010d214c..72d803fc9b 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -2,9 +2,11 @@ package alerts import ( "sort" + "sync/atomic" "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -1583,3 +1585,186 @@ func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds) } } + +// metricStatusClock drives a manager's wall and monotonic clocks together so +// recovery runs measure exactly the time the test advances. +type metricStatusClock struct { + now time.Time + tick time.Duration +} + +func (c *metricStatusClock) advance(d time.Duration) { + c.now = c.now.Add(d) + c.tick += d +} + +func newMetricStatusTestManager(t *testing.T) (*Manager, *metricStatusClock) { + t.Helper() + m := newTestManager(t) + // Recently-resolved retention prunes against the real clock, so start there. + clock := &metricStatusClock{now: time.Now().UTC()} + m.now = func() time.Time { return clock.now } + m.intentClock = func() time.Duration { return clock.tick } + m.mu.Lock() + m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75} + m.mu.Unlock() + return m, clock +} + +// fireNodeTemperatureAlert holds a breaching reading until the factory +// stability delay lets the alert open, as a real poll loop would. +func fireNodeTemperatureAlert(t *testing.T, m *Manager, clock *metricStatusClock, celsius float64) *Alert { + t.Helper() + for attempt := 0; attempt < 40; attempt++ { + checkMetricStatusNode(m, celsius) + if alert := activeNodeTemperatureAlert(t, m); alert != nil { + return alert + } + clock.advance(30 * time.Second) + } + t.Fatalf("%v°C never opened the temperature alert", celsius) + return nil +} + +func checkMetricStatusNode(m *Manager, celsius float64) { + m.CheckNode(models.Node{ + ID: "homelab-minipc", + Name: "minipc", + Instance: "homelab", + Status: "online", + CPU: 0.2, + Memory: models.Memory{Usage: 40}, + Disk: models.Disk{Usage: 40}, + Temperature: &models.Temperature{ + Available: true, + CPUPackage: celsius, + }, + }) +} + +func activeNodeTemperatureAlert(t *testing.T, m *Manager) *Alert { + t.Helper() + m.mu.RLock() + defer m.mu.RUnlock() + alert, ok := m.getActiveAlertNoLock(canonicalMetricStateID("homelab-minipc", "temperature")) + if !ok { + return nil + } + return alert.Clone() +} + +func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) { + m, clock := newMetricStatusTestManager(t) + var notified atomic.Int32 + m.SetAlertCallback(func(*Alert) { notified.Add(1) }) + + fired := fireNodeTemperatureAlert(t, m, clock, 85) + status := fired.MetricStatus + if status == nil || status.Phase != models.MetricAlertPhaseBreaching || status.Value != 85 || + status.Trigger != 80 || status.Recovery != 75 || status.Unit != "°C" { + t.Fatalf("breaching status = %+v", status) + } + if status.RecoveryDelaySeconds <= 0 { + t.Fatalf("temperature alerts carry a recovery delay, got %+v", status) + } + delay := time.Duration(status.RecoveryDelaySeconds) * time.Second + firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, notified.Load() + + // Between the clear and trigger levels the alert holds. The legacy + // snapshot keeps the breach; only the live status moves. + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 78) + held := activeNodeTemperatureAlert(t, m) + if held == nil { + t.Fatal("78°C cleared an alert that should hold until 75°C") + } + if held.Value != 85 || held.Message != firedMessage || !held.LastSeen.Equal(firedLastSeen) { + t.Fatalf("hold rewrote the breach snapshot: value=%v message=%q lastSeen=%v", held.Value, held.Message, held.LastSeen) + } + if got := held.MetricStatus; got == nil || got.Phase != models.MetricAlertPhaseLatched || got.Value != 78 || + !got.ObservedAt.Equal(clock.now) || got.RecoveryStartedAt != nil { + t.Fatalf("latched status = %+v", got) + } + + // At or below the recovery level the recovery run starts. + clock.advance(30 * time.Second) + recoveryStart := clock.now + checkMetricStatusNode(m, 72) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseRecovering || got.Value != 72 || + got.RecoveryStartedAt == nil || !got.RecoveryStartedAt.Equal(recoveryStart) || got.RecoveryElapsedSeconds != 0 { + t.Fatalf("recovery start status = %+v", got) + } + clock.advance(delay / 2) + checkMetricStatusNode(m, 75) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseRecovering || got.RecoveryElapsedSeconds != int(delay/2/time.Second) { + t.Fatalf("recovery progress status = %+v", got) + } + + // A reading back above the recovery level resets the run. + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 77) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseLatched || got.RecoveryStartedAt != nil || got.RecoveryElapsedSeconds != 0 { + t.Fatalf("status after recovery reset = %+v", got) + } + if got := notified.Load(); got != notifications { + t.Fatalf("holding and recovering sent %d notifications", got-notifications) + } + + // A full recovery delay at or below 75°C resolves the alert. + checkMetricStatusNode(m, 72) + clock.advance(delay) + checkMetricStatusNode(m, 72) + if alert := activeNodeTemperatureAlert(t, m); alert != nil { + t.Fatalf("alert still open after a full recovery delay: %+v", alert.MetricStatus) + } + m.resolvedMutex.RLock() + defer m.resolvedMutex.RUnlock() + resolved := 0 + for _, entry := range m.recentlyResolved { + if entry == nil || entry.Type != "temperature" { + continue + } + resolved++ + if entry.MetricStatus != nil { + t.Fatalf("resolved record kept the live status of an open alert: %+v", entry.MetricStatus) + } + } + if resolved != 1 { + t.Fatalf("resolved temperature records = %d, want 1", resolved) + } +} + +func TestBuildMetricAlertStatusCarriesWindowAndMonotonicRecovery(t *testing.T) { + observedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC) + status := buildMetricAlertStatus(metricStatusInput{ + value: 74, + trigger: 80, + clear: 75, + recoveryDelaySeconds: 300, + unit: "%", + window: metricWindowObservation{Value: 74, CurrentValue: 91, WindowSeconds: 600, Ready: true}, + incident: reducer.Incident{ + RecoverySince: observedAt.Add(-10 * time.Minute), + RecoveryElapsed: 90 * time.Second, + RecoveryTicksSupplied: true, + }, + observedAt: observedAt, + }) + if status.Phase != models.MetricAlertPhaseRecovering || status.Value != 74 || + status.RawValue == nil || *status.RawValue != 91 || status.EvaluationWindowSeconds != 600 { + t.Fatalf("windowed status = %+v", status) + } + // Elapsed time comes from the monotonic run, not wall-clock arithmetic. + if status.RecoveryElapsedSeconds != 90 { + t.Fatalf("recovery elapsed = %d, want 90", status.RecoveryElapsedSeconds) + } + + // The reducer clears at the trigger when no clear level is configured. + noClear := buildMetricAlertStatus(metricStatusInput{value: 85, triggered: true, trigger: 80, observedAt: observedAt}) + if noClear.Recovery != 80 || noClear.RawValue != nil { + t.Fatalf("status without clear level = %+v", noClear) + } +} diff --git a/internal/models/metrics_types_test.go b/internal/models/metrics_types_test.go index 86d4df09d0..8f5edabedc 100644 --- a/internal/models/metrics_types_test.go +++ b/internal/models/metrics_types_test.go @@ -451,3 +451,21 @@ func TestPhysicalDiskSnapshotPreservesIndependentPollingCadence(t *testing.T) { t.Fatal("snapshot mutation changed collector state") } } + +func TestMetricAlertStatusCloneIsDeep(t *testing.T) { + raw := 91.0 + startedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC) + status := &MetricAlertStatus{ + Phase: MetricAlertPhaseRecovering, Value: 74, RawValue: &raw, + RecoveryStartedAt: &startedAt, RecoveryElapsedSeconds: 90, + } + clone := status.Clone() + *clone.RawValue = 10 + *clone.RecoveryStartedAt = startedAt.Add(time.Hour) + if *status.RawValue != 91 || !status.RecoveryStartedAt.Equal(startedAt) { + t.Fatalf("clone shares pointers with the original: %+v", status) + } + if (*MetricAlertStatus)(nil).Clone() != nil { + t.Fatal("nil status must clone to nil") + } +} diff --git a/internal/models/models.go b/internal/models/models.go index 5f1f2af5b8..a08334e25a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -110,6 +110,58 @@ type Alert struct { // Metadata carries alert-engine annotations (notably resourceType) so the // frontend can classify an alert without re-deriving resource identity. Metadata map[string]interface{} `json:"metadata,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. + MetricStatus *MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// Metric alert phases. A threshold alert stays open after its reading drops +// below the trigger: it holds until the reading reaches the recovery level, +// then must stay there for the recovery delay before it clears. +const ( + MetricAlertPhaseBreaching = "breaching" + MetricAlertPhaseLatched = "latched" + MetricAlertPhaseRecovering = "recovering" +) + +// MetricAlertStatus describes the reading Pulse is evaluating now for an open +// threshold alert and why the alert is still open. The alert's Value, Message +// and LastSeen keep describing the last reading that met the trigger, which +// can be minutes or days old while the alert holds, so surfaces lead with +// this instead. It is volatile: rebuilt on every evaluation, never persisted. +type MetricAlertStatus struct { + Phase string `json:"phase"` + // Value is the value compared with the rule. For a rolling-average rule + // that is the average; RawValue then carries the latest sample. + Value float64 `json:"value"` + RawValue *float64 `json:"rawValue,omitempty"` + EvaluationWindowSeconds int `json:"evaluationWindowSeconds,omitempty"` + Unit string `json:"unit,omitempty"` + ObservedAt time.Time `json:"observedAt"` + // Trigger opens the alert at or above this value; Recovery is the level + // the value must be at or below for RecoveryDelaySeconds to clear it. + Trigger float64 `json:"trigger"` + Recovery float64 `json:"recovery"` + RecoveryDelaySeconds int `json:"recoveryDelaySeconds,omitempty"` + // Recovery progress, present only in the recovering phase. + RecoveryStartedAt *time.Time `json:"recoveryStartedAt,omitempty"` + RecoveryElapsedSeconds int `json:"recoveryElapsedSeconds,omitempty"` +} + +// Clone returns a deep copy. +func (s *MetricAlertStatus) Clone() *MetricAlertStatus { + if s == nil { + return nil + } + clone := *s + if s.RawValue != nil { + raw := *s.RawValue + clone.RawValue = &raw + } + if s.RecoveryStartedAt != nil { + startedAt := *s.RecoveryStartedAt + clone.RecoveryStartedAt = &startedAt + } + return &clone } // ResolvedAlert represents a recently resolved alert diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 498bd35b8d..9d85fedba4 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1466,3 +1466,39 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) { t.Fatal("unannounced destination received a recovery") } } + +func TestSyncAlertsToStateCarriesLiveMetricStatusOfHeldAlert(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + alertManager: alerts.NewManagerWithDataDir(t.TempDir()), + } + defer m.alertManager.Stop() + + cfg := m.alertManager.GetConfig() + cfg.TimeThresholds = map[string]int{"node": 0} + cfg.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + m.alertManager.UpdateConfig(cfg) + + node := models.Node{ + ID: "homelab-minipc", Name: "minipc", Instance: "homelab", Status: "online", + Temperature: &models.Temperature{Available: true, CPUPackage: 85}, + } + m.alertManager.CheckNode(node) + node.Temperature = &models.Temperature{Available: true, CPUPackage: 78} + m.alertManager.CheckNode(node) + + m.syncAlertsToState() + + var held *models.Alert + for i, alert := range m.state.GetSnapshot().ActiveAlerts { + if alert.Type == "temperature" { + held = &m.state.GetSnapshot().ActiveAlerts[i] + } + } + require.NotNil(t, held, "expected the temperature alert to stay open at 78°C") + require.Equal(t, 85.0, held.Value, "the legacy value keeps the last breach") + require.NotNil(t, held.MetricStatus, "websocket state must carry the live reading") + require.Equal(t, models.MetricAlertPhaseLatched, held.MetricStatus.Phase) + require.Equal(t, 78.0, held.MetricStatus.Value) + require.Equal(t, 75.0, held.MetricStatus.Recovery) +} diff --git a/internal/monitoring/monitor_alert_sync.go b/internal/monitoring/monitor_alert_sync.go index 920381c2c2..9fc1154af9 100644 --- a/internal/monitoring/monitor_alert_sync.go +++ b/internal/monitoring/monitor_alert_sync.go @@ -71,8 +71,10 @@ func (m *Monitor) activeAlertsSnapshot() []models.Alert { Acknowledged: alert.Acknowledged, AckTime: alert.AckTime, AckUser: alert.AckUser, - // GetActiveAlerts returns deep clones, so the map is already private. - Metadata: alert.Metadata, + // GetActiveAlerts returns deep clones, so the map and the live + // metric status are already private. + Metadata: alert.Metadata, + MetricStatus: alert.MetricStatus, }) } return modelAlerts