diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b51b2d57dc..408160343f 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -15,6 +15,14 @@ ## Purpose +### Alert projections carry a volatile metric status — issue #2068 + +`models.Alert` gains an optional `metricStatus` (`models.MetricAlertStatus`) +beside the alert-engine metadata. It is a live evaluation snapshot owned by +the alerts subsystem. Agent report admission, host continuity and +re-enrollment never read or write it, and it is not persisted with agent or +state records. + ### Source-record lookup preserves report admission Host and Docker report admission still performs canonical host-view matching diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index cb03c572e2..1f4f14d152 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -15,6 +15,23 @@ ## Purpose +### Open threshold alerts carry a volatile live status — issue #2068 + +`evaluateCanonicalMetricAlert` builds `MetricStatus` from the evaluator's own +inputs on every evaluation, including the hysteresis hold and the recovery +run, where `Value`, `Message` and `LastSeen` stay at the last breach and no +branch may notify. The phase comes from the trigger, the clear level handed to +the reducer (a value at or below it counts toward recovery) and the reducer's +recovery run; progress uses that run's monotonic elapsed time. The status is +stripped from the active-state checkpoint, the recovery mirror and the +degraded recovery projection, so a held alert adds no durable writes, and +after a restart it is absent until the next evaluation. Resolved records carry +no live status. Stale-alert cleanup and the retention TTLs use the latest +observation (`lastObservedAt`), so an alert held below its trigger for over a +day is not auto-resolved as unmonitored. Frontend surfaces format the status +through `features/alerts/metricAlertPresentation.ts` and never re-derive the +phase or recovery timing. + ### Retained guest observations are not alert evidence Guest memory marked `last-known`, `unavailable` or an unknown observation state diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e410966406..6c38e23650 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -20,6 +20,19 @@ ## Purpose +### Open threshold alerts expose their live evaluation — issue #2068 + +`Alert.metricStatus` (`/api/alerts/active`, the websocket state and +`/api/state`) is optional and additive. It carries `phase` (`breaching`, +`latched` or `recovering`), `value` (the value compared with the rule, the +average for a rolling-average rule) with `rawValue` and +`evaluationWindowSeconds` when windowed, `unit`, `observedAt`, `trigger`, +`recovery` (the alert clears at or below it), `recoveryDelaySeconds`, and, +while recovering, `recoveryStartedAt` and `recoveryElapsedSeconds`. Legacy +`value`, `message` and `lastSeen` keep meaning the last reading that met the +trigger. The field is absent for non-threshold alerts and until the first +evaluation after a restart, and clients then fall back to `message`. + ### Organisation deletion retains data after incomplete monitoring shutdown Authenticated organisation-owner deletion waits for tenant-loop exit and sealed diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 308d1470e2..1d8ce207c1 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -22,6 +22,14 @@ All drawer History fallbacks must distinguish current observations from retained, unavailable or freshness-unknown guest readings, using original memory evidence and filesystem read reasons. +### Drawer attention detail line and temperature alert floor — issue #2068 + +`DrawerAttentionSection` items accept an optional `detail` (a muted second +line) and `title` (hover text). `TemperatureGauge` accepts `alertSeverity` +and `title`, and `getTemperatureTextClass` treats an open alert's severity as +a floor: a reading that has dipped under an open alert's trigger keeps the +alert's tone, while the reading still wins when it is worse. + ### Canonical drawer History preserves guest read provenance The shared resource drawer passes selected memory observation state/source/time diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index f02a9c2224..2becd58456 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -17,6 +17,14 @@ ## Purpose +### Active-alert state projection carries the live metric status — issue #2068 + +`activeAlertsSnapshot` copies the alert manager's volatile `MetricStatus` +into `models.Alert.metricStatus`, so the websocket and `/api/state` alert +lists carry the reading being evaluated now for a threshold alert held below +its trigger, while `value` and `message` keep the last breach. The projection +adds no copy of its own: `GetActiveAlerts` already returns deep clones. + ### Fresh broadcast projection owns one resource/target capture Canonical live, fallback and standalone-host continuity broadcast reads prefer diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 50344121e9..44f18cda41 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -17,6 +17,13 @@ Drawer observation labels derive locally from the selected snapshot without renewing timestamps, adding reads, remounting charts or changing stored metric coordinates. +### Guest drawer alert copy formats backend status only + +Guest drawer attention rows format `alert.metricStatus` through +`getGuestDrawerAlertAttention` and the shared `getAlertAttentionCopy`. No +threshold comparison, recovery timing or resource-metric lookup runs in the +drawer, and the host-capacity memory comparison stays a suffix on that copy. + ### Drawer provenance stays local to snapshot presentation Guest and canonical resource drawer memory presentation share the same pure diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index ebbc81311a..75882a9d08 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -21,6 +21,14 @@ ## Purpose +### Storage usage headline follows the alert phase — issue #2068 + +`describeStorageAlertHeadline` says "Over N% usage limit" only while a usage +alert is breaching. When `metricStatus.phase` is `latched` or `recovering`, +the row says usage is under the limit and names the clear level, so the +headline never contradicts the usage bar beside it. Fill-forecast headlines +are unchanged. + ### Guest-memory read state stays independent of backup disk state The existing guest drawer retains Core's selected memory provenance through diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index a22df85cf1..2bd755d13a 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -23,6 +23,14 @@ and sort the complete canonical change table while startup and ingestion wait. ## Purpose +### Proxmox node temperature cell follows its own open alert — issue #2068 + +`ProxmoxNodesTable` finds the node's own temperature alert through +`hostOverrideIdCandidates` and passes its severity (unless acknowledged) and +the shared live-status copy to `TemperatureGauge`. Other node alerts never +recolour the cell. The Mail Gateway drawer formats its attention rows through +`getAlertAttentionCopy`. + ### Canonical Discovery honours VM guest-read deferrals The canonical resource drawer uses the same fixed PVE VM lock, backup and diff --git a/frontend-modern/browser-verification/4766668c29936861.json b/frontend-modern/browser-verification/4766668c29936861.json new file mode 100644 index 0000000000..af31ef6f50 --- /dev/null +++ b/frontend-modern/browser-verification/4766668c29936861.json @@ -0,0 +1,62 @@ +{ + "version": 1, + "base_sha": "92d946d057ddeefcae4d247a87e88f2245abca4a", + "verified_at": "2026-10-06T12:36:53Z", + "result": "passed", + "changed_paths": [ + "frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts", + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx", + "frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx", + "frontend-modern/src/components/Workloads/guestDrawerModel.ts", + "frontend-modern/src/components/shared/DrawerAttentionSection.tsx", + "frontend-modern/src/components/shared/TemperatureGauge.tsx", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts", + "frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx", + "frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts", + "frontend-modern/src/types/api.ts", + "frontend-modern/src/utils/temperature.ts" + ], + "content_sha256": { + "frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts": "3c6f6f01df4a13b9fb3084b398be093473b63cdb71bbc01db1dbf41531e15edd", + "frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx": "667a9b9e6fd4385a23b950f18be16fb71cbd6eccc2a4aafb57d2c4532b3af6f1", + "frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx": "5adf11bfc948cae2cca66fde5e2132c66bbb01388efdbc23cbf022b449ae3b0c", + "frontend-modern/src/components/Workloads/guestDrawerModel.ts": "a922c4c26813a676be12e06376bdb40df2aca8bfe5fc850b76009a504dada663", + "frontend-modern/src/components/shared/DrawerAttentionSection.tsx": "2a4e15e781a6158e253b6b934198d7aed71275416eb12c07ade68d5fa80c0332", + "frontend-modern/src/components/shared/TemperatureGauge.tsx": "612172a594698216b8a5fe247e5fa7361946056e02eaba9e71cfe827d0dc5023", + "frontend-modern/src/features/alerts/metricAlertPresentation.ts": "2c875f102207c36d21d2868fa6bf05be7a5ae3e1cfb0351f54c2e89d448259bc", + "frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx": "c979737f3825f12da80b25f50083b15960f7217525905c01d27edc4b26e1fbc3", + "frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx": "b3ebf1259281b7d590c4cbd0eee9ed54ad060ed0e6fd1d80f740d44c95cd3107", + "frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts": "ad5c973c1fbe64a94ad98d8f89d2ac3e51dacad1b1d06af82f77d0d32462ea16", + "frontend-modern/src/types/api.ts": "0db85005fb024e72434e35fdb7eb9ce13d70361569563579365a877fac714f79", + "frontend-modern/src/utils/temperature.ts": "3381a7948f19ae52bf0c72d6fc011ded3691aeac9fdbc9d795dc298af68c4690" + }, + "routes": [ + "/proxmox" + ], + "viewports": [ + { + "width": 1440, + "height": 900 + }, + { + "width": 375, + "height": 812 + } + ], + "states": [ + "Private mock stack built from this change (sidecar backend on 7721, Vite on 5241); pve2's CPU temperature driven through 88, 78 and 72 degrees C by a local-only, uncommitted mock overlay", + "Breaching (88C, trigger 80/clear 75): node drawer Needs attention reads 'Temperature 88°C, above the 80°C alert level' with 'Clears once it drops to 75°C or lower and stays there for 5 minutes.'", + "Held (78C): drawer reads 'Temperature 78°C now, back under the 80°C alert level' / 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.'; legacy alert message still 'Node temperature at 88.0°C' in the API; TEMP cell amber with the same explanation as hover text", + "Recovering (72C, below the 75C warning colour): drawer reads 'Temperature 72°C now, recovering' / 'Clears after 5 minutes at 75°C or lower, 1 minute so far.'; TEMP cell 72°C stays amber (text-amber-600) while 68°C on the next node is green", + "Resolved after 5 minutes at 72C: alert closed with 'Resolved: Node temperature at 72.0°C', TEMP cell green with no hover text, Needs attention section gone", + "Narrow 375px: drawer attention copy wraps inside the card, TEMP column hidden as before", + "Guest drawer, Mail Gateway drawer, storage headline and Assistant handoff copy covered by component and model tests, not exercised live" + ], + "interactions": [ + "Expanded the West Production B node row to open its inline drawer at desktop and phone widths", + "Read the TEMP cell colour class and title text through the DOM in each phase", + "Reloaded the page mid-recovery to confirm the cell and drawer render from fresh state", + "Resized between 1440x900 and 375x812" + ] +} diff --git a/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts b/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts index 9f23bdbe9a..1d9d579301 100644 --- a/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts +++ b/frontend-modern/src/components/Alerts/__tests__/alertAssistantHandoffModel.test.ts @@ -157,4 +157,35 @@ describe('alertAssistantHandoffModel', () => { const briefing = handoff.context.briefing as { detailLines: string[] }; expect(briefing.detailLines).toContain('Current value 92.5% · threshold 80.0%'); }); + + it('hands the assistant the live reading, not the breach an open alert still holds', () => { + const handoff = buildAlertAssistantHandoff({ + alert: makeAlert({ + type: 'temperature', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: '2026-05-07T10:04:50.000Z', + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + recoveryElapsedSeconds: 120, + }, + }), + now: new Date('2026-05-07T10:05:00.000Z'), + }); + + const briefing = handoff.context.briefing as { detailLines: string[] }; + expect(briefing.detailLines[0]).toContain('72.0°C'); + expect(briefing.detailLines.join('\n')).not.toContain('Node temperature at 80.0°C'); + expect(handoff.context.handoffContext).toContain('Current Value: 72.0°C'); + expect(handoff.context.handoffContext).toContain('Alert Phase: recovering'); + expect(handoff.context.handoffContext).toContain('Clears At: 75.0°C or lower for 300s'); + expect(handoff.context.handoffContext).toContain('Recovery Progress: 120s of 300s'); + expect(handoff.context.handoffContext).toContain('Last Reading At Or Above Threshold: 80.0°C'); + }); }); diff --git a/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts b/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts index 3a6a08b9c7..4ca70859cb 100644 --- a/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts +++ b/frontend-modern/src/components/Alerts/alertAssistantHandoffModel.ts @@ -2,6 +2,7 @@ import type { Alert } from '@/types/api'; import type { AIChatContext } from '@/stores/aiChat'; import { DEFAULT_LOCALE, t, type SupportedLocale } from '@/i18n'; import { getCanonicalAlertId } from '@/features/alerts/identity'; +import { getMetricAlertPresentation } from '@/features/alerts/metricAlertPresentation'; import { formatAlertValue } from '@/utils/alertFormatters'; import { isMetricAlertType } from '@/utils/alerts'; import { isPulseSystemAlert } from '@/utils/alertScope'; @@ -35,7 +36,16 @@ export function buildAlertAssistantHandoff({ // to convey what's wrong. const systemScoped = isPulseSystemAlert(alert); const hasMetricValues = !systemScoped && isMetricAlertType(alert.type); - const currentValue = hasMetricValues ? formatAlertValue(alert.value, alert.type) : ''; + // An open threshold alert keeps value/message at its last breach while it + // holds; the live status is what the metric reads now. + const metricStatus = hasMetricValues ? alert.metricStatus : undefined; + const presentation = metricStatus ? getMetricAlertPresentation(alert, now.getTime()) : null; + const currentValue = hasMetricValues + ? formatAlertValue(metricStatus ? metricStatus.value : alert.value, alert.type) + : ''; + const briefingMessage = presentation + ? `${presentation.summary}. ${presentation.detail}` + : alert.message; const thresholdValue = hasMetricValues ? formatAlertValue(alert.threshold, alert.type) : ''; const nodeLabel = !systemScoped && alert.node ? alert.nodeDisplayName || alert.node : ''; const levelLabel = formatAlertLevel(alert.level); @@ -98,8 +108,8 @@ export function buildAlertAssistantHandoff({ }) : undefined, nodeLabel ? t('alerts.assistant.detail.node', { node: nodeLabel }) : undefined, - alert.message - ? t('alerts.assistant.detail.message', { message: alert.message }) + briefingMessage + ? t('alerts.assistant.detail.message', { message: briefingMessage }) : undefined, ].filter((line): line is string => Boolean(line)), actionLabel: t('alerts.assistant.action.investigate', { alertIdentifier }), @@ -149,6 +159,7 @@ function buildAlertAssistantModelContext({ systemScoped ? undefined : formatContextLine('Resource ID', alert.resourceId), formatContextLine('Current Value', currentValue), formatContextLine('Threshold', thresholdValue), + ...metricStatusContextLines(alert), formatContextLine('Duration', durationText), formatContextLine('Node', nodeLabel), formatContextLine('Message', alert.message), @@ -158,6 +169,34 @@ function buildAlertAssistantModelContext({ .join('\n'); } +const METRIC_PHASE_CONTEXT: Record = { + breaching: 'breaching (at or above the threshold)', + latched: 'holding (below the threshold, not yet down to the clear level)', + recovering: 'recovering (at or below the clear level, waiting out the recovery delay)', +}; + +function metricStatusContextLines(alert: Alert): Array { + const status = isMetricAlertType(alert.type) ? alert.metricStatus : undefined; + if (!status) return []; + const delay = status.recoveryDelaySeconds ?? 0; + return [ + formatContextLine('Alert Phase', METRIC_PHASE_CONTEXT[status.phase] ?? status.phase), + formatContextLine( + 'Clears At', + `${formatAlertValue(status.recovery, alert.type)} or lower${delay > 0 ? ` for ${delay}s` : ''}`, + ), + status.phase === 'recovering' && status.recoveryElapsedSeconds + ? formatContextLine('Recovery Progress', `${status.recoveryElapsedSeconds}s of ${delay}s`) + : undefined, + status.phase === 'breaching' + ? undefined + : formatContextLine( + 'Last Reading At Or Above Threshold', + formatAlertValue(alert.value, alert.type), + ), + ]; +} + function formatContextLine(label: string, value?: string | number | null): string | undefined { if (value === undefined || value === null) return undefined; const text = String(value).trim(); diff --git a/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx b/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx index 443cbbd5d5..0438d3f0a8 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawer.test.tsx @@ -528,6 +528,46 @@ describe('GuestDrawer', () => { ).toBeInTheDocument(); }); + it('leads with the live reading when an open alert holds below its trigger', () => { + solidRender(() => ( + + )); + + expect(screen.getByText('Memory 87% now, back under the 90% alert level')).toBeInTheDocument(); + expect( + screen.getByText('Stays open until it reaches 85% or lower and stays there for 5 minutes.'), + ).toBeInTheDocument(); + expect(screen.queryByText('VM memory at 91.0%')).not.toBeInTheDocument(); + }); + it('keeps generic Assistant and context-copy actions out of the drawer header', () => { render(() => ); diff --git a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx index 8ad1d0c370..de41cfe2e4 100644 --- a/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/GuestDrawerOverview.tsx @@ -21,7 +21,7 @@ import { import { AvailabilityProbeSuggestionCard } from './AvailabilityProbeSuggestionCard'; import { buildWorkloadsDiskPresentation } from './diskListModel'; import { - getGuestDrawerAlertMessage, + getGuestDrawerAlertAttention, getGuestDrawerMemoryRows, getGuestDrawerMemoryReading, getGuestDrawerGuestReadPresentation, @@ -297,7 +297,7 @@ export function GuestDrawerOverview(props: GuestDrawerOverviewProps) { ({ id: alert.id, - message: getGuestDrawerAlertMessage(alert, { + ...getGuestDrawerAlertAttention(alert, { guest: props.guest, memoryDisplayBasis: props.memoryDisplayBasis, parentMemoryTotal: props.parentMemoryTotal, diff --git a/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx b/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx index 50957eac70..218cd127ba 100644 --- a/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx +++ b/frontend-modern/src/components/Workloads/NodeDrawerOverview.tsx @@ -9,6 +9,7 @@ import { } from '@/components/shared/DetailSectionTable'; import type { Alert, Disk, HostGPUSensor, Node, Temperature } from '@/types/api'; import { alertTypeDisplayLabel } from '@/features/alerts/helpers'; +import { getAlertAttentionCopy } from '@/features/alerts/metricAlertPresentation'; import { formatBytes, normalizeDiskArray } from '@/utils/format'; import type { MetricDisplayThresholds } from '@/utils/metricThresholds'; import { getProxmoxUpdateEvidencePresentation } from '@/utils/proxmoxUpdateEvidence'; @@ -370,7 +371,7 @@ export function NodeDrawerOverview(props: NodeDrawerOverviewProps) { ({ id: alert.id, - message: alert.message, + ...getAlertAttentionCopy(alert), subject: cleanText(alert.resourceName) || cleanText(alert.nodeDisplayName) || diff --git a/frontend-modern/src/components/Workloads/guestDrawerModel.ts b/frontend-modern/src/components/Workloads/guestDrawerModel.ts index f78290928c..27fa776ad3 100644 --- a/frontend-modern/src/components/Workloads/guestDrawerModel.ts +++ b/frontend-modern/src/components/Workloads/guestDrawerModel.ts @@ -7,6 +7,10 @@ import type { } from '@/api/charts'; import { formatHistoryChartTooltipValue } from '@/components/shared/historyChartModel'; +import { + getAlertAttentionCopy, + type AlertAttentionCopy, +} from '@/features/alerts/metricAlertPresentation'; import { formatBytes, formatPercent, getBackupInfo, type BackupThresholds } from '@/utils/format'; import { getWorkloadGuestDiskStatusMessage, @@ -82,12 +86,13 @@ export interface GuestDrawerBackupPresentation { export const isGuestDrawerVM = (guest: Guest): guest is VM => resolveWorkloadType(guest) === 'vm'; -export const getGuestDrawerAlertMessage = ( +export const getGuestDrawerAlertAttention = ( alert: Alert, context: Pick, -): string => { +): AlertAttentionCopy => { + const copy = getAlertAttentionCopy(alert); if (context.memoryDisplayBasis !== 'host' || alert.type.trim().toLowerCase() !== 'memory') { - return alert.message; + return copy; } const used = context.guest.memory?.used; @@ -104,7 +109,7 @@ export const getGuestDrawerAlertMessage = ( const comparison = hostShare ? `guest allocation · ${hostShare} of host capacity` : 'guest allocation · row uses host capacity'; - return `${alert.message} (${comparison})`; + return { ...copy, message: `${copy.message} (${comparison})` }; }; const getGuestDrawerDiskUsage = (guest: Guest): number | undefined => { diff --git a/frontend-modern/src/components/shared/DrawerAttentionSection.tsx b/frontend-modern/src/components/shared/DrawerAttentionSection.tsx index c0d8fd86d1..fc3d5c6b5f 100644 --- a/frontend-modern/src/components/shared/DrawerAttentionSection.tsx +++ b/frontend-modern/src/components/shared/DrawerAttentionSection.tsx @@ -10,6 +10,10 @@ import { Button } from './Button'; export interface DrawerAttentionItem { id: string; message: string; + /** Secondary line, e.g. what clears an alert whose reading has dropped. */ + detail?: string; + /** Hover text when it should say more than the message. */ + title?: string; subject?: string; metric?: string; severity?: string; @@ -73,10 +77,13 @@ export const DrawerAttentionSection: Component = (p

{item.message}

+ +

{item.detail}

+ { expect(drawerAttentionSectionSource).toContain('Show fewer alerts'); expect(drawerAttentionSectionSource).toContain('aria-expanded={expanded()}'); expect(drawerAttentionSectionSource).not.toContain('DetailSectionTable'); + // A threshold alert held under its trigger leads with the live reading and + // a second line saying what clears it, never only its stale breach text. + expect(drawerAttentionSectionSource).toContain('item.detail'); + expect(drawerAttentionSectionSource).toContain('item.title ?? item.message'); + for (const liveAlertConsumer of [ + 'src/components/Workloads/NodeDrawerOverview.tsx', + 'src/features/proxmox/ProxmoxMailGatewayDrawer.tsx', + ]) { + const source = readFrontendSource(liveAlertConsumer); + expect(source).toContain('...getAlertAttentionCopy(alert)'); + expect(source).not.toContain('message: alert.message'); + } + expect(readFrontendSource('src/components/Workloads/guestDrawerModel.ts')).toContain( + 'getAlertAttentionCopy(alert)', + ); const overviewConsumers = [ 'src/components/Infrastructure/ResourceDetailDrawerOverviewTab.tsx', diff --git a/frontend-modern/src/components/shared/TemperatureGauge.tsx b/frontend-modern/src/components/shared/TemperatureGauge.tsx index 01e0c179bc..f5622ab88b 100644 --- a/frontend-modern/src/components/shared/TemperatureGauge.tsx +++ b/frontend-modern/src/components/shared/TemperatureGauge.tsx @@ -14,6 +14,9 @@ interface TemperatureGaugeProps { warning?: number; thresholds?: MetricDisplayThresholds | null; metric?: TemperatureDisplayMetric; + /** Severity of an open alert on this reading; keeps its tone while the alert holds. */ + alertSeverity?: 'warning' | 'critical' | null; + title?: string; class?: string; } @@ -30,11 +33,14 @@ export const TemperatureGauge: Component = (props) => { }); const textColorClass = createMemo(() => - getTemperatureTextClass(props.value, explicitThresholds(), props.metric), + getTemperatureTextClass(props.value, explicitThresholds(), props.metric, props.alertSeverity), ); return ( - + {formatTemperature(props.value)} ); diff --git a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts index 160cc8da7a..676038b14f 100644 --- a/frontend-modern/src/features/alerts/__tests__/helpers.test.ts +++ b/frontend-modern/src/features/alerts/__tests__/helpers.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Alert, MetricAlertStatus } from '@/types/api'; import { clampMaxAlertsPerHour, fallbackMaxAlertsPerHour, @@ -12,6 +13,11 @@ import { alertTypeDisplayLabel, clampEscalationDelayMinutes, } from '@/features/alerts/helpers'; +import { + METRIC_ALERT_STATUS_STALE_MS, + getAlertAttentionCopy, + getMetricAlertPresentation, +} from '@/features/alerts/metricAlertPresentation'; describe('alerts helpers', () => { describe('clampEscalationDelayMinutes', () => { @@ -229,3 +235,148 @@ describe('alerts helpers', () => { }); }); }); + +const NOW = Date.parse('2026-10-06T10:52:00Z'); + +// minipc on 2026-10-06: the alert opened at 80°C, the node then read 71-76°C, +// and the drawer kept saying "Node temperature at 80.0°C". +const minipcAlert = (status?: Partial): Alert => ({ + id: 'homelab-minipc::metric-threshold:temperature', + type: 'temperature', + level: 'warning', + resourceId: 'homelab-minipc', + resourceName: 'minipc', + node: 'minipc', + instance: 'homelab', + message: 'Node temperature at 80.0°C', + value: 80, + threshold: 80, + startTime: '2026-10-06T07:57:44Z', + lastSeen: '2026-10-06T10:48:52Z', + acknowledged: false, + metricStatus: status + ? { + phase: 'latched', + value: 76, + unit: '°C', + observedAt: new Date(NOW - 10_000).toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + ...status, + } + : undefined, +}); + +describe('getMetricAlertPresentation', () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('leads with the reading now and what clears an alert held under its trigger', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ phase: 'latched', value: 76 }), + NOW, + ); + expect(presentation).toMatchObject({ + phase: 'latched', + stale: false, + summary: 'Temperature 76°C now, back under the 80°C alert level', + detail: 'Stays open until it reaches 75°C or lower and stays there for 5 minutes.', + phaseLabel: 'Alert still open', + }); + expect(presentation?.lastBreach).toMatch(/^Last reading at or above 80°C: 80°C, /); + }); + + it('says a reading at the clear level is recovering, with progress once it has some', () => { + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 0 }), + NOW, + ), + ).toMatchObject({ + summary: 'Temperature 72°C now, recovering', + detail: 'Clears after 5 minutes at 75°C or lower.', + phaseLabel: 'Recovering', + }); + expect( + getMetricAlertPresentation( + minipcAlert({ phase: 'recovering', value: 72, recoveryElapsedSeconds: 120 }), + NOW, + )?.detail, + ).toBe('Clears after 5 minutes at 75°C or lower, 2 minutes so far.'); + }); + + it('names the alert level while the reading is still over it', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ phase: 'breaching', value: 82 }), + NOW, + ); + expect(presentation).toMatchObject({ + summary: 'Temperature 82°C, above the 80°C alert level', + detail: 'Clears once it drops to 75°C or lower and stays there for 5 minutes.', + phaseLabel: 'Above 80°C', + }); + expect(presentation?.lastBreach).toBeUndefined(); + }); + + it('stops calling an old evaluation "now"', () => { + const presentation = getMetricAlertPresentation( + minipcAlert({ + phase: 'latched', + value: 76, + observedAt: new Date(NOW - METRIC_ALERT_STATUS_STALE_MS - 60_000).toISOString(), + }), + NOW, + ); + expect(presentation?.stale).toBe(true); + expect(presentation?.summary).toMatch(/^Last reading: Temperature 76°C, /); + expect(presentation?.summary).not.toContain('now'); + expect(presentation?.phaseLabel).toBe('No recent reading'); + }); + + it('shows the averaged value the rule follows beside the latest sample', () => { + const alert: Alert = { + ...minipcAlert(), + type: 'cpu', + metricStatus: { + phase: 'latched', + value: 84.2, + rawValue: 61, + evaluationWindowSeconds: 600, + unit: '%', + observedAt: new Date(NOW).toISOString(), + trigger: 90, + recovery: 80, + }, + }; + expect(getMetricAlertPresentation(alert, NOW)).toMatchObject({ + summary: 'CPU averaged 84% over 10 minutes, latest 61% now, back under the 90% alert level', + detail: 'Stays open until it reaches 80% or lower.', + }); + }); + + it('returns null without a live status so callers keep the alert message', () => { + expect(getMetricAlertPresentation(minipcAlert(), NOW)).toBeNull(); + expect(getAlertAttentionCopy(minipcAlert(), NOW)).toEqual({ + message: 'Node temperature at 80.0°C', + }); + }); +}); + +describe('getAlertAttentionCopy', () => { + it('replaces the stale breach message and keeps the breach in the hover text', () => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + const copy = getAlertAttentionCopy(minipcAlert({ phase: 'recovering', value: 72 }), NOW); + vi.useRealTimers(); + expect(copy.message).toBe('Temperature 72°C now, recovering'); + expect(copy.detail).toBe('Clears after 5 minutes at 75°C or lower.'); + expect(copy.title).toContain('Last reading at or above 80°C: 80°C'); + expect(copy.message).not.toContain('80'); + }); +}); diff --git a/frontend-modern/src/features/alerts/metricAlertPresentation.ts b/frontend-modern/src/features/alerts/metricAlertPresentation.ts new file mode 100644 index 0000000000..8d72d7c312 --- /dev/null +++ b/frontend-modern/src/features/alerts/metricAlertPresentation.ts @@ -0,0 +1,157 @@ +import type { Alert, MetricAlertPhase, MetricAlertStatus } from '@/types/api'; +import { formatRelativeTime } from '@/utils/format'; +import { formatTemperature } from '@/utils/temperature'; +import { alertTypeDisplayLabel } from './helpers'; + +/** + * A live status older than this is no longer "now": the resource stopped + * reporting, or Pulse stopped evaluating it. Generous enough for the slowest + * poll intervals. + */ +export const METRIC_ALERT_STATUS_STALE_MS = 10 * 60 * 1000; + +export interface MetricAlertPresentation { + phase: MetricAlertPhase; + /** The live status is too old to describe as the current reading. */ + stale: boolean; + /** Leads with the reading Pulse is evaluating now and where it stands. */ + summary: string; + /** What clears the alert from here. */ + detail: string; + /** Short state for compact cells and badges. */ + phaseLabel: string; + /** The last reading that met the trigger, once the live one differs. */ + lastBreach?: string; +} + +const formatMetricValue = (value: number, unit: string | undefined): string => { + if (unit === '°C') return formatTemperature(value); + if (unit === '%') { + return `${Math.abs(value) >= 10 ? Math.round(value) : Number(value.toFixed(1))}%`; + } + const rounded = Number(value.toFixed(1)); + return unit ? `${rounded} ${unit}` : String(rounded); +}; + +const formatSeconds = (seconds: number): string => { + if (seconds < 60) return `${seconds} ${seconds === 1 ? 'second' : 'seconds'}`; + if (seconds < 3600) { + const minutes = Math.round(seconds / 60); + return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`; + } + const hours = Number((seconds / 3600).toFixed(1)); + return `${hours} ${hours === 1 ? 'hour' : 'hours'}`; +}; + +const describeReading = (label: string, status: MetricAlertStatus): string => { + const value = formatMetricValue(status.value, status.unit); + const window = status.evaluationWindowSeconds ?? 0; + if (window <= 0) return `${label} ${value}`; + const latest = + typeof status.rawValue === 'number' && Number.isFinite(status.rawValue) + ? `, latest ${formatMetricValue(status.rawValue, status.unit)}` + : ''; + return `${label} averaged ${value} over ${formatSeconds(window)}${latest}`; +}; + +const describeClearRule = (status: MetricAlertStatus, lead: string): string => { + const recovery = formatMetricValue(status.recovery, status.unit); + const delay = status.recoveryDelaySeconds ?? 0; + return delay > 0 + ? `${lead} ${recovery} or lower and stays there for ${formatSeconds(delay)}.` + : `${lead} ${recovery} or lower.`; +}; + +const isUsableStatus = (status: MetricAlertStatus | undefined): status is MetricAlertStatus => + !!status && + Number.isFinite(status.value) && + Number.isFinite(status.trigger) && + Number.isFinite(status.recovery) && + (status.phase === 'breaching' || status.phase === 'latched' || status.phase === 'recovering'); + +/** + * Describes an open threshold alert from the backend's live evaluation, so + * every surface says what the reading is now and why the alert is still open + * instead of repeating the last breach. Returns null when the alert carries + * no live status (non-threshold alerts, or the moments after a restart). + */ +export function getMetricAlertPresentation( + alert: Pick, + now: number = Date.now(), +): MetricAlertPresentation | null { + const status = alert.metricStatus; + if (!isUsableStatus(status)) return null; + + const label = alertTypeDisplayLabel(alert.type); + const reading = describeReading(label, status); + const trigger = formatMetricValue(status.trigger, status.unit); + const observedAt = Date.parse(status.observedAt); + const stale = !Number.isFinite(observedAt) || now - observedAt > METRIC_ALERT_STATUS_STALE_MS; + + let summary: string; + let detail: string; + let phaseLabel: string; + switch (status.phase) { + case 'breaching': + summary = `${reading}, above the ${trigger} alert level`; + detail = describeClearRule(status, 'Clears once it drops to'); + phaseLabel = `Above ${trigger}`; + break; + case 'latched': + summary = `${reading} now, back under the ${trigger} alert level`; + detail = describeClearRule(status, 'Stays open until it reaches'); + phaseLabel = 'Alert still open'; + break; + case 'recovering': { + const delay = status.recoveryDelaySeconds ?? 0; + const elapsed = status.recoveryElapsedSeconds ?? 0; + const recovery = formatMetricValue(status.recovery, status.unit); + summary = `${reading} now, recovering`; + detail = + delay > 0 + ? `Clears after ${formatSeconds(delay)} at ${recovery} or lower${ + elapsed > 0 ? `, ${formatSeconds(Math.min(elapsed, delay))} so far` : '' + }.` + : `Clears at ${recovery} or lower.`; + phaseLabel = 'Recovering'; + break; + } + } + + if (stale) { + summary = `Last reading: ${reading}${ + Number.isFinite(observedAt) ? `, ${formatRelativeTime(observedAt)}` : '' + }`; + phaseLabel = 'No recent reading'; + } + + let lastBreach: string | undefined; + if (status.phase !== 'breaching' && Number.isFinite(alert.value)) { + const when = alert.lastSeen ? `, ${formatRelativeTime(alert.lastSeen)}` : ''; + lastBreach = `Last reading at or above ${trigger}: ${formatMetricValue(alert.value, status.unit)}${when}`; + } + + return { phase: status.phase, stale, summary, detail, phaseLabel, lastBreach }; +} + +export interface AlertAttentionCopy { + message: string; + detail?: string; + title?: string; +} + +/** Copy for an alert row in a drawer's "Needs attention" list. */ +export function getAlertAttentionCopy( + alert: Pick, + now: number = Date.now(), +): AlertAttentionCopy { + const presentation = getMetricAlertPresentation(alert, now); + if (!presentation) return { message: alert.message }; + return { + message: presentation.summary, + detail: presentation.detail, + title: [presentation.summary, presentation.detail, presentation.lastBreach] + .filter(Boolean) + .join('\n'), + }; +} diff --git a/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx b/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx index bca5931740..6211981044 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxMailGatewayDrawer.tsx @@ -1,6 +1,7 @@ import { For, Show, createMemo, createResource, type Component } from 'solid-js'; import { DrawerAttentionSection } from '@/components/shared/DrawerAttentionSection'; import { alertTypeDisplayLabel } from '@/features/alerts/helpers'; +import { getAlertAttentionCopy } from '@/features/alerts/metricAlertPresentation'; import type { Alert } from '@/types/api'; import { Card } from '@/components/shared/Card'; import { DrawerSubjectHeading } from '@/components/shared/DrawerSubjectHeading'; @@ -351,7 +352,7 @@ export const ProxmoxMailGatewayDrawer: Component<{ ({ id: alert.id, - message: alert.message, + ...getAlertAttentionCopy(alert), subject: name(), metric: alertTypeDisplayLabel(alert.type), severity: alert.level, diff --git a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx index 932f0767e3..191809f431 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxNodesTable.tsx @@ -21,6 +21,7 @@ import { StackedDiskBar } from '@/components/Workloads/StackedDiskBar'; import { MetricMiniSparkline } from '@/components/Workloads/MetricMiniSparkline'; import { TemperatureGauge } from '@/components/shared/TemperatureGauge'; import { hostOverrideIdCandidates } from '@/features/alerts/alertOverridesModel'; +import { getMetricAlertPresentation } from '@/features/alerts/metricAlertPresentation'; import { useBreakpoint } from '@/hooks/useBreakpoint'; import { TableCell, TableRow } from '@/components/shared/Table'; import { SUMMARY_ROW_ACTION_BUTTON_FOCUS_CLASS } from '@/components/shared/summaryInteractionA11y'; @@ -451,6 +452,26 @@ export const ProxmoxNodesTable: Component<{ const alertResourceIds = () => hostOverrideIdCandidates(node); const temperatureThresholds = () => alertsActivation.getMetricThresholds('node', 'temperature', alertResourceIds()); + // The node's own temperature alert holds after the reading dips + // under its trigger; the cell keeps the alert's tone and says why + // instead of reading green beside a warning row. + const temperatureAlert = createMemo(() => + getAlertsForResource(alertResourceIds(), activeAlerts, alertsEnabled()).find( + (alert) => alert.type === 'temperature', + ), + ); + const temperatureAlertSeverity = () => { + const alert = temperatureAlert(); + if (!alert || alert.acknowledged) return null; + return alert.level === 'critical' ? 'critical' : 'warning'; + }; + const temperatureAlertTitle = () => { + const alert = temperatureAlert(); + const presentation = alert ? getMetricAlertPresentation(alert) : null; + return presentation + ? `${presentation.summary}. ${presentation.detail}` + : alert?.message; + }; const cpuThresholds = () => alertsActivation.getMetricThresholds('node', 'cpu', alertResourceIds()); const memoryThresholds = () => @@ -729,6 +750,8 @@ export const ProxmoxNodesTable: Component<{ diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx index e985df726d..fc4e8b9ae9 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx @@ -46,8 +46,18 @@ vi.mock('@/components/Workloads/MetricMiniSparkline', () => ({ })); vi.mock('@/components/shared/TemperatureGauge', () => ({ - TemperatureGauge: (props: { value: number; thresholds?: unknown }) => { - temperatureGaugeMock({ value: props.value, thresholds: props.thresholds }); + TemperatureGauge: (props: { + value: number; + thresholds?: unknown; + alertSeverity?: string | null; + title?: string; + }) => { + temperatureGaugeMock({ + value: props.value, + thresholds: props.thresholds, + alertSeverity: props.alertSeverity, + title: props.title, + }); return
; }, })); @@ -358,6 +368,86 @@ describe('ProxmoxNodesTable', () => { ); }); + it('keeps an open temperature alert visible on a reading that dipped under its trigger', () => { + // minipc on 2026-10-06: the row warned "80.0°C" while the cell read a green 72°C. + activeAlertsMock.value = { + 'cpu-alert': { + id: 'cpu-alert', + resourceId: 'agent:pve-node-1', + type: 'cpu', + level: 'critical', + message: 'Node cpu at 97.0%', + value: 97, + acknowledged: false, + }, + 'temperature-alert': { + id: 'temperature-alert', + resourceId: 'agent:pve-node-1', + type: 'temperature', + level: 'warning', + message: 'Node temperature at 80.0°C', + value: 80, + acknowledged: false, + metricStatus: { + phase: 'recovering', + value: 72, + unit: '°C', + observedAt: new Date().toISOString(), + trigger: 80, + recovery: 75, + recoveryDelaySeconds: 300, + }, + }, + }; + + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + // The node's own temperature alert sets the tone; its CPU alert does not. + expect(temperatureGaugeMock).toHaveBeenLastCalledWith( + expect.objectContaining({ + value: 72, + alertSeverity: 'warning', + title: 'Temperature 72°C now, recovering. Clears after 5 minutes at 75°C or lower.', + }), + ); + }); + + it('leaves the temperature tone to the reading once the alert is acknowledged', () => { + activeAlertsMock.value = { + 'temperature-alert': { + id: 'temperature-alert', + resourceId: 'agent:pve-node-1', + type: 'temperature', + level: 'warning', + message: 'Node temperature at 80.0°C', + value: 80, + acknowledged: true, + }, + }; + + render(() => ( + } + emptyTitle="No Proxmox VE nodes" + emptyDescription="No nodes" + /> + )); + + expect(temperatureGaugeMock).toHaveBeenLastCalledWith( + expect.objectContaining({ alertSeverity: null, title: 'Node temperature at 80.0°C' }), + ); + }); + it('keeps the Proxmox cache-inclusive memory comparison label on node rows', () => { render(() => ( { expect(result.dataAlertState).toBe('acknowledged'); }); + it('does not claim usage is over the limit while the alert holds below it', () => { + const holding = (phase: 'latched' | 'recovering') => + makeAlert({ + metricStatus: { + phase, + value: phase === 'latched' ? 83 : 79, + unit: '%', + observedAt: '2026-10-04T12:30:00Z', + trigger: 85, + recovery: 80, + recoveryDelaySeconds: 300, + }, + }); + expect(describeStorageAlertHeadline(holding('latched'))).toBe( + 'Under 85% limit, clears at 80% or lower', + ); + expect(describeStorageAlertHeadline(holding('latched'), { compact: true })).toBe('Clears ≤80%'); + expect(describeStorageAlertHeadline(holding('recovering'))).toBe( + 'Recovering, clears at 80% or lower', + ); + expect(describeStorageAlertHeadline(holding('recovering'), { compact: true })).toBe( + 'Recovering', + ); + }); + it('explains a highlighted row with a fill forecast, a crossed limit or the alert text', () => { const forecast = (days: number) => makeAlert({ threshold: 100, metadata: { forecastDaysToFull: days } }); diff --git a/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts b/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts index 57c6e34cff..30d7a3aa1b 100644 --- a/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts +++ b/frontend-modern/src/features/storageBackups/storageRowAlertPresentation.ts @@ -78,6 +78,18 @@ export const describeStorageAlertHeadline = ( if (options.compact) return `Full in ~${rounded}d`; return `Full in ~${rounded} ${rounded === 1 ? 'day' : 'days'}`; } + const status = alert.type === 'usage' ? alert.metricStatus : undefined; + if (status && status.phase !== 'breaching') { + // Usage has dropped back under the limit but the alert holds until it + // reaches the clear level, so "Over" would contradict the bar beside it. + const recovery = `${Math.round(status.recovery)}%`; + if (status.phase === 'recovering') { + return options.compact ? 'Recovering' : `Recovering, clears at ${recovery} or lower`; + } + return options.compact + ? `Clears ≤${recovery}` + : `Under ${Math.round(status.trigger)}% limit, clears at ${recovery} or lower`; + } if (alert.type === 'usage' && alert.threshold > 0 && alert.threshold < 100) { return options.compact ? `Over ${alert.threshold}%` : `Over ${alert.threshold}% usage limit`; } diff --git a/frontend-modern/src/types/api.ts b/frontend-modern/src/types/api.ts index 5e7da6d6de..a455f1c4a2 100644 --- a/frontend-modern/src/types/api.ts +++ b/frontend-modern/src/types/api.ts @@ -1380,6 +1380,32 @@ export interface Alert { latestTransition?: LifecycleTransition; transitions?: LifecycleTransition[]; evidence?: EvidenceEnvelope[]; + /** + * Live evaluation behind an open threshold alert. `value`, `message` and + * `lastSeen` keep the last reading that met the trigger; this is the + * reading being evaluated now and why the alert has not cleared. + */ + metricStatus?: MetricAlertStatus; +} + +export type MetricAlertPhase = 'breaching' | 'latched' | 'recovering'; + +export interface MetricAlertStatus { + phase: MetricAlertPhase; + /** Value compared with the rule (the average for a rolling-average rule). */ + value: number; + /** Latest sample when the rule evaluates a rolling average. */ + rawValue?: number; + evaluationWindowSeconds?: number; + unit?: string; + observedAt: string; + /** Opens at or above this value. */ + trigger: number; + /** Clears after staying at or below this value for recoveryDelaySeconds. */ + recovery: number; + recoveryDelaySeconds?: number; + recoveryStartedAt?: string; + recoveryElapsedSeconds?: number; } export interface ResolvedAlert extends Alert { diff --git a/frontend-modern/src/utils/__tests__/temperature.test.ts b/frontend-modern/src/utils/__tests__/temperature.test.ts index ec4824f67f..5224735935 100644 --- a/frontend-modern/src/utils/__tests__/temperature.test.ts +++ b/frontend-modern/src/utils/__tests__/temperature.test.ts @@ -138,6 +138,22 @@ describe('temperature', () => { expect(getTemperatureTextClass(85, thresholds)).toContain('text-red-600'); }); + it("keeps an open alert's tone while the reading holds under its trigger", () => { + const thresholds = { warning: 75, critical: 80 }; + // minipc: alert open at 80°C, now reading 72°C on its way down. + expect(getTemperatureTextClass(72, thresholds)).toContain('text-green-600'); + expect(getTemperatureTextClass(72, thresholds, 'temperature', 'warning')).toContain( + 'text-amber-600', + ); + expect(getTemperatureTextClass(72, thresholds, 'temperature', 'critical')).toContain( + 'text-red-600', + ); + // The reading still wins when it is worse than the alert. + expect(getTemperatureTextClass(86, thresholds, 'temperature', 'warning')).toContain( + 'text-red-600', + ); + }); + it('uses disk temperature defaults for disk temperature displays', () => { expect(getTemperatureTextClass(49, undefined, 'diskTemperature')).toContain('text-green-600'); expect(getTemperatureTextClass(50, undefined, 'diskTemperature')).toContain('text-amber-600'); diff --git a/frontend-modern/src/utils/temperature.ts b/frontend-modern/src/utils/temperature.ts index f8b822504f..6b993af762 100644 --- a/frontend-modern/src/utils/temperature.ts +++ b/frontend-modern/src/utils/temperature.ts @@ -109,15 +109,31 @@ export type TemperatureDisplayMetric = Extract< 'temperature' | 'diskTemperature' >; +const TEMPERATURE_SEVERITY_RANK = { normal: 0, warning: 1, critical: 2 } as const; + +/** + * Colour for a temperature reading. An open alert on the reading sets a floor: + * the alert holds until the reading falls to its clear level, so a reading + * that has dipped under the trigger must not look healthy beside the alert. + */ export const getTemperatureTextClass = ( celsius: number | null | undefined, thresholds?: MetricDisplayThresholds | null, metric: TemperatureDisplayMetric = 'temperature', + openAlertSeverity?: 'warning' | 'critical' | null, ): string => { if (celsius === null || celsius === undefined || !Number.isFinite(celsius)) { return 'text-muted'; } - return TEMPERATURE_TEXT_CLASSES[ - getMetricSeverity(celsius, metric, thresholds ?? getDefaultDisplayMetricThresholds(metric)) - ]; + const readingSeverity = getMetricSeverity( + celsius, + metric, + thresholds ?? getDefaultDisplayMetricThresholds(metric), + ); + const severity = + openAlertSeverity && + TEMPERATURE_SEVERITY_RANK[openAlertSeverity] > TEMPERATURE_SEVERITY_RANK[readingSeverity] + ? openAlertSeverity + : readingSeverity; + return TEMPERATURE_TEXT_CLASSES[severity]; }; diff --git a/internal/alerts/active_cleanup.go b/internal/alerts/active_cleanup.go index 9d0a19af68..e8d24899db 100644 --- a/internal/alerts/active_cleanup.go +++ b/internal/alerts/active_cleanup.go @@ -19,7 +19,7 @@ func (m *Manager) Cleanup(maxAge time.Duration) { if alert == nil { return true } - lastSeen := alert.LastSeen + lastSeen := alert.lastObservedAt() if lastSeen.IsZero() { lastSeen = alert.StartTime } diff --git a/internal/alerts/active_persistence.go b/internal/alerts/active_persistence.go index d5c0c122a1..a1a094767c 100644 --- a/internal/alerts/active_persistence.go +++ b/internal/alerts/active_persistence.go @@ -180,6 +180,10 @@ func (m *Manager) snapshotActiveAlerts() []*Alert { clone := alert.Clone() backfillCanonicalIdentity(clone) clone.ID = exportedAlertID(clone, clone.ID) + // The live status changes on every poll while a threshold alert + // holds; keeping it out of the checkpoint keeps an unchanged alert + // set byte-identical, so the recovery mirror is not rewritten. + clone.MetricStatus = nil alerts = append(alerts, clone) } m.mu.RUnlock() @@ -308,6 +312,7 @@ func (m *Manager) setActiveRecoveryAlert(alert *Alert, fallback string) { clone := alert.Clone() backfillCanonicalIdentity(clone) clone.ID = exportedAlertID(clone, clone.ID) + clone.MetricStatus = nil storageKey := activeAlertStorageKey(clone, fallback) if storageKey == "" { return diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 68daa1b65c..f95e034cbf 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -21689,3 +21689,57 @@ func TestCleanupRetentionRequiresObservationInactivity(t *testing.T) { }) } } + +func TestMetricStatusStaysOutOfDurableAlertSnapshots(t *testing.T) { + m, clock := newMetricStatusTestManager(t) + fireNodeTemperatureAlert(t, m, clock, 85) + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 78) + + live := m.GetActiveAlerts() + if len(live) != 1 || live[0].MetricStatus == nil || live[0].MetricStatus.Value != 78 { + t.Fatalf("live alerts do not carry the current reading: %+v", live) + } + for _, alert := range m.snapshotActiveAlerts() { + if alert.MetricStatus != nil { + t.Fatalf("durable checkpoint carries volatile status for %s: %+v", alert.ID, alert.MetricStatus) + } + } + for _, alert := range m.activeRecoverySnapshot() { + if alert.MetricStatus != nil { + t.Fatalf("recovery projection carries volatile status for %s: %+v", alert.ID, alert.MetricStatus) + } + } +} + +func TestStaleCleanupKeepsThresholdAlertStillBeingEvaluated(t *testing.T) { + m := newTestManager(t) + now := time.Now() + breachedLongAgo := now.Add(-StaleTrackingThreshold - time.Hour) + + held := &Alert{ + ID: "held::metric-threshold:usage", Type: "usage", ResourceID: "held", + StartTime: breachedLongAgo, LastSeen: breachedLongAgo, + MetricStatus: &models.MetricAlertStatus{Phase: models.MetricAlertPhaseLatched, ObservedAt: now}, + } + abandoned := &Alert{ + ID: "gone::metric-threshold:usage", Type: "usage", ResourceID: "gone", + StartTime: breachedLongAgo, LastSeen: breachedLongAgo, + MetricStatus: &models.MetricAlertStatus{Phase: models.MetricAlertPhaseLatched, ObservedAt: breachedLongAgo}, + } + m.mu.Lock() + m.activeAlerts[held.ID] = held + m.activeAlerts[abandoned.ID] = abandoned + m.mu.Unlock() + + m.cleanupStaleMaps() + + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.activeAlerts[held.ID]; !ok { + t.Fatal("cleanup resolved a held alert whose condition is still being evaluated") + } + if _, ok := m.activeAlerts[abandoned.ID]; ok { + t.Fatal("cleanup kept an alert nobody has evaluated for over a day") + } +} diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index fcfa75baa5..4d56b7ca48 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -261,6 +261,17 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec alertMetadata[k] = v } } + metricStatus := buildMetricAlertStatus(metricStatusInput{ + value: value, + triggered: triggered, + trigger: spec.MetricThreshold.Trigger, + clear: clearThreshold, + recoveryDelaySeconds: stability.RecoveryDelaySeconds, + unit: metricStatusUnit(metricType), + window: windowed, + incident: incident, + observedAt: observedAt, + }) if !exists { alert := &Alert{ @@ -278,6 +289,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec StartTime: alertStartTime, LastSeen: observedAt, Metadata: alertMetadata, + MetricStatus: metricStatus, } applyCanonicalIdentity(alert, spec.ID, string(spec.Kind)) @@ -331,12 +343,16 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec } if !triggered && primary == "" { - // Hysteresis latch: below trigger but above recovery — hold - // without refreshing, as the pre-cutover engine did. + // Hysteresis hold or recovery run: Value, Message and LastSeen + // keep the last breach, and nothing here may notify. Only the + // volatile live status moves, so surfaces can say what the + // reading is now and why the alert has not cleared. + existingAlert.MetricStatus = metricStatus return } oldLevel := existingAlert.Level + existingAlert.MetricStatus = metricStatus existingAlert.LastSeen = observedAt existingAlert.Value = value existingAlert.Threshold = spec.MetricThreshold.Trigger @@ -390,6 +406,8 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec existingAlert = existingAlert.Clone() existingAlert.Value = value existingAlert.LastSeen = observedAt + // A resolved record is history; the live status describes open alerts only. + existingAlert.MetricStatus = nil message, _ := metricAlertMessage(resourceType, metricType, value, opts) existingAlert.Message = "Resolved: " + message diff --git a/internal/alerts/metric_status.go b/internal/alerts/metric_status.go new file mode 100644 index 0000000000..7bc19204b1 --- /dev/null +++ b/internal/alerts/metric_status.go @@ -0,0 +1,79 @@ +package alerts + +import ( + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +// metricStatusInput is what the canonical metric evaluator compared on one +// observation: the value after any rolling-average window, the exact rule +// handed to the reducer, and the incident the reducer holds afterwards. +type metricStatusInput struct { + value float64 + triggered bool + trigger float64 + clear float64 + recoveryDelaySeconds int + unit string + window metricWindowObservation + incident reducer.Incident + observedAt time.Time +} + +// metricStatusUnit is the unit of an evaluated metric value. CPU and memory +// share the "%" of usage even though their legacy messages omit a unit. +func metricStatusUnit(metricType string) string { + switch metricType { + case "temperature", "disk_temperature", "diskTemperature": + return "°C" + case "diskRead", "diskWrite", "networkIn", "networkOut": + return "MB/s" + default: + return "%" + } +} + +// buildMetricAlertStatus derives the live status of an open threshold alert +// from the evaluator's own inputs, so surfaces never re-derive the phase or +// the recovery timer from thresholds and a resource reading of their own. +func buildMetricAlertStatus(input metricStatusInput) *models.MetricAlertStatus { + // The reducer treats a non-positive clear as "clear at the trigger". + recovery := input.clear + if recovery <= 0 { + recovery = input.trigger + } + status := &models.MetricAlertStatus{ + Phase: models.MetricAlertPhaseBreaching, + Value: input.value, + Unit: input.unit, + ObservedAt: input.observedAt, + Trigger: input.trigger, + Recovery: recovery, + RecoveryDelaySeconds: input.recoveryDelaySeconds, + } + if input.window.WindowSeconds > 0 { + raw := input.window.CurrentValue + status.RawValue = &raw + status.EvaluationWindowSeconds = input.window.WindowSeconds + } + if input.triggered { + return status + } + if input.value > recovery || input.incident.RecoverySince.IsZero() { + status.Phase = models.MetricAlertPhaseLatched + return status + } + status.Phase = models.MetricAlertPhaseRecovering + startedAt := input.incident.RecoverySince + status.RecoveryStartedAt = &startedAt + elapsed := input.observedAt.Sub(input.incident.RecoverySince) + if input.incident.RecoveryTicksSupplied { + elapsed = input.incident.RecoveryElapsed + } + if elapsed > 0 { + status.RecoveryElapsedSeconds = int(elapsed / time.Second) + } + return status +} diff --git a/internal/alerts/model.go b/internal/alerts/model.go index 4b6324a2fd..66eecd2a29 100644 --- a/internal/alerts/model.go +++ b/internal/alerts/model.go @@ -4,6 +4,7 @@ import ( "strings" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust" ) @@ -144,6 +145,23 @@ type Alert struct { LatestTransition *operationaltrust.LifecycleTransition `json:"latestTransition,omitempty"` Transitions []operationaltrust.LifecycleTransition `json:"transitions,omitempty"` Evidence []operationaltrust.EvidenceEnvelope `json:"evidence,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. It + // is volatile and stripped from durable snapshots. + MetricStatus *models.MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// lastObservedAt is when the alert's condition was last evaluated. A +// threshold alert held below its trigger keeps LastSeen at the last breach, +// so its live status carries the newer observation. +func (a *Alert) lastObservedAt() time.Time { + if a == nil { + return time.Time{} + } + observed := a.LastSeen + if a.MetricStatus != nil && a.MetricStatus.ObservedAt.After(observed) { + observed = a.MetricStatus.ObservedAt + } + return observed } // Clone returns a deep copy of the alert so it can be safely shared across goroutines. @@ -176,6 +194,8 @@ func (a *Alert) Clone() *Alert { clone.Metadata = cloneMetadata(a.Metadata) } + clone.MetricStatus = a.MetricStatus.Clone() + if a.OperationalRecord != nil { value := a.OperationalRecord.Clone() clone.OperationalRecord = &value diff --git a/internal/alerts/tracking_cleanup.go b/internal/alerts/tracking_cleanup.go index a328600064..9e5882c2a1 100644 --- a/internal/alerts/tracking_cleanup.go +++ b/internal/alerts/tracking_cleanup.go @@ -156,7 +156,9 @@ func (m *Manager) cleanupStaleMaps() { staleAlerts := make([]string, 0) for storageKey, alert := range m.activeAlerts { alertID := effectiveAlertID(alert, storageKey) - if alert != nil && now.Sub(alert.LastSeen) > staleThreshold { + // A threshold alert holding below its trigger is still observed + // even though LastSeen stays at its last breach. + if alert != nil && now.Sub(alert.lastObservedAt()) > staleThreshold { staleAlerts = append(staleAlerts, alertID) } } diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go index de010d214c..72d803fc9b 100644 --- a/internal/alerts/unified_eval_test.go +++ b/internal/alerts/unified_eval_test.go @@ -2,9 +2,11 @@ package alerts import ( "sort" + "sync/atomic" "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer" alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -1583,3 +1585,186 @@ func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T) t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds) } } + +// metricStatusClock drives a manager's wall and monotonic clocks together so +// recovery runs measure exactly the time the test advances. +type metricStatusClock struct { + now time.Time + tick time.Duration +} + +func (c *metricStatusClock) advance(d time.Duration) { + c.now = c.now.Add(d) + c.tick += d +} + +func newMetricStatusTestManager(t *testing.T) (*Manager, *metricStatusClock) { + t.Helper() + m := newTestManager(t) + // Recently-resolved retention prunes against the real clock, so start there. + clock := &metricStatusClock{now: time.Now().UTC()} + m.now = func() time.Time { return clock.now } + m.intentClock = func() time.Duration { return clock.tick } + m.mu.Lock() + m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 75} + m.mu.Unlock() + return m, clock +} + +// fireNodeTemperatureAlert holds a breaching reading until the factory +// stability delay lets the alert open, as a real poll loop would. +func fireNodeTemperatureAlert(t *testing.T, m *Manager, clock *metricStatusClock, celsius float64) *Alert { + t.Helper() + for attempt := 0; attempt < 40; attempt++ { + checkMetricStatusNode(m, celsius) + if alert := activeNodeTemperatureAlert(t, m); alert != nil { + return alert + } + clock.advance(30 * time.Second) + } + t.Fatalf("%v°C never opened the temperature alert", celsius) + return nil +} + +func checkMetricStatusNode(m *Manager, celsius float64) { + m.CheckNode(models.Node{ + ID: "homelab-minipc", + Name: "minipc", + Instance: "homelab", + Status: "online", + CPU: 0.2, + Memory: models.Memory{Usage: 40}, + Disk: models.Disk{Usage: 40}, + Temperature: &models.Temperature{ + Available: true, + CPUPackage: celsius, + }, + }) +} + +func activeNodeTemperatureAlert(t *testing.T, m *Manager) *Alert { + t.Helper() + m.mu.RLock() + defer m.mu.RUnlock() + alert, ok := m.getActiveAlertNoLock(canonicalMetricStateID("homelab-minipc", "temperature")) + if !ok { + return nil + } + return alert.Clone() +} + +func TestOpenTemperatureAlertReportsLiveReadingWhileItHolds(t *testing.T) { + m, clock := newMetricStatusTestManager(t) + var notified atomic.Int32 + m.SetAlertCallback(func(*Alert) { notified.Add(1) }) + + fired := fireNodeTemperatureAlert(t, m, clock, 85) + status := fired.MetricStatus + if status == nil || status.Phase != models.MetricAlertPhaseBreaching || status.Value != 85 || + status.Trigger != 80 || status.Recovery != 75 || status.Unit != "°C" { + t.Fatalf("breaching status = %+v", status) + } + if status.RecoveryDelaySeconds <= 0 { + t.Fatalf("temperature alerts carry a recovery delay, got %+v", status) + } + delay := time.Duration(status.RecoveryDelaySeconds) * time.Second + firedMessage, firedLastSeen, notifications := fired.Message, fired.LastSeen, notified.Load() + + // Between the clear and trigger levels the alert holds. The legacy + // snapshot keeps the breach; only the live status moves. + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 78) + held := activeNodeTemperatureAlert(t, m) + if held == nil { + t.Fatal("78°C cleared an alert that should hold until 75°C") + } + if held.Value != 85 || held.Message != firedMessage || !held.LastSeen.Equal(firedLastSeen) { + t.Fatalf("hold rewrote the breach snapshot: value=%v message=%q lastSeen=%v", held.Value, held.Message, held.LastSeen) + } + if got := held.MetricStatus; got == nil || got.Phase != models.MetricAlertPhaseLatched || got.Value != 78 || + !got.ObservedAt.Equal(clock.now) || got.RecoveryStartedAt != nil { + t.Fatalf("latched status = %+v", got) + } + + // At or below the recovery level the recovery run starts. + clock.advance(30 * time.Second) + recoveryStart := clock.now + checkMetricStatusNode(m, 72) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseRecovering || got.Value != 72 || + got.RecoveryStartedAt == nil || !got.RecoveryStartedAt.Equal(recoveryStart) || got.RecoveryElapsedSeconds != 0 { + t.Fatalf("recovery start status = %+v", got) + } + clock.advance(delay / 2) + checkMetricStatusNode(m, 75) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseRecovering || got.RecoveryElapsedSeconds != int(delay/2/time.Second) { + t.Fatalf("recovery progress status = %+v", got) + } + + // A reading back above the recovery level resets the run. + clock.advance(30 * time.Second) + checkMetricStatusNode(m, 77) + if got := activeNodeTemperatureAlert(t, m).MetricStatus; got == nil || + got.Phase != models.MetricAlertPhaseLatched || got.RecoveryStartedAt != nil || got.RecoveryElapsedSeconds != 0 { + t.Fatalf("status after recovery reset = %+v", got) + } + if got := notified.Load(); got != notifications { + t.Fatalf("holding and recovering sent %d notifications", got-notifications) + } + + // A full recovery delay at or below 75°C resolves the alert. + checkMetricStatusNode(m, 72) + clock.advance(delay) + checkMetricStatusNode(m, 72) + if alert := activeNodeTemperatureAlert(t, m); alert != nil { + t.Fatalf("alert still open after a full recovery delay: %+v", alert.MetricStatus) + } + m.resolvedMutex.RLock() + defer m.resolvedMutex.RUnlock() + resolved := 0 + for _, entry := range m.recentlyResolved { + if entry == nil || entry.Type != "temperature" { + continue + } + resolved++ + if entry.MetricStatus != nil { + t.Fatalf("resolved record kept the live status of an open alert: %+v", entry.MetricStatus) + } + } + if resolved != 1 { + t.Fatalf("resolved temperature records = %d, want 1", resolved) + } +} + +func TestBuildMetricAlertStatusCarriesWindowAndMonotonicRecovery(t *testing.T) { + observedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC) + status := buildMetricAlertStatus(metricStatusInput{ + value: 74, + trigger: 80, + clear: 75, + recoveryDelaySeconds: 300, + unit: "%", + window: metricWindowObservation{Value: 74, CurrentValue: 91, WindowSeconds: 600, Ready: true}, + incident: reducer.Incident{ + RecoverySince: observedAt.Add(-10 * time.Minute), + RecoveryElapsed: 90 * time.Second, + RecoveryTicksSupplied: true, + }, + observedAt: observedAt, + }) + if status.Phase != models.MetricAlertPhaseRecovering || status.Value != 74 || + status.RawValue == nil || *status.RawValue != 91 || status.EvaluationWindowSeconds != 600 { + t.Fatalf("windowed status = %+v", status) + } + // Elapsed time comes from the monotonic run, not wall-clock arithmetic. + if status.RecoveryElapsedSeconds != 90 { + t.Fatalf("recovery elapsed = %d, want 90", status.RecoveryElapsedSeconds) + } + + // The reducer clears at the trigger when no clear level is configured. + noClear := buildMetricAlertStatus(metricStatusInput{value: 85, triggered: true, trigger: 80, observedAt: observedAt}) + if noClear.Recovery != 80 || noClear.RawValue != nil { + t.Fatalf("status without clear level = %+v", noClear) + } +} diff --git a/internal/models/metrics_types_test.go b/internal/models/metrics_types_test.go index 86d4df09d0..8f5edabedc 100644 --- a/internal/models/metrics_types_test.go +++ b/internal/models/metrics_types_test.go @@ -451,3 +451,21 @@ func TestPhysicalDiskSnapshotPreservesIndependentPollingCadence(t *testing.T) { t.Fatal("snapshot mutation changed collector state") } } + +func TestMetricAlertStatusCloneIsDeep(t *testing.T) { + raw := 91.0 + startedAt := time.Date(2026, 10, 6, 10, 49, 0, 0, time.UTC) + status := &MetricAlertStatus{ + Phase: MetricAlertPhaseRecovering, Value: 74, RawValue: &raw, + RecoveryStartedAt: &startedAt, RecoveryElapsedSeconds: 90, + } + clone := status.Clone() + *clone.RawValue = 10 + *clone.RecoveryStartedAt = startedAt.Add(time.Hour) + if *status.RawValue != 91 || !status.RecoveryStartedAt.Equal(startedAt) { + t.Fatalf("clone shares pointers with the original: %+v", status) + } + if (*MetricAlertStatus)(nil).Clone() != nil { + t.Fatal("nil status must clone to nil") + } +} diff --git a/internal/models/models.go b/internal/models/models.go index 5f1f2af5b8..a08334e25a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -110,6 +110,58 @@ type Alert struct { // Metadata carries alert-engine annotations (notably resourceType) so the // frontend can classify an alert without re-deriving resource identity. Metadata map[string]interface{} `json:"metadata,omitempty"` + // MetricStatus is the live evaluation behind an open threshold alert. + MetricStatus *MetricAlertStatus `json:"metricStatus,omitempty"` +} + +// Metric alert phases. A threshold alert stays open after its reading drops +// below the trigger: it holds until the reading reaches the recovery level, +// then must stay there for the recovery delay before it clears. +const ( + MetricAlertPhaseBreaching = "breaching" + MetricAlertPhaseLatched = "latched" + MetricAlertPhaseRecovering = "recovering" +) + +// MetricAlertStatus describes the reading Pulse is evaluating now for an open +// threshold alert and why the alert is still open. The alert's Value, Message +// and LastSeen keep describing the last reading that met the trigger, which +// can be minutes or days old while the alert holds, so surfaces lead with +// this instead. It is volatile: rebuilt on every evaluation, never persisted. +type MetricAlertStatus struct { + Phase string `json:"phase"` + // Value is the value compared with the rule. For a rolling-average rule + // that is the average; RawValue then carries the latest sample. + Value float64 `json:"value"` + RawValue *float64 `json:"rawValue,omitempty"` + EvaluationWindowSeconds int `json:"evaluationWindowSeconds,omitempty"` + Unit string `json:"unit,omitempty"` + ObservedAt time.Time `json:"observedAt"` + // Trigger opens the alert at or above this value; Recovery is the level + // the value must be at or below for RecoveryDelaySeconds to clear it. + Trigger float64 `json:"trigger"` + Recovery float64 `json:"recovery"` + RecoveryDelaySeconds int `json:"recoveryDelaySeconds,omitempty"` + // Recovery progress, present only in the recovering phase. + RecoveryStartedAt *time.Time `json:"recoveryStartedAt,omitempty"` + RecoveryElapsedSeconds int `json:"recoveryElapsedSeconds,omitempty"` +} + +// Clone returns a deep copy. +func (s *MetricAlertStatus) Clone() *MetricAlertStatus { + if s == nil { + return nil + } + clone := *s + if s.RawValue != nil { + raw := *s.RawValue + clone.RawValue = &raw + } + if s.RecoveryStartedAt != nil { + startedAt := *s.RecoveryStartedAt + clone.RecoveryStartedAt = &startedAt + } + return &clone } // ResolvedAlert represents a recently resolved alert diff --git a/internal/monitoring/monitor_alert_handling_test.go b/internal/monitoring/monitor_alert_handling_test.go index 498bd35b8d..9d85fedba4 100644 --- a/internal/monitoring/monitor_alert_handling_test.go +++ b/internal/monitoring/monitor_alert_handling_test.go @@ -1466,3 +1466,39 @@ func TestMonitorDelayedPartialResolutionKeepsDestinationRecovery(t *testing.T) { t.Fatal("unannounced destination received a recovery") } } + +func TestSyncAlertsToStateCarriesLiveMetricStatusOfHeldAlert(t *testing.T) { + m := &Monitor{ + state: models.NewState(), + alertManager: alerts.NewManagerWithDataDir(t.TempDir()), + } + defer m.alertManager.Stop() + + cfg := m.alertManager.GetConfig() + cfg.TimeThresholds = map[string]int{"node": 0} + cfg.NodeDefaults.Temperature = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75} + m.alertManager.UpdateConfig(cfg) + + node := models.Node{ + ID: "homelab-minipc", Name: "minipc", Instance: "homelab", Status: "online", + Temperature: &models.Temperature{Available: true, CPUPackage: 85}, + } + m.alertManager.CheckNode(node) + node.Temperature = &models.Temperature{Available: true, CPUPackage: 78} + m.alertManager.CheckNode(node) + + m.syncAlertsToState() + + var held *models.Alert + for i, alert := range m.state.GetSnapshot().ActiveAlerts { + if alert.Type == "temperature" { + held = &m.state.GetSnapshot().ActiveAlerts[i] + } + } + require.NotNil(t, held, "expected the temperature alert to stay open at 78°C") + require.Equal(t, 85.0, held.Value, "the legacy value keeps the last breach") + require.NotNil(t, held.MetricStatus, "websocket state must carry the live reading") + require.Equal(t, models.MetricAlertPhaseLatched, held.MetricStatus.Phase) + require.Equal(t, 78.0, held.MetricStatus.Value) + require.Equal(t, 75.0, held.MetricStatus.Recovery) +} diff --git a/internal/monitoring/monitor_alert_sync.go b/internal/monitoring/monitor_alert_sync.go index 920381c2c2..9fc1154af9 100644 --- a/internal/monitoring/monitor_alert_sync.go +++ b/internal/monitoring/monitor_alert_sync.go @@ -71,8 +71,10 @@ func (m *Monitor) activeAlertsSnapshot() []models.Alert { Acknowledged: alert.Acknowledged, AckTime: alert.AckTime, AckUser: alert.AckUser, - // GetActiveAlerts returns deep clones, so the map is already private. - Metadata: alert.Metadata, + // GetActiveAlerts returns deep clones, so the map and the live + // metric status are already private. + Metadata: alert.Metadata, + MetricStatus: alert.MetricStatus, }) } return modelAlerts